Skip to content

Repo manager with parsing logic #566

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dhirenmathur merged 23 commits into from
Feb 5, 2026
Merged

Repo manager with parsing logic #566

Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
425dd9f
repo manager v1
Dsantra92 Jan 20, 2026
bfe3a00
Parsing working
Dsantra92 Jan 21, 2026
9dba217
Repo manager with unique id and eviction policy update
Dsantra92 Jan 21, 2026
a9c970d
potpie working
Dsantra92 Jan 21, 2026
c4a221e
old parity + delete some files
Dsantra92 Jan 21, 2026
112c048
coderabbit fixes
Dsantra92 Jan 22, 2026
86856fa
Parsing working correctly for symlinks
Dsantra92 Jan 22, 2026
6dffafa
Merge 86856fa5d75fa6b12ad0aaa634b0cd2a98d0c44d into 921384208d4793a36…
Dsantra92 Jan 22, 2026
f4c8909
chore: Auto-fix pre-commit issues
github-actions[bot] Jan 22, 2026
e2dd43f
CodeRabbit fixes
Dsantra92 Jan 22, 2026
2ac1c8e
coderabbit fix 2
Dsantra92 Jan 22, 2026
18e143f
Use already resolved github_token
Dsantra92 Jan 22, 2026
1525055
Don't follow symlinks
Dsantra92 Jan 22, 2026
d29a1ac
exist ok for worktress
Dsantra92 Jan 28, 2026
006e76d
potpie fix
Dsantra92 Jan 28, 2026
09d35f7
CodeRabbit Fix
Dsantra92 Jan 28, 2026
c3bec4f
demo revert changes
Dsantra92 Feb 4, 2026
c7837af
update with main
Dsantra92 Feb 5, 2026
873fa2c
fix random choice
Dsantra92 Feb 5, 2026
efa2e91
async create_task
Dsantra92 Feb 5, 2026
2b027a2
ssl fix
Dsantra92 Feb 5, 2026
8332239
all requested changes
Dsantra92 Feb 5, 2026
6a2d1a4
async fix
Dsantra92 Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions app/modules/code_provider/github/github_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import asyncio
import os
import random
import secrets
import re
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Dict, List, Optional, Tuple
Expand Down Expand Up @@ -346,7 +346,7 @@ async def get_repos_for_user(self, user_id: str):
if token_list_str:
tokens = [t.strip() for t in token_list_str.split(",") if t.strip()]
if tokens:
github_oauth_token = random.choice(tokens)
github_oauth_token = secrets.choice(tokens)
logger.info("Using token from GH_TOKEN_LIST as fallback")

# Fall back to CODE_PROVIDER_TOKEN if GH_TOKEN_LIST not available
Expand Down Expand Up @@ -698,7 +698,7 @@ def get_public_github_instance(cls):

# Use factory to create provider with PAT

token = random.choice(cls.gh_token_list)
token = secrets.choice(cls.gh_token_list)
provider = GitHubProvider()
provider.authenticate({"token": token}, AuthMethod.PERSONAL_ACCESS_TOKEN)
return provider.client
Expand Down
9 changes: 3 additions & 6 deletions app/modules/code_provider/provider_factory.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
import secrets
from enum import Enum
from typing import Any, Dict, Optional

Expand Down Expand Up @@ -159,13 +160,11 @@ def create_provider(
# Fallback to legacy GH_TOKEN_LIST
token_list_str = os.getenv("GH_TOKEN_LIST", "")
if token_list_str:
import random

tokens = [
t.strip() for t in token_list_str.split(",") if t.strip()
]
if tokens:
token = random.choice(tokens)
token = secrets.choice(tokens)
logger.info(
"Authenticating with GH_TOKEN_LIST (legacy PAT pool)"
)
Expand Down Expand Up @@ -347,8 +346,6 @@ def create_provider_with_fallback(repo_name: str) -> ICodeProvider:
logger.debug(f" - Repr: {token_repr[:50]}...")

if token_list_str:
import random

tokens = [t.strip() for t in token_list_str.split(",") if t.strip()]
logger.debug(f"Parsed {len(tokens)} token(s) from GH_TOKEN_LIST")
if tokens:
Expand All @@ -359,7 +356,7 @@ def create_provider_with_fallback(repo_name: str) -> ICodeProvider:
# Always use GitHub's API endpoint when using GH_TOKEN_LIST
base_url = "https://api.github.com"
provider = GitHubProvider(base_url=base_url)
token = random.choice(tokens)
token = secrets.choice(tokens)

provider.authenticate(
{"token": token}, AuthMethod.PERSONAL_ACCESS_TOKEN
Expand Down
4 changes: 2 additions & 2 deletions app/modules/intelligence/tools/web_tools/code_provider_add_pr_comment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import os
import secrets
from app.modules.utils.logger import setup_logger

import random
from typing import Dict, Any, Optional, Type, List
from pydantic import BaseModel, Field
from github import Github
Expand Down Expand Up @@ -92,7 +92,7 @@ def __init__(self, sql_db: Session, user_id: str):
def get_public_github_instance(cls):
if not cls.gh_token_list:
cls.initialize_tokens()
token = random.choice(cls.gh_token_list)
token = secrets.choice(cls.gh_token_list)
return Github(token)

def _get_github_client(self, repo_name: str) -> Github:
Expand Down
4 changes: 2 additions & 2 deletions app/modules/intelligence/tools/web_tools/code_provider_create_branch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import os
import secrets
from app.modules.utils.logger import setup_logger

logger = setup_logger(__name__)
import random
from typing import Dict, Any, List, Optional, Type
from pydantic import BaseModel, Field
from github import Github
Expand Down Expand Up @@ -59,7 +59,7 @@ def initialize_tokens(cls):
def get_public_github_instance(cls):
if not cls.gh_token_list:
cls.initialize_tokens()
token = random.choice(cls.gh_token_list)
token = secrets.choice(cls.gh_token_list)
return Github(token)

def _get_github_client(self, repo_name: str) -> Github:
Expand Down
4 changes: 2 additions & 2 deletions app/modules/intelligence/tools/web_tools/code_provider_create_pr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import os
import secrets
from app.modules.utils.logger import setup_logger

logger = setup_logger(__name__)
import random
from typing import Dict, Any, Optional, Type, List
from pydantic import BaseModel, Field
from github import Github
Expand Down Expand Up @@ -71,7 +71,7 @@ def __init__(self, sql_db: Session, user_id: str):
def get_public_github_instance(cls):
if not cls.gh_token_list:
cls.initialize_tokens()
token = random.choice(cls.gh_token_list)
token = secrets.choice(cls.gh_token_list)
return Github(token)

def _get_github_client(self, repo_name: str) -> Github:
Expand Down
4 changes: 2 additions & 2 deletions app/modules/intelligence/tools/web_tools/code_provider_tool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import asyncio
import os
import secrets
from app.modules.utils.logger import setup_logger

logger = setup_logger(__name__)
import random
from typing import Any, Dict, List, Optional

from github import Github
Expand Down Expand Up @@ -105,7 +105,7 @@ def run(
def get_public_github_instance(cls):
if not cls.gh_token_list:
cls.initialize_tokens()
token = random.choice(cls.gh_token_list)
token = secrets.choice(cls.gh_token_list)
return Github(token)

def _get_github_client(self, repo_name: str) -> Github:
Expand Down
4 changes: 2 additions & 2 deletions app/modules/intelligence/tools/web_tools/code_provider_update_file.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import os
import secrets
from app.modules.utils.logger import setup_logger

logger = setup_logger(__name__)
import random
from typing import Dict, Any, List, Optional, Type
from pydantic import BaseModel, Field
from github import Github
Expand Down Expand Up @@ -62,7 +62,7 @@ def __init__(self, sql_db: Session, user_id: str):
def get_public_github_instance(cls):
if not cls.gh_token_list:
cls.initialize_tokens()
token = random.choice(cls.gh_token_list)
token = secrets.choice(cls.gh_token_list)
return Github(token)

def _get_github_client(self, repo_name: str) -> Github:
Expand Down
56 changes: 28 additions & 28 deletions app/modules/parsing/graph_construction/parsing_controller.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,21 @@
import asyncio
import os
from asyncio import create_task
from typing import Any, Dict

from dotenv import load_dotenv
from fastapi import HTTPException
from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, or_
from sqlalchemy.orm import Session
from uuid6 import uuid7

from app.celery.tasks.parsing_tasks import process_parsing
from app.core.config_provider import config_provider
from app.modules.code_provider.code_provider_service import CodeProviderService
from app.modules.conversations.conversation.conversation_model import (
Conversation,
Visibility,
)
from app.modules.parsing.graph_construction.parsing_helper import ParseHelper
from app.modules.parsing.graph_construction.parsing_schema import (
ParsingRequest,
Expand All @@ -22,14 +26,12 @@
validate_parsing_input,
)
from app.modules.parsing.utils.repo_name_normalizer import normalize_repo_name
from app.modules.projects.projects_model import Project
from app.modules.projects.projects_schema import ProjectStatusEnum
from app.modules.projects.projects_service import ProjectService
from app.modules.utils.email_helper import EmailHelper
from app.modules.utils.posthog_helper import PostHogClient
from app.modules.conversations.conversation.conversation_model import Conversation
from app.modules.conversations.conversation.conversation_model import Visibility
from app.modules.projects.projects_model import Project
from app.modules.utils.logger import setup_logger
from app.modules.utils.posthog_helper import PostHogClient

logger = setup_logger(__name__)

Expand All @@ -40,7 +42,7 @@ class ParsingController:
@staticmethod
@validate_parsing_input
async def parse_directory(
repo_details: ParsingRequest, db: AsyncSession, user: Dict[str, Any]
repo_details: ParsingRequest, db: Session, user: Dict[str, Any]
):
if "email" not in user:
user_email = None
Expand Down Expand Up @@ -98,21 +100,10 @@ async def parse_directory(
project_manager,
db,
)

demo_repos = [
"Portkey-AI/gateway",
"crewAIInc/crewAI",
"AgentOps-AI/agentops",
"calcom/cal.com",
"langchain-ai/langchain",
"AgentOps-AI/AgentStack",
"formbricks/formbricks",
]

try:
# Normalize repository name for consistent database lookups
normalized_repo_name = normalize_repo_name(repo_name)
logger.info(
logger.debug(
f"Original repo_name: {repo_name}, Normalized: {normalized_repo_name}"
)

Expand All @@ -123,8 +114,15 @@ async def parse_directory(
repo_path=repo_details.repo_path,
commit_id=repo_details.commit_id,
)

# First check if this is a demo project that hasn't been accessed by this user yet
demo_repos = [
"Portkey-AI/gateway",
"crewAIInc/crewAI",
"AgentOps-AI/agentops",
"calcom/cal.com",
"langchain-ai/langchain",
"AgentOps-AI/AgentStack",
"formbricks/formbricks",
]
if not project and repo_details.repo_name in demo_repos:
existing_project = await project_manager.get_global_project_from_db(
normalized_repo_name,
Expand Down Expand Up @@ -210,8 +208,10 @@ async def parse_directory(
# If project exists but commit doesn't match or status is not READY, reparse
cleanup_graph = True
logger.info(
f"Submitting parsing task for existing project {project_id} "
f"(is_latest={is_latest}, status={project.status})"
"Submitting parsing task for existing project.",
project_id=project_id,
is_latest=is_latest,
status=project.status,
)
process_parsing.delay(
repo_details.model_dump(),
Expand Down Expand Up @@ -258,10 +258,10 @@ async def parse_directory(
async def handle_new_project(
repo_details: ParsingRequest,
user_id: str,
user_email: str,
user_email: str | None,
new_project_id: str,
project_manager: ProjectService,
db: AsyncSession,
db: Session,
):
Comment on lines 286 to 292
Copy link
Contributor

@coderabbitai coderabbitai bot Feb 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Unused db parameter.

If this is intentionally unused, consider renaming to _db to avoid lint noise.

🧹 Suggested fix 
-        db: Session,
+        _db: Session,
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
repo_details: ParsingRequest,
user_id: str,
user_email: str,
user_email: str | None,
new_project_id: str,
project_manager: ProjectService,
db: AsyncSession,
db: Session,
):
repo_details: ParsingRequest,
user_id: str,
user_email: str | None,
new_project_id: str,
project_manager: ProjectService,
_db: Session,
):
🧰 Tools
🪛 Ruff (0.14.14)

[warning] 291-291: Unused static method argument: db

(ARG004)

🤖 Prompt for AI Agents 
In `@app/modules/parsing/graph_construction/parsing_controller.py` around lines
286 - 292, The parameter named db: Session is unused in the function that takes
repo_details: ParsingRequest, user_id, user_email, new_project_id,
project_manager: ProjectService, db: Session; rename db to _db to silence lint
warnings (or if callers can be updated, remove the parameter entirely) and
update any call sites or references accordingly so the signature and callers
remain consistent.

response = {
"project_id": new_project_id,
Expand All @@ -278,9 +278,9 @@ async def handle_new_project(
repo_details.commit_id,
repo_details.repo_path,
)
asyncio.create_task(
CodeProviderService(db).get_project_structure_async(new_project_id)
)
# asyncio.create_task(
# CodeProviderService(db).get_project_structure_async(new_project_id)
# )
if not user_email:
user_email = None

Expand Down
Loading