From 11aa77e1b7348340892870efc491b2c6b472a96a Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 30 Jan 2026 12:01:46 +0100 Subject: [PATCH 1/3] chore: deprecate insightsProject.repositories and use repositories instead --- backend/src/services/integrationService.ts | 60 +--------------------- 1 file changed, 1 insertion(+), 59 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 3ed0279c91..62df8f408b 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -16,7 +16,6 @@ import { } from '@crowd/common' import { CommonIntegrationService, getGithubInstallationToken } from '@crowd/common_services' import { ICreateInsightsProject } from '@crowd/data-access-layer/src/collections' -import { findRepositoriesForSegment } from '@crowd/data-access-layer/src/integrations' import { ICreateRepository, IRepository, @@ -193,28 +192,11 @@ export default class IntegrationService { const { segmentId, id: insightsProjectId } = insightsProject const { platform } = data - let repositories = [] - - if (IntegrationService.isCodePlatform(platform)) { - const qx = SequelizeRepository.getQueryExecutor(txOptions) - await CommonIntegrationService.syncGithubRepositoriesToInsights( - qx, - this.options.redis, - integration.id, - ) - - // Get the updated repositories for git integration - const updatedProject = await collectionService.findInsightsProjectsBySegmentId(segmentId) - repositories = updatedProject[0]?.repositories || [] - } else { - repositories = insightsProject.repositories || [] - } await this.updateInsightsProject({ insightsProjectId, isFirstUpdate: true, platform, - repositories, segmentId, transaction, }) @@ -241,49 +223,17 @@ export default class IntegrationService { integration.segmentId, ) - let repositories = [] const { platform } = data if (insightsProject) { const { segmentId, id: insightsProjectId } = insightsProject - if (IntegrationService.isCodePlatform(platform)) { - const qx = SequelizeRepository.getQueryExecutor(txOptions) - await CommonIntegrationService.syncGithubRepositoriesToInsights( - qx, - this.options.redis, - integration.id, - ) - // Get the updated repositories for git integration - const updatedProject = await collectionService.findInsightsProjectsBySegmentId(segmentId) - repositories = updatedProject[0]?.repositories || [] - } else { - repositories = insightsProject.repositories || [] - } - await this.updateInsightsProject({ insightsProjectId, platform, - repositories, segmentId, transaction, }) - } else { - const qx = SequelizeRepository.getQueryExecutor(txOptions) - const currentRepositories = await findRepositoriesForSegment(qx, integration.segmentId) - repositories = Object.values(currentRepositories).flatMap((repos) => - repos.map((repo) => repo.url), - ) - } - - if (IntegrationService.isCodePlatform(platform) && platform !== PlatformType.GIT) { - await this.gitConnectOrUpdate( - { - remotes: repositories.map((url) => ({ url, forkedFrom: null })), - }, - txOptions, - platform, - ) } return integration @@ -301,21 +251,18 @@ export default class IntegrationService { platform, segmentId, transaction, - repositories, }: { insightsProjectId: string isFirstUpdate?: boolean platform: PlatformType segmentId: string transaction: Transaction - repositories: string[] }) { const collectionService = new CollectionService({ ...this.options, transaction }) const data: Partial = {} const { widgets } = await collectionService.findSegmentsWidgetsById(segmentId) data.widgets = widgets - data.repositories = repositories if ( (platform === PlatformType.GITHUB || platform === PlatformType.GITHUB_NANGO) && @@ -533,13 +480,8 @@ export default class IntegrationService { // Note: Repos are soft-deleted in public.repositories via mapUnifiedRepositories above } - const insightsRepo = insightsProject?.repositories ?? [] - const filteredRepos = insightsRepo.filter((repo) => !toRemoveRepo.has(repo)) - // remove duplicates - const repositories = [...new Set(filteredRepos)] - if (insightsProject) { - await collectionService.updateInsightsProject(insightsProject.id, { widgets, repositories }) + await collectionService.updateInsightsProject(insightsProject.id, { widgets }) } await SequelizeRepository.commitTransaction(transaction) From c6fb6917c41529e5331e6c2865af6b430124ba04 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 30 Jan 2026 12:03:04 +0100 Subject: [PATCH 2/3] chore: deprecate insightsProject.repositories and use repositories instead --- .../insight-project-helper.ts | 15 ++-- services/apps/nango_worker/src/activities.ts | 2 - .../src/activities/nangoActivities.ts | 9 +- .../src/workflows/syncGithubIntegration.ts | 3 - .../src/services/integration.service.ts | 86 ------------------- .../src/collections/index.ts | 11 ++- .../src/integrations/index.ts | 15 ++-- .../src/repositories/index.ts | 3 +- .../data-access-layer/src/segments/index.ts | 12 +-- services/libs/data-access-layer/src/utils.ts | 3 +- 10 files changed, 33 insertions(+), 126 deletions(-) diff --git a/frontend/src/modules/admin/modules/insights-projects/insight-project-helper.ts b/frontend/src/modules/admin/modules/insights-projects/insight-project-helper.ts index 173bc38e0a..85569c653f 100644 --- a/frontend/src/modules/admin/modules/insights-projects/insight-project-helper.ts +++ b/frontend/src/modules/admin/modules/insights-projects/insight-project-helper.ts @@ -44,13 +44,7 @@ export const buildForm = ( keywords: result.keywords || [], searchKeywords: result.searchKeywords || [], repositoryGroups: result.repositoryGroups || [], - repositories: - repositories?.map((repository) => ({ - ...repository, - enabled: - result.repositories?.some((repo: string) => repo === repository.url) - || false, - })) || [], + repositories: repositories || [], widgets: Object.fromEntries( getDefaultWidgets().map((key) => [ key, @@ -62,7 +56,7 @@ export const buildForm = ( }); export const buildRepositories = ( - res: Record>, + res: Record>, ) => { const urlMap = new Map< string, @@ -82,12 +76,15 @@ export const buildRepositories = ( // If URL exists, add the platform to its platforms array const existing = urlMap.get(repo.url)!; existing.platforms.push(platform); + if (repo.enabled) { + existing.enabled = true; + } } else { // If URL is new, create a new entry urlMap.set(repo.url, { url: repo.url, label: repo.label, - enabled: true, + enabled: repo.enabled, platforms: [platform], }); } diff --git a/services/apps/nango_worker/src/activities.ts b/services/apps/nango_worker/src/activities.ts index 73171ee9ef..6e06f54a39 100644 --- a/services/apps/nango_worker/src/activities.ts +++ b/services/apps/nango_worker/src/activities.ts @@ -9,7 +9,6 @@ import { removeGithubConnection, setGithubConnection, startNangoSync, - syncGithubReposToInsights, unmapGithubRepo, updateGitIntegrationWithRepo, } from './activities/nangoActivities' @@ -26,6 +25,5 @@ export { unmapGithubRepo, canCreateGithubConnection, updateGitIntegrationWithRepo, - syncGithubReposToInsights, logInfo, } diff --git a/services/apps/nango_worker/src/activities/nangoActivities.ts b/services/apps/nango_worker/src/activities/nangoActivities.ts index c055b2f3e5..caf3385ce2 100644 --- a/services/apps/nango_worker/src/activities/nangoActivities.ts +++ b/services/apps/nango_worker/src/activities/nangoActivities.ts @@ -1,6 +1,6 @@ import { IS_DEV_ENV, IS_STAGING_ENV, singleOrDefault } from '@crowd/common' import { generateUUIDv4 as uuid } from '@crowd/common' -import { CommonIntegrationService, GithubIntegrationService } from '@crowd/common_services' +import { GithubIntegrationService } from '@crowd/common_services' import { addGithubNangoConnection, addRepoToGitIntegration, @@ -501,13 +501,6 @@ function parseGithubUrl(url: string): IGithubRepoData { throw new Error('Invalid GitHub URL format') } -export async function syncGithubReposToInsights(integrationId: string): Promise { - svc.log.info({ integrationId }, `Syncing GitHub repositories to insights!`) - - const qx = dbStoreQx(svc.postgres.writer) - await CommonIntegrationService.syncGithubRepositoriesToInsights(qx, svc.redis, integrationId) -} - export async function logInfo(message: string, serializedParams?: string): Promise { svc.log.info(serializedParams ? JSON.parse(serializedParams) : {}, message) } diff --git a/services/apps/nango_worker/src/workflows/syncGithubIntegration.ts b/services/apps/nango_worker/src/workflows/syncGithubIntegration.ts index 33164bc9c9..3575cbdf9c 100644 --- a/services/apps/nango_worker/src/workflows/syncGithubIntegration.ts +++ b/services/apps/nango_worker/src/workflows/syncGithubIntegration.ts @@ -61,8 +61,5 @@ export async function syncGithubIntegration(args: ISyncGithubIntegrationArgument // start nango sync await activity.startNangoSync(integrationId, result.providerConfigKey, connectionId) - - // sync repositories to segmentRepositories and insightsProjects after processing all repos - await activity.syncGithubReposToInsights(integrationId) } } diff --git a/services/libs/common_services/src/services/integration.service.ts b/services/libs/common_services/src/services/integration.service.ts index 40c22df72d..4cd9ebb922 100644 --- a/services/libs/common_services/src/services/integration.service.ts +++ b/services/libs/common_services/src/services/integration.service.ts @@ -1,18 +1,5 @@ import { decryptData } from '@crowd/common' -import { - InsightsProjectField, - queryInsightsProjects, - updateInsightsProject, -} from '@crowd/data-access-layer/src/collections' -import { - fetchIntegrationById, - findNangoRepositoriesToBeRemoved, - findRepositoriesForSegment, -} from '@crowd/data-access-layer/src/integrations' -import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' -import { getRepoUrlsMappedToOtherSegments } from '@crowd/data-access-layer/src/segments' import { getServiceChildLogger } from '@crowd/logging' -import { RedisClient } from '@crowd/redis' import { PlatformType } from '@crowd/types' export class CommonIntegrationService { @@ -70,77 +57,4 @@ export class CommonIntegrationService { return settings } } - - /** - * Syncs GitHub repositories to insightsProject.repositories field - * @param qx - Query executor for database operations - * @param redis - Redis client for cache invalidation - * @param integrationId - The integration ID to sync repositories for - */ - public static async syncGithubRepositoriesToInsights( - qx: QueryExecutor, - redis: RedisClient, - integrationId: string, - ): Promise { - // Fetch integration to get segmentId - const integration = await fetchIntegrationById(qx, integrationId) - - if (!integration) { - CommonIntegrationService.log.warn(`Integration ${integrationId} not found`) - return - } - - const segmentId = integration.segmentId - - // Query insights project for this segment - const insightsProjects = await queryInsightsProjects(qx, { - filter: { - segmentId: { eq: segmentId }, - }, - fields: [ - InsightsProjectField.ID, - InsightsProjectField.SEGMENT_ID, - InsightsProjectField.REPOSITORIES, - ], - }) - - const insightsProject = insightsProjects[0] - - if (!insightsProject) { - CommonIntegrationService.log.info( - `The segmentId: ${segmentId} does not have any InsightsProject related`, - ) - return - } - - const insightsProjectId = insightsProject.id - - // Get repositories to be removed - const reposToBeRemoved = await findNangoRepositoriesToBeRemoved(qx, integrationId) - - // Get current repositories for segment - const currentRepositories = await findRepositoriesForSegment(qx, segmentId) - - const currentUrls = Object.values(currentRepositories).flatMap((repos) => - repos.map((repo) => repo.url), - ) - - // Find repos already mapped to other segments (conflicts) - const alreadyMappedRepos = await getRepoUrlsMappedToOtherSegments(qx, currentUrls, segmentId) - - // Filter valid repositories (dedupe, remove deleted, remove already mapped to other segments) - const repositories = [...new Set(currentUrls)].filter( - (url) => !reposToBeRemoved.includes(url) && !alreadyMappedRepos.includes(url), - ) - - // Update insightsProject.repositories field (this also sets updatedAt automatically) - // Note: Writes to public.repositories happen earlier via mapGithubRepoToRepositories() - await updateInsightsProject(qx, insightsProjectId, { - repositories, - }) - - CommonIntegrationService.log.info( - `Synced ${repositories.length} repositories for integration ${integrationId} to segment ${segmentId}`, - ) - } } diff --git a/services/libs/data-access-layer/src/collections/index.ts b/services/libs/data-access-layer/src/collections/index.ts index 933d5153c6..a1124e5333 100644 --- a/services/libs/data-access-layer/src/collections/index.ts +++ b/services/libs/data-access-layer/src/collections/index.ts @@ -299,22 +299,25 @@ export async function updateInsightsProject( id: string, project: Partial, ) { + // Exclude repositories from update; only used to sync public.repositories.enabled + const { repositories, ...projectWithoutRepositories } = project + const updated = await updateTableById( qx, 'insightsProjects', id, Object.values(InsightsProjectField), - prepareProject(project), + prepareProject(projectWithoutRepositories), ) if (!updated) { throw new Error(`Update failed or project with id ${id} not found`) } - // Sync repositories.enabled status when repositories field is updated + // Sync repositories.enabled status when repositories field is provided // Disables repos not in the new list (new repos are enabled by default on insert) - if (project.repositories !== undefined) { - const enabledUrls = normalizeRepositoriesToUrls(project.repositories) + if (repositories !== undefined) { + const enabledUrls = normalizeRepositoriesToUrls(repositories) await syncRepositoriesEnabledStatus(qx, id, enabledUrls) } diff --git a/services/libs/data-access-layer/src/integrations/index.ts b/services/libs/data-access-layer/src/integrations/index.ts index 7be2333f55..7ac31d301c 100644 --- a/services/libs/data-access-layer/src/integrations/index.ts +++ b/services/libs/data-access-layer/src/integrations/index.ts @@ -660,17 +660,18 @@ export async function findNangoRepositoriesToBeRemoved( export async function findRepositoriesForSegment( qx: QueryExecutor, segmentId: string, -): Promise>> { +): Promise>> { // Get all repos grouped by platform (github-nango merged into github) const reposByPlatform = await getReposBySegmentGroupedByPlatform(qx, segmentId, true) - // Transform to include normalized URLs and labels - const result: Record> = {} + // Transform to include normalized URLs, labels, and enabled status + const result: Record> = {} - for (const [platform, urls] of Object.entries(reposByPlatform)) { - result[platform] = urls.map((url) => ({ - url: normalizeRepoUrl(url), - label: extractLabelFromUrl(url), + for (const [platform, repos] of Object.entries(reposByPlatform)) { + result[platform] = repos.map((repo) => ({ + url: normalizeRepoUrl(repo.url), + label: extractLabelFromUrl(repo.url), + enabled: repo.enabled, })) } diff --git a/services/libs/data-access-layer/src/repositories/index.ts b/services/libs/data-access-layer/src/repositories/index.ts index 446005eb54..bcb27a1bca 100644 --- a/services/libs/data-access-layer/src/repositories/index.ts +++ b/services/libs/data-access-layer/src/repositories/index.ts @@ -555,7 +555,8 @@ export async function findSegmentsForRepos( } /** - * Syncs repositories.enabled to match insightsProject.repositories list. + * Updates repositories.enabled based on the provided list of enabled URLs. + * Called when user toggles repository enabled status in the UI. */ export async function syncRepositoriesEnabledStatus( qx: QueryExecutor, diff --git a/services/libs/data-access-layer/src/segments/index.ts b/services/libs/data-access-layer/src/segments/index.ts index 5ed3018029..9174044baa 100644 --- a/services/libs/data-access-layer/src/segments/index.ts +++ b/services/libs/data-access-layer/src/segments/index.ts @@ -213,6 +213,7 @@ export async function getMappedRepos( export interface IRepoByPlatform { url: string platform: string + enabled: boolean } /** @@ -222,18 +223,19 @@ export interface IRepoByPlatform { * @param qx - Query executor * @param segmentId - The segment ID to get repos for * @param mergeGithubNango - If true, merges 'github-nango' platform into 'github' (default: true) - * @returns Record of platform -> array of repo URLs + * @returns Record of platform -> array of repo objects with url and enabled status */ export async function getReposBySegmentGroupedByPlatform( qx: QueryExecutor, segmentId: string, mergeGithubNango = true, -): Promise> { +): Promise>> { const rows: IRepoByPlatform[] = await qx.select( ` SELECT DISTINCT r.url, - i.platform + i.platform, + r.enabled FROM public.repositories r JOIN integrations i ON r."sourceIntegrationId" = i.id WHERE r."segmentId" = $(segmentId) @@ -244,7 +246,7 @@ export async function getReposBySegmentGroupedByPlatform( { segmentId }, ) - const result: Record = {} + const result: Record> = {} for (const row of rows) { let platform = row.platform @@ -258,7 +260,7 @@ export async function getReposBySegmentGroupedByPlatform( result[platform] = [] } - result[platform].push(row.url) + result[platform].push({ url: row.url, enabled: row.enabled }) } return result diff --git a/services/libs/data-access-layer/src/utils.ts b/services/libs/data-access-layer/src/utils.ts index 2717571786..86471fa477 100644 --- a/services/libs/data-access-layer/src/utils.ts +++ b/services/libs/data-access-layer/src/utils.ts @@ -158,11 +158,12 @@ export async function updateTableById( data: Partial<{ [K in T]: unknown }>, ) { const fields = columns.filter((col) => col in data) + const fieldsSql = fields.map((key, i) => `$(fields.col${i}:name) = $(data.${key})`).join(',\n') return qx.selectOneOrNone( ` UPDATE $(table:name) SET - ${fields.map((key, i) => `$(fields.col${i}:name) = $(data.${key})`).join(',\n')}, + ${fieldsSql ? `${fieldsSql},` : ''} "updatedAt" = now() WHERE id = $(id) RETURNING * From 3fb69ee9be1bd7bfa548f995798e559000f1f097 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 30 Jan 2026 12:22:06 +0100 Subject: [PATCH 3/3] fix: disable url normalization to avoid conflicts on update --- services/libs/data-access-layer/src/integrations/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/libs/data-access-layer/src/integrations/index.ts b/services/libs/data-access-layer/src/integrations/index.ts index 7ac31d301c..6792304cad 100644 --- a/services/libs/data-access-layer/src/integrations/index.ts +++ b/services/libs/data-access-layer/src/integrations/index.ts @@ -664,12 +664,12 @@ export async function findRepositoriesForSegment( // Get all repos grouped by platform (github-nango merged into github) const reposByPlatform = await getReposBySegmentGroupedByPlatform(qx, segmentId, true) - // Transform to include normalized URLs, labels, and enabled status + // Transform to include URLs, labels, and enabled status const result: Record> = {} for (const [platform, repos] of Object.entries(reposByPlatform)) { result[platform] = repos.map((repo) => ({ - url: normalizeRepoUrl(repo.url), + url: repo.url, label: extractLabelFromUrl(repo.url), enabled: repo.enabled, }))