diff --git a/backend/config/custom-environment-variables.json b/backend/config/custom-environment-variables.json index 494a1ccc03..3b06c9cfc9 100644 --- a/backend/config/custom-environment-variables.json +++ b/backend/config/custom-environment-variables.json @@ -222,5 +222,13 @@ "clientSecret": "CROWD_GITLAB_CLIENT_SECRET", "callbackUrl": "CROWD_GITLAB_CALLBACK_URL", "webhookToken": "CROWD_GITLAB_WEBHOOK_TOKEN" + }, + "snowflake": { + "privateKey": "CROWD_SNOWFLAKE_PRIVATE_KEY", + "account": "CROWD_SNOWFLAKE_ACCOUNT", + "username": "CROWD_SNOWFLAKE_USERNAME", + "database": "CROWD_SNOWFLAKE_DATABASE", + "warehouse": "CROWD_SNOWFLAKE_WAREHOUSE", + "role": "CROWD_SNOWFLAKE_ROLE" } } diff --git a/backend/package.json b/backend/package.json index d715dd79ae..5e207dce34 100644 --- a/backend/package.json +++ b/backend/package.json @@ -65,6 +65,7 @@ "@crowd/telemetry": "workspace:*", "@crowd/temporal": "workspace:*", "@crowd/types": "workspace:*", + "@crowd/snowflake": "workspace:*", "@google-cloud/storage": "5.3.0", "@octokit/auth-app": "^3.6.1", "@octokit/core": "^6.1.2", diff --git a/backend/src/api/integration/helpers/getGithubInstallations.ts b/backend/src/api/integration/helpers/getGithubInstallations.ts deleted file mode 100644 index 08448c107d..0000000000 --- a/backend/src/api/integration/helpers/getGithubInstallations.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) - const payload = await new IntegrationService(req).getGithubInstallations() - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/helpers/githubAuthenticate.ts b/backend/src/api/integration/helpers/githubAuthenticate.ts deleted file mode 100644 index 36b3d6e242..0000000000 --- a/backend/src/api/integration/helpers/githubAuthenticate.ts +++ /dev/null @@ -1,13 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) - const payload = await new IntegrationService(req).connectGithub( - req.params.code, - req.body.installId, - req.body.setupAction, - ) - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/helpers/githubConnectInstallation.ts b/backend/src/api/integration/helpers/githubConnectInstallation.ts deleted file mode 100644 index d7da9b0f43..0000000000 --- a/backend/src/api/integration/helpers/githubConnectInstallation.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) - const payload = await new IntegrationService(req).connectGithubInstallation(req.body.installId) - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/helpers/githubMapRepos.ts b/backend/src/api/integration/helpers/githubMapRepos.ts index 7edce4b9d3..b76e43da7b 100644 --- a/backend/src/api/integration/helpers/githubMapRepos.ts +++ b/backend/src/api/integration/helpers/githubMapRepos.ts @@ -4,6 +4,11 @@ import PermissionChecker from '../../../services/user/permissionChecker' export default async (req, res) => { new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) - const payload = await new IntegrationService(req).mapGithubRepos(req.params.id, req.body.mapping) + const payload = await new IntegrationService(req).mapGithubRepos( + req.params.id, + req.body.mapping, + true, + req.body?.isUpdateTransaction ?? false, + ) await req.responseHandler.success(req, res, payload) } diff --git a/backend/src/api/integration/helpers/githubOrgRepos.ts b/backend/src/api/integration/helpers/githubOrgRepos.ts new file mode 100644 index 0000000000..24a1942822 --- /dev/null +++ b/backend/src/api/integration/helpers/githubOrgRepos.ts @@ -0,0 +1,10 @@ +import Permissions from '@/security/permissions' +import GithubIntegrationService from '@/services/githubIntegrationService' +import PermissionChecker from '@/services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.integrationEdit) + + const payload = await GithubIntegrationService.getOrgRepos(req.params.org) + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/integration/helpers/githubSearchOrgs.ts b/backend/src/api/integration/helpers/githubSearchOrgs.ts new file mode 100644 index 0000000000..a8d3ff1541 --- /dev/null +++ b/backend/src/api/integration/helpers/githubSearchOrgs.ts @@ -0,0 +1,10 @@ +import Permissions from '@/security/permissions' +import GithubIntegrationService from '@/services/githubIntegrationService' +import PermissionChecker from '@/services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.integrationEdit) + + const payload = await GithubIntegrationService.findOrgs(req.query.query) + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/integration/helpers/githubSearchRepos.ts b/backend/src/api/integration/helpers/githubSearchRepos.ts new file mode 100644 index 0000000000..3a6cdac3cd --- /dev/null +++ b/backend/src/api/integration/helpers/githubSearchRepos.ts @@ -0,0 +1,10 @@ +import Permissions from '@/security/permissions' +import GithubIntegrationService from '@/services/githubIntegrationService' +import PermissionChecker from '@/services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.integrationEdit) + + const payload = await new GithubIntegrationService(req).findGithubRepos(req.query.query) + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/integration/index.ts b/backend/src/api/integration/index.ts index 3ffc448443..23ae1cdffe 100644 --- a/backend/src/api/integration/index.ts +++ b/backend/src/api/integration/index.ts @@ -37,10 +37,6 @@ export default (app) => { app.get(`/tenant/:tenantId/integration`, safeWrap(require('./integrationList').default)) app.get(`/tenant/:tenantId/integration/:id`, safeWrap(require('./integrationFind').default)) - app.put( - `/authenticate/:tenantId/:code`, - safeWrap(require('./helpers/githubAuthenticate').default), - ) app.put( `/tenant/:tenantId/integration/:id/github/repos`, safeWrap(require('./helpers/githubMapRepos').default), @@ -49,6 +45,18 @@ export default (app) => { `/tenant/:tenantId/integration/:id/github/repos`, safeWrap(require('./helpers/githubMapReposGet').default), ) + app.get( + `/tenant/:tenantId/integration/github/search/orgs`, + safeWrap(require('./helpers/githubSearchOrgs').default), + ) + app.get( + `/tenant/:tenantId/integration/github/search/repos`, + safeWrap(require('./helpers/githubSearchRepos').default), + ) + app.get( + `/tenant/:tenantId/integration/github/orgs/:org/repos`, + safeWrap(require('./helpers/githubOrgRepos').default), + ) app.put( `/discord-authenticate/:tenantId/:guild_id`, safeWrap(require('./helpers/discordAuthenticate').default), @@ -202,16 +210,6 @@ export default (app) => { safeWrap(require('./helpers/jiraConnectOrUpdate').default), ) - app.get( - '/tenant/:tenantId/github-installations', - safeWrap(require('./helpers/getGithubInstallations').default), - ) - - app.post( - '/tenant/:tenantId/github-connect-installation', - safeWrap(require('./helpers/githubConnectInstallation').default), - ) - app.get('/gitlab/:tenantId/connect', safeWrap(require('./helpers/gitlabAuthenticate').default)) app.get( diff --git a/backend/src/bin/jobs/index.ts b/backend/src/bin/jobs/index.ts index f984ee7e38..f24af9095e 100644 --- a/backend/src/bin/jobs/index.ts +++ b/backend/src/bin/jobs/index.ts @@ -4,7 +4,7 @@ import autoImportGroups from './autoImportGroupsioGroups' import checkStuckIntegrationRuns from './checkStuckIntegrationRuns' import cleanUp from './cleanUp' import integrationTicks from './integrationTicks' -import refreshGithubRepoSettings from './refreshGithubRepoSettings' +import refreshGithubRepoSettingsJob from './refreshGithubRepoSettings' import refreshGitlabToken from './refreshGitlabToken' import refreshGroupsioToken from './refreshGroupsioToken' import refreshMaterializedViews from './refreshMaterializedViews' @@ -17,7 +17,7 @@ const jobs: CrowdJob[] = [ checkStuckIntegrationRuns, refreshGroupsioToken, refreshGitlabToken, - refreshGithubRepoSettings, + refreshGithubRepoSettingsJob, autoImportGroups, syncActivitiesJob, ] diff --git a/backend/src/bin/jobs/refreshGithubRepoSettings.ts b/backend/src/bin/jobs/refreshGithubRepoSettings.ts index f1f4f66495..67c027c2ac 100644 --- a/backend/src/bin/jobs/refreshGithubRepoSettings.ts +++ b/backend/src/bin/jobs/refreshGithubRepoSettings.ts @@ -1,56 +1,144 @@ +/* eslint-disable no-continue */ import cronGenerator from 'cron-time-generator' import { timeout } from '@crowd/common' import { getServiceChildLogger } from '@crowd/logging' import SequelizeRepository from '../../database/repositories/sequelizeRepository' +import GithubIntegrationService from '../../services/githubIntegrationService' import IntegrationService from '../../services/integrationService' import { CrowdJob } from '../../types/jobTypes' const log = getServiceChildLogger('refreshGithubRepoSettings') -const job: CrowdJob = { - name: 'Refresh Github repo settings', - // every day - cronTime: cronGenerator.every(1).days(), - onTrigger: async () => { - log.info('Updating Github repo settings.') - const dbOptions = await SequelizeRepository.getDefaultIRepositoryOptions() +interface Integration { + id: string + tenantId: string + segmentId: string + integrationIdentifier: string + settings: { + orgs: Array<{ + name: string + logo: string + url: string + fullSync: boolean + updatedAt: string + repos: Array<{ + name: string + url: string + updatedAt: string + }> + }> + } +} - interface Integration { - id: string - tenantId: string - integrationIdentifier: string - } +export const refreshGithubRepoSettings = async () => { + log.info('Updating Github repo settings.') + const dbOptions = await SequelizeRepository.getDefaultIRepositoryOptions() - const githubIntegrations = await dbOptions.database.sequelize.query( - `SELECT id, "tenantId", "integrationIdentifier" FROM integrations + const githubIntegrations = await dbOptions.database.sequelize.query( + `SELECT id, "tenantId", settings, "segmentId" FROM integrations WHERE platform = 'github' AND "deletedAt" IS NULL AND "createdAt" < NOW() - INTERVAL '1 day'`, - ) + ) + + for (const integration of githubIntegrations[0] as Integration[]) { + log.info(`Updating repo settings for Github integration: ${integration.id}`) + + try { + const options = await SequelizeRepository.getDefaultIRepositoryOptions() + options.currentTenant = { id: integration.tenantId } + options.currentSegments = [ + // @ts-ignore + { + id: integration.segmentId, + }, + ] + + const integrationService = new IntegrationService(options) + + if (!integration.settings.orgs) { + log.info(`No orgs found for Github integration: ${integration.id}`) + continue + } + + // Get all orgs with fullSync enabled + const fullSyncOrgs = integration.settings.orgs.filter((org) => org.fullSync) + const currentRepos = new Set( + integration.settings.orgs.flatMap((org) => org.repos.map((r) => r.url)), + ) + const newRepoMappings: Record = {} - for (const integration of githubIntegrations[0] as Integration[]) { - log.info(`Updating repo settings for Github integration: ${integration.id}`) + // Fetch new repos for each org + for (const org of fullSyncOrgs) { + log.info(`Fetching repos for org ${org.name} with fullSync enabled`) + const githubRepos = await GithubIntegrationService.getOrgRepos(org.name) - try { - const options = await SequelizeRepository.getDefaultIRepositoryOptions() - options.currentTenant = { id: integration.tenantId } + // Find new repos that aren't in current settings + const newRepos = githubRepos.filter((repo) => !currentRepos.has(repo.url)) - const integrationService = new IntegrationService(options) - // newly discovered repos will be mapped to default segment of the integration - await integrationService.updateGithubIntegrationSettings(integration.integrationIdentifier) + // Map new repos to the integration's segment + newRepos.forEach((repo) => { + newRepoMappings[repo.url] = integration.segmentId + }) - log.info(`Successfully updated repo settings for Github integration: ${integration.id}`) - } catch (err) { - log.error( - `Error updating repo settings for Github integration ${integration.id}: ${err.message}`, + // Update org's repos list directly in the integration settings + const orgIndex = integration.settings.orgs.findIndex((o) => o.name === org.name) + if (orgIndex !== -1) { + integration.settings.orgs[orgIndex].repos = [ + ...integration.settings.orgs[orgIndex].repos, + ...newRepos.map((repo) => ({ + name: repo.name, + url: repo.url, + updatedAt: new Date().toISOString(), + })), + ] + integration.settings.orgs[orgIndex].updatedAt = new Date().toISOString() + } + } + + if (Object.keys(newRepoMappings).length > 0) { + log.info(`Found ${Object.keys(newRepoMappings).length} new repos to add`) + + // Update integration with modified settings object + await integrationService.update(integration.id, { + settings: integration.settings, + status: 'in-progress', + }) + + // Map new repos + await integrationService.mapGithubRepos( + integration.id, + newRepoMappings, + true, + // this will fire onboarding only for new repos + true, ) - } finally { - await timeout(1000) + + log.info(`Successfully mapped ${Object.keys(newRepoMappings).length} new repos`) } + + log.info(`Successfully updated repo settings for Github integration: ${integration.id}`) + } catch (err) { + log.error( + `Error updating repo settings for Github integration ${integration.id}: ${err.message}`, + ) + // that's probably a rate limit error, let's sleep for a minute + await timeout(60000) + } finally { + await timeout(10000) } + } - log.info('Finished updating Github repo settings.') + log.info('Finished updating Github repo settings.') +} + +const job: CrowdJob = { + name: 'Refresh Github repo settings', + // every day + cronTime: cronGenerator.every(1).days(), + onTrigger: async () => { + await refreshGithubRepoSettings() }, } diff --git a/backend/src/bin/scripts/refresh-github-repo-settings.ts b/backend/src/bin/scripts/refresh-github-repo-settings.ts new file mode 100644 index 0000000000..97e729c3dd --- /dev/null +++ b/backend/src/bin/scripts/refresh-github-repo-settings.ts @@ -0,0 +1,22 @@ +import { getServiceChildLogger } from '@crowd/logging' + +import { refreshGithubRepoSettings } from '../jobs/refreshGithubRepoSettings' + +const logger = getServiceChildLogger('refreshGithubRepoSettings') + +setImmediate(async () => { + try { + const startTime = Date.now() + logger.info('Starting refresh of Github repo settings') + + await refreshGithubRepoSettings() + + const duration = Date.now() - startTime + logger.info(`Completed refresh of Github repo settings in ${duration}ms`) + + process.exit(0) + } catch (error) { + logger.error(`Error refreshing Github repo settings: ${error.message}`) + process.exit(1) + } +}) diff --git a/backend/src/conf/configTypes.ts b/backend/src/conf/configTypes.ts index d7b3266620..46e6596ce3 100644 --- a/backend/src/conf/configTypes.ts +++ b/backend/src/conf/configTypes.ts @@ -256,3 +256,12 @@ export interface IRedditConfig { clientId: string clientSecret: string } + +export interface SnowflakeConfiguration { + privateKey: string + account: string + username: string + database: string + warehouse: string + role: string +} diff --git a/backend/src/conf/index.ts b/backend/src/conf/index.ts index a40f25867b..2d397bf07b 100644 --- a/backend/src/conf/index.ts +++ b/backend/src/conf/index.ts @@ -39,6 +39,7 @@ import { SlackAlertingConfiguration, SlackConfiguration, SlackNotifierConfiguration, + SnowflakeConfiguration, StackExchangeConfiguration, TenantMode, TwitterConfiguration, @@ -172,3 +173,6 @@ export const OPEN_STATUS_API_CONFIG: IOpenStatusApiConfig = export const GITLAB_CONFIG: GitlabConfiguration = config.get('gitlab') export const REDDIT_CONFIG: IRedditConfig = config.get('reddit') + +export const SNOWFLAKE_CONFIG: SnowflakeConfiguration = + config.get('snowflake') diff --git a/backend/src/database/migrations/U1732613678__migrate-github-integration-settings.sql b/backend/src/database/migrations/U1732613678__migrate-github-integration-settings.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/database/migrations/V1732613678__migrate-github-integration-settings.sql b/backend/src/database/migrations/V1732613678__migrate-github-integration-settings.sql new file mode 100644 index 0000000000..2da91eb4bd --- /dev/null +++ b/backend/src/database/migrations/V1732613678__migrate-github-integration-settings.sql @@ -0,0 +1,101 @@ +DO $$ +DECLARE + _record RECORD; + _new_settings JSONB; + _org JSONB; +BEGIN + FOR _record IN SELECT * FROM integrations WHERE platform = 'github' LOOP + /* + old settings: + { + "repos": [ + { + "url": "https://github.com/camaraproject/Governance", + "fork": false, + "name": "Governance", + "owner": "camaraproject", + "private": false, + "cloneUrl": "https://github.com/camaraproject/Governance.git", + "createdAt": "2021-06-28T04:15:55Z" + }, + { + "url": "https://github.com/camaraproject/QualityOnDemand", + "fork": false, + "name": "QualityOnDemand", + "owner": "camaraproject", + "private": false, + "cloneUrl": "https://github.com/camaraproject/QualityOnDemand.git", + "createdAt": "2022-02-22T10:05:50Z" + }, + ... + ], + "orgAvatar": "https://avatars.githubusercontent.com/u/91603532?v=4", + "unavailableRepos": [], + "updateMemberAttributes": false + } + */ + /* + new settings: + { + orgs: [ + { + name: "ASWF", + logo: "https://blah...png", + url: "https://github.com/ASWF", + fullSync: true|false, + updatedAt: "2024-11-18T12:03:09", + repos: [ + { + name: "foundation", + url: "https://github.com/ASWF/foundation", + updatedAt: "2024-11-18T12:03:09", + }, + { + name: "something-else", + url: "https://github.com/ASWF/something-else", + updatedAt: "2024-11-18T12:03:09", + } + ] + }, + ... + ] + } + */ + + -- Skip if settings already have orgs key + IF _record.settings ? 'orgs' THEN + RAISE NOTICE 'integration: %, skipping', _record.id; + CONTINUE; + END IF; + + -- Extract org name and avatar from first repo if available + IF jsonb_array_length(_record.settings->'repos') > 0 THEN + -- Create org object with repos + _org = jsonb_build_object( + 'name', (_record.settings->'repos'->0->>'owner'), + 'logo', _record.settings->>'orgAvatar', + 'url', 'https://github.com/' || (_record.settings->'repos'->0->>'owner'), + 'fullSync', true, + 'updatedAt', CURRENT_TIMESTAMP, + 'repos', ( + SELECT jsonb_agg( + jsonb_build_object( + 'name', repo->>'name', + 'url', repo->>'url', + 'updatedAt', repo->>'createdAt' + ) + ) + FROM jsonb_array_elements(_record.settings->'repos') repo + ) + ); + + -- Create new settings object with orgs array + _new_settings = jsonb_build_object('orgs', jsonb_build_array(_org)); + + UPDATE integrations + SET settings = _new_settings + WHERE id = _record.id; + END IF; + END LOOP; +END; +$$; diff --git a/backend/src/database/repositories/conversationRepository.ts b/backend/src/database/repositories/conversationRepository.ts index ef92a5e70f..bca86a0908 100644 --- a/backend/src/database/repositories/conversationRepository.ts +++ b/backend/src/database/repositories/conversationRepository.ts @@ -1,6 +1,6 @@ import lodash from 'lodash' -import { Error404, distinct } from '@crowd/common' +import { Error404, distinct, single } from '@crowd/common' import { DEFAULT_COLUMNS_TO_SELECT, IQueryActivityResult, @@ -264,10 +264,22 @@ class ConversationRepository { } } - output.activities = [...results.rows].sort( - (a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime(), + // find the conversation starter + const firstActivity = single( + results.rows, + (a) => a.conversationId === conversation.id && a.parentId === null, ) + const remainingActivities = results.rows + .filter((a) => a.parentId !== null) + .sort( + (a, b) => + // from oldest to newest + new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime(), + ) + + output.activities = [firstActivity, ...remainingActivities] + output.memberCount = results.rows .map((row) => row.memberId) .filter((item, index, arr) => arr.indexOf(item) === index).length diff --git a/backend/src/database/repositories/githubReposRepository.ts b/backend/src/database/repositories/githubReposRepository.ts index 759cebc15a..73c16007ee 100644 --- a/backend/src/database/repositories/githubReposRepository.ts +++ b/backend/src/database/repositories/githubReposRepository.ts @@ -28,7 +28,8 @@ export default class GithubReposRepository { VALUES ${placeholders.join(', ')} ON CONFLICT ("tenantId", "url") DO UPDATE SET "segmentId" = EXCLUDED."segmentId", - "integrationId" = EXCLUDED."integrationId" + "integrationId" = EXCLUDED."integrationId", + "deletedAt" = NULL `, { replacements, @@ -37,21 +38,80 @@ export default class GithubReposRepository { ) } - static async updateMapping(integrationId, mapping, options: IRepositoryOptions) { + static async updateMapping( + integrationId, + newMapping: Record, + oldMapping: { + url: string + segment: { + id: string + name: string + } + }[], + options: IRepositoryOptions, + ) { const tenantId = options.currentTenant.id + const transaction = SequelizeRepository.getTransaction(options) + const seq = SequelizeRepository.getSequelize(options) - await GithubReposRepository.bulkInsert( - 'githubRepos', - ['tenantId', 'integrationId', 'segmentId', 'url'], - (idx) => `(:tenantId_${idx}, :integrationId_${idx}, :segmentId_${idx}, :url_${idx})`, - Object.entries(mapping).map(([url, segmentId], idx) => ({ - [`tenantId_${idx}`]: tenantId, - [`integrationId_${idx}`]: integrationId, - [`segmentId_${idx}`]: segmentId, - [`url_${idx}`]: url, - })), - options, - ) + console.log('Old mapping:', oldMapping) + console.log('New mapping:', newMapping) + + // Create maps for efficient lookup + const oldMappingMap = new Map(oldMapping.map((m) => [m.url, m.segment.id])) + const newMappingEntries = Object.entries(newMapping) + + // Find repos to insert or update (where they didn't exist or segment changed) + const reposToUpsert = newMappingEntries.filter(([url, segmentId]) => { + const oldSegmentId = oldMappingMap.get(url) + return !oldSegmentId || oldSegmentId !== segmentId + }) + + console.log('Repos to upsert:', reposToUpsert) + + if (reposToUpsert.length > 0) { + const result = await GithubReposRepository.bulkInsert( + 'githubRepos', + ['tenantId', 'integrationId', 'segmentId', 'url'], + (idx) => `(:tenantId_${idx}, :integrationId_${idx}, :segmentId_${idx}, :url_${idx})`, + reposToUpsert.map(([url, segmentId], idx) => ({ + [`tenantId_${idx}`]: tenantId, + [`integrationId_${idx}`]: integrationId, + [`segmentId_${idx}`]: segmentId, + [`url_${idx}`]: url, + })), + options, + ) + console.log('Bulk insert result:', result) + } + + // Find repos that were removed (exist in old but not in new) + const newUrlSet = new Set(Object.keys(newMapping)) + const urlsToRemove = oldMapping.filter((m) => !newUrlSet.has(m.url)).map((m) => m.url) + + console.log('URLs to remove:', urlsToRemove) + + if (urlsToRemove.length > 0) { + const result = await seq.query( + ` + UPDATE "githubRepos" + SET "deletedAt" = NOW() + WHERE "tenantId" = :tenantId + AND "integrationId" = :integrationId + AND "url" IN (:urls) + AND "deletedAt" IS NULL + `, + { + replacements: { + tenantId, + integrationId, + urls: urlsToRemove, + }, + transaction, + }, + ) + console.log('Delete result:', result) + } } static async getMapping(integrationId, options: IRepositoryOptions) { @@ -70,6 +130,7 @@ export default class GithubReposRepository { JOIN segments s ON s.id = r."segmentId" WHERE r."integrationId" = :integrationId AND r."tenantId" = :tenantId + AND r."deletedAt" is null `, { replacements: { diff --git a/backend/src/serverless/integrations/types/regularTypes.ts b/backend/src/serverless/integrations/types/regularTypes.ts index 4b93498be8..f7c5f1f8cb 100644 --- a/backend/src/serverless/integrations/types/regularTypes.ts +++ b/backend/src/serverless/integrations/types/regularTypes.ts @@ -3,12 +3,7 @@ import { PlatformType } from '@crowd/types' export type Repo = { url: string name: string - createdAt: string - owner: string - available?: boolean - fork?: boolean - private?: boolean - cloneUrl?: string + updatedAt: string } export type Repos = Array diff --git a/backend/src/serverless/integrations/usecases/github/graphql/baseQuery.ts b/backend/src/serverless/integrations/usecases/github/graphql/baseQuery.ts deleted file mode 100644 index a6203a2829..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/baseQuery.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { graphql } from '@octokit/graphql' -import { GraphQlQueryResponseData } from '@octokit/graphql/dist-types/types' -import moment from 'moment' - -import { RateLimitError } from '../../../../../types/integration/rateLimitError' -import { GraphQlQueryResponse } from '../../../types/messageTypes' - -class BaseQuery { - static BASE_URL = 'https://api.github.com/graphql' - - static USER_SELECT = `{ - login - name - avatarUrl - id - isHireable - twitterUsername - url - websiteUrl - email - bio - company - location - followers { - totalCount - } - }` - - static ORGANIZATION_SELECT = `{ - email - url - location - name - twitterUsername - websiteUrl - description - avatarUrl - }` - - static PAGE_SELECT = `{ - hasPreviousPage - startCursor - }` - - graphQL - - query - - githubToken - - additionalHeaders - - perPage - - eventType - - constructor(githubToken: string, query: string, eventType: string, perPage: number) { - this.githubToken = githubToken - this.query = query - this.perPage = perPage - this.eventType = eventType - this.graphQL = graphql.defaults({ - headers: { - authorization: `token ${this.githubToken}`, - }, - }) - } - - /** - * Substitutes a variable like string $var with given variable - * in a string. Useful when reusing the same string template - * for multiple graphql paging requests. - * $var in the string is substituted with obj[var] - * @param str string to make the substitution - * @param obj object containing variable to interpolate - * @returns interpolated string - */ - static interpolate(str: string, obj: any): string { - return str.replace(/\${([^}]+)}/g, (_, prop) => obj[prop]) - } - - /** - * Gets a single page result given a cursor. - * Single page before the given cursor will be fetched. - * @param beforeCursor Cursor to paginate records before it - * @returns parsed graphQl result - */ - async getSinglePage(beforeCursor: string): Promise { - const paginatedQuery = BaseQuery.interpolate(this.query, { - beforeCursor: BaseQuery.getPagination(beforeCursor), - }) - - try { - const result = await this.graphQL(paginatedQuery) - return this.getEventData(result) - } catch (err) { - throw BaseQuery.processGraphQLError(err) - } - } - - /** - * Parses graphql result into an object. - * Object contains information about paging, and fetched data. - * @param result from graphql query - * @returns parsed result into paging and data values. - */ - getEventData(result: GraphQlQueryResponseData): GraphQlQueryResponse { - return { - hasPreviousPage: result.repository[this.eventType].pageInfo?.hasPreviousPage, - startCursor: result.repository[this.eventType].pageInfo?.startCursor, - data: [{}], - } - } - - /** - * Returns pagination string given cursor. - * @param beforeCursor cursor to use for the pagination - * @returns pagination string that can be injected into a graphql query. - */ - static getPagination(beforeCursor: string): string { - if (beforeCursor) { - return `before: "${beforeCursor}"` - } - return '' - } - - static processGraphQLError(err: any): any { - if (err.errors && err.errors[0].type === 'RATE_LIMITED') { - if (err.headers && err.headers['x-ratelimit-reset']) { - const query = - err.request && err.request.query ? err.request.query : 'Unknown GraphQL query!' - - const epochReset = parseInt(err.headers['x-ratelimit-reset'], 10) - const resetDate = moment.unix(epochReset) - const diffInSeconds = resetDate.diff(moment(), 'seconds') - - return new RateLimitError(diffInSeconds + 5, query, err) - } - } - - return err - } -} - -export default BaseQuery diff --git a/backend/src/serverless/integrations/usecases/github/graphql/discussionComments.ts b/backend/src/serverless/integrations/usecases/github/graphql/discussionComments.ts deleted file mode 100644 index b8dabd2a1c..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/discussionComments.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { Repo } from '../../../types/regularTypes' - -import BaseQuery from './baseQuery' - -/* eslint class-methods-use-this: 0 */ -class DiscussionCommentsQuery extends BaseQuery { - repo: Repo - - constructor( - repo: Repo, - discussionNumber: string, - githubToken: string, - perPage: number = 100, - maxRepliesPerComment: number = 100, - ) { - const discussionCommentsQuery = `{ - repository(name: "${repo.name}", owner: "${repo.owner}") { - discussion(number: ${discussionNumber}) { - comments(first: ${perPage}, \${beforeCursor}) { - pageInfo ${BaseQuery.PAGE_SELECT} - nodes { - author { - ... on User ${BaseQuery.USER_SELECT} - } - bodyText - url - id - createdAt - isAnswer - replies(first: ${maxRepliesPerComment}) { - nodes { - author { - ... on User ${BaseQuery.USER_SELECT} - } - bodyText - url - id - createdAt - } - } - discussion { - url - id - title - } - } - - } - } - } - }` - - super(githubToken, discussionCommentsQuery, 'discussionComments', perPage) - - this.repo = repo - } - - getEventData(result) { - return { - hasPreviousPage: result.repository?.discussion?.comments?.pageInfo?.hasPreviousPage, - startCursor: result.repository?.discussion?.comments?.pageInfo?.startCursor, - data: result.repository?.discussion?.comments?.nodes, - } - } -} - -export default DiscussionCommentsQuery diff --git a/backend/src/serverless/integrations/usecases/github/graphql/discussions.ts b/backend/src/serverless/integrations/usecases/github/graphql/discussions.ts deleted file mode 100644 index 3ee712ea77..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/discussions.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { Repo } from '../../../types/regularTypes' - -import BaseQuery from './baseQuery' - -/* eslint class-methods-use-this: 0 */ -class DiscussionsQuery extends BaseQuery { - repo: Repo - - constructor(repo: Repo, githubToken: string, perPage: number = 100) { - const discussionsQuery = `{ - repository(owner: "${repo.owner}", name: "${repo.name}") { - discussions(last: ${perPage}, \${beforeCursor}) { - pageInfo ${BaseQuery.PAGE_SELECT} - nodes { - author { - ... on User ${BaseQuery.USER_SELECT} - } - number - bodyText - title - id - url - createdAt - comments { - totalCount - } - category { - id - isAnswerable - name - slug - emoji - description - } - } - } - } - }` - - super(githubToken, discussionsQuery, 'discussions', perPage) - - this.repo = repo - } - - getEventData(result) { - return { ...super.getEventData(result), data: result.repository?.discussions?.nodes } - } -} - -export default DiscussionsQuery diff --git a/backend/src/serverless/integrations/usecases/github/graphql/forks.ts b/backend/src/serverless/integrations/usecases/github/graphql/forks.ts deleted file mode 100644 index 4a3ee2f9a0..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/forks.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Repo } from '../../../types/regularTypes' - -import BaseQuery from './baseQuery' - -/* eslint class-methods-use-this: 0 */ -class ForksQuery extends BaseQuery { - repo: Repo - - constructor(repo: Repo, githubToken: string, perPage: number = 100) { - const forksQuery = `{ - repository(owner: "${repo.owner}", name: "${repo.name}") { - forks(last: ${perPage}, \${beforeCursor}) { - pageInfo ${BaseQuery.PAGE_SELECT} - nodes { - owner { - ... on User ${BaseQuery.USER_SELECT} - } - name - url - id - createdAt - } - } - } - }` - - super(githubToken, forksQuery, 'forks', perPage) - - this.repo = repo - } - - getEventData(result) { - return { ...super.getEventData(result), data: result.repository?.forks?.nodes } - } -} - -export default ForksQuery diff --git a/backend/src/serverless/integrations/usecases/github/graphql/issueComments.ts b/backend/src/serverless/integrations/usecases/github/graphql/issueComments.ts deleted file mode 100644 index 3ad715f33d..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/issueComments.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Repo } from '../../../types/regularTypes' - -import BaseQuery from './baseQuery' - -/* eslint class-methods-use-this: 0 */ -class IssueCommentsQuery extends BaseQuery { - repo: Repo - - constructor(repo: Repo, issueNumber: string, githubToken: string, perPage: number = 100) { - const issueCommentsQuery = `{ - repository(name: "${repo.name}", owner: "${repo.owner}") { - issue(number: ${issueNumber}) { - comments(first: ${perPage}, \${beforeCursor}) { - pageInfo ${BaseQuery.PAGE_SELECT} - nodes { - author { - ... on User ${BaseQuery.USER_SELECT} - } - bodyText - url - id - createdAt - issue { - url - id - title - } - repository { - url - } - } - } - } - } - }` - - super(githubToken, issueCommentsQuery, 'issueComments', perPage) - - this.repo = repo - } - - getEventData(result) { - return { - hasPreviousPage: result.repository?.issue?.comments?.pageInfo?.hasPreviousPage, - startCursor: result.repository?.issue?.comments?.pageInfo?.startCursor, - data: result.repository?.issue?.comments?.nodes, - } - } -} - -export default IssueCommentsQuery diff --git a/backend/src/serverless/integrations/usecases/github/graphql/issues.ts b/backend/src/serverless/integrations/usecases/github/graphql/issues.ts deleted file mode 100644 index 2b19a80d2a..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/issues.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { Repo } from '../../../types/regularTypes' - -import BaseQuery from './baseQuery' - -/* eslint class-methods-use-this: 0 */ -class IssuesQuery extends BaseQuery { - repo: Repo - - constructor(repo: Repo, githubToken: string, perPage: number = 100) { - const issuesQuery = `{ - repository(owner: "${repo.owner}", name: "${repo.name}") { - issues(last: ${perPage}, \${beforeCursor}) { - pageInfo ${BaseQuery.PAGE_SELECT} - nodes { - author { - ... on User ${BaseQuery.USER_SELECT} - } - bodyText - state - id - title - url - createdAt - number - timelineItems(first: 100, itemTypes: [CLOSED_EVENT]) { - nodes { - ... on ClosedEvent { - __typename - id - actor { - ... on User ${BaseQuery.USER_SELECT} - } - createdAt - } - } - } - } - } - } - }` - - super(githubToken, issuesQuery, 'issues', perPage) - - this.repo = repo - } - - getEventData(result) { - return { ...super.getEventData(result), data: result.repository?.issues?.nodes } - } -} - -export default IssuesQuery diff --git a/backend/src/serverless/integrations/usecases/github/graphql/members.ts b/backend/src/serverless/integrations/usecases/github/graphql/members.ts deleted file mode 100644 index 079a8d893f..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/members.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { graphql } from '@octokit/graphql' - -import BaseQuery from './baseQuery' - -/** - * Get information from a member using the GitHub GraphQL API. - * @param username GitHub username - * @param token GitHub personal access token - * @returns Information from member - */ -const getMember = async (username: string, token: string): Promise => { - let user: string | null - try { - const graphqlWithAuth = graphql.defaults({ - headers: { - authorization: `token ${token}`, - }, - }) - - user = ( - (await graphqlWithAuth(`{ - user(login: "${username}") ${BaseQuery.USER_SELECT} - } - `)) as any - ).user - } catch (err) { - // It may be that the user was not found, if for example it is a bot - // In that case we want to return null instead of throwing an error - if (err.errors && err.errors[0].type === 'NOT_FOUND') { - user = null - } else { - throw BaseQuery.processGraphQLError(err) - } - } - return user -} - -export default getMember diff --git a/backend/src/serverless/integrations/usecases/github/graphql/organizations.ts b/backend/src/serverless/integrations/usecases/github/graphql/organizations.ts deleted file mode 100644 index c45b2b6143..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/organizations.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { graphql } from '@octokit/graphql' - -import { getServiceChildLogger } from '@crowd/logging' - -import BaseQuery from './baseQuery' - -const logger = getServiceChildLogger('github.getOrganization') - -/** - * Get information from a organization using the GitHub GraphQL API. - * @param name Name of the organization in GitHub - * @param token GitHub personal access token - * @returns Information from organization - */ -const getOrganization = async (name: string, token: string): Promise => { - let organization: string | null - try { - const graphqlWithAuth = graphql.defaults({ - headers: { - authorization: `token ${token}`, - }, - }) - - const sanitizedName = name.replaceAll('\\', '').replaceAll('"', '') - - const organizationsQuery = `{ - search(query: "type:org ${sanitizedName}", type: USER, first: 10) { - nodes { - ... on Organization ${BaseQuery.ORGANIZATION_SELECT} - } - } - rateLimit { - limit - cost - remaining - resetAt - } - }` - - organization = (await graphqlWithAuth(organizationsQuery)) as any - - organization = - (organization as any).search.nodes.length > 0 ? (organization as any).search.nodes[0] : null - } catch (err) { - logger.error(err, { name }, 'Error getting organization!') - // It may be that the organization was not found, if for example it is a bot - // In that case we want to return null instead of throwing an error - if (err.errors && err.errors[0].type === 'NOT_FOUND') { - organization = null - } else { - throw BaseQuery.processGraphQLError(err) - } - } - return organization -} - -export default getOrganization diff --git a/backend/src/serverless/integrations/usecases/github/graphql/pullRequestComments.ts b/backend/src/serverless/integrations/usecases/github/graphql/pullRequestComments.ts deleted file mode 100644 index a8f54d7764..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/pullRequestComments.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Repo } from '../../../types/regularTypes' - -import BaseQuery from './baseQuery' - -/* eslint class-methods-use-this: 0 */ -class PullRequestCommentsQuery extends BaseQuery { - repo: Repo - - constructor(repo: Repo, pullRequestNumber: string, githubToken: string, perPage: number = 100) { - const pullRequestCommentsQuery = `{ - repository(name: "${repo.name}", owner: "${repo.owner}") { - pullRequest(number: ${pullRequestNumber}) { - comments(first: ${perPage}, \${beforeCursor}) { - pageInfo ${BaseQuery.PAGE_SELECT} - nodes { - author { - ... on User ${BaseQuery.USER_SELECT} - } - bodyText - url - id - createdAt - pullRequest { - url - id - title - } - repository { - url - } - } - } - } - } - }` - - super(githubToken, pullRequestCommentsQuery, 'pullRequestComments', perPage) - - this.repo = repo - } - - getEventData(result) { - return { - hasPreviousPage: result.repository?.pullRequest?.comments?.pageInfo?.hasPreviousPage, - startCursor: result.repository?.pullRequest?.comments?.pageInfo?.startCursor, - data: result.repository?.pullRequest?.comments?.nodes, - } - } -} - -export default PullRequestCommentsQuery diff --git a/backend/src/serverless/integrations/usecases/github/graphql/pullRequestCommits.ts b/backend/src/serverless/integrations/usecases/github/graphql/pullRequestCommits.ts deleted file mode 100644 index e0b6578cc5..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/pullRequestCommits.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { Repo } from '../../../types/regularTypes' - -import BaseQuery from './baseQuery' - -export interface PullRequestCommit { - repository: { - pullRequest: { - id: string - number: number - baseRefName: string - headRefName: string - commits: { - pageInfo: { - hasPreviousPage: boolean - startCursor: string - } - nodes: { - commit: { - authoredDate: string - committedDate: string - additions: number - changedFilesIfAvailable: number - deletions: number - oid: string - message: string - url: string - parents: { - totalCount: number - } - authors: { - nodes: { - user: { - login: string - name: string - avatarUrl: string - id: string - isHireable: boolean - twitterUsername: string | null - url: string - websiteUrl: string | null - email: string - bio: string - company: string - location: string | null - followers: { - totalCount: number - } - } - }[] - } - } - }[] - } - } - } -} - -/* eslint class-methods-use-this: 0 */ -class PullRequestCommitsQuery extends BaseQuery { - repo: Repo - - constructor( - repo: Repo, - pullRequestNumber: string, - githubToken: string, - perPage: number = 100, - maxAuthors: number = 1, - ) { - const pullRequestCommitsQuery = `{ - repository(name: "${repo.name}", owner: "${repo.owner}") { - pullRequest(number: ${pullRequestNumber}) { - id - number - baseRefName - headRefName - commits(first: ${perPage}, \${beforeCursor}) { - pageInfo ${BaseQuery.PAGE_SELECT} - nodes { - commit { - authoredDate - committedDate - additions - changedFilesIfAvailable - deletions - oid - message - url - parents(first: 2) { - totalCount - } - authors(first: ${maxAuthors}) { - nodes { - user ${BaseQuery.USER_SELECT} - } - } - } - } - } - } - } - }` - - super(githubToken, pullRequestCommitsQuery, 'pullRequestCommits', perPage) - - this.repo = repo - } - - // Override the getEventData method to process commit details - getEventData(result) { - const commitData = result as PullRequestCommit - - return { - hasPreviousPage: result.repository?.pullRequest?.commits?.pageInfo?.hasPreviousPage, - startCursor: result.repository?.pullRequest?.commits?.pageInfo?.startCursor, - data: [commitData], // returning an array to match the parseActivities function - } - } -} - -export default PullRequestCommitsQuery diff --git a/backend/src/serverless/integrations/usecases/github/graphql/pullRequestCommitsNoAdditions.ts b/backend/src/serverless/integrations/usecases/github/graphql/pullRequestCommitsNoAdditions.ts deleted file mode 100644 index 76f20b8d48..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/pullRequestCommitsNoAdditions.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { Repo } from '../../../types/regularTypes' - -import BaseQuery from './baseQuery' - -export interface PullRequestCommitNoAdditions { - repository: { - pullRequest: { - id: string - number: number - baseRefName: string - headRefName: string - commits: { - pageInfo: { - hasPreviousPage: boolean - startCursor: string - } - nodes: { - commit: { - authoredDate: string - committedDate: string - changedFilesIfAvailable: number - oid: string - message: string - url: string - parents: { - totalCount: number - } - authors: { - nodes: { - user: { - login: string - name: string - avatarUrl: string - id: string - isHireable: boolean - twitterUsername: string | null - url: string - websiteUrl: string | null - email: string - bio: string - company: string - location: string | null - followers: { - totalCount: number - } - } - }[] - } - } - }[] - } - } - } -} - -/* eslint class-methods-use-this: 0 */ -class PullRequestCommitsQueryNoAdditions extends BaseQuery { - repo: Repo - - constructor( - repo: Repo, - pullRequestNumber: string, - githubToken: string, - perPage: number = 100, - maxAuthors: number = 1, - ) { - const pullRequestCommitsQuery = `{ - repository(name: "${repo.name}", owner: "${repo.owner}") { - pullRequest(number: ${pullRequestNumber}) { - id - number - baseRefName - headRefName - commits(first: ${perPage}, \${beforeCursor}) { - pageInfo ${BaseQuery.PAGE_SELECT} - nodes { - commit { - authoredDate - committedDate - changedFilesIfAvailable - oid - message - url - parents(first: 2) { - totalCount - } - authors(first: ${maxAuthors}) { - nodes { - user ${BaseQuery.USER_SELECT} - } - } - } - } - } - } - } - }` - - super(githubToken, pullRequestCommitsQuery, 'pullRequestCommits', perPage) - - this.repo = repo - } - - // Override the getEventData method to process commit details - getEventData(result) { - const commitData = result as PullRequestCommitNoAdditions - - return { - hasPreviousPage: result.repository?.pullRequest?.commits?.pageInfo?.hasPreviousPage, - startCursor: result.repository?.pullRequest?.commits?.pageInfo?.startCursor, - data: [commitData], // returning an array to match the parseActivities function - } - } -} - -export default PullRequestCommitsQueryNoAdditions diff --git a/backend/src/serverless/integrations/usecases/github/graphql/pullRequestReviewThreadComments.ts b/backend/src/serverless/integrations/usecases/github/graphql/pullRequestReviewThreadComments.ts deleted file mode 100644 index ea8b4ad6ad..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/pullRequestReviewThreadComments.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { Repo } from '../../../types/regularTypes' - -import BaseQuery from './baseQuery' - -/* eslint class-methods-use-this: 0 */ -class PullRequestReviewThreadCommentsQuery extends BaseQuery { - repo: Repo - - constructor(repo: Repo, reviewThreadId: string, githubToken: string, perPage: number = 50) { - const pullRequestReviewThreadCommentsQuery = `{ - node(id: "${reviewThreadId}") { - ... on PullRequestReviewThread { - comments(first: ${perPage}, \${beforeCursor}) { - pageInfo ${BaseQuery.PAGE_SELECT} - nodes { - author { - ... on User ${BaseQuery.USER_SELECT} - } - pullRequestReview { - submittedAt - author { - ... on User ${BaseQuery.USER_SELECT} - } - } - bodyText - url - id - createdAt - pullRequest { - url - id - title - additions - deletions - changedFiles - authorAssociation - state - repository{ - url - } - } - } - } - } - } - }` - - super( - githubToken, - pullRequestReviewThreadCommentsQuery, - 'pullRequestReviewThreadComments', - perPage, - ) - - this.repo = repo - } - - getEventData(result) { - return { - hasPreviousPage: result.node?.comments?.pageInfo?.hasPreviousPage, - startCursor: result.node?.comments?.pageInfo?.startCursor, - data: result.node?.comments?.nodes, - } - } -} - -export default PullRequestReviewThreadCommentsQuery diff --git a/backend/src/serverless/integrations/usecases/github/graphql/pullRequestReviewThreads.ts b/backend/src/serverless/integrations/usecases/github/graphql/pullRequestReviewThreads.ts deleted file mode 100644 index 258b780e20..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/pullRequestReviewThreads.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { Repo } from '../../../types/regularTypes' - -import BaseQuery from './baseQuery' - -/* eslint class-methods-use-this: 0 */ -class PullRequestReviewThreadsQuery extends BaseQuery { - repo: Repo - - constructor(repo: Repo, pullRequestNumber: string, githubToken: string, perPage: number = 100) { - const pullRequestReviewThreadsQuery = `{ - repository(name: "${repo.name}", owner: "${repo.owner}") { - pullRequest(number: ${pullRequestNumber}) { - id - reviewDecision - reviewThreads(first: ${perPage}, \${beforeCursor}) { - pageInfo ${BaseQuery.PAGE_SELECT} - nodes { - id - } - } - } - } - }` - - super(githubToken, pullRequestReviewThreadsQuery, 'pullRequestReviewThreads', perPage) - - this.repo = repo - } - - getEventData(result) { - return { - hasPreviousPage: result.repository?.pullRequest?.reviewThreads?.pageInfo?.hasPreviousPage, - startCursor: result.repository?.pullRequest?.reviewThreads?.pageInfo?.startCursor, - data: result.repository?.pullRequest?.reviewThreads?.nodes, - } - } -} - -export default PullRequestReviewThreadsQuery diff --git a/backend/src/serverless/integrations/usecases/github/graphql/pullRequests.ts b/backend/src/serverless/integrations/usecases/github/graphql/pullRequests.ts deleted file mode 100644 index 9a011c0f63..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/pullRequests.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { Repo } from '../../../types/regularTypes' - -import BaseQuery from './baseQuery' - -/* eslint class-methods-use-this: 0 */ -class PullRequestsQuery extends BaseQuery { - repo: Repo - - constructor(repo: Repo, githubToken: string, perPage: number = 20) { - const pullRequestsQuery = `{ - repository(owner: "${repo.owner}", name: "${repo.name}") { - pullRequests(last: ${perPage}, \${beforeCursor}) { - pageInfo ${BaseQuery.PAGE_SELECT} - nodes { - author { - ... on User ${BaseQuery.USER_SELECT} - } - bodyText - state - title - id - url - createdAt - number - additions - deletions - changedFiles - authorAssociation - labels(first: 10) { - nodes { - name - } - } - timelineItems( - first: 100 - itemTypes: [PULL_REQUEST_REVIEW, MERGED_EVENT, ASSIGNED_EVENT, REVIEW_REQUESTED_EVENT, CLOSED_EVENT] - ) { - nodes { - ... on ReviewRequestedEvent { - __typename - id - createdAt - actor { - ... on User ${BaseQuery.USER_SELECT} - } - requestedReviewer { - ... on User ${BaseQuery.USER_SELECT} - } - } - ... on PullRequestReview { - __typename - id - state - submittedAt - body - author { - ... on User ${BaseQuery.USER_SELECT} - } - } - ... on AssignedEvent { - __typename - id - assignee { - ... on User ${BaseQuery.USER_SELECT} - } - actor { - ... on User ${BaseQuery.USER_SELECT} - } - createdAt - } - ... on MergedEvent { - __typename - id - createdAt - actor { - ... on User ${BaseQuery.USER_SELECT} - } - createdAt - } - ... on ClosedEvent{ - __typename - id - actor { - ... on User ${BaseQuery.USER_SELECT} - } - createdAt - } - } - } - } - } - } - }` - - super(githubToken, pullRequestsQuery, 'pullRequests', perPage) - - this.repo = repo - } - - getEventData(result) { - return { ...super.getEventData(result), data: result.repository?.pullRequests?.nodes } - } -} - -export default PullRequestsQuery diff --git a/backend/src/serverless/integrations/usecases/github/graphql/stargazers.ts b/backend/src/serverless/integrations/usecases/github/graphql/stargazers.ts deleted file mode 100644 index 9df5e905cd..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/stargazers.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Repo } from '../../../types/regularTypes' - -import BaseQuery from './baseQuery' - -/* eslint class-methods-use-this: 0 */ -class StargazersQuery extends BaseQuery { - repo: Repo - - constructor(repo: Repo, githubToken: string, perPage: number = 100) { - const stargazersQuery = `{ - repository(owner: "${repo.owner}", name: "${repo.name}") { - stargazers(last: ${perPage}, \${beforeCursor}) { - pageInfo ${BaseQuery.PAGE_SELECT} - totalCount - edges { - starredAt - node ${BaseQuery.USER_SELECT} - } - } - } - }` - - super(githubToken, stargazersQuery, 'stargazers', perPage) - - this.repo = repo - } - - getEventData(result) { - return { ...super.getEventData(result), data: result.repository?.stargazers?.edges } - } -} - -export default StargazersQuery diff --git a/backend/src/serverless/integrations/usecases/github/graphql/teams.ts b/backend/src/serverless/integrations/usecases/github/graphql/teams.ts deleted file mode 100644 index 5878f4c5ea..0000000000 --- a/backend/src/serverless/integrations/usecases/github/graphql/teams.ts +++ /dev/null @@ -1,24 +0,0 @@ -import BaseQuery from './baseQuery' - -/* eslint class-methods-use-this: 0 */ -class TeamsQuery extends BaseQuery { - constructor(teamNodeId: string, githubToken: string, perPage: number = 50) { - const teamsQuery = `{ - node(id: "${teamNodeId}") { - ... on Team { - members { - nodes ${BaseQuery.USER_SELECT} - } - } - } - }` - - super(githubToken, teamsQuery, 'teams', perPage) - } - - getEventData(result) { - return { hasPreviousPage: false, startCursor: '', data: result.node?.members?.nodes } - } -} - -export default TeamsQuery diff --git a/backend/src/serverless/integrations/usecases/github/rest/getAppToken.ts b/backend/src/serverless/integrations/usecases/github/rest/getAppToken.ts deleted file mode 100644 index 15180a0a4b..0000000000 --- a/backend/src/serverless/integrations/usecases/github/rest/getAppToken.ts +++ /dev/null @@ -1,38 +0,0 @@ -import axios, { AxiosRequestConfig } from 'axios' - -import { getServiceChildLogger } from '@crowd/logging' - -const log = getServiceChildLogger('getAppToken') - -export interface AppTokenResponse { - token: string - expiresAt: string -} - -export const getAppToken = async ( - jwt: string, - installationId: number, -): Promise => { - try { - const config = { - method: 'post', - url: `https://api.github.com/app/installations/${installationId}/access_tokens`, - headers: { - Authorization: `Bearer ${jwt}`, - Accept: 'application/vnd.github+json', - }, - } as AxiosRequestConfig - - const response = await axios(config) - - const data = response.data as any - - return { - token: data.token, - expiresAt: data.expires_at, - } - } catch (err: any) { - log.error(err, { installationId }, 'Error fetching app token!') - throw err - } -} diff --git a/backend/src/serverless/integrations/usecases/github/rest/getInstalledRepositories.ts b/backend/src/serverless/integrations/usecases/github/rest/getInstalledRepositories.ts deleted file mode 100644 index 0a56c41806..0000000000 --- a/backend/src/serverless/integrations/usecases/github/rest/getInstalledRepositories.ts +++ /dev/null @@ -1,67 +0,0 @@ -import axios, { AxiosRequestConfig } from 'axios' - -import { getServiceChildLogger } from '@crowd/logging' - -import { GITHUB_CONFIG } from '../../../../../conf' -import { Repos } from '../../../types/regularTypes' - -const IS_GITHUB_COMMIT_DATA_ENABLED = GITHUB_CONFIG.isCommitDataEnabled === 'true' - -const log = getServiceChildLogger('getInstalledRepositories') - -const getRepositoriesFromGH = async (page: number, installToken: string): Promise => { - const REPOS_PER_PAGE = 100 - - const requestConfig = { - method: 'get', - url: `https://api.github.com/installation/repositories?page=${page}&per_page=${REPOS_PER_PAGE}`, - headers: { - Authorization: `Bearer ${installToken}`, - }, - } as AxiosRequestConfig - - const response = await axios(requestConfig) - return response.data -} - -const parseRepos = (repositories: any): Repos => { - const repos: Repos = [] - - for (const repo of repositories) { - repos.push({ - url: repo.html_url, - owner: repo.owner.login, - createdAt: repo.created_at, - name: repo.name, - fork: repo.fork, - private: repo.private, - cloneUrl: repo.clone_url, - }) - } - - return repos -} - -export const getInstalledRepositories = async (installToken: string): Promise => { - try { - let page = 1 - let hasMorePages = true - - const repos: Repos = [] - - while (hasMorePages) { - const data = await getRepositoriesFromGH(page, installToken) - - if (data.repositories) { - repos.push(...parseRepos(data.repositories)) - } - - hasMorePages = data.total_count && data.total_count > 0 && data.total_count > repos.length - page += 1 - } - return repos.filter((repo) => !IS_GITHUB_COMMIT_DATA_ENABLED || !(repo.fork || repo.private)) - } catch (err: any) { - log.error(err, 'Error fetching installed repositories!') - throw err - } -} diff --git a/backend/src/services/conversationService.ts b/backend/src/services/conversationService.ts index 88fba062d9..19d24ac7db 100644 --- a/backend/src/services/conversationService.ts +++ b/backend/src/services/conversationService.ts @@ -570,17 +570,22 @@ export default class ConversationService extends LoggerBase { for (const conversation of results.rows) { const data = conversation as any - data.activities = activities.rows - .filter((a) => a.conversationId === conversation.id) + + const firstActivity = single( + activities.rows, + (a) => a.conversationId === conversation.id && a.parentId === null, + ) + + const remainingActivities = activities.rows + .filter((a) => a.conversationId === conversation.id && a.parentId !== null) .sort( (a, b) => // from oldest to newest new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime(), ) - // TODO questdb: This should not be needed. Front-end must be updated to - // only get activities array. - data.conversationStarter = data.activities[0] + data.activities = [firstActivity, ...remainingActivities] + data.conversationStarter = data.activities[0] ?? null data.lastReplies = data.activities.slice(1) } diff --git a/backend/src/services/githubIntegrationService.ts b/backend/src/services/githubIntegrationService.ts new file mode 100644 index 0000000000..0a2db0fa07 --- /dev/null +++ b/backend/src/services/githubIntegrationService.ts @@ -0,0 +1,82 @@ +import { request } from '@octokit/request' + +import { GithubSnowflakeClient, SnowflakeClient } from '@crowd/snowflake' + +import { GITHUB_TOKEN_CONFIG } from '@/conf' + +import { IServiceOptions } from './IServiceOptions' + +export default class GithubIntegrationService { + constructor(private readonly options: IServiceOptions) {} + + public async getGithubRepositories(org: string) { + const client = SnowflakeClient.fromEnv() + this.options.log.info(`Getting GitHub repositories for org: ${org}`) + const githubClient = new GithubSnowflakeClient(client) + return githubClient.getOrgRepositories({ org, perPage: 10000 }) + } + + public async findGithubRepos(query: string) { + const auth = GITHUB_TOKEN_CONFIG.token + + const [orgRepos, repos] = await Promise.all([ + request('GET /search/repositories', { + q: `owner:${query}`, + headers: { + authorization: `bearer ${auth}`, + }, + }).catch((err) => { + this.options.log.error(`Error getting GitHub repositories for org: ${query}`, err) + return { data: { items: [] } } + }), + request('GET /search/repositories', { + q: query, + headers: { + authorization: `bearer ${auth}`, + }, + }).catch((err) => { + this.options.log.error(`Error getting GitHub repositories for org: ${query}`, err) + return { data: { items: [] } } + }), + ]) + + return [...orgRepos.data.items, ...repos.data.items].map((item) => ({ + name: item.name, + url: item.html_url, + org: { + name: item.owner.login, + url: item.owner.html_url, + logo: item.owner.avatar_url, + }, + })) + } + + public static async findOrgs(query: string) { + const auth = GITHUB_TOKEN_CONFIG.token + const response = await request('GET /search/users', { + q: query, + headers: { + authorization: `bearer ${auth}`, + }, + }) + return response.data.items.map((item) => ({ + name: item.login, + url: item.html_url, + logo: item.avatar_url, + })) + } + + public static async getOrgRepos(org: string) { + const auth = GITHUB_TOKEN_CONFIG.token + const response = await request('GET /orgs/{org}/repos', { + org, + headers: { + authorization: `bearer ${auth}`, + }, + }) + return response.data.map((repo) => ({ + name: repo.name, + url: repo.html_url, + })) + } +} diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 120c631c13..33e9e638d6 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -1,11 +1,10 @@ /* eslint-disable no-promise-executor-return */ import { createAppAuth } from '@octokit/auth-app' -import { request } from '@octokit/request' import axios, { AxiosRequestConfig, AxiosResponse } from 'axios' import lodash from 'lodash' import moment from 'moment' -import { EDITION, Error400, Error404, Error542 } from '@crowd/common' +import { EDITION, Error400, Error404 } from '@crowd/common' import { MemberField, findMemberById } from '@crowd/data-access-layer/src/members' import { HubspotEndpoint, @@ -24,7 +23,6 @@ import { RedisCache } from '@crowd/redis' import { Edition, PlatformType } from '@crowd/types' import { IRepositoryOptions } from '@/database/repositories/IRepositoryOptions' -import GithubInstallationsRepository from '@/database/repositories/githubInstallationsRepository' import GitlabReposRepository from '@/database/repositories/gitlabReposRepository' import IntegrationProgressRepository from '@/database/repositories/integrationProgressRepository' import MemberSyncRemoteRepository from '@/database/repositories/memberSyncRemoteRepository' @@ -47,6 +45,7 @@ import { import { DISCORD_CONFIG, GITHUB_CONFIG, + GITHUB_TOKEN_CONFIG, GITLAB_CONFIG, IS_TEST_ENV, KUBE_MODE, @@ -60,7 +59,6 @@ import TenantRepository from '../database/repositories/tenantRepository' import telemetryTrack from '../segment/telemetryTrack' import track from '../segment/track' import { ILinkedInOrganization } from '../serverless/integrations/types/linkedinTypes' -import { getInstalledRepositories } from '../serverless/integrations/usecases/github/rest/getInstalledRepositories' import { GitHubStats, getGitHubRemoteStats, @@ -399,175 +397,6 @@ export default class IntegrationService { return installationAuthentication.token } - /** - * Adds GitHub integration to a tenant and calls the onboarding SOA endpoint - * @param code Temporary code generated by GitHub after authorize - * @param installId Install id of the Crowd GitHub app - * @param setupAction - * @returns integration object - */ - async connectGithub(code, installId, setupAction = 'install') { - if (setupAction === 'request') { - return this.createOrUpdate( - { - platform: PlatformType.GITHUB, - status: 'waiting-approval', - }, - await SequelizeRepository.createTransaction(this.options), - ) - } - - const GITHUB_AUTH_ACCESSTOKEN_URL = 'https://github.com/login/oauth/access_token' - const CLIENT_ID = GITHUB_CONFIG.clientId - const CLIENT_SECRET = GITHUB_CONFIG.clientSecret - - const tokenResponse = await axios({ - method: 'post', - url: GITHUB_AUTH_ACCESSTOKEN_URL, - data: { - client_id: CLIENT_ID, - client_secret: CLIENT_SECRET, - code, - }, - }) - - let token = tokenResponse.data - token = token.slice(token.search('=') + 1, token.search('&')) - - try { - const requestWithAuth = request.defaults({ - headers: { - authorization: `token ${token}`, - }, - }) - await requestWithAuth('GET /user') - } catch { - throw new Error542( - `Invalid token for GitHub integration. Code: ${code}, setupAction: ${setupAction}. Token: ${token}`, - ) - } - - const installToken = await IntegrationService.getInstallToken(installId) - const repos = await getInstalledRepositories(installToken) - const githubOwner = IntegrationService.extractOwner(repos, this.options) - - let orgAvatar - try { - const response = await request('GET /users/{user}', { - user: githubOwner, - }) - orgAvatar = response.data.avatar_url - } catch (err) { - this.options.log.warn(err, 'Error while fetching GitHub user!') - } - - const integration = await this.createOrUpdateGithubIntegration( - { - platform: PlatformType.GITHUB, - token, - settings: { updateMemberAttributes: true, orgAvatar }, - integrationIdentifier: installId, - status: 'mapping', - }, - repos, - ) - - return integration - } - - async connectGithubInstallation(installId: string) { - const installToken = await IntegrationService.getInstallToken(installId) - const repos = await getInstalledRepositories(installToken) - const githubOwner = IntegrationService.extractOwner(repos, this.options) - - let orgAvatar - try { - const response = await request('GET /users/{user}', { - user: githubOwner, - }) - orgAvatar = response.data.avatar_url - } catch (err) { - this.options.log.warn(err, 'Error while fetching GitHub user!') - } - - const integration = await this.createOrUpdateGithubIntegration( - { - platform: PlatformType.GITHUB, - token: installToken, - settings: { updateMemberAttributes: true, orgAvatar }, - integrationIdentifier: installId, - status: 'mapping', - }, - repos, - ) - - return integration - } - - async getGithubInstallations() { - return GithubInstallationsRepository.getInstallations(this.options) - } - - /** - * Creates or updates a GitHub integration, handling large repos data - * @param integrationData The integration data to create or update - * @param repos The repositories data - */ - private async createOrUpdateGithubIntegration(integrationData, repos) { - let integration - const transaction = await SequelizeRepository.createTransaction(this.options) - - this.options.log.error(repos.length) - - try { - // First, create or update the integration without the repos data - integration = await this.createOrUpdate(integrationData, transaction) - - await SequelizeRepository.commitTransaction(transaction) - } catch (err) { - await SequelizeRepository.rollbackTransaction(transaction) - throw err - } - - // Then, update the repos data in chunks to avoid query timeout - const chunkSize = 100 // Adjust this value based on your specific needs - for (let i = 0; i < repos.length; i += chunkSize) { - const reposChunk = repos.slice(i, i + chunkSize) - await this.upsertGitHubRepos(integration.id, reposChunk) - } - - return integration - } - - private async upsertGitHubRepos(integrationId, repos) { - const transaction = await SequelizeRepository.createTransaction(this.options) - const sequelize = SequelizeRepository.getSequelize(this.options) - - try { - const query = ` - UPDATE integrations - SET settings = jsonb_set( - COALESCE(settings, '{}'::jsonb), - '{repos}', - COALESCE(settings->'repos', '[]'::jsonb) || ?::jsonb - ) - WHERE id = ? - ` - - const values = [JSON.stringify(repos), integrationId] - - await sequelize.query(query, { - replacements: values, - transaction, - }) - - await SequelizeRepository.commitTransaction(transaction) - } catch (error) { - await SequelizeRepository.rollbackTransaction(transaction) - throw error - } - } - static extractOwner(repos, options) { const owners = lodash.countBy(repos, 'owner') @@ -581,7 +410,7 @@ export default class IntegrationService { return lodash.maxBy(Object.keys(owners), (owner) => owners[owner]) } - async mapGithubRepos(integrationId, mapping, fireOnboarding = true) { + async mapGithubRepos(integrationId, mapping, fireOnboarding = true, isUpdateTransaction = false) { const transaction = await SequelizeRepository.createTransaction(this.options) const txOptions = { @@ -589,23 +418,119 @@ export default class IntegrationService { transaction, } + let reposToAdd: Record = {} + let reposToRemove: Record = {} + let urlsWithChangedSegment: { url: string; oldSegmentId: string; newSegmentId: unknown }[] = [] + try { - await GithubReposRepository.updateMapping(integrationId, mapping, txOptions) + const oldMapping = await GithubReposRepository.getMapping(integrationId, txOptions) + await GithubReposRepository.updateMapping(integrationId, mapping, oldMapping, txOptions) // add the repos to the git integration if (EDITION === Edition.LFX) { - const repos: Record = Object.entries(mapping).reduce( - (acc, [url, segmentId]) => { - if (!acc[segmentId as string]) { - acc[segmentId as string] = [] - } - acc[segmentId as string].push(url) - return acc - }, - {}, - ) + // Convert old mapping to URL -> segmentId format + const oldMappingDict = oldMapping.reduce((acc, item) => { + acc[item.url] = item.segment.id + return acc + }, {}) + + // Find URLs to add, remove and those that changed segments + const urlsToAdd = Object.entries(mapping).filter(([url]) => !oldMappingDict[url]) + const urlsToRemove = oldMapping + .filter((item) => !mapping[item.url]) + .map((item) => ({ url: item.url, segmentId: item.segment.id })) + urlsWithChangedSegment = Object.entries(mapping) + .filter(([url, segmentId]) => oldMappingDict[url] && oldMappingDict[url] !== segmentId) + .map(([url, newSegmentId]) => ({ + url, + oldSegmentId: oldMappingDict[url], + newSegmentId, + })) + + // Group new URLs by segment + reposToAdd = urlsToAdd.reduce((acc, [url, segmentId]) => { + if (!acc[segmentId as string]) { + acc[segmentId as string] = [] + } + acc[segmentId as string].push(url) + return acc + }, {}) + + // Group URLs to remove by segment + reposToRemove = urlsToRemove.reduce((acc, { url, segmentId }) => { + if (!acc[segmentId]) { + acc[segmentId] = [] + } + acc[segmentId].push(url) + return acc + }, {}) + + // Handle segment changes + for (const { url, oldSegmentId, newSegmentId } of urlsWithChangedSegment) { + // Remove from old segment + const oldSegmentOptions: IRepositoryOptions = { + ...this.options, + currentSegments: [ + { + ...this.options.currentSegments[0], + id: oldSegmentId, + }, + ], + } - for (const [segmentId, urls] of Object.entries(repos)) { + try { + const oldGitInfo = await this.gitGetRemotes(oldSegmentOptions) + const oldGitRemotes = oldGitInfo[oldSegmentId].remotes + await this.gitConnectOrUpdate( + { + remotes: oldGitRemotes.filter((remote) => remote !== url), + }, + oldSegmentOptions, + ) + } catch (err) { + this.options.log.error(err, `Failed to remove repo from old segment ${oldSegmentId}`) + } + + // Add to new segment + const newSegmentOptions: IRepositoryOptions = { + ...this.options, + currentSegments: [ + { + ...this.options.currentSegments[0], + id: newSegmentId as string, + }, + ], + } + + let isGitIntegrationConfigured + try { + await IntegrationRepository.findByPlatform(PlatformType.GIT, newSegmentOptions) + isGitIntegrationConfigured = true + } catch (err) { + isGitIntegrationConfigured = false + } + + if (isGitIntegrationConfigured) { + const gitInfo = await this.gitGetRemotes(newSegmentOptions) + const gitRemotes = gitInfo[newSegmentId as string].remotes + await this.gitConnectOrUpdate( + { + remotes: Array.from(new Set([...gitRemotes, url])), + }, + newSegmentOptions, + ) + } else { + await this.gitConnectOrUpdate( + { + remotes: [url], + }, + newSegmentOptions, + ) + } + } + + // Handle additions + for (const [segmentId, urls] of Object.entries(reposToAdd)) { let isGitintegrationConfigured const segmentOptions: IRepositoryOptions = { ...this.options, @@ -618,7 +543,6 @@ export default class IntegrationService { } try { await IntegrationRepository.findByPlatform(PlatformType.GIT, segmentOptions) - isGitintegrationConfigured = true } catch (err) { isGitintegrationConfigured = false @@ -642,9 +566,40 @@ export default class IntegrationService { ) } } + + // Handle removals + for (const [segmentId, urls] of Object.entries(reposToRemove)) { + const segmentOptions: IRepositoryOptions = { + ...this.options, + currentSegments: [ + { + ...this.options.currentSegments[0], + id: segmentId, + }, + ], + } + + try { + const gitInfo = await this.gitGetRemotes(segmentOptions) + const gitRemotes = gitInfo[segmentId].remotes + const remainingRemotes = gitRemotes.filter((remote) => !urls.includes(remote)) + + await this.gitConnectOrUpdate( + { + remotes: remainingRemotes, + }, + segmentOptions, + ) + } catch (err) { + this.options.log.error(err, `Failed to remove repos for segment ${segmentId}`) + } + } } - if (fireOnboarding) { + const hasAddedRepos = Object.values(reposToAdd).some((urls) => urls.length > 0) + const hasChangedSegments = urlsWithChangedSegment.length > 0 + + if (fireOnboarding && (hasAddedRepos || hasChangedSegments)) { const integration = await IntegrationRepository.update( integrationId, { status: 'in-progress' }, @@ -661,6 +616,13 @@ export default class IntegrationService { integration.platform, integration.id, true, + null, + null, + isUpdateTransaction ? { messageSentAt: new Date().toISOString() } : null, + ) + } else if (hasAddedRepos || hasChangedSegments) { + this.options.log.debug( + 'No onboarding message sent because no repos to add or change segments for!', ) } @@ -2040,11 +2002,13 @@ export default class IntegrationService { } } - const githubToken = await IntegrationService.getInstallToken( - integration.integrationIdentifier, - ) + const githubToken = GITHUB_TOKEN_CONFIG.token - const repos = await getInstalledRepositories(githubToken) + const repos = integration.settings.orgs.flatMap((org) => org.repos) as { + url: string + name: string + updatedAt: string + }[] const cacheRemote = new RedisCache( 'github-progress-remote', this.options.redis, @@ -2451,73 +2415,4 @@ export default class IntegrationService { throw err } } - - async updateGithubIntegrationSettings(installId: string) { - this.options.log.info(`Updating GitHub integration settings for installId: ${installId}`) - - // Find the integration by installId - const integration: any = await IntegrationRepository.findByIdentifier( - installId, - PlatformType.GITHUB, - ) - - if (!integration || integration.platform !== PlatformType.GITHUB) { - this.options.log.warn(`GitHub integration not found for installId: ${installId}`) - throw new Error404('GitHub integration not found') - } - - this.options.log.info(`Found integration: ${integration.id}`) - - // Get the install token - const installToken = await IntegrationService.getInstallToken(installId) - this.options.log.info(`Obtained install token for installId: ${installId}`) - - // Fetch all installed repositories - const repos = await getInstalledRepositories(installToken) - this.options.log.info(`Fetched ${repos.length} installed repositories`) - - // Update integration settings - const currentSettings = integration.settings || {} - const currentRepos = currentSettings.repos || [] - const newRepos = repos.filter((repo) => !currentRepos.some((r) => r.url === repo.url)) - this.options.log.info(`Found ${newRepos.length} new repositories`) - - const updatedSettings = { - ...currentSettings, - repos: [...currentRepos, ...newRepos], - } - - this.options = { - ...this.options, - currentSegments: [ - { - id: integration.segmentId, - } as any, - ], - } - - // Update the integration with new settings - await this.update(integration.id, { settings: updatedSettings }) - - this.options.log.info(`Updated integration settings for integration id: ${integration.id}`) - - // Update GitHub repos mapping - const defaultSegmentId = integration.segmentId - const mapping = {} - for (const repo of newRepos) { - mapping[repo.url] = defaultSegmentId - } - if (Object.keys(mapping).length > 0) { - // false - not firing onboarding - await this.mapGithubRepos(integration.id, mapping, false) - this.options.log.info(`Updated GitHub repos mapping for integration id: ${integration.id}`) - } else { - this.options.log.info(`No new repos to map for integration id: ${integration.id}`) - } - - this.options.log.info( - `Completed updating GitHub integration settings for installId: ${installId}`, - ) - return integration - } } diff --git a/frontend/public/images/integrations/github-search.png b/frontend/public/images/integrations/github-search.png new file mode 100644 index 0000000000..edf809c731 Binary files /dev/null and b/frontend/public/images/integrations/github-search.png differ diff --git a/frontend/src/config/integrations/github/components/connect/github-connect-finishing-modal.vue b/frontend/src/config/integrations/github/components/connect/github-connect-finishing-modal.vue deleted file mode 100644 index b90681e324..0000000000 --- a/frontend/src/config/integrations/github/components/connect/github-connect-finishing-modal.vue +++ /dev/null @@ -1,39 +0,0 @@ - - - - - diff --git a/frontend/src/config/integrations/github/components/connect/github-connect-modal.vue b/frontend/src/config/integrations/github/components/connect/github-connect-modal.vue deleted file mode 100644 index 41f50df2d7..0000000000 --- a/frontend/src/config/integrations/github/components/connect/github-connect-modal.vue +++ /dev/null @@ -1,220 +0,0 @@ - - - - - diff --git a/frontend/src/config/integrations/github/components/connect/github-connect.vue b/frontend/src/config/integrations/github/components/connect/github-connect.vue index bb6a355448..de2e6f2009 100644 --- a/frontend/src/config/integrations/github/components/connect/github-connect.vue +++ b/frontend/src/config/integrations/github/components/connect/github-connect.vue @@ -29,10 +29,6 @@ import LfButton from '@/ui-kit/button/Button.vue'; import LfIcon from '@/ui-kit/icon/Icon.vue'; import { useRoute } from 'vue-router'; import { mapActions } from '@/shared/vuex/vuex.helpers'; -import LfGithubConnectFinishingModal - from '@/config/integrations/github/components/connect/github-connect-finishing-modal.vue'; -// import LfGithubDetailsModal from '@/config/integrations/github/components/github-details-modal.vue'; -import LfGithubConnectModal from './github-connect-modal.vue'; const route = useRoute(); const { doGithubConnect } = mapActions('integration'); diff --git a/frontend/src/config/integrations/github/components/github-action.vue b/frontend/src/config/integrations/github/components/github-action.vue deleted file mode 100644 index a661f18b09..0000000000 --- a/frontend/src/config/integrations/github/components/github-action.vue +++ /dev/null @@ -1,30 +0,0 @@ - - - - - diff --git a/frontend/src/config/integrations/github/components/github-connect.vue b/frontend/src/config/integrations/github/components/github-connect.vue new file mode 100644 index 0000000000..6fe24b502f --- /dev/null +++ b/frontend/src/config/integrations/github/components/github-connect.vue @@ -0,0 +1,30 @@ + + + + + diff --git a/frontend/src/config/integrations/github/components/github-dropdown.vue b/frontend/src/config/integrations/github/components/github-dropdown.vue new file mode 100644 index 0000000000..900f8be6a4 --- /dev/null +++ b/frontend/src/config/integrations/github/components/github-dropdown.vue @@ -0,0 +1,30 @@ + + + + + diff --git a/frontend/src/config/integrations/github/components/github-status.vue b/frontend/src/config/integrations/github/components/github-status.vue deleted file mode 100644 index 66d2b9d4af..0000000000 --- a/frontend/src/config/integrations/github/components/github-status.vue +++ /dev/null @@ -1,23 +0,0 @@ - - - - - diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-add-repository-modal.vue b/frontend/src/config/integrations/github/components/settings/github-settings-add-repository-modal.vue new file mode 100644 index 0000000000..771b14d064 --- /dev/null +++ b/frontend/src/config/integrations/github/components/settings/github-settings-add-repository-modal.vue @@ -0,0 +1,263 @@ + + + + + diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-drawer.vue b/frontend/src/config/integrations/github/components/settings/github-settings-drawer.vue index d2402ab47f..e9f8ac77b8 100644 --- a/frontend/src/config/integrations/github/components/settings/github-settings-drawer.vue +++ b/frontend/src/config/integrations/github/components/settings/github-settings-drawer.vue @@ -1,200 +1,121 @@ diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-empty.vue b/frontend/src/config/integrations/github/components/settings/github-settings-empty.vue new file mode 100644 index 0000000000..4657322b66 --- /dev/null +++ b/frontend/src/config/integrations/github/components/settings/github-settings-empty.vue @@ -0,0 +1,31 @@ + + + + + diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-mapping.vue b/frontend/src/config/integrations/github/components/settings/github-settings-mapping.vue new file mode 100644 index 0000000000..7b996026aa --- /dev/null +++ b/frontend/src/config/integrations/github/components/settings/github-settings-mapping.vue @@ -0,0 +1,146 @@ + + + + + diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-org-item.vue b/frontend/src/config/integrations/github/components/settings/github-settings-org-item.vue new file mode 100644 index 0000000000..a3412a6f8d --- /dev/null +++ b/frontend/src/config/integrations/github/components/settings/github-settings-org-item.vue @@ -0,0 +1,129 @@ + + + + + diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-repo-item.vue b/frontend/src/config/integrations/github/components/settings/github-settings-repo-item.vue new file mode 100644 index 0000000000..22e0a3d719 --- /dev/null +++ b/frontend/src/config/integrations/github/components/settings/github-settings-repo-item.vue @@ -0,0 +1,110 @@ + + + + + diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-bulk-select.vue b/frontend/src/config/integrations/github/components/settings/github-settings-repositories-bulk-select.vue similarity index 98% rename from frontend/src/config/integrations/github/components/settings/github-settings-bulk-select.vue rename to frontend/src/config/integrations/github/components/settings/github-settings-repositories-bulk-select.vue index 4028814ac8..142ad2603c 100644 --- a/frontend/src/config/integrations/github/components/settings/github-settings-bulk-select.vue +++ b/frontend/src/config/integrations/github/components/settings/github-settings-repositories-bulk-select.vue @@ -142,6 +142,6 @@ const mapRepos = () => { diff --git a/frontend/src/config/integrations/github/config.ts b/frontend/src/config/integrations/github/config.ts index c15137ba68..ef0a6854f0 100644 --- a/frontend/src/config/integrations/github/config.ts +++ b/frontend/src/config/integrations/github/config.ts @@ -1,8 +1,7 @@ import { IntegrationConfig } from '@/config/integrations'; -import GithubConnect from './components/connect/github-connect.vue'; -import GithubStatus from './components/github-status.vue'; -import GithubAction from './components/github-action.vue'; +import GithubConnect from './components/github-connect.vue'; import GithubParams from './components/github-params.vue'; +import GithubDropdown from './components/github-dropdown.vue'; const github: IntegrationConfig = { key: 'github', @@ -10,8 +9,7 @@ const github: IntegrationConfig = { image: '/images/integrations/github.png', description: 'Connect GitHub to sync profile information, stars, forks, pull requests, issues, and discussions.', connectComponent: GithubConnect, - statusComponent: GithubStatus, - actionComponent: GithubAction, + dropdownComponent: GithubDropdown, connectedParamsComponent: GithubParams, }; diff --git a/frontend/src/config/integrations/github/services/github.api.service.ts b/frontend/src/config/integrations/github/services/github.api.service.ts new file mode 100644 index 0000000000..af09c021ac --- /dev/null +++ b/frontend/src/config/integrations/github/services/github.api.service.ts @@ -0,0 +1,45 @@ +import authAxios from '@/shared/axios/auth-axios'; +import { AuthService } from '@/modules/auth/services/auth.service'; +import { GitHubOrganization, GitHubRepository } from '@/config/integrations/github/types/GithubSettings'; + +export class GithubApiService { + static async searchRepositories(query: string): Promise { + const tenantId = AuthService.getTenantId(); + + const response = await authAxios.get( + `/tenant/${tenantId}/integration/github/search/repos`, + { + params: { + query, + }, + }, + ); + + return response.data; + } + + static async searchOrganizations(query: string): Promise { + const tenantId = AuthService.getTenantId(); + + const response = await authAxios.get( + `/tenant/${tenantId}/integration/github/search/orgs`, + { + params: { + query, + }, + }, + ); + + return response.data; + } + + static async getOrganizationRepositories(name: string): Promise { + const tenantId = AuthService.getTenantId(); + + const response = await authAxios.get( + `/tenant/${tenantId}/integration/github/orgs/${name}/repos`, + ); + + return response.data; + } +} diff --git a/frontend/src/config/integrations/github/types/GithubSettings.ts b/frontend/src/config/integrations/github/types/GithubSettings.ts new file mode 100644 index 0000000000..264bec30e5 --- /dev/null +++ b/frontend/src/config/integrations/github/types/GithubSettings.ts @@ -0,0 +1,25 @@ +export interface GitHubOrganization { + logo: string; + name: string; + url: string; + updatedAt?: string; +} + +export interface GitHubRepository { + name: string; + url: string; + org?: GitHubOrganization; +} + +export interface GitHubSettingsRepository extends GitHubRepository { + updatedAt?: string; +} + +export interface GitHubSettingsOrganization extends GitHubOrganization { + fullSync: boolean; + repos: GitHubSettingsRepository[]; +} +export interface GitHubSettings { + orgs: GitHubSettingsOrganization[]; + updateMemberAttributes: boolean; +} diff --git a/frontend/src/modules/admin/modules/integration/types/Integration.ts b/frontend/src/modules/admin/modules/integration/types/Integration.ts new file mode 100644 index 0000000000..6ad68c2214 --- /dev/null +++ b/frontend/src/modules/admin/modules/integration/types/Integration.ts @@ -0,0 +1,5 @@ +export interface Integration{ + id: string; + status: string; + settings: Settings; +} diff --git a/frontend/src/modules/integration/components/integration-progress-wrapper.vue b/frontend/src/modules/integration/components/integration-progress-wrapper.vue index 474b3ea7ca..615e6e0013 100644 --- a/frontend/src/modules/integration/components/integration-progress-wrapper.vue +++ b/frontend/src/modules/integration/components/integration-progress-wrapper.vue @@ -8,6 +8,7 @@ import { } from 'vue'; import { IntegrationService } from '@/modules/integration/integration-service'; import { IntegrationProgress } from '@/modules/integration/types/IntegrationProgress'; +import { useTimeoutPoll } from '@vueuse/core'; const props = withDefaults(defineProps<{ interval?: number @@ -20,36 +21,34 @@ const props = withDefaults(defineProps<{ const progress = ref(null); const progressError = ref(false); -const intervalInstance = ref(null); - const fetchUpdates = () => { IntegrationService.fetchIntegrationsProgress(props.segments) .then((data: IntegrationProgress) => { progress.value = data; if (data.length === 0) { - clearInterval(intervalInstance.value); + pause(); } }) .catch(() => { progress.value = null; progressError.value = true; - clearInterval(intervalInstance.value); + pause(); }); }; +const { pause, resume } = useTimeoutPoll(fetchUpdates, props.interval * 1000); + watch(() => props.segments, () => { fetchUpdates(); }, { deep: true }); onMounted(() => { - fetchUpdates(); - intervalInstance.value = setInterval(() => { - fetchUpdates(); - }, props.interval * 1000); + console.log('Component mounted'); + resume(); }); onUnmounted(() => { - clearInterval(intervalInstance.value); + pause(); }); diff --git a/frontend/src/modules/integration/integration-service.js b/frontend/src/modules/integration/integration-service.js index b9daffc4ae..e60bc24436 100644 --- a/frontend/src/modules/integration/integration-service.js +++ b/frontend/src/modules/integration/integration-service.js @@ -7,7 +7,7 @@ const getSegments = () => ({ segments: [router.currentRoute.value.params.id] }); export class IntegrationService { static async update(id, data) { const body = { - data, + ...data, ...getSegments(), }; @@ -38,7 +38,7 @@ export class IntegrationService { static async create(data) { const body = { - data, + ...data, ...getSegments(), }; @@ -147,13 +147,14 @@ export class IntegrationService { return response.data; } - static async githubMapRepos(integrationId, mapping, segments) { + static async githubMapRepos(integrationId, mapping, segments, isUpdateTransaction = false) { const tenantId = AuthService.getTenantId(); const response = await authAxios.put( `/tenant/${tenantId}/integration/${integrationId}/github/repos`, { mapping, segments, + isUpdateTransaction, }, ); return response.data; diff --git a/frontend/src/modules/integration/integration-store.js b/frontend/src/modules/integration/integration-store.js index 8271159cac..087db1d5bd 100644 --- a/frontend/src/modules/integration/integration-store.js +++ b/frontend/src/modules/integration/integration-store.js @@ -216,7 +216,7 @@ export default { } }, - async doDestroy({ commit }, integrationId) { + async doDestroy({ commit, dispatch }, integrationId) { try { commit('DESTROY_STARTED'); @@ -224,6 +224,7 @@ export default { Message.success('Integration was disconnected successfully'); commit('DESTROY_SUCCESS', integrationId); + dispatch('doFetch'); } catch (error) { Errors.handle(error); commit('DESTROY_ERROR'); diff --git a/frontend/src/ui-kit/drawer/Drawer.stories.ts b/frontend/src/ui-kit/drawer/Drawer.stories.ts new file mode 100644 index 0000000000..06ccbb21f8 --- /dev/null +++ b/frontend/src/ui-kit/drawer/Drawer.stories.ts @@ -0,0 +1,49 @@ +import LfDrawer from './Drawer.vue'; + +export default { + title: 'LinuxFoundation/Drawer', + component: LfDrawer, + tags: ['autodocs'], + argTypes: { + // Props + modelValue: { + description: 'Is drawer open', + defaultValue: 'false', + control: 'boolean', + }, + + width: { + description: 'Specifies drawer width', + defaultValue: '37.5rem', + control: 'text', + }, + + closeFunction: { + description: 'Function that prevents drawer from closing', + control: { + type: null, + }, + }, + + // Slots + default: { + description: 'Any content belonging to drawer', + control: { + type: null, + }, + }, + + // Events + 'update:modelValue': { + description: 'Event triggered when drawer open state changes', + control: { + type: null, + }, + }, + }, +}; + +export const Regular = { + args: { + }, +}; diff --git a/frontend/src/ui-kit/drawer/Drawer.vue b/frontend/src/ui-kit/drawer/Drawer.vue new file mode 100644 index 0000000000..d897bdfe89 --- /dev/null +++ b/frontend/src/ui-kit/drawer/Drawer.vue @@ -0,0 +1,66 @@ + + + + + diff --git a/frontend/src/ui-kit/drawer/drawer.scss b/frontend/src/ui-kit/drawer/drawer.scss new file mode 100644 index 0000000000..980961d035 --- /dev/null +++ b/frontend/src/ui-kit/drawer/drawer.scss @@ -0,0 +1,11 @@ +.c-drawer { + @apply fixed top-0 left-0 w-full h-full z-40; + background: rgba(0, 0, 0, 0.1); + + .c-drawer__content { + @apply bg-white h-full absolute top-0 right-0 w-full; + box-shadow: -4px 0px 10px 0px #0000000D; + } +} + + diff --git a/frontend/src/ui-kit/index.scss b/frontend/src/ui-kit/index.scss index 3b2c244263..da22da20a8 100644 --- a/frontend/src/ui-kit/index.scss +++ b/frontend/src/ui-kit/index.scss @@ -3,6 +3,7 @@ @import 'button/button'; @import 'card/card'; @import 'checkbox/checkbox'; +@import 'drawer/drawer'; @import 'dropdown/dropdown'; @import 'field/field'; @import 'icon/icon'; diff --git a/frontend/src/ui-kit/search/Search.vue b/frontend/src/ui-kit/search/Search.vue index 3e74daa0b9..05c19ce801 100644 --- a/frontend/src/ui-kit/search/Search.vue +++ b/frontend/src/ui-kit/search/Search.vue @@ -5,7 +5,7 @@ @@ -24,7 +24,7 @@ const props = defineProps<{ const emit = defineEmits<{(e: 'update:modelValue', value: string | number): any }>(); -const valueProxy = ref(props.modelValue); +const valueProxy = ref(props.modelValue); const emitValue = (value: string | number) => emit('update:modelValue', value); @@ -37,6 +37,10 @@ watch(valueProxy, (newVal) => { emitValue(newVal); } }); + +watch(() => props.modelValue, (newVal) => { + valueProxy.value = newVal; +});