diff --git a/backend/config/custom-environment-variables.json b/backend/config/custom-environment-variables.json index 739611c36e..470fb3c8ae 100644 --- a/backend/config/custom-environment-variables.json +++ b/backend/config/custom-environment-variables.json @@ -104,7 +104,8 @@ "callbackUrl": "CROWD_GITHUB_CALLBACK_URL", "privateKey": "CROWD_GITHUB_PRIVATE_KEY", "webhookSecret": "CROWD_GITHUB_WEBHOOK_SECRET", - "isCommitDataEnabled": "CROWD_GITHUB_IS_COMMIT_DATA_ENABLED" + "isCommitDataEnabled": "CROWD_GITHUB_IS_COMMIT_DATA_ENABLED", + "isSnowflakeEnabled": "CROWD_GITHUB_IS_SNOWFLAKE_ENABLED" }, "githubIssueReporter": { "appId": "CROWD_GITHUB_ISSUE_REPORTER_APP_ID", diff --git a/backend/package.json b/backend/package.json index 85bd759f52..f228fe9c6b 100644 --- a/backend/package.json +++ b/backend/package.json @@ -32,7 +32,8 @@ "script:purge-tenants-and-data": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/purge-tenants-and-data.ts", "script:import-lfx-memberships": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/import-lfx-memberships.ts", "script:fix-missing-org-displayName": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/fix-missing-org-displayName.ts", - "script:syncActivities": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/syncActivities.ts" + "script:syncActivities": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/syncActivities.ts", + "script:refreshGithubRepoSettings": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/refresh-github-repo-settings.ts" }, "dependencies": { "@aws-sdk/client-comprehend": "^3.159.0", diff --git a/backend/src/api/integration/helpers/githubAuthenticate.ts b/backend/src/api/integration/helpers/githubAuthenticate.ts new file mode 100644 index 0000000000..36b3d6e242 --- /dev/null +++ b/backend/src/api/integration/helpers/githubAuthenticate.ts @@ -0,0 +1,13 @@ +import Permissions from '../../../security/permissions' +import IntegrationService from '../../../services/integrationService' +import PermissionChecker from '../../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) + const payload = await new IntegrationService(req).connectGithub( + req.params.code, + req.body.installId, + req.body.setupAction, + ) + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/integration/helpers/githubConnectInstallation.ts b/backend/src/api/integration/helpers/githubConnectInstallation.ts new file mode 100644 index 0000000000..d7da9b0f43 --- /dev/null +++ b/backend/src/api/integration/helpers/githubConnectInstallation.ts @@ -0,0 +1,9 @@ +import Permissions from '../../../security/permissions' +import IntegrationService from '../../../services/integrationService' +import PermissionChecker from '../../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) + const payload = await new IntegrationService(req).connectGithubInstallation(req.body.installId) + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/integration/helpers/githubGetInstallations.ts b/backend/src/api/integration/helpers/githubGetInstallations.ts new file mode 100644 index 0000000000..08448c107d --- /dev/null +++ b/backend/src/api/integration/helpers/githubGetInstallations.ts @@ -0,0 +1,9 @@ +import Permissions from '../../../security/permissions' +import IntegrationService from '../../../services/integrationService' +import PermissionChecker from '../../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) + const payload = await new IntegrationService(req).getGithubInstallations() + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/integration/helpers/githubMapRepos.ts b/backend/src/api/integration/helpers/githubMapRepos.ts index b76e43da7b..33487b15b3 100644 --- a/backend/src/api/integration/helpers/githubMapRepos.ts +++ b/backend/src/api/integration/helpers/githubMapRepos.ts @@ -1,14 +1,27 @@ +import { GITHUB_CONFIG } from '@/conf' + import Permissions from '../../../security/permissions' import IntegrationService from '../../../services/integrationService' import PermissionChecker from '../../../services/user/permissionChecker' +const isSnowflakeEnabled = GITHUB_CONFIG.isSnowflakeEnabled === 'true' + export default async (req, res) => { new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) - const payload = await new IntegrationService(req).mapGithubRepos( - req.params.id, - req.body.mapping, - true, - req.body?.isUpdateTransaction ?? false, - ) + let payload + if (isSnowflakeEnabled) { + payload = await new IntegrationService(req).mapGithubReposSnowflake( + req.params.id, + req.body.mapping, + true, + req.body?.isUpdateTransaction ?? false, + ) + } else { + payload = await new IntegrationService(req).mapGithubRepos( + req.params.id, + req.body.mapping, + true, + ) + } await req.responseHandler.success(req, res, payload) } diff --git a/backend/src/api/integration/index.ts b/backend/src/api/integration/index.ts index 401d204010..fbc3f874b4 100644 --- a/backend/src/api/integration/index.ts +++ b/backend/src/api/integration/index.ts @@ -26,9 +26,22 @@ export default (app) => { app.get(`/integration/autocomplete`, safeWrap(require('./integrationAutocomplete').default)) app.get(`/integration/global`, safeWrap(require('./integrationGlobal').default)) app.get(`/integration/global/status`, safeWrap(require('./integrationGlobalStatus').default)) + + app.get( + '/integration/github-installations', + safeWrap(require('./helpers/githubGetInstallations').default), + ) + + app.post( + '/integration/github-connect-installation', + safeWrap(require('./helpers/githubConnectInstallation').default), + ) + app.get(`/integration`, safeWrap(require('./integrationList').default)) app.get(`/integration/:id`, safeWrap(require('./integrationFind').default)) + app.put(`/authenticate/:code`, safeWrap(require('./helpers/githubAuthenticate').default)) + app.put(`/integration/:id/github/repos`, safeWrap(require('./helpers/githubMapRepos').default)) app.get(`/integration/:id/github/repos`, safeWrap(require('./helpers/githubMapReposGet').default)) app.get( diff --git a/backend/src/bin/jobs/refreshGithubRepoSettings.ts b/backend/src/bin/jobs/refreshGithubRepoSettings.ts index 5f7070221a..383182cd80 100644 --- a/backend/src/bin/jobs/refreshGithubRepoSettings.ts +++ b/backend/src/bin/jobs/refreshGithubRepoSettings.ts @@ -4,12 +4,15 @@ import cronGenerator from 'cron-time-generator' import { timeout } from '@crowd/common' import { getServiceChildLogger } from '@crowd/logging' +import { GITHUB_CONFIG } from '../../conf' import GithubReposRepository from '../../database/repositories/githubReposRepository' import SequelizeRepository from '../../database/repositories/sequelizeRepository' import GithubIntegrationService from '../../services/githubIntegrationService' import IntegrationService from '../../services/integrationService' import { CrowdJob } from '../../types/jobTypes' +const IS_GITHUB_SNOWFLAKE_ENABLED = GITHUB_CONFIG.isSnowflakeEnabled === 'true' + const log = getServiceChildLogger('refreshGithubRepoSettings') interface Integration { @@ -33,8 +36,7 @@ interface Integration { } } -export const refreshGithubRepoSettings = async () => { - log.info('Updating Github repo settings.') +const refreshForSnowflake = async () => { const dbOptions = await SequelizeRepository.getDefaultIRepositoryOptions() const githubIntegrations = await dbOptions.database.sequelize.query( @@ -124,7 +126,7 @@ export const refreshGithubRepoSettings = async () => { } // Map new repos - await integrationService.mapGithubRepos( + await integrationService.mapGithubReposSnowflake( integration.id, newFullMapping, true, @@ -150,6 +152,56 @@ export const refreshGithubRepoSettings = async () => { log.info('Finished updating Github repo settings.') } +const refreshForGitHub = async () => { + log.info('Updating Github repo settings.') + const dbOptions = await SequelizeRepository.getDefaultIRepositoryOptions() + + interface Integration { + id: string + tenantId: string + integrationIdentifier: string + } + + const githubIntegrations = await dbOptions.database.sequelize.query( + `SELECT id, "tenantId", "integrationIdentifier" FROM integrations + WHERE platform = 'github' AND "deletedAt" IS NULL + AND "createdAt" < NOW() - INTERVAL '1 minute' AND "integrationIdentifier" IS NOT NULL`, + ) + + for (const integration of githubIntegrations[0] as Integration[]) { + log.info(`Updating repo settings for Github integration: ${integration.id}`) + + try { + const options = await SequelizeRepository.getDefaultIRepositoryOptions() + options.currentTenant = { id: integration.tenantId } + + const integrationService = new IntegrationService(options) + // newly discovered repos will be mapped to default segment of the integration + await integrationService.updateGithubIntegrationSettings(integration.integrationIdentifier) + + log.info(`Successfully updated repo settings for Github integration: ${integration.id}`) + } catch (err) { + log.error( + `Error updating repo settings for Github integration ${integration.id}: ${err.message}`, + ) + } finally { + await timeout(1000) + } + } + + log.info('Finished updating Github repo settings.') +} + +export const refreshGithubRepoSettings = async () => { + log.info('Updating Github repo settings.') + + if (IS_GITHUB_SNOWFLAKE_ENABLED) { + await refreshForSnowflake() + } else { + await refreshForGitHub() + } +} + const job: CrowdJob = { name: 'Refresh Github repo settings', // every day diff --git a/backend/src/conf/configTypes.ts b/backend/src/conf/configTypes.ts index fa5a108683..893f7762fe 100644 --- a/backend/src/conf/configTypes.ts +++ b/backend/src/conf/configTypes.ts @@ -120,6 +120,7 @@ export interface GithubConfiguration { privateKey: string webhookSecret: string isCommitDataEnabled: string + isSnowflakeEnabled: string globalLimit?: number callbackUrl: string } diff --git a/backend/src/database/repositories/githubReposRepository.ts b/backend/src/database/repositories/githubReposRepository.ts index c53971ac9e..30f0d843b4 100644 --- a/backend/src/database/repositories/githubReposRepository.ts +++ b/backend/src/database/repositories/githubReposRepository.ts @@ -38,7 +38,24 @@ export default class GithubReposRepository { ) } - static async updateMapping( + static async updateMapping(integrationId, mapping, options: IRepositoryOptions) { + const tenantId = options.currentTenant.id + + await GithubReposRepository.bulkInsert( + 'githubRepos', + ['tenantId', 'integrationId', 'segmentId', 'url'], + (idx) => `(:tenantId_${idx}, :integrationId_${idx}, :segmentId_${idx}, :url_${idx})`, + Object.entries(mapping).map(([url, segmentId], idx) => ({ + [`tenantId_${idx}`]: tenantId, + [`integrationId_${idx}`]: integrationId, + [`segmentId_${idx}`]: segmentId, + [`url_${idx}`]: url, + })), + options, + ) + } + + static async updateMappingSnowflake( integrationId, newMapping: Record, oldMapping: { diff --git a/backend/src/serverless/integrations/types/regularTypes.ts b/backend/src/serverless/integrations/types/regularTypes.ts index f7c5f1f8cb..b2509c5c81 100644 --- a/backend/src/serverless/integrations/types/regularTypes.ts +++ b/backend/src/serverless/integrations/types/regularTypes.ts @@ -3,7 +3,13 @@ import { PlatformType } from '@crowd/types' export type Repo = { url: string name: string - updatedAt: string + updatedAt?: string + createdAt?: string + owner?: string + available?: boolean + fork?: boolean + private?: boolean + cloneUrl?: string } export type Repos = Array diff --git a/backend/src/serverless/integrations/usecases/github/rest/getInstalledRepositories.ts b/backend/src/serverless/integrations/usecases/github/rest/getInstalledRepositories.ts new file mode 100644 index 0000000000..1e00f75dba --- /dev/null +++ b/backend/src/serverless/integrations/usecases/github/rest/getInstalledRepositories.ts @@ -0,0 +1,64 @@ +import axios, { AxiosRequestConfig } from 'axios' + +import { getServiceChildLogger } from '@crowd/logging' + +import { Repos } from '../../../types/regularTypes' + +const log = getServiceChildLogger('getInstalledRepositories') + +const getRepositoriesFromGH = async (page: number, installToken: string): Promise => { + const REPOS_PER_PAGE = 100 + + const requestConfig = { + method: 'get', + url: `https://api.github.com/installation/repositories?page=${page}&per_page=${REPOS_PER_PAGE}`, + headers: { + Authorization: `Bearer ${installToken}`, + }, + } as AxiosRequestConfig + + const response = await axios(requestConfig) + return response.data +} + +const parseRepos = (repositories: any): Repos => { + const repos: Repos = [] + + for (const repo of repositories) { + repos.push({ + url: repo.html_url, + owner: repo.owner.login, + createdAt: repo.created_at, + name: repo.name, + fork: repo.fork, + private: repo.private, + cloneUrl: repo.clone_url, + }) + } + + return repos +} + +export const getInstalledRepositories = async (installToken: string): Promise => { + try { + let page = 1 + let hasMorePages = true + + const repos: Repos = [] + + while (hasMorePages) { + const data = await getRepositoriesFromGH(page, installToken) + + if (data.repositories) { + repos.push(...parseRepos(data.repositories)) + } + + hasMorePages = data.total_count && data.total_count > 0 && data.total_count > repos.length + page += 1 + } + return repos.filter((repo) => !repo.private && !repo.fork) + } catch (err: any) { + log.error(err, 'Error fetching installed repositories!') + throw err + } +} diff --git a/backend/src/serverless/integrations/usecases/github/rest/getRemoteStats.ts b/backend/src/serverless/integrations/usecases/github/rest/getRemoteStats.ts index 4071dd5cc2..491ada86dc 100644 --- a/backend/src/serverless/integrations/usecases/github/rest/getRemoteStats.ts +++ b/backend/src/serverless/integrations/usecases/github/rest/getRemoteStats.ts @@ -28,9 +28,10 @@ const checkHeaders = (response: AxiosResponse, defaultValue = 0): number => } const getStatsForRepo = async (repoUrl: string, token: string): Promise => { - const [owner, repo] = repoUrl.split('/').slice(-2) + try { + const [owner, repo] = repoUrl.split('/').slice(-2) - const query = ` + const query = ` query { repository(owner: "${owner}", name: "${repo}") { starCount: stargazers { @@ -49,44 +50,53 @@ const getStatsForRepo = async (repoUrl: string, token: string): Promise owners[owner]) } - async mapGithubRepos(integrationId, mapping, fireOnboarding = true, isUpdateTransaction = false) { + async connectGithub(code, installId, setupAction = 'install') { + if (setupAction === 'request') { + return this.createOrUpdate( + { + platform: PlatformType.GITHUB, + status: 'waiting-approval', + }, + await SequelizeRepository.createTransaction(this.options), + ) + } + + const GITHUB_AUTH_ACCESSTOKEN_URL = 'https://github.com/login/oauth/access_token' + const CLIENT_ID = GITHUB_CONFIG.clientId + const CLIENT_SECRET = GITHUB_CONFIG.clientSecret + + const tokenResponse = await axios({ + method: 'post', + url: GITHUB_AUTH_ACCESSTOKEN_URL, + data: { + client_id: CLIENT_ID, + client_secret: CLIENT_SECRET, + code, + }, + }) + + let token = tokenResponse.data + token = token.slice(token.search('=') + 1, token.search('&')) + + try { + const requestWithAuth = request.defaults({ + headers: { + authorization: `token ${token}`, + }, + }) + await requestWithAuth('GET /user') + } catch { + throw new Error542( + `Invalid token for GitHub integration. Code: ${code}, setupAction: ${setupAction}. Token: ${token}`, + ) + } + + const installToken = await IntegrationService.getInstallToken(installId) + const repos = await getInstalledRepositories(installToken) + const githubOwner = IntegrationService.extractOwner(repos, this.options) + + let orgAvatar + try { + const response = await request('GET /users/{user}', { + user: githubOwner, + }) + orgAvatar = response.data.avatar_url + } catch (err) { + this.options.log.warn(err, 'Error while fetching GitHub user!') + } + + const integration = await this.createOrUpdateGithubIntegration( + { + platform: PlatformType.GITHUB, + token, + settings: { updateMemberAttributes: true, orgAvatar }, + integrationIdentifier: installId, + status: 'mapping', + }, + repos, + ) + + return integration + } + + async connectGithubInstallation(installId: string) { + const installToken = await IntegrationService.getInstallToken(installId) + const repos = await getInstalledRepositories(installToken) + const githubOwner = IntegrationService.extractOwner(repos, this.options) + + let orgAvatar + try { + const response = await request('GET /users/{user}', { + user: githubOwner, + }) + orgAvatar = response.data.avatar_url + } catch (err) { + this.options.log.warn(err, 'Error while fetching GitHub user!') + } + + const integration = await this.createOrUpdateGithubIntegration( + { + platform: PlatformType.GITHUB, + token: installToken, + settings: { updateMemberAttributes: true, orgAvatar }, + integrationIdentifier: installId, + status: 'mapping', + }, + repos, + ) + + return integration + } + + async getGithubInstallations() { + return GithubInstallationsRepository.getInstallations(this.options) + } + + /** + * Creates or updates a GitHub integration, handling large repos data + * @param integrationData The integration data to create or update + * @param repos The repositories data + */ + private async createOrUpdateGithubIntegration(integrationData, repos: Repos) { + let integration + const transaction = await SequelizeRepository.createTransaction(this.options) + + try { + // Get the first repo's owner since we know all repos are from same installation + const orgName = repos[0]?.owner + + // Create initial integration with org structure but empty repos + const initialOrg = { + name: orgName, + logo: integrationData.settings.orgAvatar, + url: `https://github.com/${orgName}`, + fullSync: true, + updatedAt: new Date().toISOString(), + repos: [], + } + + integration = await this.createOrUpdate( + { + ...integrationData, + settings: { + ...integrationData.settings, + orgs: [initialOrg], + }, + }, + transaction, + ) + + await SequelizeRepository.commitTransaction(transaction) + + // Transform repos into the new format + const transformedRepos = repos.map((repo) => ({ + name: repo.name, + url: repo.url, + updatedAt: repo.createdAt || new Date().toISOString(), + })) + + // Add repos in chunks + const chunkSize = 100 // Process 100 repos at a time + for (let i = 0; i < transformedRepos.length; i += chunkSize) { + const reposChunk = transformedRepos.slice(i, i + chunkSize) + await this.appendGitHubReposToOrg(integration.id, reposChunk) + } + + return integration + } catch (err) { + await SequelizeRepository.rollbackTransaction(transaction) + throw err + } + } + + private async appendGitHubReposToOrg(integrationId: string, repos: any[]) { + const transaction = await SequelizeRepository.createTransaction(this.options) + const sequelize = SequelizeRepository.getSequelize(this.options) + + try { + // Append repos to the first (and only) org's repos array + const query = ` + UPDATE integrations + SET settings = jsonb_set( + settings, + '{orgs,0,repos}', + COALESCE(settings->'orgs'->0->'repos', '[]'::jsonb) || ?::jsonb + ) + WHERE id = ? + ` + + const values = [JSON.stringify(repos), integrationId] + + await sequelize.query(query, { + replacements: values, + transaction, + }) + + await SequelizeRepository.commitTransaction(transaction) + } catch (error) { + await SequelizeRepository.rollbackTransaction(transaction) + throw error + } + } + + async mapGithubReposSnowflake( + integrationId, + mapping, + fireOnboarding = true, + isUpdateTransaction = false, + ) { const transaction = await SequelizeRepository.createTransaction(this.options) const txOptions = { @@ -398,7 +595,12 @@ export default class IntegrationService { try { const oldMapping = await GithubReposRepository.getMapping(integrationId, txOptions) - await GithubReposRepository.updateMapping(integrationId, mapping, oldMapping, txOptions) + await GithubReposRepository.updateMappingSnowflake( + integrationId, + mapping, + oldMapping, + txOptions, + ) // add the repos to the git integration if (EDITION === Edition.LFX) { @@ -607,6 +809,96 @@ export default class IntegrationService { } } + async mapGithubRepos(integrationId, mapping, fireOnboarding = true) { + const transaction = await SequelizeRepository.createTransaction(this.options) + + const txOptions = { + ...this.options, + transaction, + } + + try { + await GithubReposRepository.updateMapping(integrationId, mapping, txOptions) + + // add the repos to the git integration + if (EDITION === Edition.LFX) { + const repos: Record = Object.entries(mapping).reduce( + (acc, [url, segmentId]) => { + if (!acc[segmentId as string]) { + acc[segmentId as string] = [] + } + acc[segmentId as string].push(url) + return acc + }, + {}, + ) + + for (const [segmentId, urls] of Object.entries(repos)) { + let isGitintegrationConfigured + const segmentOptions: IRepositoryOptions = { + ...this.options, + currentSegments: [ + { + ...this.options.currentSegments[0], + id: segmentId as string, + }, + ], + } + try { + await IntegrationRepository.findByPlatform(PlatformType.GIT, segmentOptions) + + isGitintegrationConfigured = true + } catch (err) { + isGitintegrationConfigured = false + } + + if (isGitintegrationConfigured) { + const gitInfo = await this.gitGetRemotes(segmentOptions) + const gitRemotes = gitInfo[segmentId as string].remotes + await this.gitConnectOrUpdate( + { + remotes: Array.from(new Set([...gitRemotes, ...urls])), + }, + segmentOptions, + ) + } else { + await this.gitConnectOrUpdate( + { + remotes: urls, + }, + segmentOptions, + ) + } + } + } + + if (fireOnboarding) { + const integration = await IntegrationRepository.update( + integrationId, + { status: 'in-progress' }, + txOptions, + ) + + this.options.log.info( + { tenantId: integration.tenantId }, + 'Sending GitHub message to int-run-worker!', + ) + const emitter = await getIntegrationRunWorkerEmitter() + await emitter.triggerIntegrationRun( + integration.tenantId, + integration.platform, + integration.id, + true, + ) + } + + await SequelizeRepository.commitTransaction(transaction) + } catch (err) { + await SequelizeRepository.rollbackTransaction(transaction) + throw err + } + } + async getGithubRepos(integrationId): Promise { const transaction = await SequelizeRepository.createTransaction(this.options) @@ -1886,4 +2178,103 @@ export default class IntegrationService { throw err } } + + async updateGithubIntegrationSettings(installId: string) { + this.options.log.info(`Updating GitHub integration settings for installId: ${installId}`) + + // Find the integration by installId + const integration: any = await IntegrationRepository.findByIdentifier( + installId, + PlatformType.GITHUB, + ) + + if (!integration || integration.platform !== PlatformType.GITHUB) { + this.options.log.warn(`GitHub integration not found for installId: ${installId}`) + throw new Error404('GitHub integration not found') + } + + this.options.log.info(`Found integration: ${integration.id}`) + + // Get the install token + const installToken = await IntegrationService.getInstallToken(installId) + this.options.log.info(`Obtained install token for installId: ${installId}`) + + // Fetch all installed repositories + const repos = await getInstalledRepositories(installToken) + this.options.log.info(`Fetched ${repos.length} installed repositories`) + + // Update integration settings + const currentSettings: { + orgs: Array<{ + name: string + logo: string + url: string + fullSync: boolean + updatedAt: string + repos: Array<{ + name: string + url: string + updatedAt: string + }> + }> + } = integration.settings || { orgs: [] } + + if (currentSettings.orgs.length !== 1) { + throw new Error('Integration settings must have exactly one organization') + } + + const currentRepos = currentSettings.orgs[0].repos || [] + const newRepos = repos.filter((repo) => !currentRepos.some((r) => r.url === repo.url)) + this.options.log.info(`Found ${newRepos.length} new repositories`) + + const updatedSettings = { + ...currentSettings, + orgs: [ + { + ...currentSettings.orgs[0], + repos: [ + ...currentRepos, + ...newRepos.map((repo) => ({ + name: repo.name, + url: repo.url, + updatedAt: repo.updatedAt || new Date().toISOString(), + })), + ], + }, + ], + } + + this.options = { + ...this.options, + currentSegments: [ + { + id: integration.segmentId, + } as any, + ], + } + + // Update the integration with new settings + await this.update(integration.id, { settings: updatedSettings }) + + this.options.log.info(`Updated integration settings for integration id: ${integration.id}`) + + // Update GitHub repos mapping + const defaultSegmentId = integration.segmentId + const mapping = {} + for (const repo of newRepos) { + mapping[repo.url] = defaultSegmentId + } + if (Object.keys(mapping).length > 0) { + // false - not firing onboarding + await this.mapGithubRepos(integration.id, mapping, false) + this.options.log.info(`Updated GitHub repos mapping for integration id: ${integration.id}`) + } else { + this.options.log.info(`No new repos to map for integration id: ${integration.id}`) + } + + this.options.log.info( + `Completed updating GitHub integration settings for installId: ${installId}`, + ) + return integration + } } diff --git a/frontend/scripts/docker-entrypoint.sh b/frontend/scripts/docker-entrypoint.sh index c3cea261d2..7eedb3d44f 100755 --- a/frontend/scripts/docker-entrypoint.sh +++ b/frontend/scripts/docker-entrypoint.sh @@ -34,6 +34,7 @@ declare -a ENV_VARIABLES=( "VUE_APP_DATADOG_RUM_APPLICATION_ID" "VUE_APP_DATADOG_RUM_CLIENT_TOKEN" "VUE_APP_TEAM_USER_IDS" + "VUE_APP_IS_GITHUB_ARCHIVE_ENABLED" ) for ENV_VAR in "${ENV_VARIABLES[@]}" diff --git a/frontend/src/config.js b/frontend/src/config.js index d9e39dd61b..bce5a9019a 100644 --- a/frontend/src/config.js +++ b/frontend/src/config.js @@ -55,6 +55,7 @@ const defaultConfig = { lf: { tenantId: import.meta.env.VUE_APP_LF_TENANT_ID, }, + isGithubArchiveEnabled: import.meta.env.VUE_APP_IS_GITHUB_ARCHIVE_ENABLED, isGitEnabled: import.meta.env.VUE_APP_IS_GIT_ENABLED, isGroupsioEnabled: import.meta.env.VUE_APP_IS_GROUPSIO_ENABLED, isConfluenceEnabled: import.meta.env.VUE_APP_IS_CONFLUENCE_ENABLED, @@ -116,6 +117,7 @@ const composedConfig = { lf: { tenantId: 'CROWD_VUE_APP_LF_TENANT_ID', }, + isGithubArchiveEnabled: 'CROWD_VUE_APP_IS_GITHUB_ARCHIVE_ENABLED', isGitEnabled: 'CROWD_VUE_APP_IS_GIT_ENABLED', isGroupsioEnabled: 'CROWD_VUE_APP_IS_GROUPSIO_ENABLED', isTwitterEnabled: 'CROWD_VUE_APP_IS_TWITTER_ENABLED', diff --git a/frontend/src/config/integrations/github-archive/components/connect/github-connect.vue b/frontend/src/config/integrations/github-archive/components/connect/github-connect.vue new file mode 100644 index 0000000000..fbf455d262 --- /dev/null +++ b/frontend/src/config/integrations/github-archive/components/connect/github-connect.vue @@ -0,0 +1,76 @@ + + + + + diff --git a/frontend/src/config/integrations/github-archive/components/github-connect.vue b/frontend/src/config/integrations/github-archive/components/github-connect.vue new file mode 100644 index 0000000000..e16b9b5279 --- /dev/null +++ b/frontend/src/config/integrations/github-archive/components/github-connect.vue @@ -0,0 +1,54 @@ + + + + + diff --git a/frontend/src/config/integrations/github-archive/components/github-details-modal.vue b/frontend/src/config/integrations/github-archive/components/github-details-modal.vue new file mode 100644 index 0000000000..29fd10c473 --- /dev/null +++ b/frontend/src/config/integrations/github-archive/components/github-details-modal.vue @@ -0,0 +1,112 @@ + + + + + diff --git a/frontend/src/config/integrations/github/components/github-dropdown.vue b/frontend/src/config/integrations/github-archive/components/github-dropdown.vue similarity index 93% rename from frontend/src/config/integrations/github/components/github-dropdown.vue rename to frontend/src/config/integrations/github-archive/components/github-dropdown.vue index ce508a7b96..7126c09568 100644 --- a/frontend/src/config/integrations/github/components/github-dropdown.vue +++ b/frontend/src/config/integrations/github-archive/components/github-dropdown.vue @@ -16,7 +16,7 @@ import { defineProps, ref } from 'vue'; import LfDropdownItem from '@/ui-kit/dropdown/DropdownItem.vue'; import LfIcon from '@/ui-kit/icon/Icon.vue'; -import LfGithubSettingsDrawer from '@/config/integrations/github/components/settings/github-settings-drawer.vue'; +import LfGithubSettingsDrawer from '@/config/integrations/github-archive/components/settings/github-settings-drawer.vue'; const props = defineProps<{ integration: any, diff --git a/frontend/src/config/integrations/github-archive/components/github-params.vue b/frontend/src/config/integrations/github-archive/components/github-params.vue new file mode 100644 index 0000000000..408f93b475 --- /dev/null +++ b/frontend/src/config/integrations/github-archive/components/github-params.vue @@ -0,0 +1,76 @@ + + + + + diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-add-repository-modal.vue b/frontend/src/config/integrations/github-archive/components/settings/github-settings-add-repository-modal.vue similarity index 98% rename from frontend/src/config/integrations/github/components/settings/github-settings-add-repository-modal.vue rename to frontend/src/config/integrations/github-archive/components/settings/github-settings-add-repository-modal.vue index 771b14d064..19ed448496 100644 --- a/frontend/src/config/integrations/github/components/settings/github-settings-add-repository-modal.vue +++ b/frontend/src/config/integrations/github-archive/components/settings/github-settings-add-repository-modal.vue @@ -160,8 +160,8 @@ import { GitHubOrganization, GitHubRepository, GitHubSettingsRepository, -} from '@/config/integrations/github/types/GithubSettings'; -import { GithubApiService } from '@/config/integrations/github/services/github.api.service'; +} from '@/config/integrations/github-archive/types/GithubSettings'; +import { GithubApiService } from '@/config/integrations/github-archive/services/github.api.service'; import Message from '@/shared/message/message'; import dayjs from 'dayjs'; diff --git a/frontend/src/config/integrations/github-archive/components/settings/github-settings-drawer.vue b/frontend/src/config/integrations/github-archive/components/settings/github-settings-drawer.vue new file mode 100644 index 0000000000..0051b4832a --- /dev/null +++ b/frontend/src/config/integrations/github-archive/components/settings/github-settings-drawer.vue @@ -0,0 +1,293 @@ + + + + + diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-empty.vue b/frontend/src/config/integrations/github-archive/components/settings/github-settings-empty.vue similarity index 100% rename from frontend/src/config/integrations/github/components/settings/github-settings-empty.vue rename to frontend/src/config/integrations/github-archive/components/settings/github-settings-empty.vue diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-mapping.vue b/frontend/src/config/integrations/github-archive/components/settings/github-settings-mapping.vue similarity index 93% rename from frontend/src/config/integrations/github/components/settings/github-settings-mapping.vue rename to frontend/src/config/integrations/github-archive/components/settings/github-settings-mapping.vue index 7b996026aa..07463dbea1 100644 --- a/frontend/src/config/integrations/github/components/settings/github-settings-mapping.vue +++ b/frontend/src/config/integrations/github-archive/components/settings/github-settings-mapping.vue @@ -73,14 +73,14 @@ import LfIcon from '@/ui-kit/icon/Icon.vue'; import { computed, ref } from 'vue'; import LfSearch from '@/ui-kit/search/Search.vue'; import LfGithubSettingsRepositoriesBulkSelect - from '@/config/integrations/github/components/settings/github-settings-repositories-bulk-select.vue'; + from '@/config/integrations/github-archive/components/settings/github-settings-repositories-bulk-select.vue'; import { GitHubOrganization, GitHubSettingsRepository, -} from '@/config/integrations/github/types/GithubSettings'; -import LfGithubSettingsOrgItem from '@/config/integrations/github/components/settings/github-settings-org-item.vue'; +} from '@/config/integrations/github-archive/types/GithubSettings'; +import LfGithubSettingsOrgItem from '@/config/integrations/github-archive/components/settings/github-settings-org-item.vue'; import LfTable from '@/ui-kit/table/Table.vue'; -import LfGithubSettingsRepoItem from '@/config/integrations/github/components/settings/github-settings-repo-item.vue'; +import LfGithubSettingsRepoItem from '@/config/integrations/github-archive/components/settings/github-settings-repo-item.vue'; import LfIconOld from '@/ui-kit/icon/IconOld.vue'; const props = defineProps<{ diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-org-item.vue b/frontend/src/config/integrations/github-archive/components/settings/github-settings-org-item.vue similarity index 96% rename from frontend/src/config/integrations/github/components/settings/github-settings-org-item.vue rename to frontend/src/config/integrations/github-archive/components/settings/github-settings-org-item.vue index a3412a6f8d..7828b191eb 100644 --- a/frontend/src/config/integrations/github/components/settings/github-settings-org-item.vue +++ b/frontend/src/config/integrations/github-archive/components/settings/github-settings-org-item.vue @@ -68,7 +68,7 @@ import { GitHubOrganization, GitHubRepository, GitHubSettingsRepository, -} from '@/config/integrations/github/types/GithubSettings'; +} from '@/config/integrations/github-archive/types/GithubSettings'; import LfAvatar from '@/ui-kit/avatar/Avatar.vue'; import LfIconOld from '@/ui-kit/icon/IconOld.vue'; import LfButton from '@/ui-kit/button/Button.vue'; @@ -78,7 +78,7 @@ import { computed } from 'vue'; import LfDropdown from '@/ui-kit/dropdown/Dropdown.vue'; import LfDropdownItem from '@/ui-kit/dropdown/DropdownItem.vue'; import dayjs from 'dayjs'; -import { GithubApiService } from '@/config/integrations/github/services/github.api.service'; +import { GithubApiService } from '@/config/integrations/github-archive/services/github.api.service'; const props = defineProps<{ organizations: GitHubOrganization[]; diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-repo-item.vue b/frontend/src/config/integrations/github-archive/components/settings/github-settings-repo-item.vue similarity index 97% rename from frontend/src/config/integrations/github/components/settings/github-settings-repo-item.vue rename to frontend/src/config/integrations/github-archive/components/settings/github-settings-repo-item.vue index 22e0a3d719..b3e9785f26 100644 --- a/frontend/src/config/integrations/github/components/settings/github-settings-repo-item.vue +++ b/frontend/src/config/integrations/github-archive/components/settings/github-settings-repo-item.vue @@ -49,7 +49,7 @@ + + diff --git a/frontend/src/config/integrations/github/components/connect/github-connect-modal.vue b/frontend/src/config/integrations/github/components/connect/github-connect-modal.vue new file mode 100644 index 0000000000..0d5b574025 --- /dev/null +++ b/frontend/src/config/integrations/github/components/connect/github-connect-modal.vue @@ -0,0 +1,221 @@ + + + + + diff --git a/frontend/src/config/integrations/github/components/connect/github-connect.vue b/frontend/src/config/integrations/github/components/connect/github-connect.vue index fbf455d262..bb6a355448 100644 --- a/frontend/src/config/integrations/github/components/connect/github-connect.vue +++ b/frontend/src/config/integrations/github/components/connect/github-connect.vue @@ -5,18 +5,10 @@ - - - - - Connect - - - + + + Connect + + + Map repositories + + + + + + + diff --git a/frontend/src/config/integrations/github/components/github-status.vue b/frontend/src/config/integrations/github/components/github-status.vue new file mode 100644 index 0000000000..66d2b9d4af --- /dev/null +++ b/frontend/src/config/integrations/github/components/github-status.vue @@ -0,0 +1,23 @@ + + + + + diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-bulk-select.vue b/frontend/src/config/integrations/github/components/settings/github-settings-bulk-select.vue new file mode 100644 index 0000000000..4028814ac8 --- /dev/null +++ b/frontend/src/config/integrations/github/components/settings/github-settings-bulk-select.vue @@ -0,0 +1,147 @@ + + + + + diff --git a/frontend/src/config/integrations/github/components/settings/github-settings-drawer.vue b/frontend/src/config/integrations/github/components/settings/github-settings-drawer.vue index 402322270d..1b1da42452 100644 --- a/frontend/src/config/integrations/github/components/settings/github-settings-drawer.vue +++ b/frontend/src/config/integrations/github/components/settings/github-settings-drawer.vue @@ -1,129 +1,200 @@