Github archive

backend/config/custom-environment-variables.json

@@ -222,5 +222,13 @@
     "clientSecret": "CROWD_GITLAB_CLIENT_SECRET",
     "callbackUrl": "CROWD_GITLAB_CALLBACK_URL",
     "webhookToken": "CROWD_GITLAB_WEBHOOK_TOKEN"
+  },
+  "snowflake": {
+    "privateKey": "CROWD_SNOWFLAKE_PRIVATE_KEY",
+    "account": "CROWD_SNOWFLAKE_ACCOUNT",
+    "username": "CROWD_SNOWFLAKE_USERNAME",
+    "database": "CROWD_SNOWFLAKE_DATABASE",
+    "warehouse": "CROWD_SNOWFLAKE_WAREHOUSE",
+    "role": "CROWD_SNOWFLAKE_ROLE"
   }
 }

backend/package.json

@@ -65,6 +65,7 @@
     "@crowd/telemetry": "workspace:*",
     "@crowd/temporal": "workspace:*",
     "@crowd/types": "workspace:*",
+    "@crowd/snowflake": "workspace:*",
     "@google-cloud/storage": "5.3.0",
     "@octokit/auth-app": "^3.6.1",
     "@octokit/core": "^6.1.2",

backend/src/api/integration/helpers/githubMapRepos.ts

@@ -4,6 +4,6 @@ import PermissionChecker from '../../../services/user/permissionChecker'
 
 export default async (req, res) => {
   new PermissionChecker(req).validateHas(Permissions.values.tenantEdit)
-  const payload = await new IntegrationService(req).mapGithubRepos(req.params.id, req.body.mapping)
+  const payload = await new IntegrationService(req).mapGithubRepos(req.params.id, req.body.mapping, true, req.body?.isUpdateTransaction ?? false)
   await req.responseHandler.success(req, res, payload)
 }

backend/src/api/integration/helpers/githubOrgRepos.ts

@@ -0,0 +1,10 @@
+import Permissions from '@/security/permissions'
+import GithubIntegrationService from '@/services/githubIntegrationService'
+import PermissionChecker from '@/services/user/permissionChecker'
+
+export default async (req, res) => {
+  new PermissionChecker(req).validateHas(Permissions.values.integrationEdit)
+
+  const payload = await GithubIntegrationService.getOrgRepos(req.params.org)
+  await req.responseHandler.success(req, res, payload)
+}

backend/src/api/integration/helpers/githubSearchOrgs.ts

@@ -0,0 +1,10 @@
+import Permissions from '@/security/permissions'
+import GithubIntegrationService from '@/services/githubIntegrationService'
+import PermissionChecker from '@/services/user/permissionChecker'
+
+export default async (req, res) => {
+  new PermissionChecker(req).validateHas(Permissions.values.integrationEdit)
+
+  const payload = await GithubIntegrationService.findOrgs(req.query.query)
+  await req.responseHandler.success(req, res, payload)
+}

backend/src/api/integration/helpers/githubSearchRepos.ts

@@ -0,0 +1,10 @@
+import Permissions from '@/security/permissions'
+import GithubIntegrationService from '@/services/githubIntegrationService'
+import PermissionChecker from '@/services/user/permissionChecker'
+
+export default async (req, res) => {
+  new PermissionChecker(req).validateHas(Permissions.values.integrationEdit)
+
+  const payload = await new GithubIntegrationService(req).findGithubRepos(req.query.query)
+  await req.responseHandler.success(req, res, payload)
+}

backend/src/api/integration/index.ts

@@ -37,10 +37,6 @@ export default (app) => {
   app.get(`/tenant/:tenantId/integration`, safeWrap(require('./integrationList').default))
   app.get(`/tenant/:tenantId/integration/:id`, safeWrap(require('./integrationFind').default))
 
-  app.put(
-    `/authenticate/:tenantId/:code`,
-    safeWrap(require('./helpers/githubAuthenticate').default),
-  )
   app.put(
     `/tenant/:tenantId/integration/:id/github/repos`,
     safeWrap(require('./helpers/githubMapRepos').default),
@@ -49,6 +45,18 @@ export default (app) => {
     `/tenant/:tenantId/integration/:id/github/repos`,
     safeWrap(require('./helpers/githubMapReposGet').default),
   )
+  app.get(
+    `/tenant/:tenantId/integration/github/search/orgs`,
+    safeWrap(require('./helpers/githubSearchOrgs').default),
+  )
+  app.get(
+    `/tenant/:tenantId/integration/github/search/repos`,
+    safeWrap(require('./helpers/githubSearchRepos').default),
+  )
+  app.get(
+    `/tenant/:tenantId/integration/github/orgs/:org/repos`,
+    safeWrap(require('./helpers/githubOrgRepos').default),
+  )
   app.put(
     `/discord-authenticate/:tenantId/:guild_id`,
     safeWrap(require('./helpers/discordAuthenticate').default),
@@ -202,16 +210,6 @@ export default (app) => {
     safeWrap(require('./helpers/jiraConnectOrUpdate').default),
   )
 
-  app.get(
-    '/tenant/:tenantId/github-installations',
-    safeWrap(require('./helpers/getGithubInstallations').default),
-  )
-
-  app.post(
-    '/tenant/:tenantId/github-connect-installation',
-    safeWrap(require('./helpers/githubConnectInstallation').default),
-  )
-
   app.get('/gitlab/:tenantId/connect', safeWrap(require('./helpers/gitlabAuthenticate').default))
 
   app.get(

backend/src/bin/jobs/refreshGithubRepoSettings.ts

@@ -1,56 +1,144 @@
+/* eslint-disable no-continue */
 import cronGenerator from 'cron-time-generator'
 
 import { timeout } from '@crowd/common'
 import { getServiceChildLogger } from '@crowd/logging'
 
 import SequelizeRepository from '../../database/repositories/sequelizeRepository'
+import GithubIntegrationService from '../../services/githubIntegrationService'
 import IntegrationService from '../../services/integrationService'
 import { CrowdJob } from '../../types/jobTypes'
 
 const log = getServiceChildLogger('refreshGithubRepoSettings')
 
-const job: CrowdJob = {
-  name: 'Refresh Github repo settings',
-  // every day
-  cronTime: cronGenerator.every(1).days(),
-  onTrigger: async () => {
-    log.info('Updating Github repo settings.')
-    const dbOptions = await SequelizeRepository.getDefaultIRepositoryOptions()
+interface Integration {
+  id: string
+  tenantId: string
+  segmentId: string
+  integrationIdentifier: string
+  settings: {
+    orgs: Array<{
+      name: string
+      logo: string
+      url: string
+      fullSync: boolean
+      updatedAt: string
+      repos: Array<{
+        name: string
+        url: string
+        updatedAt: string
+      }>
+    }>
+  }
+}
 
-    interface Integration {
-      id: string
-      tenantId: string
-      integrationIdentifier: string
-    }
+export const refreshGithubRepoSettings = async () => {
+  log.info('Updating Github repo settings.')
+  const dbOptions = await SequelizeRepository.getDefaultIRepositoryOptions()
 
-    const githubIntegrations = await dbOptions.database.sequelize.query(
-      `SELECT id, "tenantId", "integrationIdentifier" FROM integrations 
+  const githubIntegrations = await dbOptions.database.sequelize.query(
+    `SELECT id, "tenantId", settings, "segmentId" FROM integrations 
        WHERE platform = 'github' AND "deletedAt" IS NULL
        AND "createdAt" < NOW() - INTERVAL '1 day'`,
-    )
+  )
+
+  for (const integration of githubIntegrations[0] as Integration[]) {
+    log.info(`Updating repo settings for Github integration: ${integration.id}`)
+
+    try {
+      const options = await SequelizeRepository.getDefaultIRepositoryOptions()
+      options.currentTenant = { id: integration.tenantId }
+      options.currentSegments = [
+        // @ts-ignore
+        {
+          id: integration.segmentId,
+        },
+      ]
+
+      const integrationService = new IntegrationService(options)
+
+      if (!integration.settings.orgs) {
+        log.info(`No orgs found for Github integration: ${integration.id}`)
+        continue
+      }
+
+      // Get all orgs with fullSync enabled
+      const fullSyncOrgs = integration.settings.orgs.filter((org) => org.fullSync)
+      const currentRepos = new Set(
+        integration.settings.orgs.flatMap((org) => org.repos.map((r) => r.url)),
+      )
+      const newRepoMappings: Record<string, string> = {}
 
-    for (const integration of githubIntegrations[0] as Integration[]) {
-      log.info(`Updating repo settings for Github integration: ${integration.id}`)
+      // Fetch new repos for each org
+      for (const org of fullSyncOrgs) {
+        log.info(`Fetching repos for org ${org.name} with fullSync enabled`)
+        const githubRepos = await GithubIntegrationService.getOrgRepos(org.name)
 
-      try {
-        const options = await SequelizeRepository.getDefaultIRepositoryOptions()
-        options.currentTenant = { id: integration.tenantId }
+        // Find new repos that aren't in current settings
+        const newRepos = githubRepos.filter((repo) => !currentRepos.has(repo.url))
 
-        const integrationService = new IntegrationService(options)
-        // newly discovered repos will be mapped to default segment of the integration
-        await integrationService.updateGithubIntegrationSettings(integration.integrationIdentifier)
+        // Map new repos to the integration's segment
+        newRepos.forEach((repo) => {
+          newRepoMappings[repo.url] = integration.segmentId
+        })
 
-        log.info(`Successfully updated repo settings for Github integration: ${integration.id}`)
-      } catch (err) {
-        log.error(
-          `Error updating repo settings for Github integration ${integration.id}: ${err.message}`,
+        // Update org's repos list directly in the integration settings
+        const orgIndex = integration.settings.orgs.findIndex((o) => o.name === org.name)
+        if (orgIndex !== -1) {
+          integration.settings.orgs[orgIndex].repos = [
+            ...integration.settings.orgs[orgIndex].repos,
+            ...newRepos.map((repo) => ({
+              name: repo.name,
+              url: repo.url,
+              updatedAt: new Date().toISOString(),
+            })),
+          ]
+          integration.settings.orgs[orgIndex].updatedAt = new Date().toISOString()
+        }
+      }
+
+      if (Object.keys(newRepoMappings).length > 0) {
+        log.info(`Found ${Object.keys(newRepoMappings).length} new repos to add`)
+
+        // Update integration with modified settings object
+        await integrationService.update(integration.id, {
+          settings: integration.settings,
+          status: 'in-progress',
+        })
+
+        // Map new repos
+        await integrationService.mapGithubRepos(
+          integration.id,
+          newRepoMappings,
+          true,
+          // this will fire onboarding only for new repos
+          true,
         )
-      } finally {
-        await timeout(1000)
+
+        log.info(`Successfully mapped ${Object.keys(newRepoMappings).length} new repos`)
       }
+
+      log.info(`Successfully updated repo settings for Github integration: ${integration.id}`)
+    } catch (err) {
+      log.error(
+        `Error updating repo settings for Github integration ${integration.id}: ${err.message}`,
+      )
+      // that's probably a rate limit error, let's sleep for a minute
+      await timeout(60000)
+    } finally {
+      await timeout(10000)
     }
+  }
 
-    log.info('Finished updating Github repo settings.')
+  log.info('Finished updating Github repo settings.')
+}
+
+const job: CrowdJob = {
+  name: 'Refresh Github repo settings',
+  // every day
+  cronTime: cronGenerator.every(1).days(),
+  onTrigger: async () => {
+    await refreshGithubRepoSettings()
   },
 }
 

backend/src/bin/scripts/refresh-github-repo-settings.ts

@@ -0,0 +1,22 @@
+import { getServiceChildLogger } from '@crowd/logging'
+
+import { refreshGithubRepoSettings } from '../jobs/refreshGithubRepoSettings'
+
+const logger = getServiceChildLogger('refreshGithubRepoSettings')
+
+setImmediate(async () => {
+  try {
+    const startTime = Date.now()
+    logger.info('Starting refresh of Github repo settings')
+
+    await refreshGithubRepoSettings()
+
+    const duration = Date.now() - startTime
+    logger.info(`Completed refresh of Github repo settings in ${duration}ms`)
+
+    process.exit(0)
+  } catch (error) {
+    logger.error(`Error refreshing Github repo settings: ${error.message}`)
+    process.exit(1)
+  }
+})

backend/src/conf/configTypes.ts

@@ -256,3 +256,12 @@ export interface IRedditConfig {
   clientId: string
   clientSecret: string
 }
+
+export interface SnowflakeConfiguration {
+  privateKey: string
+  account: string
+  username: string
+  database: string
+  warehouse: string
+  role: string
+}

backend/src/conf/index.ts

@@ -44,6 +44,7 @@ import {
   TwitterConfiguration,
   UnleashConfiguration,
   WeeklyEmailsConfiguration,
+  SnowflakeConfiguration,
 } from './configTypes'
 
 // TODO-kube
@@ -172,3 +173,7 @@ export const OPEN_STATUS_API_CONFIG: IOpenStatusApiConfig =
 export const GITLAB_CONFIG: GitlabConfiguration = config.get<GitlabConfiguration>('gitlab')
 
 export const REDDIT_CONFIG: IRedditConfig = config.get<IRedditConfig>('reddit')
+
+
+export const SNOWFLAKE_CONFIG: SnowflakeConfiguration =
+  config.get<SnowflakeConfiguration>('snowflake')