Merge suggestions quick wins

services/apps/merge_suggestions_worker/src/activities.ts

@@ -12,17 +12,18 @@ import {
   getMembers,
   getMembersForLLMConsumption,
   getRawMemberMergeSuggestions,
-  removeRawMemberMergeSuggestions,
+  removeMemberMergeSuggestion,
   updateMemberMergeSuggestionsLastGeneratedAt,
 } from './activities/memberMergeSuggestions'
 import {
+  addOrganizationSuggestionToNoMerge,
   addOrganizationToMerge,
   findTenantsLatestOrganizationSuggestionGeneratedAt,
   getOrganizationMergeSuggestions,
   getOrganizations,
   getOrganizationsForLLMConsumption,
   getRawOrganizationMergeSuggestions,
-  removeRawOrganizationMergeSuggestions,
+  removeOrganizationMergeSuggestions,
   updateOrganizationMergeSuggestionsLastGeneratedAt,
 } from './activities/organizationMergeSuggestions'
 
@@ -42,10 +43,11 @@ export {
   getMembersForLLMConsumption,
   getOrganizationsForLLMConsumption,
   getRawOrganizationMergeSuggestions,
-  removeRawOrganizationMergeSuggestions,
+  removeOrganizationMergeSuggestions,
   getRawMemberMergeSuggestions,
-  removeRawMemberMergeSuggestions,
+  removeMemberMergeSuggestion,
   saveLLMVerdict,
   mergeMembers,
   mergeOrganizations,
+  addOrganizationSuggestionToNoMerge,
 }

services/apps/merge_suggestions_worker/src/activities/memberMergeSuggestions.ts

@@ -1,4 +1,5 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
+import { addMemberNoMerge } from '@crowd/data-access-layer/src/member_merge'
 import { MemberField, queryMembers } from '@crowd/data-access-layer/src/members'
 import MemberMergeSuggestionsRepository from '@crowd/data-access-layer/src/old/apps/merge_suggestions_worker/memberMergeSuggestions.repo'
 import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor'
@@ -351,10 +352,23 @@ export async function getRawMemberMergeSuggestions(
   return memberMergeSuggestionsRepo.getRawMemberSuggestions(similarityFilter, limit)
 }
 
-export async function removeRawMemberMergeSuggestions(suggestion: string[]): Promise<void> {
+export async function removeMemberMergeSuggestion(
+  suggestion: string[],
+  table: MemberMergeSuggestionTable,
+): Promise<void> {
   const memberMergeSuggestionsRepo = new MemberMergeSuggestionsRepository(
     svc.postgres.writer.connection(),
     svc.log,
   )
-  await memberMergeSuggestionsRepo.removeRawMemberSuggestions(suggestion)
+  await memberMergeSuggestionsRepo.removeMemberMergeSuggestion(suggestion, table)
+}
+
+export async function addMemberSuggestionToNoMerge(suggestion: string[]): Promise<void> {
+  if (suggestion.length !== 2) {
+    svc.log.debug(`Suggestions array must have two ids!`)
+    return
+  }
+  const qx = pgpQx(svc.postgres.writer.connection())
+
+  await addMemberNoMerge(qx, suggestion[0], suggestion[1])
 }

services/apps/merge_suggestions_worker/src/activities/organizationMergeSuggestions.ts

@@ -1,6 +1,7 @@
 import { OrganizationField, findOrgById, queryOrgs } from '@crowd/data-access-layer'
 import { hasLfxMembership } from '@crowd/data-access-layer/src/lfx_memberships'
 import OrganizationMergeSuggestionsRepository from '@crowd/data-access-layer/src/old/apps/merge_suggestions_worker/organizationMergeSuggestions.repo'
+import { addOrgNoMerge } from '@crowd/data-access-layer/src/org_merge'
 import { fetchOrgIdentities, findOrgAttributes } from '@crowd/data-access-layer/src/organizations'
 import { QueryExecutor, pgpQx } from '@crowd/data-access-layer/src/queryExecutor'
 import { buildFullOrgForMergeSuggestions } from '@crowd/opensearch'
@@ -428,10 +429,23 @@ export async function getRawOrganizationMergeSuggestions(
   return suggestions
 }
 
-export async function removeRawOrganizationMergeSuggestions(suggestion: string[]): Promise<void> {
+export async function removeOrganizationMergeSuggestions(
+  suggestion: string[],
+  table: OrganizationMergeSuggestionTable,
+): Promise<void> {
   const organizationMergeSuggestionsRepo = new OrganizationMergeSuggestionsRepository(
     svc.postgres.writer.connection(),
     svc.log,
   )
-  await organizationMergeSuggestionsRepo.removeRawOrganizationMergeSuggestions(suggestion)
+  await organizationMergeSuggestionsRepo.removeOrganizationMergeSuggestions(suggestion, table)
+}
+
+export async function addOrganizationSuggestionToNoMerge(suggestion: string[]): Promise<void> {
+  if (suggestion.length !== 2) {
+    svc.log.debug(`Suggestions array must have two ids!`)
+    return
+  }
+  const qx = pgpQx(svc.postgres.writer.connection())
+
+  await addOrgNoMerge(qx, suggestion[0], suggestion[1])
 }

services/apps/merge_suggestions_worker/src/workflows/generateMemberMergeSuggestions.ts

@@ -17,7 +17,7 @@ export async function generateMemberMergeSuggestions(
 ): Promise<void> {
   const PAGE_SIZE = 50
   const PARALLEL_SUGGESTION_PROCESSING = 10
-  const SIMILARITY_CONFIDENCE_SCORE_THRESHOLD = 0.5
+  const SIMILARITY_CONFIDENCE_SCORE_THRESHOLD = 0.75
 
   let lastUuid: string = args.lastUuid || null
 

services/apps/merge_suggestions_worker/src/workflows/generateOrganizationMergeSuggestions.ts

@@ -17,7 +17,7 @@ export async function generateOrganizationMergeSuggestions(
 ): Promise<void> {
   const PAGE_SIZE = 25
   const PARALLEL_SUGGESTION_PROCESSING = 50
-  const SIMILARITY_CONFIDENCE_SCORE_THRESHOLD = 0.5
+  const SIMILARITY_CONFIDENCE_SCORE_THRESHOLD = 0.75
 
   let lastUuid: string = args.lastUuid || null
 

services/apps/merge_suggestions_worker/src/workflows/mergeMembersWithLLM.ts

@@ -1,6 +1,6 @@
 import { continueAsNew, proxyActivities } from '@temporalio/workflow'
 
-import { LLMSuggestionVerdictType } from '@crowd/types'
+import { LLMSuggestionVerdictType, MemberMergeSuggestionTable } from '@crowd/types'
 
 import * as commonActivities from '../activities/common'
 import * as memberActivities from '../activities/memberMergeSuggestions'
@@ -63,7 +63,14 @@ export async function mergeMembersWithLLM(
 
     if (members.length !== 2) {
       console.log(`Failed getting members data in suggestion. Skipping suggestion: ${suggestion}`)
-      await memberActivitiesProxy.removeRawMemberMergeSuggestions(suggestion)
+      await memberActivitiesProxy.removeMemberMergeSuggestion(
+        suggestion,
+        MemberMergeSuggestionTable.MEMBER_TO_MERGE_RAW,
+      )
+      await memberActivitiesProxy.removeMemberMergeSuggestion(
+        suggestion,
+        MemberMergeSuggestionTable.MEMBER_TO_MERGE_FILTERED,
+      )
       continue
     }
 
@@ -89,6 +96,19 @@ export async function mergeMembersWithLLM(
 
     if (llmResult.body.content[0].text === 'true') {
       await commonActivitiesProxy.mergeMembers(suggestion[0], suggestion[1], args.tenantId)
+    } else {
+      console.log(
+        `LLM doesn't think these members are the same. Removing from suggestions and adding to no merge: ${suggestion[0]} and ${suggestion[1]}!`,
+      )
+      await memberActivitiesProxy.removeMemberMergeSuggestion(
+        suggestion,
+        MemberMergeSuggestionTable.MEMBER_TO_MERGE_FILTERED,
+      )
+      await memberActivitiesProxy.removeMemberMergeSuggestion(
+        suggestion,
+        MemberMergeSuggestionTable.MEMBER_TO_MERGE_RAW,
+      )
+      await memberActivitiesProxy.addMemberSuggestionToNoMerge(suggestion)
     }
   }
 

services/apps/merge_suggestions_worker/src/workflows/mergeOrganizationsWithLLM.ts

@@ -1,6 +1,6 @@
 import { continueAsNew, proxyActivities } from '@temporalio/workflow'
 
-import { LLMSuggestionVerdictType } from '@crowd/types'
+import { LLMSuggestionVerdictType, OrganizationMergeSuggestionTable } from '@crowd/types'
 
 import * as commonActivities from '../activities/common'
 import * as organizationActivities from '../activities/organizationMergeSuggestions'
@@ -51,7 +51,14 @@ export async function mergeOrganizationsWithLLM(
       console.log(
         `Failed getting organization data in suggestion. Skipping suggestion: ${suggestion}`,
       )
-      await organizationActivitiesProxy.removeRawOrganizationMergeSuggestions(suggestion)
+      await organizationActivitiesProxy.removeOrganizationMergeSuggestions(
+        suggestion,
+        OrganizationMergeSuggestionTable.ORGANIZATION_TO_MERGE_FILTERED,
+      )
+      await organizationActivitiesProxy.removeOrganizationMergeSuggestions(
+        suggestion,
+        OrganizationMergeSuggestionTable.ORGANIZATION_TO_MERGE_RAW,
+      )
       continue
     }
 
@@ -80,6 +87,19 @@ export async function mergeOrganizationsWithLLM(
         `LLM verdict says these two orgs are the same. Merging organizations: ${suggestion[0]} and ${suggestion[1]}!`,
       )
       await commonActivitiesProxy.mergeOrganizations(suggestion[0], suggestion[1], args.tenantId)
+    } else {
+      console.log(
+        `LLM doesn't think these orgs are the same. Removing from suggestions and adding to no merge: ${suggestion[0]} and ${suggestion[1]}!`,
+      )
+      await organizationActivitiesProxy.removeOrganizationMergeSuggestions(
+        suggestion,
+        OrganizationMergeSuggestionTable.ORGANIZATION_TO_MERGE_FILTERED,
+      )
+      await organizationActivitiesProxy.removeOrganizationMergeSuggestions(
+        suggestion,
+        OrganizationMergeSuggestionTable.ORGANIZATION_TO_MERGE_RAW,
+      )
+      await organizationActivitiesProxy.addOrganizationSuggestionToNoMerge(suggestion)
     }
   }
 

services/libs/data-access-layer/src/member_merge/index.ts

@@ -28,6 +28,7 @@ export async function addMemberNoMerge(
     `
       INSERT INTO "memberNoMerge" ("memberId", "noMergeId", "createdAt", "updatedAt")
       VALUES ($(memberId), $(noMergeId), $(createdAt), $(updatedAt))
+      on conflict ("memberId", "noMergeId") do nothing
     `,
     {
       memberId,

services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/memberMergeSuggestions.repo.ts

@@ -284,9 +284,12 @@ class MemberMergeSuggestionsRepository {
     return results.map((r) => [r.memberId, r.toMergeId])
   }
 
-  async removeRawMemberSuggestions(suggestion: string[]): Promise<void> {
+  async removeMemberMergeSuggestion(
+    suggestion: string[],
+    table: MemberMergeSuggestionTable,
+  ): Promise<void> {
     const query = `
-      delete from "memberToMergeRaw" 
+      delete from "${table}" 
       where 
         ("memberId" = $(memberId) and "toMergeId" = $(toMergeId))
         or 
@@ -301,7 +304,7 @@ class MemberMergeSuggestionsRepository {
     try {
       await this.connection.none(query, replacements)
     } catch (error) {
-      this.log.error('Error removing raw member suggestions', error)
+      this.log.error(`Error removing member suggestions from ${table}`, error)
       throw error
     }
   }

services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/organizationMergeSuggestions.repo.ts

@@ -301,9 +301,12 @@ class OrganizationMergeSuggestionsRepository {
     return results.map((r) => [r.organizationId, r.toMergeId])
   }
 
-  async removeRawOrganizationMergeSuggestions(suggestion: string[]): Promise<void> {
+  async removeOrganizationMergeSuggestions(
+    suggestion: string[],
+    table: OrganizationMergeSuggestionTable,
+  ): Promise<void> {
     const query = `
-      delete from "organizationToMergeRaw" 
+      delete from "${table}" 
       where 
         ("organizationId" = $(organizationId) and "toMergeId" = $(toMergeId))
         or 
@@ -318,7 +321,7 @@ class OrganizationMergeSuggestionsRepository {
     try {
       await this.connection.none(query, replacements)
     } catch (error) {
-      this.log.error('Error removing raw organization suggestions', error)
+      this.log.error(`Error removing organization suggestions rom ${table}!`, error)
       throw error
     }
   }

services/libs/data-access-layer/src/org_merge/index.ts

@@ -18,3 +18,24 @@ export async function findOrgNoMergeIds(
 
   return rows.map((row: { noMergeId: string }) => row.noMergeId)
 }
+
+export async function addOrgNoMerge(
+  qx: QueryExecutor,
+  organizationId: string,
+  noMergeId: string,
+): Promise<void> {
+  const currentTime = new Date()
+  await qx.result(
+    `
+      INSERT INTO "organizationNoMerge" ("organizationId", "noMergeId", "createdAt", "updatedAt")
+      VALUES ($(organizationId), $(noMergeId), $(createdAt), $(updatedAt))
+      on conflict ("organizationId", "noMergeId") do nothing
+    `,
+    {
+      organizationId,
+      noMergeId,
+      createdAt: currentTime,
+      updatedAt: currentTime,
+    },
+  )
+}