From c50ab0df04fd936f3b38e37e3337ec0d1ac5a186 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Thu, 28 Nov 2024 16:05:26 +0100
Subject: [PATCH 01/40] add some logging

---
 .../apps/premium/members_enrichment_worker/index.ts | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/services/libs/data-access-layer/src/old/apps/premium/members_enrichment_worker/index.ts b/services/libs/data-access-layer/src/old/apps/premium/members_enrichment_worker/index.ts
index cc211bbd03..77ccbedca0 100644
--- a/services/libs/data-access-layer/src/old/apps/premium/members_enrichment_worker/index.ts
+++ b/services/libs/data-access-layer/src/old/apps/premium/members_enrichment_worker/index.ts
@@ -514,6 +514,17 @@ export async function updateMemberOrg(
     delete params.dateStart
   }
 
+  const query = `select 1 from "memberOrganizations"
+      where "memberId" = $(memberId)
+      and "organizationId" = $(organizationId)
+      ${dateStartFilter}
+      ${dateEndFilter}
+      and "deletedAt" is null
+      and id <> $(id)`
+
+  console.log(query)
+  console.log(params)
+
   const existing = await tx.oneOrNone(
     `
       select 1 from "memberOrganizations"
@@ -527,6 +538,8 @@ export async function updateMemberOrg(
     params,
   )
 
+  console.log('existing', existing)
+
   if (existing) {
     // we should just delete the row
     await tx.none(

From e66da1e317a52a57c1d7ac8a9574e1c0589e2b23 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Thu, 28 Nov 2024 16:18:23 +0100
Subject: [PATCH 02/40] log more

---
 .../members_enrichment_worker/index.ts        | 40 ++++++-------------
 1 file changed, 13 insertions(+), 27 deletions(-)

diff --git a/services/libs/data-access-layer/src/old/apps/premium/members_enrichment_worker/index.ts b/services/libs/data-access-layer/src/old/apps/premium/members_enrichment_worker/index.ts
index 77ccbedca0..f672dff99d 100644
--- a/services/libs/data-access-layer/src/old/apps/premium/members_enrichment_worker/index.ts
+++ b/services/libs/data-access-layer/src/old/apps/premium/members_enrichment_worker/index.ts
@@ -1,18 +1,7 @@
-import { generateUUIDv4, redactNullByte } from '@crowd/common'
-import { DbConnOrTx, DbStore, DbTransaction } from '@crowd/database'
-import {
-  IAttributes,
-  IEnrichableMember,
-  IMemberEnrichmentCache,
-  IMemberEnrichmentSourceQueryInput,
-  IMemberIdentity,
-  IMemberOrganizationData,
-  IMemberOriginalData,
-  IOrganizationIdentity,
-  MemberEnrichmentSource,
-  MemberIdentityType,
-  OrganizationSource,
-} from '@crowd/types'
+import { generateUUIDv4, redactNullByte } from '@crowd/common';
+import { DbConnOrTx, DbStore, DbTransaction } from '@crowd/database';
+import { IAttributes, IEnrichableMember, IMemberEnrichmentCache, IMemberEnrichmentSourceQueryInput, IMemberIdentity, IMemberOrganizationData, IMemberOriginalData, IOrganizationIdentity, MemberEnrichmentSource, MemberIdentityType, OrganizationSource } from '@crowd/types';
+
 
 export async function fetchMemberDataForLLMSquashing(
   db: DbConnOrTx,
@@ -486,6 +475,12 @@ export async function updateMemberOrg(
   original: IMemberOrganizationData,
   toUpdate: Record<string, unknown>,
 ) {
+  // generate a random hash
+  const hash = generateUUIDv4()
+  console.log(`[${hash}] - Original: ${JSON.stringify(original)}`)
+  console.log(`[${hash}] - To Update: ${JSON.stringify(toUpdate)}`)
+  console.log(`[${hash}] - Member ID: ${memberId}`)
+
   const keys = Object.keys(toUpdate)
   if (keys.length === 0) {
     return
@@ -514,16 +509,7 @@ export async function updateMemberOrg(
     delete params.dateStart
   }
 
-  const query = `select 1 from "memberOrganizations"
-      where "memberId" = $(memberId)
-      and "organizationId" = $(organizationId)
-      ${dateStartFilter}
-      ${dateEndFilter}
-      and "deletedAt" is null
-      and id <> $(id)`
-
-  console.log(query)
-  console.log(params)
+  console.log(`[${hash}] - Params: ${JSON.stringify(params)}`)
 
   const existing = await tx.oneOrNone(
     `
@@ -538,7 +524,7 @@ export async function updateMemberOrg(
     params,
   )
 
-  console.log('existing', existing)
+  console.log(`[${hash}] - Existing: ${existing}`)
 
   if (existing) {
     // we should just delete the row
@@ -724,4 +710,4 @@ export async function findMemberEnrichmentCacheForAllSourcesDb<T>(
   )
 
   return result ?? []
-}
+}
\ No newline at end of file

From eace62d91139cdf481e70a18447fbbee0d854af8 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Thu, 28 Nov 2024 17:08:28 +0100
Subject: [PATCH 03/40] test new version

---
 .../src/activities/enrichment.ts                      | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
index 2ed2bad4c8..d8cb353be0 100644
--- a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
@@ -796,21 +796,12 @@ function prepareWorkExperiences(
   // we iterate through the existing version experiences to see if update is needed
   for (const current of orderedCurrentVersion) {
     // try and find a matching experience in the new versions by title
-    let match = orderedNewVersion.find(
+    const match = orderedNewVersion.find(
       (e) =>
         e.title === current.jobTitle &&
         e.identities &&
         e.identities.some((e) => e.organizationId === current.orgId),
     )
-    if (!match) {
-      // if we didn't find a match by title we should check dates
-      match = orderedNewVersion.find(
-        (e) =>
-          dateIntersects(current.dateStart, current.dateEnd, e.startDate, e.endDate) &&
-          e.identities &&
-          e.identities.some((e) => e.organizationId === current.orgId),
-      )
-    }
 
     // if we found a match we can check if we need something to update
     if (match) {

From 8a4bb549f4e23a5b4f83297510004144345dbc42 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Thu, 28 Nov 2024 17:47:00 +0100
Subject: [PATCH 04/40] only update manual changes when dateEnd is null

---
 .../src/activities/enrichment.ts               | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
index d8cb353be0..02a2214d42 100644
--- a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
@@ -804,8 +804,13 @@ function prepareWorkExperiences(
     )
 
     // if we found a match we can check if we need something to update
-    if (match) {
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    if (
+      match &&
+      (current.source !== OrganizationSource.UI ||
+        (current.dateStart === match.startDate &&
+          current.dateEnd === null &&
+          match.endDate !== null))
+    ) {
       const toUpdateInner: Record<string, any> = {}
 
       // lets check if the dates and title are the same otherwise we need to update them
@@ -827,6 +832,15 @@ function prepareWorkExperiences(
 
       // remove the match from the new version array so we later don't process it again
       orderedNewVersion = orderedNewVersion.filter((e) => e.id !== match.id)
+    } else if (
+      match &&
+      current.source === OrganizationSource.UI &&
+      (current.dateStart !== match.startDate || current.dateEnd !== null || match.endDate === null)
+    ) {
+      // there's an incoming work experiences, but it's conflicting with the existing manually updated data
+      // we shouldn't add or update anything when this happens
+      // we can only update dateEnd of existing manually changed data, when it has a null dateEnd
+      orderedNewVersion = orderedNewVersion.filter((e) => e.id !== match.id)
     }
     // if we didn't find a match we should just leave it as it is in the database since it was manual input
   }

From 547393857c779c16c74a9c159c9872d097a8ed5a Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Thu, 28 Nov 2024 18:18:42 +0100
Subject: [PATCH 05/40] profile finder and work expfixes

---
 .../src/activities/enrichment.ts                   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
index 02a2214d42..b575f02030 100644
--- a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
@@ -608,7 +608,7 @@ export async function findWhichLinkedinProfileToUseAmongScraperResult(
     }
   }
 
-  if (!categorized.selected && profilesFromUnverfiedIdentities.length > 0) {
+  if (profilesFromUnverfiedIdentities.length > 0) {
     const result = await findRelatedLinkedinProfilesWithLLM(
       memberId,
       memberData,
@@ -617,7 +617,9 @@ export async function findWhichLinkedinProfileToUseAmongScraperResult(
 
     // check if empty object
     if (result.profileIndex !== null) {
-      categorized.selected = profilesFromUnverfiedIdentities[result.profileIndex]
+      if (!categorized.selected) {
+        categorized.selected = profilesFromUnverfiedIdentities[result.profileIndex]
+      }
       // add profiles not selected to discarded
       for (let i = 0; i < profilesFromUnverfiedIdentities.length; i++) {
         if (i !== result.profileIndex) {
@@ -806,10 +808,9 @@ function prepareWorkExperiences(
     // if we found a match we can check if we need something to update
     if (
       match &&
-      (current.source !== OrganizationSource.UI ||
-        (current.dateStart === match.startDate &&
-          current.dateEnd === null &&
-          match.endDate !== null))
+      current.dateStart === match.startDate &&
+      current.dateEnd === null &&
+      match.endDate !== null
     ) {
       const toUpdateInner: Record<string, any> = {}
 
@@ -834,7 +835,6 @@ function prepareWorkExperiences(
       orderedNewVersion = orderedNewVersion.filter((e) => e.id !== match.id)
     } else if (
       match &&
-      current.source === OrganizationSource.UI &&
       (current.dateStart !== match.startDate || current.dateEnd !== null || match.endDate === null)
     ) {
       // there's an incoming work experiences, but it's conflicting with the existing manually updated data

From 1175d84f2b04e4b9105792bd020b77489bafae81 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Thu, 28 Nov 2024 18:21:59 +0100
Subject: [PATCH 06/40] simplify a bit

---
 .../src/activities/enrichment.ts               | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
index b575f02030..ec9fcd3b85 100644
--- a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
@@ -814,22 +814,8 @@ function prepareWorkExperiences(
     ) {
       const toUpdateInner: Record<string, any> = {}
 
-      // lets check if the dates and title are the same otherwise we need to update them
-      if (current.dateStart !== match.startDate) {
-        toUpdateInner.dateStart = match.startDate
-      }
-
-      if (current.dateEnd !== match.endDate) {
-        toUpdateInner.dateEnd = match.endDate
-      }
-
-      if (current.jobTitle !== match.title) {
-        toUpdateInner.title = match.title
-      }
-
-      if (Object.keys(toUpdateInner).length > 0) {
-        toUpdate.set(current, toUpdateInner)
-      }
+      toUpdateInner.dateEnd = match.endDate
+      toUpdate.set(current, toUpdateInner)
 
       // remove the match from the new version array so we later don't process it again
       orderedNewVersion = orderedNewVersion.filter((e) => e.id !== match.id)

From ca1f8d49fb23acec9a50f866586ae1dad580014e Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Thu, 28 Nov 2024 19:40:22 +0100
Subject: [PATCH 07/40] clean incoming attributes from llm

---
 .../src/workflows/processMemberSources.ts                       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/apps/premium/members_enrichment_worker/src/workflows/processMemberSources.ts b/services/apps/premium/members_enrichment_worker/src/workflows/processMemberSources.ts
index d573c1ac14..231919c5e0 100644
--- a/services/apps/premium/members_enrichment_worker/src/workflows/processMemberSources.ts
+++ b/services/apps/premium/members_enrichment_worker/src/workflows/processMemberSources.ts
@@ -233,7 +233,7 @@ export async function processMemberSources(args: IProcessMemberSourcesArgs): Pro
       for (const attribute of Object.keys(multipleValueAttributesSquashed)) {
         if (multipleValueAttributesSquashed[attribute]) {
           attributesSquashed[attribute] = {
-            enrichment: multipleValueAttributesSquashed[attribute],
+            enrichment: await cleanAttributeValue(multipleValueAttributesSquashed[attribute]),
           }
         }
       }

From 8faed6c3fc3c27c2c13d768e3aa09ae2280fa5f7 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Thu, 28 Nov 2024 19:51:17 +0100
Subject: [PATCH 08/40] fix

---
 services/libs/common/src/utils.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/libs/common/src/utils.ts b/services/libs/common/src/utils.ts
index 7bc376b7b4..b2ec5de6c0 100644
--- a/services/libs/common/src/utils.ts
+++ b/services/libs/common/src/utils.ts
@@ -73,7 +73,7 @@ export const redactNullByte = (str: string | null | undefined): string =>
   str ? str.replace(/\\u0000|\0/g, '[NULL]') : ''
 
 export const replaceDoubleQuotes = (str: string | null | undefined): string =>
-  str ? str.replace(/"/g, "'") : ''
+  str ? str.replace(/["“”]/g, "'") : ''
 
 export const dateEqualityChecker = (a, b) => {
   if (a instanceof Date) {

From 23e6cdbaf5761a0eea0f5d035a71b3f80be2c640 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Thu, 28 Nov 2024 20:26:20 +0100
Subject: [PATCH 09/40] update log levels

---
 .../src/activities/enrichment.ts                         | 9 ++++-----
 .../libs/common_services/src/services/llm.service.ts     | 2 +-
 .../data-access-layer/src/organizations/organizations.ts | 2 +-
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
index ec9fcd3b85..a18b9a04a0 100644
--- a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts
@@ -228,7 +228,7 @@ export async function updateMemberUsingSquashedPayload(
 
     // process identities
     if (squashedPayload.identities.length > 0) {
-      svc.log.info({ memberId }, 'Adding to member identities!')
+      svc.log.debug({ memberId }, 'Adding to member identities!')
       for (const i of squashedPayload.identities) {
         updated = true
         promises.push(
@@ -247,7 +247,7 @@ export async function updateMemberUsingSquashedPayload(
     // process contributions
     // if squashed payload has data from progai, we should fetch contributions here
     // it's ommited from the payload because it takes a lot of space
-    svc.log.info('Processing contributions! ', { memberId, hasContributions })
+    svc.log.debug('Processing contributions! ', { memberId, hasContributions })
     if (hasContributions) {
       promises.push(
         findMemberEnrichmentCache([MemberEnrichmentSource.PROGAI], memberId)
@@ -265,7 +265,6 @@ export async function updateMemberUsingSquashedPayload(
           .then((normalized) => {
             if (normalized) {
               const typed = normalized as IMemberEnrichmentDataNormalized
-              svc.log.info('Normalized contributions: ', { contributions: typed.contributions })
 
               if (typed.contributions) {
                 updated = true
@@ -280,7 +279,7 @@ export async function updateMemberUsingSquashedPayload(
     let attributes = existingMemberData.attributes as Record<string, unknown>
 
     if (squashedPayload.attributes) {
-      svc.log.info({ memberId }, 'Updating member attributes!')
+      svc.log.debug({ memberId }, 'Updating member attributes!')
 
       attributes = _.merge({}, attributes, squashedPayload.attributes)
 
@@ -298,7 +297,7 @@ export async function updateMemberUsingSquashedPayload(
 
     // process reach
     if (squashedPayload.reach && Object.keys(squashedPayload.reach).length > 0) {
-      svc.log.info({ memberId }, 'Updating member reach!')
+      svc.log.debug({ memberId }, 'Updating member reach!')
       let reach: IMemberReach
 
       if (existingMemberData.reach && existingMemberData.reach.total) {
diff --git a/services/libs/common_services/src/services/llm.service.ts b/services/libs/common_services/src/services/llm.service.ts
index 5b4ba5a26f..ca60a1690a 100644
--- a/services/libs/common_services/src/services/llm.service.ts
+++ b/services/libs/common_services/src/services/llm.service.ts
@@ -123,7 +123,7 @@ export class LlmService extends LoggerBase {
     const outputCost = (outputTokenCount / 1000) * pricing.costPer1000OutputTokens
     const totalCost = inputCost + outputCost
 
-    this.log.info({ type, entityId, inputCost, outputCost, totalCost }, 'Estimated LLM cost!')
+    this.log.debug({ type, entityId, inputCost, outputCost, totalCost }, 'Estimated LLM cost!')
 
     const result = {
       prompt,
diff --git a/services/libs/data-access-layer/src/organizations/organizations.ts b/services/libs/data-access-layer/src/organizations/organizations.ts
index 2b2bb5f3da..87e8995925 100644
--- a/services/libs/data-access-layer/src/organizations/organizations.ts
+++ b/services/libs/data-access-layer/src/organizations/organizations.ts
@@ -513,7 +513,7 @@ export async function findOrCreateOrganization(
     let id
 
     if (!existing && verifiedIdentities.length === 0) {
-      log.warn(
+      log.debug(
         { tenantId },
         'Organization does not have any verified identities and was not found by name so we will not create it.',
       )

From 00c7dc22a360a81034f0f30d939d156b79d28fc3 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Thu, 28 Nov 2024 22:35:51 +0100
Subject: [PATCH 10/40] fix  prog normalization

---
 .../members_enrichment_worker/src/sources/progai/service.ts     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts
index 65b07427b4..bfebc9c9ec 100644
--- a/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts
+++ b/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts
@@ -270,7 +270,7 @@ export default class EnrichmentServiceProgAI extends LoggerBase implements IEnri
           if (
             normalizedDomain &&
             !workExperience.companyUrl.toLowerCase().includes('github') &&
-            !workExperience.company.toLowerCase().includes('github')
+            !(workExperience.company || '').toLowerCase().includes('github')
           ) {
             identities.push({
               platform: PlatformType.LINKEDIN,

From 21c76ba5ca7fac182b5e39836af395bd72982c23 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Thu, 28 Nov 2024 23:10:40 +0100
Subject: [PATCH 11/40] fix serp linkedin handle parse

---
 .../members_enrichment_worker/src/sources/serp/service.ts | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts
index 6a93a08042..5cca50b3dd 100644
--- a/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts
+++ b/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts
@@ -128,13 +128,19 @@ export default class EnrichmentServiceSerpApi extends LoggerBase implements IEnr
           platform: PlatformType.LINKEDIN,
           type: MemberIdentityType.USERNAME,
           verified: false,
-          value: this.normalizeLinkedUrl(data.linkedinUrl).split('/').pop(),
+          value: this.getLinkedInProfileHandle(this.normalizeLinkedUrl(data.linkedinUrl)),
         },
       ],
     }
     return normalized
   }
 
+  private getLinkedInProfileHandle(url: string): string | null {
+    const regex = /in\/([^/]+)/
+    const match = url.match(regex)
+    return match ? match[1] : null
+  }
+
   private normalizeLinkedUrl(url: string): string {
     try {
       const parsedUrl = new URL(url)

From c0abfa3099fd1b9dd10ac159578a5de8632dc9f6 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 29 Nov 2024 01:01:19 +0100
Subject: [PATCH 12/40] process all members

---
 .../premium/members_enrichment_worker/src/bin/onboarding.ts     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/apps/premium/members_enrichment_worker/src/bin/onboarding.ts b/services/apps/premium/members_enrichment_worker/src/bin/onboarding.ts
index 31f1e6daa4..64df3cb9a6 100644
--- a/services/apps/premium/members_enrichment_worker/src/bin/onboarding.ts
+++ b/services/apps/premium/members_enrichment_worker/src/bin/onboarding.ts
@@ -27,7 +27,7 @@ const tenantId = processArguments[0]
 
 const minMemberActivities = 100
 const maxConcurrentProcessing = 5
-const maxMembersToProcess = 1000
+const maxMembersToProcess = Infinity
 
 async function getEnrichableMembers(limit: number): Promise<string[]> {
   const query = `

From 763480f89de0a18f47298d6da72c6d458cbd79e7 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 29 Nov 2024 09:48:38 +0100
Subject: [PATCH 13/40] when primary domain exists don't send the additional
 identities as verified to prevent multiple org matches

---
 .../members_enrichment_worker/src/sources/progai/service.ts   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts
index bfebc9c9ec..ec8641e210 100644
--- a/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts
+++ b/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts
@@ -262,6 +262,7 @@ export default class EnrichmentServiceProgAI extends LoggerBase implements IEnri
     if (data.work_experiences) {
       for (const workExperience of data.work_experiences) {
         const identities = []
+        let hasPrimaryDomainIdentity = false
 
         if (workExperience.companyUrl) {
           const normalizedDomain = websiteNormalizer(workExperience.companyUrl, false)
@@ -278,6 +279,7 @@ export default class EnrichmentServiceProgAI extends LoggerBase implements IEnri
               type: OrganizationIdentityType.PRIMARY_DOMAIN,
               verified: true,
             })
+            hasPrimaryDomainIdentity = true
           }
         }
 
@@ -286,7 +288,7 @@ export default class EnrichmentServiceProgAI extends LoggerBase implements IEnri
             platform: PlatformType.LINKEDIN,
             value: `company:${workExperience.companyLinkedInUrl.split('/').pop()}`,
             type: OrganizationIdentityType.USERNAME,
-            verified: true,
+            verified: !hasPrimaryDomainIdentity,
           })
         }
 

From a36056c2e5284e371cff555b1ef228ef317a1acb Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 29 Nov 2024 10:26:09 +0100
Subject: [PATCH 14/40] improve progai work exp normalization

---
 .../src/sources/progai/service.ts             | 74 ++++++++++---------
 1 file changed, 40 insertions(+), 34 deletions(-)

diff --git a/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts
index ec8641e210..8bf0c4ea87 100644
--- a/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts
+++ b/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts
@@ -261,47 +261,53 @@ export default class EnrichmentServiceProgAI extends LoggerBase implements IEnri
   ): IMemberEnrichmentDataNormalized {
     if (data.work_experiences) {
       for (const workExperience of data.work_experiences) {
-        const identities = []
-        let hasPrimaryDomainIdentity = false
-
-        if (workExperience.companyUrl) {
-          const normalizedDomain = websiteNormalizer(workExperience.companyUrl, false)
-
-          // sometimes companyUrl is a github link, we don't want to add it as a primary domain
-          if (
-            normalizedDomain &&
-            !workExperience.companyUrl.toLowerCase().includes('github') &&
-            !(workExperience.company || '').toLowerCase().includes('github')
-          ) {
+        if (
+          workExperience.company !== null ||
+          workExperience.companyUrl !== null ||
+          workExperience.companyLinkedInUrl !== null
+        ) {
+          const identities = []
+          let hasPrimaryDomainIdentity = false
+
+          if (workExperience.companyUrl) {
+            const normalizedDomain = websiteNormalizer(workExperience.companyUrl, false)
+
+            // sometimes companyUrl is a github link, we don't want to add it as a primary domain
+            if (
+              normalizedDomain &&
+              !workExperience.companyUrl.toLowerCase().includes('github') &&
+              !(workExperience.company || '').toLowerCase().includes('github')
+            ) {
+              identities.push({
+                platform: PlatformType.LINKEDIN,
+                value: normalizedDomain,
+                type: OrganizationIdentityType.PRIMARY_DOMAIN,
+                verified: true,
+              })
+              hasPrimaryDomainIdentity = true
+            }
+          }
+
+          if (workExperience.companyLinkedInUrl) {
             identities.push({
               platform: PlatformType.LINKEDIN,
-              value: normalizedDomain,
-              type: OrganizationIdentityType.PRIMARY_DOMAIN,
-              verified: true,
+              value: `company:${workExperience.companyLinkedInUrl.split('/').pop()}`,
+              type: OrganizationIdentityType.USERNAME,
+              verified: !hasPrimaryDomainIdentity,
             })
-            hasPrimaryDomainIdentity = true
           }
-        }
 
-        if (workExperience.companyLinkedInUrl) {
-          identities.push({
-            platform: PlatformType.LINKEDIN,
-            value: `company:${workExperience.companyLinkedInUrl.split('/').pop()}`,
-            type: OrganizationIdentityType.USERNAME,
-            verified: !hasPrimaryDomainIdentity,
+          normalized.memberOrganizations.push({
+            name: workExperience.company,
+            source: OrganizationSource.ENRICHMENT_PROGAI,
+            identities,
+            title: workExperience.title,
+            startDate: workExperience.startDate
+              ? workExperience.startDate.replace('Z', '+00:00')
+              : null,
+            endDate: workExperience.endDate ? workExperience.endDate.replace('Z', '+00:00') : null,
           })
         }
-
-        normalized.memberOrganizations.push({
-          name: workExperience.company,
-          source: OrganizationSource.ENRICHMENT_PROGAI,
-          identities,
-          title: workExperience.title,
-          startDate: workExperience.startDate
-            ? workExperience.startDate.replace('Z', '+00:00')
-            : null,
-          endDate: workExperience.endDate ? workExperience.endDate.replace('Z', '+00:00') : null,
-        })
       }
     }
 

From 6c19087f0b75cef8e78785c99771a557fdcf6c55 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 29 Nov 2024 13:13:06 +0100
Subject: [PATCH 15/40] better replaceDoubleQuotes

---
 services/libs/common/src/utils.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/libs/common/src/utils.ts b/services/libs/common/src/utils.ts
index b2ec5de6c0..0234a7fc6c 100644
--- a/services/libs/common/src/utils.ts
+++ b/services/libs/common/src/utils.ts
@@ -73,7 +73,7 @@ export const redactNullByte = (str: string | null | undefined): string =>
   str ? str.replace(/\\u0000|\0/g, '[NULL]') : ''
 
 export const replaceDoubleQuotes = (str: string | null | undefined): string =>
-  str ? str.replace(/["“”]/g, "'") : ''
+  str ? str.replace(/[\u201C\u201D\u0022\u201E\u201F\u2033\u2036"]/g, "'") : ''
 
 export const dateEqualityChecker = (a, b) => {
   if (a instanceof Date) {

From e8f761858c8ae8a4fb7ce292056dc97020c23ca3 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 29 Nov 2024 13:45:14 +0100
Subject: [PATCH 16/40] some logging

---
 services/libs/common/src/utils.ts | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/services/libs/common/src/utils.ts b/services/libs/common/src/utils.ts
index 0234a7fc6c..e62d125628 100644
--- a/services/libs/common/src/utils.ts
+++ b/services/libs/common/src/utils.ts
@@ -72,8 +72,12 @@ export const escapeNullByte = (str: string | null | undefined): string =>
 export const redactNullByte = (str: string | null | undefined): string =>
   str ? str.replace(/\\u0000|\0/g, '[NULL]') : ''
 
-export const replaceDoubleQuotes = (str: string | null | undefined): string =>
-  str ? str.replace(/[\u201C\u201D\u0022\u201E\u201F\u2033\u2036"]/g, "'") : ''
+export const replaceDoubleQuotes = (str: string | null | undefined): string => {
+  const cleanedValue = str ? str.replace(/[\u201C\u201D\u0022\u201E\u201F\u2033\u2036"]/g, "'") : ''
+
+  console.log(`String ${str} cleaned to ${cleanedValue}`)
+  return cleanedValue
+}
 
 export const dateEqualityChecker = (a, b) => {
   if (a instanceof Date) {

From d59ac6b9740a508dd9c3ebf0df7f53017e8fa752 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 29 Nov 2024 13:51:18 +0100
Subject: [PATCH 17/40] remove logging

---
 services/libs/common/src/utils.ts | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/services/libs/common/src/utils.ts b/services/libs/common/src/utils.ts
index e62d125628..0234a7fc6c 100644
--- a/services/libs/common/src/utils.ts
+++ b/services/libs/common/src/utils.ts
@@ -72,12 +72,8 @@ export const escapeNullByte = (str: string | null | undefined): string =>
 export const redactNullByte = (str: string | null | undefined): string =>
   str ? str.replace(/\\u0000|\0/g, '[NULL]') : ''
 
-export const replaceDoubleQuotes = (str: string | null | undefined): string => {
-  const cleanedValue = str ? str.replace(/[\u201C\u201D\u0022\u201E\u201F\u2033\u2036"]/g, "'") : ''
-
-  console.log(`String ${str} cleaned to ${cleanedValue}`)
-  return cleanedValue
-}
+export const replaceDoubleQuotes = (str: string | null | undefined): string =>
+  str ? str.replace(/[\u201C\u201D\u0022\u201E\u201F\u2033\u2036"]/g, "'") : ''
 
 export const dateEqualityChecker = (a, b) => {
   if (a instanceof Date) {

From 31ee17b903a5b6aa5c8a90334784b571b82255ff Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 29 Nov 2024 19:30:03 +0100
Subject: [PATCH 18/40] org identity squash fix

---
 .../src/workflows/processMemberSources.ts     | 29 ++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts b/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
index 231919c5e0..bf445fb9f5 100644
--- a/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
+++ b/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
@@ -1,6 +1,6 @@
 import { proxyActivities } from '@temporalio/workflow'
 
-import { MemberEnrichmentSource, PlatformType } from '@crowd/types'
+import { MemberEnrichmentSource, OrganizationIdentityType, PlatformType } from '@crowd/types'
 
 import * as activities from '../activities/enrichment'
 import { IMemberEnrichmentDataNormalized, IProcessMemberSourcesArgs } from '../types'
@@ -265,6 +265,33 @@ export async function processMemberSources(args: IProcessMemberSourcesArgs): Pro
           args.memberId,
           workExperienceDataInDifferentSources,
         )
+        // make sure only one version of same-value primary-domain identities exist in org identities
+        // also if there are multiple different-value primary-domain identities, mark one of them as unverified
+        // because these two can belong to different organizations
+        workExperiencesSquashedByLLM.forEach((we) => {
+          let found = false
+          let foundValue
+          const checkedIdentities = []
+          we.identities.forEach((i) => {
+            if (i.type === OrganizationIdentityType.PRIMARY_DOMAIN) {
+              if (!found) {
+                found = true
+                foundValue = i.value
+                checkedIdentities.push(i)
+              } else {
+                if (i.value !== foundValue) {
+                  checkedIdentities.push({
+                    ...i,
+                    verified: false,
+                  })
+                }
+              }
+            } else {
+              checkedIdentities.push(i)
+            }
+          })
+          we.identities = checkedIdentities
+        })
         squashedPayload.memberOrganizations = workExperiencesSquashedByLLM
       }
     }

From 98105dd9e595bb64e82af502c736c7f950f63657 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 29 Nov 2024 20:18:34 +0100
Subject: [PATCH 19/40] fix

---
 .../src/workflows/processMemberSources.ts     | 28 ++++++-------------
 1 file changed, 9 insertions(+), 19 deletions(-)

diff --git a/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts b/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
index bf445fb9f5..d593ff2f84 100644
--- a/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
+++ b/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
@@ -265,29 +265,19 @@ export async function processMemberSources(args: IProcessMemberSourcesArgs): Pro
           args.memberId,
           workExperienceDataInDifferentSources,
         )
-        // make sure only one version of same-value primary-domain identities exist in org identities
-        // also if there are multiple different-value primary-domain identities, mark one of them as unverified
-        // because these two can belong to different organizations
+        // make sure there's only one incoming verified identity and
         workExperiencesSquashedByLLM.forEach((we) => {
           let found = false
-          let foundValue
           const checkedIdentities = []
           we.identities.forEach((i) => {
-            if (i.type === OrganizationIdentityType.PRIMARY_DOMAIN) {
-              if (!found) {
-                found = true
-                foundValue = i.value
-                checkedIdentities.push(i)
-              } else {
-                if (i.value !== foundValue) {
-                  checkedIdentities.push({
-                    ...i,
-                    verified: false,
-                  })
-                }
-              }
-            } else {
-              checkedIdentities.push(i)
+            if (i.verified && !found) {
+              found = true
+              checkedIdentities.push(i.value)
+            } else if (i.verified && found) {
+              checkedIdentities.push({
+                ...i,
+                verified: false,
+              })
             }
           })
           we.identities = checkedIdentities

From d4e7bb0c8666f8725cd8e3a5ed281f6ab3f1e360 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 29 Nov 2024 22:27:58 +0100
Subject: [PATCH 20/40] fix for identities inside workExperiencesSquashedByLLM

---
 .../src/workflows/processMemberSources.ts      | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts b/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
index d593ff2f84..c45bf78ad2 100644
--- a/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
+++ b/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
@@ -265,22 +265,20 @@ export async function processMemberSources(args: IProcessMemberSourcesArgs): Pro
           args.memberId,
           workExperienceDataInDifferentSources,
         )
-        // make sure there's only one incoming verified identity and
+        // if there are multiple verified identities in work experiences, we reduce it
+        // to one because in our db they might exist in different organizations and
+        // might need a merge. To avoid this, we'll only send the org with one verified identity
         workExperiencesSquashedByLLM.forEach((we) => {
           let found = false
-          const checkedIdentities = []
-          we.identities.forEach((i) => {
+          we.identities = we.identities.map((i) => {
             if (i.verified && !found) {
               found = true
-              checkedIdentities.push(i.value)
-            } else if (i.verified && found) {
-              checkedIdentities.push({
-                ...i,
-                verified: false,
-              })
+              return i
+            } else if (i.verified) {
+              return { ...i, verified: false }
             }
+            return i
           })
-          we.identities = checkedIdentities
         })
         squashedPayload.memberOrganizations = workExperiencesSquashedByLLM
       }

From 73add515f49daec162f53cb1ea6ccb976baa51e7 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 29 Nov 2024 22:44:58 +0100
Subject: [PATCH 21/40] bit more fixing

---
 .../src/workflows/processMemberSources.ts                       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts b/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
index c45bf78ad2..5f0f4532ac 100644
--- a/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
+++ b/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
@@ -270,7 +270,7 @@ export async function processMemberSources(args: IProcessMemberSourcesArgs): Pro
         // might need a merge. To avoid this, we'll only send the org with one verified identity
         workExperiencesSquashedByLLM.forEach((we) => {
           let found = false
-          we.identities = we.identities.map((i) => {
+          we.identities = (we.identities || []).map((i) => {
             if (i.verified && !found) {
               found = true
               return i

From ab189a6dceb6a8315ffdc48024e519ce51beb83f Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Sat, 30 Nov 2024 10:49:58 +0100
Subject: [PATCH 22/40] use replaceDoubleQuotes on work experience free text
 fields

---
 .../src/sources/clearbit/service.ts                    |  5 +++--
 .../src/sources/crustdata/service.ts                   | 10 ++++++----
 .../src/sources/progai/service.ts                      |  6 +++---
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/services/apps/members_enrichment_worker/src/sources/clearbit/service.ts b/services/apps/members_enrichment_worker/src/sources/clearbit/service.ts
index 0345481114..60efc4c8dc 100644
--- a/services/apps/members_enrichment_worker/src/sources/clearbit/service.ts
+++ b/services/apps/members_enrichment_worker/src/sources/clearbit/service.ts
@@ -1,5 +1,6 @@
 import axios from 'axios'
 
+import { replaceDoubleQuotes } from '@crowd/common'
 import { Logger, LoggerBase } from '@crowd/logging'
 import {
   MemberAttributeName,
@@ -216,10 +217,10 @@ export default class EnrichmentServiceClearbit extends LoggerBase implements IEn
       }
 
       normalized.memberOrganizations.push({
-        name: data.employment.name,
+        name: replaceDoubleQuotes(data.employment.name),
         source: OrganizationSource.ENRICHMENT_CLEARBIT,
         identities: orgIdentities,
-        title: data.employment.title,
+        title: replaceDoubleQuotes(data.employment.title),
         startDate: null,
         endDate: null,
       })
diff --git a/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts
index 195fe8ce82..f0c5bc5f23 100644
--- a/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts
+++ b/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts
@@ -1,6 +1,6 @@
 import axios from 'axios'
 
-import { isEmail } from '@crowd/common'
+import { isEmail, replaceDoubleQuotes } from '@crowd/common'
 import { Logger, LoggerBase } from '@crowd/logging'
 import {
   IMemberEnrichmentCache,
@@ -352,13 +352,15 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE
         }
 
         normalized.memberOrganizations.push({
-          name: workExperience.employer_name,
+          name: replaceDoubleQuotes(workExperience.employer_name),
           source: OrganizationSource.ENRICHMENT_CRUSTDATA,
           identities,
-          title: workExperience.employee_title,
+          title: replaceDoubleQuotes(workExperience.employee_title),
           startDate: workExperience?.start_date ?? null,
           endDate: workExperience?.end_date ?? null,
-          organizationDescription: workExperience.employer_linkedin_description,
+          organizationDescription: replaceDoubleQuotes(
+            workExperience.employer_linkedin_description,
+          ),
         })
       }
     }
diff --git a/services/apps/members_enrichment_worker/src/sources/progai/service.ts b/services/apps/members_enrichment_worker/src/sources/progai/service.ts
index 8bf0c4ea87..8d7719f919 100644
--- a/services/apps/members_enrichment_worker/src/sources/progai/service.ts
+++ b/services/apps/members_enrichment_worker/src/sources/progai/service.ts
@@ -1,7 +1,7 @@
 import axios from 'axios'
 import lodash from 'lodash'
 
-import { websiteNormalizer } from '@crowd/common'
+import { replaceDoubleQuotes, websiteNormalizer } from '@crowd/common'
 import { Logger, LoggerBase } from '@crowd/logging'
 import {
   MemberAttributeName,
@@ -298,10 +298,10 @@ export default class EnrichmentServiceProgAI extends LoggerBase implements IEnri
           }
 
           normalized.memberOrganizations.push({
-            name: workExperience.company,
+            name: replaceDoubleQuotes(workExperience.company),
             source: OrganizationSource.ENRICHMENT_PROGAI,
             identities,
-            title: workExperience.title,
+            title: replaceDoubleQuotes(workExperience.title),
             startDate: workExperience.startDate
               ? workExperience.startDate.replace('Z', '+00:00')
               : null,

From cf6f43af321a7bbb17e0ce5679e5802658d9945c Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Sat, 30 Nov 2024 13:56:02 +0100
Subject: [PATCH 23/40] switch to sonnet 3.5v2 because of longer output token
 support

---
 services/libs/types/src/enums/llm.ts |  1 +
 services/libs/types/src/llm.ts       | 11 ++++++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/services/libs/types/src/enums/llm.ts b/services/libs/types/src/enums/llm.ts
index 28f707c36a..7d28970db5 100644
--- a/services/libs/types/src/enums/llm.ts
+++ b/services/libs/types/src/enums/llm.ts
@@ -1,5 +1,6 @@
 export enum LlmModelType {
   CLAUDE_3_5_SONNET = 'anthropic.claude-3-5-sonnet-20240620-v1:0',
+  CLAUDE_3_5_SONNET_V2 = 'anthropic.claude-3-5-sonnet-20241022-v2:0',
   CLAUDE_3_OPUS = 'anthropic.claude-3-opus-20240229-v1:0',
 }
 
diff --git a/services/libs/types/src/llm.ts b/services/libs/types/src/llm.ts
index f2bf310839..677a0fe952 100644
--- a/services/libs/types/src/llm.ts
+++ b/services/libs/types/src/llm.ts
@@ -30,6 +30,7 @@ export interface ILlmPricing {
 export const LLM_MODEL_REGION_MAP: Record<LlmModelType, string> = {
   [LlmModelType.CLAUDE_3_OPUS]: 'us-west-2',
   [LlmModelType.CLAUDE_3_5_SONNET]: 'us-east-1',
+  [LlmModelType.CLAUDE_3_5_SONNET_V2]: 'us-west-2',
 }
 
 // to estimate costs - these numbers can change
@@ -42,6 +43,10 @@ export const LLM_MODEL_PRICING_MAP: Record<LlmModelType, ILlmPricing> = {
     costPer1000InputTokens: 0.003,
     costPer1000OutputTokens: 0.015,
   },
+  [LlmModelType.CLAUDE_3_5_SONNET_V2]: {
+    costPer1000InputTokens: 0.003,
+    costPer1000OutputTokens: 0.015,
+  },
 }
 
 export const LLM_SETTINGS: Record<LlmQueryType, ILlmSettings> = {
@@ -54,7 +59,7 @@ export const LLM_SETTINGS: Record<LlmQueryType, ILlmSettings> = {
     },
   },
   [LlmQueryType.MEMBER_ENRICHMENT_FIND_RELATED_LINKEDIN_PROFILES]: {
-    modelId: LlmModelType.CLAUDE_3_5_SONNET,
+    modelId: LlmModelType.CLAUDE_3_5_SONNET_V2,
     arguments: {
       max_tokens: 200000,
       anthropic_version: 'bedrock-2023-05-31',
@@ -62,7 +67,7 @@ export const LLM_SETTINGS: Record<LlmQueryType, ILlmSettings> = {
     },
   },
   [LlmQueryType.MEMBER_ENRICHMENT_SQUASH_MULTIPLE_VALUE_ATTRIBUTES]: {
-    modelId: LlmModelType.CLAUDE_3_5_SONNET,
+    modelId: LlmModelType.CLAUDE_3_5_SONNET_V2,
     arguments: {
       max_tokens: 200000,
       anthropic_version: 'bedrock-2023-05-31',
@@ -70,7 +75,7 @@ export const LLM_SETTINGS: Record<LlmQueryType, ILlmSettings> = {
     },
   },
   [LlmQueryType.MEMBER_ENRICHMENT_SQUASH_WORK_EXPERIENCES_FROM_MULTIPLE_SOURCES]: {
-    modelId: LlmModelType.CLAUDE_3_5_SONNET,
+    modelId: LlmModelType.CLAUDE_3_5_SONNET_V2,
     arguments: {
       max_tokens: 200000,
       anthropic_version: 'bedrock-2023-05-31',

From d8b9762d9e37cc070f74edcb2c73d7813e450b09 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Mon, 2 Dec 2024 09:40:05 +0100
Subject: [PATCH 24/40] final fixes, some cleaning

---
 .../src/activities/enrichment.ts                       |  2 +-
 .../src/workflows/enrichMember.ts                      |  7 +++++++
 .../src/old/apps/members_enrichment_worker/index.ts    | 10 ++--------
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
index 2808851a71..19a50d1e64 100644
--- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
@@ -65,7 +65,7 @@ export async function getEnrichmentData(
   input: IEnrichmentSourceInput,
 ): Promise<IMemberEnrichmentData | null> {
   const service = EnrichmentSourceServiceFactory.getEnrichmentSourceService(source, svc.log)
-  if ((await service.isEnrichableBySource(input)) && (await hasRemainingCredits(source))) {
+  if (await service.isEnrichableBySource(input)) {
     return service.getData(input)
   }
   return null
diff --git a/services/apps/members_enrichment_worker/src/workflows/enrichMember.ts b/services/apps/members_enrichment_worker/src/workflows/enrichMember.ts
index 16088ac8f5..e8a39308b2 100644
--- a/services/apps/members_enrichment_worker/src/workflows/enrichMember.ts
+++ b/services/apps/members_enrichment_worker/src/workflows/enrichMember.ts
@@ -8,6 +8,7 @@ import {
 import { IEnrichableMember, MemberEnrichmentSource } from '@crowd/types'
 
 import * as activities from '../activities'
+import { hasRemainingCredits } from '../activities/enrichment'
 import { IEnrichmentSourceInput } from '../types'
 import { sourceHasDifferentDataComparedToCache } from '../utils/common'
 
@@ -46,6 +47,12 @@ export async function enrichMember(
     if (await isCacheObsolete(source, cache)) {
       const enrichmentInput: IEnrichmentSourceInput = await getEnrichmentInput(input)
 
+      if (!(await hasRemainingCredits(source))) {
+        // no credits remaining, only update cache.updatedAt and keep the old data
+        await touchMemberEnrichmentCacheUpdatedAt(source, input.id)
+        continue
+      }
+
       const data = await getEnrichmentData(source, enrichmentInput)
 
       if (!cache) {
diff --git a/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts b/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
index f672dff99d..da14782ed0 100644
--- a/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
+++ b/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
@@ -125,8 +125,9 @@ export async function fetchMembersForEnrichment(
          INNER JOIN tenants ON tenants.id = members."tenantId"
          INNER JOIN "memberIdentities" mi ON mi."memberId" = members.id
          LEFT JOIN "membersGlobalActivityCount" ON "membersGlobalActivityCount"."memberId" = members.id
-    WHERE 
+    WHERE
       ${enrichableBySqlJoined}
+      AND coalesce((m.attributes ->'isBot'->>'default')::boolean, false) = false 
       AND tenants."deletedAt" IS NULL
       AND members."deletedAt" IS NULL
       AND (${cacheAgeInnerQueryItems.join(' OR ')})
@@ -477,9 +478,6 @@ export async function updateMemberOrg(
 ) {
   // generate a random hash
   const hash = generateUUIDv4()
-  console.log(`[${hash}] - Original: ${JSON.stringify(original)}`)
-  console.log(`[${hash}] - To Update: ${JSON.stringify(toUpdate)}`)
-  console.log(`[${hash}] - Member ID: ${memberId}`)
 
   const keys = Object.keys(toUpdate)
   if (keys.length === 0) {
@@ -509,8 +507,6 @@ export async function updateMemberOrg(
     delete params.dateStart
   }
 
-  console.log(`[${hash}] - Params: ${JSON.stringify(params)}`)
-
   const existing = await tx.oneOrNone(
     `
       select 1 from "memberOrganizations"
@@ -524,8 +520,6 @@ export async function updateMemberOrg(
     params,
   )
 
-  console.log(`[${hash}] - Existing: ${existing}`)
-
   if (existing) {
     // we should just delete the row
     await tx.none(

From 4c24a9b73c26fe56b17fc46c6fe96c6acc1fe0d1 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Mon, 2 Dec 2024 09:43:38 +0100
Subject: [PATCH 25/40] remove unused code

---
 .../src/old/apps/members_enrichment_worker/index.ts            | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts b/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
index da14782ed0..6ab625f8ac 100644
--- a/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
+++ b/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
@@ -476,9 +476,6 @@ export async function updateMemberOrg(
   original: IMemberOrganizationData,
   toUpdate: Record<string, unknown>,
 ) {
-  // generate a random hash
-  const hash = generateUUIDv4()
-
   const keys = Object.keys(toUpdate)
   if (keys.length === 0) {
     return

From d8a63b497a52696def5f6edf7388d22c818182fd Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Mon, 2 Dec 2024 10:08:35 +0100
Subject: [PATCH 26/40] formatting

---
 .../apps/members_enrichment_worker/index.ts   | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts b/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
index 6ab625f8ac..295e5eaefe 100644
--- a/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
+++ b/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
@@ -1,7 +1,18 @@
-import { generateUUIDv4, redactNullByte } from '@crowd/common';
-import { DbConnOrTx, DbStore, DbTransaction } from '@crowd/database';
-import { IAttributes, IEnrichableMember, IMemberEnrichmentCache, IMemberEnrichmentSourceQueryInput, IMemberIdentity, IMemberOrganizationData, IMemberOriginalData, IOrganizationIdentity, MemberEnrichmentSource, MemberIdentityType, OrganizationSource } from '@crowd/types';
-
+import { generateUUIDv4, redactNullByte } from '@crowd/common'
+import { DbConnOrTx, DbStore, DbTransaction } from '@crowd/database'
+import {
+  IAttributes,
+  IEnrichableMember,
+  IMemberEnrichmentCache,
+  IMemberEnrichmentSourceQueryInput,
+  IMemberIdentity,
+  IMemberOrganizationData,
+  IMemberOriginalData,
+  IOrganizationIdentity,
+  MemberEnrichmentSource,
+  MemberIdentityType,
+  OrganizationSource,
+} from '@crowd/types'
 
 export async function fetchMemberDataForLLMSquashing(
   db: DbConnOrTx,
@@ -701,4 +712,4 @@ export async function findMemberEnrichmentCacheForAllSourcesDb<T>(
   )
 
   return result ?? []
-}
\ No newline at end of file
+}

From 64aee6c44d29dd9ada9e5c9ca34ebca0dd3d3b80 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Mon, 2 Dec 2024 10:35:57 +0100
Subject: [PATCH 27/40] fix linting

---
 .../src/activities/enrichment.ts              | 22 -------------------
 .../src/workflows/processMemberSources.ts     |  2 +-
 2 files changed, 1 insertion(+), 23 deletions(-)

diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
index 19a50d1e64..79442ed7e6 100644
--- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
@@ -840,28 +840,6 @@ function prepareWorkExperiences(
   }
 }
 
-function dateIntersects(
-  d1Start?: string | null,
-  d1End?: string | null,
-  d2Start?: string | null,
-  d2End?: string | null,
-): boolean {
-  // If both periods have no dates at all, we can't determine intersection
-  if ((!d1Start && !d1End) || (!d2Start && !d2End)) {
-    return false
-  }
-
-  // Convert strings to timestamps, using fallbacks for missing dates
-  const start1 = d1Start ? new Date(d1Start).getTime() : -Infinity
-  const end1 = d1End ? new Date(d1End).getTime() : Infinity
-  const start2 = d2Start ? new Date(d2Start).getTime() : -Infinity
-  const end2 = d2End ? new Date(d2End).getTime() : Infinity
-
-  // Periods intersect if one period's start is before other period's end
-  // and that same period's end is after the other period's start
-  return start1 <= end2 && end1 >= start2
-}
-
 export async function cleanAttributeValue(
   attributeValue: string | string[] | Record<string, any>,
 ): Promise<string | string[] | Record<string, any>> {
diff --git a/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts b/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
index 5f0f4532ac..5c5f4f5959 100644
--- a/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
+++ b/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
@@ -1,6 +1,6 @@
 import { proxyActivities } from '@temporalio/workflow'
 
-import { MemberEnrichmentSource, OrganizationIdentityType, PlatformType } from '@crowd/types'
+import { MemberEnrichmentSource, PlatformType } from '@crowd/types'
 
 import * as activities from '../activities/enrichment'
 import { IMemberEnrichmentDataNormalized, IProcessMemberSourcesArgs } from '../types'

From a859861867d3d5668ecacac5fb20437c452e2d70 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Tue, 3 Dec 2024 14:36:16 +0100
Subject: [PATCH 28/40] fix progai linkedin normalization

---
 .../src/sources/progai/service.ts             | 25 +++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/services/apps/members_enrichment_worker/src/sources/progai/service.ts b/services/apps/members_enrichment_worker/src/sources/progai/service.ts
index 8d7719f919..584fa4d5d6 100644
--- a/services/apps/members_enrichment_worker/src/sources/progai/service.ts
+++ b/services/apps/members_enrichment_worker/src/sources/progai/service.ts
@@ -288,10 +288,13 @@ export default class EnrichmentServiceProgAI extends LoggerBase implements IEnri
             }
           }
 
-          if (workExperience.companyLinkedInUrl) {
+          if (
+            workExperience.companyLinkedInUrl &&
+            this.getLinkedInProfileHandle(workExperience.companyLinkedInUrl)
+          ) {
             identities.push({
               platform: PlatformType.LINKEDIN,
-              value: `company:${workExperience.companyLinkedInUrl.split('/').pop()}`,
+              value: this.getLinkedInProfileHandle(workExperience.companyLinkedInUrl),
               type: OrganizationIdentityType.USERNAME,
               verified: !hasPrimaryDomainIdentity,
             })
@@ -314,6 +317,24 @@ export default class EnrichmentServiceProgAI extends LoggerBase implements IEnri
     return normalized
   }
 
+  private getLinkedInProfileHandle(url: string): string | null {
+    let regex = /company\/([^/]+)/
+    let match = url.match(regex)
+
+    if (match) {
+      return `company:${match[1]}`
+    }
+
+    regex = /school\/([^/]+)/
+    match = url.match(regex)
+
+    if (match) {
+      return `school:${match[1]}`
+    }
+
+    return null
+  }
+
   async getDataUsingGitHubHandle(githubUsername: string): Promise<IMemberEnrichmentDataProgAI> {
     const url = `${process.env['CROWD_ENRICHMENT_PROGAI_URL']}/get_profile`
     const config = {

From 5e2fcb0acad5f6366e26db36d5c26128c619ad0e Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Thu, 5 Dec 2024 11:36:10 +0100
Subject: [PATCH 29/40] checking existent work experiences when mapping
 incoming new work experiences from enrichment

---
 .../src/activities.ts                         |  2 +
 .../src/activities/enrichment.ts              | 63 ++++++++++++++++++-
 .../src/workflows/enrichMember.ts             |  2 +-
 .../src/workflows/processMemberSources.ts     |  1 +
 4 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/services/apps/members_enrichment_worker/src/activities.ts b/services/apps/members_enrichment_worker/src/activities.ts
index 3853e32133..1686b5c1d5 100644
--- a/services/apps/members_enrichment_worker/src/activities.ts
+++ b/services/apps/members_enrichment_worker/src/activities.ts
@@ -8,6 +8,7 @@ import {
   getEnrichmentInput,
   getObsoleteSourcesOfMember,
   getTenantPriorityArray,
+  hasRemainingCredits,
   insertMemberEnrichmentCache,
   isCacheObsolete,
   isEnrichableBySource,
@@ -75,4 +76,5 @@ export {
   updateMemberUsingSquashedPayload,
   getTenantPriorityArray,
   cleanAttributeValue,
+  hasRemainingCredits,
 }
diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
index 79442ed7e6..6d6424ae8e 100644
--- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
@@ -220,6 +220,7 @@ export async function updateMemberUsingSquashedPayload(
   existingMemberData: IMemberOriginalData,
   squashedPayload: IMemberEnrichmentDataNormalized,
   hasContributions: boolean,
+  isHighConfidenceSourceSelectedForWorkExperiences: boolean,
 ): Promise<boolean> {
   return await svc.postgres.writer.transactionally(async (tx) => {
     let updated = false
@@ -318,7 +319,54 @@ export async function updateMemberUsingSquashedPayload(
 
     if (squashedPayload.memberOrganizations.length > 0) {
       const orgPromises = []
+
+      // try matching member's existing organizations with the new ones
+      // we'll be using displayName, title, dates
       for (const org of squashedPayload.memberOrganizations) {
+        if (!org.organizationId) {
+          // Check if any similar in existing work experiences
+          const existingOrg = existingMemberData.organizations.find((o) => {
+            const incomingOrgStartDate = org.startDate ? new Date(org.startDate) : null
+            const incomingOrgEndDate = org.endDate ? new Date(org.endDate) : null
+            const existingOrgStartDate = o.dateStart ? new Date(o.dateStart) : null
+            const existingOrgEndEndDate = o.dateEnd ? new Date(o.dateEnd) : null
+
+            const isSameStartMonthYear =
+              (!incomingOrgStartDate && !existingOrgStartDate) || // Both start dates are null
+              (incomingOrgStartDate &&
+                existingOrgStartDate &&
+                incomingOrgStartDate.getMonth() === existingOrgStartDate.getMonth() &&
+                incomingOrgStartDate.getFullYear() === existingOrgStartDate.getFullYear())
+
+            const isSameEndMonthYear =
+              (!incomingOrgEndDate && !existingOrgEndEndDate) || // Both end dates are null
+              (incomingOrgEndDate &&
+                existingOrgEndEndDate &&
+                incomingOrgEndDate.getMonth() === existingOrgEndEndDate.getMonth() &&
+                incomingOrgEndDate.getFullYear() === existingOrgEndEndDate.getFullYear())
+
+            return (
+              (o.orgName.toLowerCase().includes(org.name.toLowerCase()) ||
+                org.name.toLowerCase().includes(o.orgName.toLowerCase())) &&
+              ((isSameStartMonthYear && isSameEndMonthYear) || org.title === o.jobTitle)
+            )
+          })
+
+          if (existingOrg) {
+            // Get all orgs with the same name as the current one
+            const matchingOrgs = squashedPayload.memberOrganizations.filter(
+              (otherOrg) => otherOrg.name === org.name,
+            )
+
+            // Set organizationId for all matching orgs
+            for (const matchingOrg of matchingOrgs) {
+              matchingOrg.organizationId = existingOrg.orgId
+            }
+          }
+        }
+      }
+
+      for (const org of squashedPayload.memberOrganizations.filter((o) => !o.organizationId)) {
         orgPromises.push(
           findOrCreateOrganization(
             qx,
@@ -350,6 +398,7 @@ export async function updateMemberUsingSquashedPayload(
       const results = prepareWorkExperiences(
         existingMemberData.organizations,
         squashedPayload.memberOrganizations,
+        isHighConfidenceSourceSelectedForWorkExperiences,
       )
 
       if (results.toDelete.length > 0) {
@@ -759,14 +808,25 @@ interface IWorkExperienceChanges {
 function prepareWorkExperiences(
   oldVersion: IMemberOrganizationData[],
   newVersion: IMemberEnrichmentDataNormalizedOrganization[],
+  isHighConfidenceSourceSelectedForWorkExperiences: boolean,
 ): IWorkExperienceChanges {
   // we delete all the work experiences that were not manually created
-  const toDelete = oldVersion.filter((c) => c.source !== OrganizationSource.UI)
+  let toDelete = oldVersion.filter((c) => c.source !== OrganizationSource.UI)
 
   const toCreate: IMemberEnrichmentDataNormalizedOrganization[] = []
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const toUpdate: Map<IMemberOrganizationData, Record<string, any>> = new Map()
 
+  if (isHighConfidenceSourceSelectedForWorkExperiences) {
+    toDelete = oldVersion
+    toCreate.push(...newVersion)
+    return {
+      toDelete,
+      toCreate,
+      toUpdate,
+    }
+  }
+
   // sort both versions by start date and only use manual changes from the current version
   const orderedCurrentVersion = oldVersion
     .filter((c) => c.source === OrganizationSource.UI)
@@ -779,6 +839,7 @@ function prepareWorkExperiences(
       // Compare dates if both values exist
       return new Date(a.dateStart as string).getTime() - new Date(b.dateStart as string).getTime()
     })
+
   let orderedNewVersion = newVersion.sort((a, b) => {
     // If either value is null/undefined, move it to the beginning
     if (!a.startDate && !b.startDate) return 0
diff --git a/services/apps/members_enrichment_worker/src/workflows/enrichMember.ts b/services/apps/members_enrichment_worker/src/workflows/enrichMember.ts
index e8a39308b2..88cbae8787 100644
--- a/services/apps/members_enrichment_worker/src/workflows/enrichMember.ts
+++ b/services/apps/members_enrichment_worker/src/workflows/enrichMember.ts
@@ -8,7 +8,6 @@ import {
 import { IEnrichableMember, MemberEnrichmentSource } from '@crowd/types'
 
 import * as activities from '../activities'
-import { hasRemainingCredits } from '../activities/enrichment'
 import { IEnrichmentSourceInput } from '../types'
 import { sourceHasDifferentDataComparedToCache } from '../utils/common'
 
@@ -22,6 +21,7 @@ const {
   updateMemberEnrichmentCache,
   isCacheObsolete,
   getEnrichmentInput,
+  hasRemainingCredits,
 } = proxyActivities<typeof activities>({
   startToCloseTimeout: '1 minute',
   retry: {
diff --git a/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts b/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
index 5c5f4f5959..0e0dfb2e15 100644
--- a/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
+++ b/services/apps/members_enrichment_worker/src/workflows/processMemberSources.ts
@@ -294,6 +294,7 @@ export async function processMemberSources(args: IProcessMemberSourcesArgs): Pro
       existingMemberData,
       squashedPayload,
       progaiLinkedinScraperProfileSelected && hasContributions,
+      !!crustDataProfileSelected,
     )
 
     return memberUpdated

From 22121ea12a4bff9b83c97be9e438d1e3e3f783c0 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 6 Dec 2024 10:40:30 +0100
Subject: [PATCH 30/40] sync member and created orgs on enrchment to opensearch

---
 .../src/activities/enrichment.ts              | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
index 6d6424ae8e..526a92704c 100644
--- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
@@ -26,6 +26,7 @@ import {
 import { findOrCreateOrganization } from '@crowd/data-access-layer/src/organizations'
 import { dbStoreQx } from '@crowd/data-access-layer/src/queryExecutor'
 import { refreshMaterializedView } from '@crowd/data-access-layer/src/utils'
+import { SearchSyncApiClient } from '@crowd/opensearch'
 import { RedisCache } from '@crowd/redis'
 import {
   IEnrichableMember,
@@ -425,6 +426,8 @@ export async function updateMemberUsingSquashedPayload(
               org.source,
             ),
           )
+
+          await syncOrganization(org.organizationId)
         }
       }
 
@@ -438,6 +441,7 @@ export async function updateMemberUsingSquashedPayload(
 
     if (updated) {
       await setMemberEnrichmentUpdateDateDb(tx.transaction(), memberId)
+      await syncMember(memberId)
     } else {
       await setMemberEnrichmentTryDateDb(tx.transaction(), memberId)
     }
@@ -901,6 +905,22 @@ function prepareWorkExperiences(
   }
 }
 
+export async function syncMember(memberId: string): Promise<void> {
+  const syncApi = new SearchSyncApiClient({
+    baseUrl: process.env['CROWD_SEARCH_SYNC_API_URL'],
+  })
+
+  await syncApi.triggerMemberSync(memberId, { withAggs: false })
+}
+
+export async function syncOrganization(organizationId: string): Promise<void> {
+  const syncApi = new SearchSyncApiClient({
+    baseUrl: process.env['CROWD_SEARCH_SYNC_API_URL'],
+  })
+
+  await syncApi.triggerOrganizationSync(organizationId, undefined, { withAggs: false })
+}
+
 export async function cleanAttributeValue(
   attributeValue: string | string[] | Record<string, any>,
 ): Promise<string | string[] | Record<string, any>> {

From 52549a093919dd7043e23580f40cc2ced09a9460 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 6 Dec 2024 12:11:17 +0100
Subject: [PATCH 31/40] also return unverified identities on getting existing
 member data while enriching

---
 .../src/old/apps/members_enrichment_worker/index.ts            | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts b/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
index 295e5eaefe..11aebb8d29 100644
--- a/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
+++ b/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
@@ -46,8 +46,7 @@ export async function fetchMemberDataForLLMSquashing(
                                                 mi.value) r)
                           )
                     from "memberIdentities" mi
-                    where mi."memberId" = m.id
-                      and verified = true), '[]'::json) as identities,
+                    where mi."memberId" = m.id), '[]'::json) as identities,
           case
               when exists (select 1 from member_orgs where "memberId" = m.id)
                   then (

From 3a5281806f9c32314b04a1f71ecee4b3e3c7e0b2 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 6 Dec 2024 16:41:34 +0100
Subject: [PATCH 32/40] check org existence in all platforms if the type is
 primary domain

---
 .../src/organizations/organizations.ts        | 41 +++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/services/libs/data-access-layer/src/organizations/organizations.ts b/services/libs/data-access-layer/src/organizations/organizations.ts
index 87e8995925..0f64dc8c39 100644
--- a/services/libs/data-access-layer/src/organizations/organizations.ts
+++ b/services/libs/data-access-layer/src/organizations/organizations.ts
@@ -156,6 +156,42 @@ export async function findOrgByName(
   return result
 }
 
+export async function findOrgByVerifiedDomain(
+  qx: QueryExecutor,
+  tenantId: string,
+  identity: IDbOrgIdentity,
+): Promise<IDbOrganization | null> {
+  if (identity.type !== OrganizationIdentityType.PRIMARY_DOMAIN) {
+    throw new Error('Invalid identity type')
+  }
+  const result = await qx.selectOneOrNone(
+    `
+    with "organizationsWithIdentity" as (
+              select oi."organizationId"
+              from "organizationIdentities" oi
+              where
+                    oi."tenantId" = $(tenantId)
+                    and lower(oi.value) = lower($(value))
+                    and oi.type = $(type)
+                    and oi.verified = true
+          )
+          select  ${prepareSelectColumns(ORG_SELECT_COLUMNS, 'o')}
+          from organizations o
+          where o."tenantId" = $(tenantId)
+          and o.id in (select distinct "organizationId" from "organizationsWithIdentity")
+          limit 1;
+    `,
+    {
+      tenantId,
+      value: identity.value,
+      platform: identity.platform,
+      type: identity.type,
+    },
+  )
+
+  return result
+}
+
 export async function findOrgByVerifiedIdentity(
   qx: QueryExecutor,
   tenantId: string,
@@ -501,6 +537,11 @@ export async function findOrCreateOrganization(
     // find existing org by sent verified identities
     for (const identity of verifiedIdentities) {
       existing = await findOrgByVerifiedIdentity(qe, tenantId, identity)
+
+      if (!existing && identity.type === OrganizationIdentityType.PRIMARY_DOMAIN) {
+        // if primary domain isn't found in the incoming platform, check if the domain exists in any platform
+        existing = await findOrgByVerifiedDomain(qe, tenantId, identity)
+      }
       if (existing) {
         break
       }

From 82aed4f0c11596271db6eac5b4490ad15f562e6c Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 6 Dec 2024 18:07:03 +0100
Subject: [PATCH 33/40] cross-checking verified domains in existing & incoming
 org for existence optimization

---
 .../src/activities/enrichment.ts              | 80 ++++++++++++-------
 services/libs/common/src/array.ts             |  5 ++
 .../apps/members_enrichment_worker/index.ts   | 24 +++---
 services/libs/types/src/enrichment.ts         |  2 +
 4 files changed, 74 insertions(+), 37 deletions(-)

diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
index 526a92704c..850601ca2f 100644
--- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
@@ -1,6 +1,11 @@
 import _ from 'lodash'
 
-import { generateUUIDv1, replaceDoubleQuotes, setAttributesDefaultValues } from '@crowd/common'
+import {
+  generateUUIDv1,
+  hasIntersection,
+  replaceDoubleQuotes,
+  setAttributesDefaultValues,
+} from '@crowd/common'
 import { LlmService } from '@crowd/common_services'
 import {
   updateMemberAttributes,
@@ -38,6 +43,7 @@ import {
   MemberEnrichmentSource,
   MemberIdentityType,
   OrganizationAttributeSource,
+  OrganizationIdentityType,
   OrganizationSource,
   PlatformType,
 } from '@crowd/types'
@@ -326,32 +332,9 @@ export async function updateMemberUsingSquashedPayload(
       for (const org of squashedPayload.memberOrganizations) {
         if (!org.organizationId) {
           // Check if any similar in existing work experiences
-          const existingOrg = existingMemberData.organizations.find((o) => {
-            const incomingOrgStartDate = org.startDate ? new Date(org.startDate) : null
-            const incomingOrgEndDate = org.endDate ? new Date(org.endDate) : null
-            const existingOrgStartDate = o.dateStart ? new Date(o.dateStart) : null
-            const existingOrgEndEndDate = o.dateEnd ? new Date(o.dateEnd) : null
-
-            const isSameStartMonthYear =
-              (!incomingOrgStartDate && !existingOrgStartDate) || // Both start dates are null
-              (incomingOrgStartDate &&
-                existingOrgStartDate &&
-                incomingOrgStartDate.getMonth() === existingOrgStartDate.getMonth() &&
-                incomingOrgStartDate.getFullYear() === existingOrgStartDate.getFullYear())
-
-            const isSameEndMonthYear =
-              (!incomingOrgEndDate && !existingOrgEndEndDate) || // Both end dates are null
-              (incomingOrgEndDate &&
-                existingOrgEndEndDate &&
-                incomingOrgEndDate.getMonth() === existingOrgEndEndDate.getMonth() &&
-                incomingOrgEndDate.getFullYear() === existingOrgEndEndDate.getFullYear())
-
-            return (
-              (o.orgName.toLowerCase().includes(org.name.toLowerCase()) ||
-                org.name.toLowerCase().includes(o.orgName.toLowerCase())) &&
-              ((isSameStartMonthYear && isSameEndMonthYear) || org.title === o.jobTitle)
-            )
-          })
+          const existingOrg = existingMemberData.organizations.find((o) =>
+            doesIncomingOrgExistInExistingOrgs(o, org),
+          )
 
           if (existingOrg) {
             // Get all orgs with the same name as the current one
@@ -453,6 +436,49 @@ export async function updateMemberUsingSquashedPayload(
   })
 }
 
+export function doesIncomingOrgExistInExistingOrgs(
+  existingOrg: IMemberOrganizationData,
+  incomingOrg: IMemberEnrichmentDataNormalizedOrganization,
+): boolean {
+  // Check if any similar in existing work experiences
+  const incomingVerifiedPrimaryDomainIdentityValues = incomingOrg.identities
+    .filter((i) => i.type === OrganizationIdentityType.PRIMARY_DOMAIN && i.verified)
+    .map((i) => i.value)
+
+  const existingVerifiedPrimaryDomainIdentityValues = existingOrg.identities
+    .filter((i) => i.type === OrganizationIdentityType.PRIMARY_DOMAIN && i.verified)
+    .map((i) => i.value)
+
+  const incomingOrgStartDate = incomingOrg.startDate ? new Date(incomingOrg.startDate) : null
+  const incomingOrgEndDate = incomingOrg.endDate ? new Date(incomingOrg.endDate) : null
+  const existingOrgStartDate = existingOrg.dateStart ? new Date(existingOrg.dateStart) : null
+  const existingOrgEndEndDate = existingOrg.dateEnd ? new Date(existingOrg.dateEnd) : null
+
+  const isSameStartMonthYear =
+    (!incomingOrgStartDate && !existingOrgStartDate) || // Both start dates are null
+    (incomingOrgStartDate &&
+      existingOrgStartDate &&
+      incomingOrgStartDate.getMonth() === existingOrgStartDate.getMonth() &&
+      incomingOrgStartDate.getFullYear() === existingOrgStartDate.getFullYear())
+
+  const isSameEndMonthYear =
+    (!incomingOrgEndDate && !existingOrgEndEndDate) || // Both end dates are null
+    (incomingOrgEndDate &&
+      existingOrgEndEndDate &&
+      incomingOrgEndDate.getMonth() === existingOrgEndEndDate.getMonth() &&
+      incomingOrgEndDate.getFullYear() === existingOrgEndEndDate.getFullYear())
+
+  return (
+    hasIntersection(
+      incomingVerifiedPrimaryDomainIdentityValues,
+      existingVerifiedPrimaryDomainIdentityValues,
+    ) ||
+    ((existingOrg.orgName.toLowerCase().includes(incomingOrg.name.toLowerCase()) ||
+      incomingOrg.name.toLowerCase().includes(existingOrg.orgName.toLowerCase())) &&
+      ((isSameStartMonthYear && isSameEndMonthYear) || incomingOrg.title === existingOrg.jobTitle))
+  )
+}
+
 export async function setMemberEnrichmentTryDate(memberId: string): Promise<void> {
   await setMemberEnrichmentTryDateDb(svc.postgres.writer.connection(), memberId)
 }
diff --git a/services/libs/common/src/array.ts b/services/libs/common/src/array.ts
index 7bb53f773a..5f84d630ee 100644
--- a/services/libs/common/src/array.ts
+++ b/services/libs/common/src/array.ts
@@ -129,3 +129,8 @@ export const areArraysEqual = <T>(a: T[], b: T[]): boolean => {
 export const firstArrayContainsSecondArray = <T>(array1: T[], array2: T[]): boolean => {
   return array2.every((val) => array1.includes(val))
 }
+
+export const hasIntersection = (arr1: string[], arr2: string[]): boolean => {
+  const set1 = new Set(arr1)
+  return arr2.some((item) => set1.has(item))
+}
diff --git a/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts b/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
index 11aebb8d29..e091c25b3d 100644
--- a/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
+++ b/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
@@ -20,19 +20,23 @@ export async function fetchMemberDataForLLMSquashing(
 ): Promise<IMemberOriginalData | null> {
   const result = await db.oneOrNone(
     `
-    with member_orgs as (select distinct mo."memberId",
-                                        mo."organizationId" as "orgId",
-                                        o."displayName"     as "orgName",
-                                        mo.title            as "jobTitle",
-                                        mo.id,
-                                        mo."dateStart",
-                                        mo."dateEnd",
-                                        mo.source
+    with member_orgs as (select
+                            distinct mo."memberId",
+                            mo."organizationId" as "orgId",
+                            o."displayName"     as "orgName",
+                            mo.title            as "jobTitle",
+                            mo.id,
+                            mo."dateStart",
+                            mo."dateEnd",
+                            mo.source,
+                            jsonb_agg(oi)
                         from "memberOrganizations" mo
-                                  inner join organizations o on mo."organizationId" = o.id
+                            inner join organizations o on mo."organizationId" = o.id
+                            inner join "organizationIdentities" oi on oi."organizationId" = o.id
                         where mo."memberId" = $(memberId)
                           and mo."deletedAt" is null
-                          and o."deletedAt" is null)
+                          and o."deletedAt" is null
+                        group by mo."memberId", mo."organizationId", o."displayName", mo.id)
     select m."displayName",
           m.attributes,
           m."manuallyChangedFields",
diff --git a/services/libs/types/src/enrichment.ts b/services/libs/types/src/enrichment.ts
index 07005a8767..a5e67668f0 100644
--- a/services/libs/types/src/enrichment.ts
+++ b/services/libs/types/src/enrichment.ts
@@ -1,5 +1,6 @@
 import { MemberEnrichmentSource } from './enums'
 import { IMemberIdentity, IMemberReach } from './members'
+import { IOrganizationIdentity } from './organizations'
 
 export interface IMemberEnrichmentCache<T> {
   createdAt: string
@@ -39,6 +40,7 @@ export interface IMemberOrganizationData {
   dateStart: string
   dateEnd: string
   source: string
+  identities?: IOrganizationIdentity[]
 }
 
 export interface IMemberOriginalData {

From 71a297d43e33e6469aada0d8afb04f077ea65307 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 6 Dec 2024 18:18:32 +0100
Subject: [PATCH 34/40] fix returning identities within existing work exps

---
 .../src/old/apps/members_enrichment_worker/index.ts          | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts b/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
index e091c25b3d..5b663051f5 100644
--- a/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
+++ b/services/libs/data-access-layer/src/old/apps/members_enrichment_worker/index.ts
@@ -29,7 +29,7 @@ export async function fetchMemberDataForLLMSquashing(
                             mo."dateStart",
                             mo."dateEnd",
                             mo.source,
-                            jsonb_agg(oi)
+                            jsonb_agg(oi) as identities
                         from "memberOrganizations" mo
                             inner join organizations o on mo."organizationId" = o.id
                             inner join "organizationIdentities" oi on oi."organizationId" = o.id
@@ -62,7 +62,8 @@ export async function fetchMemberDataForLLMSquashing(
                                                 mo."jobTitle",
                                                 mo."dateStart",
                                                 mo."dateEnd",
-                                                mo.source) r)
+                                                mo.source,
+                                                mo.identities) r)
                           )
                   from member_orgs mo
                   where mo."memberId" = m.id

From b8d5d2b84c8d559d7aa8ade8324ac580161f962e Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 6 Dec 2024 18:42:08 +0100
Subject: [PATCH 35/40] fix consistenct issue when syncing bcs of Promise.all

---
 .../src/activities/enrichment.ts                      | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
index 850601ca2f..a691990fce 100644
--- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
@@ -324,6 +324,8 @@ export async function updateMemberUsingSquashedPayload(
       }
     }
 
+    const orgIdsToSync: string[] = []
+
     if (squashedPayload.memberOrganizations.length > 0) {
       const orgPromises = []
 
@@ -410,7 +412,7 @@ export async function updateMemberUsingSquashedPayload(
             ),
           )
 
-          await syncOrganization(org.organizationId)
+          orgIdsToSync.push(org.organizationId)
         }
       }
 
@@ -422,6 +424,8 @@ export async function updateMemberUsingSquashedPayload(
       }
     }
 
+    await Promise.all(promises)
+
     if (updated) {
       await setMemberEnrichmentUpdateDateDb(tx.transaction(), memberId)
       await syncMember(memberId)
@@ -429,7 +433,10 @@ export async function updateMemberUsingSquashedPayload(
       await setMemberEnrichmentTryDateDb(tx.transaction(), memberId)
     }
 
-    await Promise.all(promises)
+    for (const orgId of orgIdsToSync) {
+      await syncOrganization(orgId)
+    }
+
     svc.log.debug({ memberId }, 'Member sources processed successfully!')
 
     return updated

From 7318e166bb726aa14578bc1bef838ae738d512b1 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 6 Dec 2024 20:01:30 +0100
Subject: [PATCH 36/40] fix

---
 .../members_enrichment_worker/src/activities/enrichment.ts     | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
index a691990fce..2bcebd9225 100644
--- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
@@ -371,6 +371,7 @@ export async function updateMemberUsingSquashedPayload(
                 i.organizationId = orgId
               }
             }
+            orgIdsToSync.push(orgId)
           }),
         )
       }
@@ -411,8 +412,6 @@ export async function updateMemberUsingSquashedPayload(
               org.source,
             ),
           )
-
-          orgIdsToSync.push(org.organizationId)
         }
       }
 

From 47f926368e1a3a6cff0c1090503929565d9c2b3f Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 6 Dec 2024 20:08:28 +0100
Subject: [PATCH 37/40] check sent orgid exists

---
 .../opensearch/src/service/organization.sync.service.ts  | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/services/libs/opensearch/src/service/organization.sync.service.ts b/services/libs/opensearch/src/service/organization.sync.service.ts
index 402621db13..91bc88db27 100644
--- a/services/libs/opensearch/src/service/organization.sync.service.ts
+++ b/services/libs/opensearch/src/service/organization.sync.service.ts
@@ -414,9 +414,12 @@ export class OrganizationSyncService {
           OrganizationField.LOCATION,
           OrganizationField.INDUSTRY,
         ])
-        const data = await buildFullOrgForMergeSuggestions(qx, base)
-        const prefixed = OrganizationSyncService.prefixData(data)
-        await this.openSearchService.index(orgId, OpenSearchIndex.ORGANIZATIONS, prefixed)
+
+        if (base) {
+          const data = await buildFullOrgForMergeSuggestions(qx, base)
+          const prefixed = OrganizationSyncService.prefixData(data)
+          await this.openSearchService.index(orgId, OpenSearchIndex.ORGANIZATIONS, prefixed)
+        }
       }
 
       return {

From 1606a84e69134db22effe3a54874dabf89b26127 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Sat, 7 Dec 2024 13:23:36 +0100
Subject: [PATCH 38/40] sync fix

---
 .../src/activities/enrichment.ts              | 32 +++++++++++--------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
index 2bcebd9225..735a237ea4 100644
--- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
@@ -363,16 +363,26 @@ export async function updateMemberUsingSquashedPayload(
               description: org.organizationDescription,
               identities: org.identities ? org.identities : [],
             },
-          ).then((orgId) => {
-            // set the organization id for later use
-            org.organizationId = orgId
-            if (org.identities) {
-              for (const i of org.identities) {
-                i.organizationId = orgId
+          )
+            .then((orgId) => {
+              // set the organization id for later use
+              org.organizationId = orgId
+              if (org.identities) {
+                for (const i of org.identities) {
+                  i.organizationId = orgId
+                }
               }
-            }
-            orgIdsToSync.push(orgId)
-          }),
+              orgIdsToSync.push(orgId)
+            })
+            .then(() =>
+              Promise.all(
+                orgIdsToSync.map((orgId) =>
+                  syncOrganization(orgId).catch((error) => {
+                    console.error(`Failed to sync organization with ID ${orgId}:`, error)
+                  }),
+                ),
+              ),
+            ),
         )
       }
 
@@ -432,10 +442,6 @@ export async function updateMemberUsingSquashedPayload(
       await setMemberEnrichmentTryDateDb(tx.transaction(), memberId)
     }
 
-    for (const orgId of orgIdsToSync) {
-      await syncOrganization(orgId)
-    }
-
     svc.log.debug({ memberId }, 'Member sources processed successfully!')
 
     return updated

From c87f7328479c5216e23e16369050249eb22c55fd Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Sat, 7 Dec 2024 13:31:51 +0100
Subject: [PATCH 39/40] fix

---
 .../members_enrichment_worker/src/activities/enrichment.ts    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
index 735a237ea4..01756a5b82 100644
--- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
@@ -372,7 +372,9 @@ export async function updateMemberUsingSquashedPayload(
                   i.organizationId = orgId
                 }
               }
-              orgIdsToSync.push(orgId)
+              if (orgId) {
+                orgIdsToSync.push(orgId)
+              }
             })
             .then(() =>
               Promise.all(

From 6ae3a91eb7bee46f638fa362127bb451b3ce9be0 Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Mon, 9 Dec 2024 15:19:03 +0100
Subject: [PATCH 40/40] update linkedin discarding logic to discard all
 unverified when there's already a profile from verified

---
 .../src/activities/enrichment.ts              | 39 +++++++++++--------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
index 01756a5b82..aca34c5622 100644
--- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts
+++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts
@@ -701,26 +701,31 @@ export async function findWhichLinkedinProfileToUseAmongScraperResult(
   }
 
   if (profilesFromUnverfiedIdentities.length > 0) {
-    const result = await findRelatedLinkedinProfilesWithLLM(
-      memberId,
-      memberData,
-      profilesFromUnverfiedIdentities,
-    )
+    if (categorized.selected) {
+      // we already found a match from verified identities, discard all profiles from unverified identities
+      categorized.discarded = profilesFromUnverfiedIdentities
+    } else {
+      const result = await findRelatedLinkedinProfilesWithLLM(
+        memberId,
+        memberData,
+        profilesFromUnverfiedIdentities,
+      )
 
-    // check if empty object
-    if (result.profileIndex !== null) {
-      if (!categorized.selected) {
-        categorized.selected = profilesFromUnverfiedIdentities[result.profileIndex]
-      }
-      // add profiles not selected to discarded
-      for (let i = 0; i < profilesFromUnverfiedIdentities.length; i++) {
-        if (i !== result.profileIndex) {
-          categorized.discarded.push(profilesFromUnverfiedIdentities[i])
+      // check if empty object
+      if (result.profileIndex !== null) {
+        if (!categorized.selected) {
+          categorized.selected = profilesFromUnverfiedIdentities[result.profileIndex]
+        }
+        // add profiles not selected to discarded
+        for (let i = 0; i < profilesFromUnverfiedIdentities.length; i++) {
+          if (i !== result.profileIndex) {
+            categorized.discarded.push(profilesFromUnverfiedIdentities[i])
+          }
         }
+      } else {
+        // if no match found, we should discard all profiles from verified identities
+        categorized.discarded = profilesFromUnverfiedIdentities
       }
-    } else {
-      // if no match found, we should discard all profiles from verified identities
-      categorized.discarded = profilesFromUnverfiedIdentities
     }
   }