Serp linkedin finder enrichment source

services/apps/premium/members_enrichment_worker/src/activities.ts

@@ -8,7 +8,7 @@ import {
   touchMemberEnrichmentCacheUpdatedAt,
   updateMemberEnrichmentCache,
 } from './activities/enrichment'
-import { getMembers } from './activities/getMembers'
+import { getEnrichableMembers } from './activities/getMembers'
 import { refreshToken } from './activities/lf-auth0/authenticateLFAuth0'
 import {
   getIdentitiesExistInOtherMembers,
@@ -29,7 +29,7 @@ import {
 } from './activities/syncEnrichedData'
 
 export {
-  getMembers,
+  getEnrichableMembers,
   getEnrichmentData,
   normalizeEnrichmentData,
   findMemberEnrichmentCache,

services/apps/premium/members_enrichment_worker/src/activities/getMembers.ts

@@ -1,21 +1,25 @@
 import { fetchMembersForEnrichment } from '@crowd/data-access-layer/src/old/apps/premium/members_enrichment_worker'
-import { IMember, IMemberEnrichmentSourceQueryInput, MemberEnrichmentSource } from '@crowd/types'
+import {
+  IEnrichableMember,
+  IMemberEnrichmentSourceQueryInput,
+  MemberEnrichmentSource,
+} from '@crowd/types'
 
 import { EnrichmentSourceServiceFactory } from '../factory'
 import { svc } from '../main'
 
-export async function getMembers(
+export async function getEnrichableMembers(
   limit: number,
   sources: MemberEnrichmentSource[],
   afterId: string,
-): Promise<IMember[]> {
-  let rows: IMember[] = []
+): Promise<IEnrichableMember[]> {
+  let rows: IEnrichableMember[] = []
   const sourceInputs: IMemberEnrichmentSourceQueryInput[] = sources.map((s) => {
     const srv = EnrichmentSourceServiceFactory.getEnrichmentSourceService(s, svc.log)
     return {
       source: s,
       cacheObsoleteAfterSeconds: srv.cacheObsoleteAfterSeconds,
-      enrichableBy: srv.enrichableBy,
+      enrichableBySql: srv.enrichableBySql,
     }
   })
   const db = svc.postgres.reader

services/apps/premium/members_enrichment_worker/src/factory.ts

@@ -4,6 +4,7 @@ import { MemberEnrichmentSource } from '@crowd/types'
 
 import EnrichmentServiceClearbit from './sources/clearbit/service'
 import EnrichmentServiceProgAI from './sources/progai/service'
+import EnrichmentServiceSerpApi from './sources/serp/service'
 import { IEnrichmentService } from './types'
 import { ALSO_USE_EMAIL_IDENTITIES_FOR_ENRICHMENT, ENRICH_EMAIL_IDENTITIES } from './utils/config'
 
@@ -21,6 +22,8 @@ export class EnrichmentSourceServiceFactory {
         )
       case MemberEnrichmentSource.CLEARBIT:
         return new EnrichmentServiceClearbit(log)
+      case MemberEnrichmentSource.SERP:
+        return new EnrichmentServiceSerpApi(log)
       default:
         throw new Error(`Enrichment service for ${source} is not found!`)
     }

services/apps/premium/members_enrichment_worker/src/main.ts

@@ -10,6 +10,8 @@ const config: Config = {
     'CROWD_ENRICHMENT_PROGAI_API_KEY',
     'CROWD_ENRICHMENT_CLEARBIT_URL',
     'CROWD_ENRICHMENT_CLEARBIT_API_KEY',
+    'CROWD_ENRICHMENT_SERP_API_URL',
+    'CROWD_ENRICHMENT_SERP_API_KEY',
   ],
   producer: {
     enabled: false,

services/apps/premium/members_enrichment_worker/src/sources/clearbit/service.ts

@@ -2,7 +2,6 @@ import axios from 'axios'
 
 import { Logger, LoggerBase } from '@crowd/logging'
 import {
-  IMemberEnrichmentSourceEnrichableBy,
   MemberAttributeName,
   MemberEnrichmentSource,
   MemberIdentityType,
@@ -28,11 +27,7 @@ import {
 export default class EnrichmentServiceClearbit extends LoggerBase implements IEnrichmentService {
   public source: MemberEnrichmentSource = MemberEnrichmentSource.CLEARBIT
   public platform = `enrichment-${this.source}`
-  public enrichableBy: IMemberEnrichmentSourceEnrichableBy[] = [
-    {
-      type: MemberIdentityType.EMAIL,
-    },
-  ]
+  public enrichableBySql = `mi.type = 'email' and mi.verified`
 
   // bust cache after 120 days
   public cacheObsoleteAfterSeconds = 60 * 60 * 24 * 120
@@ -60,7 +55,7 @@ export default class EnrichmentServiceClearbit extends LoggerBase implements IEn
   }
 
   isEnrichableBySource(input: IEnrichmentSourceInput): boolean {
-    return !!input.email?.value
+    return !!input.email?.value && input.email?.verified
   }
 
   async getData(input: IEnrichmentSourceInput): Promise<IMemberEnrichmentDataClearbit | null> {

services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts

@@ -3,7 +3,6 @@ import lodash from 'lodash'
 
 import { Logger, LoggerBase } from '@crowd/logging'
 import {
-  IMemberEnrichmentSourceEnrichableBy,
   MemberAttributeName,
   MemberEnrichmentSource,
   MemberIdentityType,
@@ -33,15 +32,8 @@ import {
 export default class EnrichmentServiceProgAI extends LoggerBase implements IEnrichmentService {
   public source: MemberEnrichmentSource = MemberEnrichmentSource.PROGAI
   public platform = `enrichment-${this.source}`
-  public enrichableBy: IMemberEnrichmentSourceEnrichableBy[] = [
-    {
-      type: MemberIdentityType.USERNAME,
-      platform: PlatformType.GITHUB,
-    },
-    {
-      type: MemberIdentityType.EMAIL,
-    },
-  ]
+
+  enrichableBySql = `mi.verified and ((mi.type = 'username' AND mi.platform = 'github') OR (mi.type = 'email'))`
 
   // bust cache after 90 days
   public cacheObsoleteAfterSeconds = 60 * 60 * 24 * 90

services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts

@@ -0,0 +1,133 @@
+import axios from 'axios'
+
+import { Logger, LoggerBase } from '@crowd/logging'
+import { MemberEnrichmentSource, MemberIdentityType, PlatformType } from '@crowd/types'
+
+import {
+  IEnrichmentService,
+  IEnrichmentSourceInput,
+  IMemberEnrichmentDataNormalized,
+} from '../../types'
+
+import { IMemberEnrichmentDataSerp, IMemberEnrichmentSerpApiResponse } from './types'
+
+export default class EnrichmentServiceSerpApi extends LoggerBase implements IEnrichmentService {
+  public source: MemberEnrichmentSource = MemberEnrichmentSource.SERP
+  public platform = `enrichment-${this.source}`
+  public enrichMembersWithActivityMoreThan = 10
+
+  public enrichableBySql = `
+  ("activitySummary".total_count > ${this.enrichMembersWithActivityMoreThan}) AND
+  (members."displayName" like '% %') AND 
+  (members.attributes->'location'->>'default' is not null and members.attributes->'location'->>'default' <> '') AND
+  ((members.attributes->'websiteUrl'->>'default' is not null and members.attributes->'websiteUrl'->>'default' <> '') OR 
+   (mi.verified AND mi.type = 'username' and mi.platform = 'github') OR 
+   (mi.verified AND mi.type = 'email')
+  )`
+
+  // bust cache after 120 days
+  public cacheObsoleteAfterSeconds = 60 * 60 * 24 * 120
+
+  constructor(public readonly log: Logger) {
+    super(log)
+  }
+
+  isEnrichableBySource(input: IEnrichmentSourceInput): boolean {
+    const displayNameSplit = input.displayName?.split(' ')
+    return (
+      displayNameSplit?.length > 1 &&
+      !!input.location &&
+      ((!!input.email && input.email.verified) ||
+        (!!input.github && input.github.verified) ||
+        !!input.website)
+    )
+  }
+
+  async getData(input: IEnrichmentSourceInput): Promise<IMemberEnrichmentDataSerp | null> {
+    let enriched: IMemberEnrichmentDataSerp = null
+
+    if (input.displayName && input.location && input.website) {
+      enriched = await this.querySerpApi(input.displayName, input.location, input.website)
+    }
+
+    if (!enriched && input.displayName && input.location && input.github && input.github.value) {
+      enriched = await this.querySerpApi(input.displayName, input.location, input.github.value)
+    }
+
+    if (!enriched && input.displayName && input.location && input.email && input.email.value) {
+      enriched = await this.querySerpApi(input.displayName, input.location, input.email.value)
+    }
+    return enriched
+  }
+
+  private async querySerpApi(
+    displayName: string,
+    location: string,
+    identifier: string,
+  ): Promise<IMemberEnrichmentDataSerp> {
+    const config = {
+      method: 'get',
+      url: process.env['CROWD_ENRICHMENT_SERP_API_URL'],
+      params: {
+        api_key: process.env['CROWD_ENRICHMENT_SERP_API_KEY'],
+        q: `"${displayName}" ${location} "${identifier}" site:linkedin.com/in`,
+        num: 3,
+        engine: 'google',
+      },
+    }
+
+    const response: IMemberEnrichmentSerpApiResponse = (await axios(config)).data
+
+    if (response.search_information.total_results > 0) {
+      if (
+        response.organic_results.length > 0 &&
+        response.organic_results[0].link &&
+        !response.search_information.spelling_fix &&
+        !response.search_information.spelling_fix_type
+      ) {
+        return {
+          linkedinUrl: response.organic_results[0].link,
+        }
+      }
+    }
+
+    return null
+  }
+
+  normalize(data: IMemberEnrichmentDataSerp): IMemberEnrichmentDataNormalized {
+    const normalized: IMemberEnrichmentDataNormalized = {
+      identities: [
+        {
+          platform: PlatformType.LINKEDIN,
+          type: MemberIdentityType.USERNAME,
+          verified: false,
+          value: this.normalizeLinkedUrl(data.linkedinUrl),
+        },
+      ],
+    }
+    return normalized
+  }
+
+  private normalizeLinkedUrl(url: string): string {
+    try {
+      const parsedUrl = new URL(url)
+
+      if (parsedUrl.hostname.endsWith('linkedin.com')) {
+        parsedUrl.hostname = 'linkedin.com'
+        parsedUrl.search = ''
+
+        let path = parsedUrl.pathname
+        if (path.endsWith('/')) {
+          path = path.slice(0, -1)
+        }
+
+        return parsedUrl.origin + path
+      }
+
+      return url
+    } catch (error) {
+      this.log.error(`Error while normalizing linkedin url: ${url}`, error)
+      throw error
+    }
+  }
+}

services/apps/premium/members_enrichment_worker/src/sources/serp/tests.ts

@@ -0,0 +1,69 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import axios from 'axios'
+
+// import { writeFileSync } from 'fs'
+// import { Parser } from 'json2csv'
+import { timeout } from '@crowd/common'
+
+const testSerpApi = async () => {
+  const members = [] as any[]
+
+  for (const mem of members) {
+    const url = `https://serpapi.com/search.json`
+    const config = {
+      method: 'get',
+      url,
+      params: {
+        api_key: process.env['CROWD_SERP_API_KEY'],
+        q: `"${mem.displayName}" ${mem.location} "${mem.website}" site:linkedin.com/in`,
+        num: 3,
+        engine: 'google',
+      },
+    }
+
+    const response = (await axios(config)).data
+
+    if (response.search_information.total_results > 0) {
+      if (
+        response.organic_results.length > 0 &&
+        response.organic_results[0].link &&
+        !response.search_information.spelling_fix &&
+        !response.search_information.spelling_fix_type
+      ) {
+        console.log(`Found LinkedIn for ${mem.displayName}: ${response.organic_results[0].link}`)
+        console.log(response.search_information)
+        mem.linkedinFromSerp = response.organic_results[0].link
+      } else {
+        console.log(`No LinkedIn found for ${mem.displayName}`)
+      }
+    } else {
+      console.log(`No LinkedIn found for ${mem.displayName}`)
+    }
+
+    await timeout(1000)
+  }
+
+  try {
+    // const fields = [
+    //   'id',
+    //   'displayName',
+    //   'location',
+    //   'profileUrl',
+    //   'website',
+    //   'linkedinFromClearbit',
+    //   'linkedinFromProgai',
+    //   'linkedinFromSerp',
+    // ]
+    // const json2csvParser = new Parser({ fields })
+    // const csv = json2csvParser.parse(members)
+    // writeFileSync('output.csv', csv)
+    // console.log('CSV file has been successfully written.')
+  } catch (err) {
+    console.error('Error writing CSV file:', err)
+  }
+}
+
+setImmediate(async () => {
+  await testSerpApi()
+  process.exit(0)
+})

services/apps/premium/members_enrichment_worker/src/sources/serp/types.ts

@@ -0,0 +1,37 @@
+export interface IMemberEnrichmentDataSerp {
+  linkedinUrl: string
+}
+
+export interface IMemberEnrichmentSerpApiResponse {
+  organic_results: IMemberEnrichmentSerpApiResponseOrganicResult[]
+  search_information: IMemberEnrichmentSerpApiResponseSearchInformation
+}
+
+export interface IMemberEnrichmentSerpApiResponseSearchInformation {
+  query_displayed: string
+  total_results: number
+  time_taken_displayed: number
+  organic_results_state: string
+  spelling_fix?: string
+  spelling_fix_type?: string
+}
+
+export interface IMemberEnrichmentSerpApiResponseOrganicResult {
+  position: number
+  title: string
+  link: string
+  redirect_link: string
+  displayed_link: string
+  favicon: string
+  snippet: string
+  snippet_highlighted_words: string[]
+  sitelinks: {
+    inline: IMemberEnrichmentSerpApiResponseOrganicResultSitelinkInline[]
+  }
+  source: string
+}
+
+export interface IMemberEnrichmentSerpApiResponseOrganicResultSitelinkInline {
+  title: string
+  link: string
+}