Serp linkedin finder enrichment source

services/apps/premium/members_enrichment_worker/src/activities/getMembers.ts

@@ -1,5 +1,9 @@
 import { fetchMembersForEnrichment } from '@crowd/data-access-layer/src/old/apps/premium/members_enrichment_worker'
-import { IMember, IMemberEnrichmentSourceQueryInput, MemberEnrichmentSource } from '@crowd/types'
+import {
+  IEnrichableMember,
+  IMemberEnrichmentSourceQueryInput,
+  MemberEnrichmentSource,
+} from '@crowd/types'
 
 import { EnrichmentSourceServiceFactory } from '../factory'
 import { svc } from '../main'
@@ -8,14 +12,14 @@ export async function getMembers(
   limit: number,
   sources: MemberEnrichmentSource[],
   afterId: string,
-): Promise<IMember[]> {
-  let rows: IMember[] = []
+): Promise<IEnrichableMember[]> {
+  let rows: IEnrichableMember[] = []
   const sourceInputs: IMemberEnrichmentSourceQueryInput[] = sources.map((s) => {
     const srv = EnrichmentSourceServiceFactory.getEnrichmentSourceService(s, svc.log)
     return {
       source: s,
       cacheObsoleteAfterSeconds: srv.cacheObsoleteAfterSeconds,
-      enrichableBy: srv.enrichableBy,
+      enrichableBySql: srv.enrichableBySql,
     }
   })
   const db = svc.postgres.reader

services/apps/premium/members_enrichment_worker/src/factory.ts

@@ -4,6 +4,7 @@ import { MemberEnrichmentSource } from '@crowd/types'
 
 import EnrichmentServiceClearbit from './sources/clearbit/service'
 import EnrichmentServiceProgAI from './sources/progai/service'
+import EnrichmentServiceSerpApi from './sources/serp/service'
 import { IEnrichmentService } from './types'
 import { ALSO_USE_EMAIL_IDENTITIES_FOR_ENRICHMENT, ENRICH_EMAIL_IDENTITIES } from './utils/config'
 
@@ -21,6 +22,8 @@ export class EnrichmentSourceServiceFactory {
         )
       case MemberEnrichmentSource.CLEARBIT:
         return new EnrichmentServiceClearbit(log)
+      case MemberEnrichmentSource.SERP:
+        return new EnrichmentServiceSerpApi(log)
       default:
         throw new Error(`Enrichment service for ${source} is not found!`)
     }

services/apps/premium/members_enrichment_worker/src/sources/clearbit/service.ts

@@ -2,7 +2,6 @@ import axios from 'axios'
 
 import { Logger, LoggerBase } from '@crowd/logging'
 import {
-  IMemberEnrichmentSourceEnrichableBy,
   MemberAttributeName,
   MemberEnrichmentSource,
   MemberIdentityType,
@@ -28,11 +27,7 @@ import {
 export default class EnrichmentServiceClearbit extends LoggerBase implements IEnrichmentService {
   public source: MemberEnrichmentSource = MemberEnrichmentSource.CLEARBIT
   public platform = `enrichment-${this.source}`
-  public enrichableBy: IMemberEnrichmentSourceEnrichableBy[] = [
-    {
-      type: MemberIdentityType.EMAIL,
-    },
-  ]
+  public enrichableBySql = `mi.type = 'email' and mi.verified`
 
   // bust cache after 120 days
   public cacheObsoleteAfterSeconds = 60 * 60 * 24 * 120

services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts

@@ -3,7 +3,6 @@ import lodash from 'lodash'
 
 import { Logger, LoggerBase } from '@crowd/logging'
 import {
-  IMemberEnrichmentSourceEnrichableBy,
   MemberAttributeName,
   MemberEnrichmentSource,
   MemberIdentityType,
@@ -33,15 +32,8 @@ import {
 export default class EnrichmentServiceProgAI extends LoggerBase implements IEnrichmentService {
   public source: MemberEnrichmentSource = MemberEnrichmentSource.PROGAI
   public platform = `enrichment-${this.source}`
-  public enrichableBy: IMemberEnrichmentSourceEnrichableBy[] = [
-    {
-      type: MemberIdentityType.USERNAME,
-      platform: PlatformType.GITHUB,
-    },
-    {
-      type: MemberIdentityType.EMAIL,
-    },
-  ]
+
+  enrichableBySql = `mi.verified and ((mi.type = 'username' AND mi.platform = 'github') OR (mi.type = 'email'))`
 
   // bust cache after 90 days
   public cacheObsoleteAfterSeconds = 60 * 60 * 24 * 90

services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts

@@ -0,0 +1,209 @@
+import axios from 'axios'
+
+import { Logger, LoggerBase } from '@crowd/logging'
+import { MemberEnrichmentSource, MemberIdentityType, PlatformType } from '@crowd/types'
+
+import {
+  IEnrichmentService,
+  IEnrichmentSourceInput,
+  IMemberEnrichmentDataNormalized,
+} from '../../types'
+
+import { IMemberEnrichmentDataSerp, IMemberEnrichmentSerpApiResponse } from './types'
+
+export default class EnrichmentServiceSerpApi extends LoggerBase implements IEnrichmentService {
+  public source: MemberEnrichmentSource = MemberEnrichmentSource.SERP
+  public platform = `enrichment-${this.source}`
+
+  public enrichableBySql = `
+  (members."displayName" like '% %') AND 
+  (members.attributes->'location'->>'default' is not null and members.attributes->'location'->>'default' <> '') AND
+  ((members.attributes->'websiteUrl'->>'default' is not null and members.attributes->'websiteUrl'->>'default' <> '') OR 
+   (mi.verified AND mi.type = 'username' and mi.platform = 'github') OR 
+   (mi.verified AND mi.type = 'email')
+  )`
+
+  // bust cache after 60 days
+  public cacheObsoleteAfterSeconds = 60 * 60 * 24 * 60
+
+  constructor(public readonly log: Logger) {
+    super(log)
+  }
+
+  isEnrichableBySource(input: IEnrichmentSourceInput): boolean {
+    const displayNameSplit = input.displayName?.split(' ')
+    return (
+      displayNameSplit?.length > 1 &&
+      !!input.location &&
+      ((!!input.email && input.email.verified) ||
+        (!!input.github && input.github.verified) ||
+        !!input.website)
+    )
+  }
+
+  async getData(input: IEnrichmentSourceInput): Promise<IMemberEnrichmentDataSerp | null> {
+    let enriched: IMemberEnrichmentDataSerp = null
+
+    if (input.displayName && input.location && input.website) {
+      enriched = await this.queryWithWebsite(input.displayName, input.location, input.website)
+    }
+
+    if (!enriched && input.displayName && input.location && input.github) {
+      enriched = await this.queryWithGithubIdentity(
+        input.displayName,
+        input.location,
+        input.github.value,
+      )
+    }
+
+    if (!enriched && input.displayName && input.location && input.email) {
+      enriched = await this.queryWithEmail(input.displayName, input.location, input.email.value)
+    }
+    return enriched
+  }
+
+  private async queryWithWebsite(
+    displayName: string,
+    location: string,
+    website: string,
+  ): Promise<IMemberEnrichmentDataSerp> {
+    const url = `https://serpapi.com/search.json`
+
+    const config = {
+      method: 'get',
+      url,
+      params: {
+        api_key: process.env['CROWD_SERP_API_KEY'],
+        q: `"${displayName}" ${location} "${website}" site:linkedin.com/in`,
+        num: 3,
+        engine: 'google',
+      },
+    }
+
+    const response: IMemberEnrichmentSerpApiResponse = (await axios(config)).data
+
+    if (response.search_information.total_results > 0) {
+      if (
+        response.organic_results.length > 0 &&
+        response.organic_results[0].link &&
+        !response.search_information.spelling_fix &&
+        !response.search_information.spelling_fix_type
+      ) {
+        return {
+          linkedinUrl: response.organic_results[0].link,
+        }
+      }
+    }
+
+    return null
+  }
+
+  private async queryWithGithubIdentity(
+    displayName: string,
+    location: string,
+    githubIdentity: string,
+  ): Promise<IMemberEnrichmentDataSerp> {
+    const url = `https://serpapi.com/search.json`
+
+    const config = {
+      method: 'get',
+      url,
+      params: {
+        api_key: process.env['CROWD_SERP_API_KEY'],
+        q: `"${displayName}" ${location} "${githubIdentity}" site:linkedin.com/in`,
+        num: 3,
+        engine: 'google',
+      },
+    }
+
+    const response: IMemberEnrichmentSerpApiResponse = (await axios(config)).data
+
+    if (response.search_information.total_results > 0) {
+      if (
+        response.organic_results.length > 0 &&
+        response.organic_results[0].link &&
+        !response.search_information.spelling_fix &&
+        !response.search_information.spelling_fix_type
+      ) {
+        return {
+          linkedinUrl: response.organic_results[0].link,
+        }
+      }
+    }
+
+    return null
+  }
+
+  private async queryWithEmail(
+    displayName: string,
+    location: string,
+    email: string,
+  ): Promise<IMemberEnrichmentDataSerp> {
+    const url = `https://serpapi.com/search.json`
+
+    const config = {
+      method: 'get',
+      url,
+      params: {
+        api_key: process.env['CROWD_SERP_API_KEY'],
+        q: `"${displayName}" ${location} "${email}" site:linkedin.com/in`,
+        num: 3,
+        engine: 'google',
+      },
+    }
+
+    const response: IMemberEnrichmentSerpApiResponse = (await axios(config)).data
+
+    if (response.search_information.total_results > 0) {
+      if (
+        response.organic_results.length > 0 &&
+        response.organic_results[0].link &&
+        !response.search_information.spelling_fix &&
+        !response.search_information.spelling_fix_type
+      ) {
+        return {
+          linkedinUrl: response.organic_results[0].link,
+        }
+      }
+    }
+
+    return null
+  }
+
+  normalize(data: IMemberEnrichmentDataSerp): IMemberEnrichmentDataNormalized {
+    const normalized: IMemberEnrichmentDataNormalized = {
+      identities: [
+        {
+          platform: PlatformType.LINKEDIN,
+          type: MemberIdentityType.USERNAME,
+          verified: false,
+          value: this.normalizeLinkedUrl(data.linkedinUrl),
+        },
+      ],
+    }
+    return normalized
+  }
+
+  private normalizeLinkedUrl(url: string): string {
+    try {
+      const parsedUrl = new URL(url)
+
+      if (parsedUrl.hostname.endsWith('linkedin.com')) {
+        parsedUrl.hostname = 'linkedin.com'
+        parsedUrl.search = ''
+
+        let path = parsedUrl.pathname
+        if (path.endsWith('/')) {
+          path = path.slice(0, -1)
+        }
+
+        return parsedUrl.origin + path
+      }
+
+      return url
+    } catch (error) {
+      this.log.error(`Error while normalizing linkedin url: ${url}`, error)
+      throw error
+    }
+  }
+}

services/apps/premium/members_enrichment_worker/src/sources/serp/types.ts

@@ -0,0 +1,37 @@
+export interface IMemberEnrichmentDataSerp {
+  linkedinUrl: string
+}
+
+export interface IMemberEnrichmentSerpApiResponse {
+  organic_results: IMemberEnrichmentSerpApiResponseOrganicResult[]
+  search_information: IMemberEnrichmentSerpApiResponseSearchInformation
+}
+
+export interface IMemberEnrichmentSerpApiResponseSearchInformation {
+  query_displayed: string
+  total_results: number
+  time_taken_displayed: number
+  organic_results_state: string
+  spelling_fix?: string
+  spelling_fix_type?: string
+}
+
+export interface IMemberEnrichmentSerpApiResponseOrganicResult {
+  position: number
+  title: string
+  link: string
+  redirect_link: string
+  displayed_link: string
+  favicon: string
+  snippet: string
+  snippet_highlighted_words: string[]
+  sitelinks: {
+    inline: IMemberEnrichmentSerpApiResponseOrganicResultSitelinkInline[]
+  }
+  source: string
+}
+
+export interface IMemberEnrichmentSerpApiResponseOrganicResultSitelinkInline {
+  title: string
+  link: string
+}

services/apps/premium/members_enrichment_worker/src/types.ts

@@ -1,7 +1,6 @@
 import {
   IAttributes,
   IMemberContribution,
-  IMemberEnrichmentSourceEnrichableBy,
   IMemberIdentity,
   IOrganizationIdentity,
   MemberAttributeName,
@@ -12,14 +11,21 @@ import {
 
 import { IMemberEnrichmentDataClearbit } from './sources/clearbit/types'
 import { IMemberEnrichmentDataProgAI } from './sources/progai/types'
+import { IMemberEnrichmentDataSerp } from './sources/serp/types'
 
 export interface IEnrichmentSourceInput {
   github?: IMemberIdentity
   linkedin?: IMemberIdentity
   email?: IMemberIdentity
+  website?: string
+  location?: string
+  displayName?: string
 }
 
-export type IMemberEnrichmentData = IMemberEnrichmentDataProgAI | IMemberEnrichmentDataClearbit
+export type IMemberEnrichmentData =
+  | IMemberEnrichmentDataProgAI
+  | IMemberEnrichmentDataClearbit
+  | IMemberEnrichmentDataSerp
 
 export interface IEnrichmentService {
   source: MemberEnrichmentSource
@@ -30,8 +36,8 @@ export interface IEnrichmentService {
   // can the source enrich using this input
   isEnrichableBySource(input: IEnrichmentSourceInput): boolean
 
-  // what kind of identities can this source use as input
-  enrichableBy: IMemberEnrichmentSourceEnrichableBy[]
+  // what kind of custom sql should this source use as input
+  enrichableBySql: string
 
   // should either return the data or null if it's a miss
   getData(input: IEnrichmentSourceInput): Promise<IMemberEnrichmentData | null>