From 842aa4f956733f961b8538fce6a231c07f89570a Mon Sep 17 00:00:00 2001 From: anilb Date: Mon, 4 Nov 2024 09:38:54 +0100 Subject: [PATCH 1/6] enrich members with 100 activities serp api --- .../members_enrichment_worker/src/sources/serp/service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts index 79cc9e4ff3..3074d97054 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts @@ -14,7 +14,7 @@ import { IMemberEnrichmentDataSerp, IMemberEnrichmentSerpApiResponse } from './t export default class EnrichmentServiceSerpApi extends LoggerBase implements IEnrichmentService { public source: MemberEnrichmentSource = MemberEnrichmentSource.SERP public platform = `enrichment-${this.source}` - public enrichMembersWithActivityMoreThan = 10 + public enrichMembersWithActivityMoreThan = 100 public enrichableBySql = ` ("activitySummary".total_count > ${this.enrichMembersWithActivityMoreThan}) AND From 3d7f7b893538fbada5da7c2ef47eab11bf33638a Mon Sep 17 00:00:00 2001 From: anilb Date: Mon, 4 Nov 2024 09:44:39 +0100 Subject: [PATCH 2/6] more strict --- .../members_enrichment_worker/src/sources/serp/service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts index 3074d97054..2d5c350355 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts @@ -14,7 +14,7 @@ import { IMemberEnrichmentDataSerp, IMemberEnrichmentSerpApiResponse } from './t export default class EnrichmentServiceSerpApi extends LoggerBase implements IEnrichmentService { public source: MemberEnrichmentSource = MemberEnrichmentSource.SERP public platform = `enrichment-${this.source}` - public enrichMembersWithActivityMoreThan = 100 + public enrichMembersWithActivityMoreThan = 500 public enrichableBySql = ` ("activitySummary".total_count > ${this.enrichMembersWithActivityMoreThan}) AND From 52ecd05112f78ddd6430b5f07f4c2281f07dbec4 Mon Sep 17 00:00:00 2001 From: anilb Date: Mon, 4 Nov 2024 11:10:30 +0100 Subject: [PATCH 3/6] support for credit checking, stricter serp api conditions, add activity count filter to clearbit --- .../src/activities/enrichment.ts | 38 ++++++++++++++++++- .../src/sources/clearbit/service.ts | 8 +++- .../src/sources/progai/service.ts | 4 ++ .../src/sources/serp/service.ts | 20 +++++++++- .../src/sources/serp/types.ts | 18 +++++++++ .../members_enrichment_worker/src/types.ts | 4 ++ services/libs/redis/src/cache.ts | 6 +++ 7 files changed, 95 insertions(+), 3 deletions(-) diff --git a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts index 8b321b37f6..4fe5a8e82f 100644 --- a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts +++ b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts @@ -4,6 +4,7 @@ import { touchMemberEnrichmentCacheUpdatedAtDb, updateMemberEnrichmentCacheDb, } from '@crowd/data-access-layer/src/old/apps/premium/members_enrichment_worker' +import { RedisCache } from '@crowd/redis' import { IMemberEnrichmentCache, MemberEnrichmentSource } from '@crowd/types' import { EnrichmentSourceServiceFactory } from '../factory' @@ -27,7 +28,7 @@ export async function getEnrichmentData( input: IEnrichmentSourceInput, ): Promise { const service = EnrichmentSourceServiceFactory.getEnrichmentSourceService(source, svc.log) - if (service.isEnrichableBySource(input)) { + if (service.isEnrichableBySource(input) && (await hasRemainingCredits(source))) { return service.getData(input) } return null @@ -52,6 +53,41 @@ export async function isCacheObsolete( ) } +export async function setHasRemainingCredits( + source: MemberEnrichmentSource, + hasCredits: boolean, +): Promise { + const redisCache = new RedisCache(`enrichment-${source}`, svc.redis, svc.log) + if (hasCredits) { + await redisCache.set('hasRemainingCredits', 'true', 60) + } else { + await redisCache.set('hasRemainingCredits', 'false', 60) + } +} + +export async function getHasRemainingCredits(source: MemberEnrichmentSource): Promise { + const redisCache = new RedisCache(`enrichment-${source}`, svc.redis, svc.log) + return (await redisCache.get('hasRemainingCredits')) === 'true' +} + +export async function hasRemainingCreditsExists(source: MemberEnrichmentSource): Promise { + const redisCache = new RedisCache(`enrichment-${source}`, svc.redis, svc.log) + return await redisCache.exists('hasRemainingCredits') +} + +export async function hasRemainingCredits(source: MemberEnrichmentSource): Promise { + const service = EnrichmentSourceServiceFactory.getEnrichmentSourceService(source, svc.log) + + if (await hasRemainingCreditsExists(source)) { + return getHasRemainingCredits(source) + } + + const hasCredits = await service.hasRemainingCredits() + + await setHasRemainingCredits(source, hasCredits) + return hasCredits +} + export async function findMemberEnrichmentCache( source: MemberEnrichmentSource, memberId: string, diff --git a/services/apps/premium/members_enrichment_worker/src/sources/clearbit/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/clearbit/service.ts index 4264f8a4ca..7af642ac91 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/clearbit/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/clearbit/service.ts @@ -27,7 +27,9 @@ import { export default class EnrichmentServiceClearbit extends LoggerBase implements IEnrichmentService { public source: MemberEnrichmentSource = MemberEnrichmentSource.CLEARBIT public platform = `enrichment-${this.source}` - public enrichableBySql = `mi.type = 'email' and mi.verified` + public enrichMembersWithActivityMoreThan = 10 + + public enrichableBySql = `"activitySummary".total_count > ${this.enrichMembersWithActivityMoreThan} AND mi.type = 'email' and mi.verified` // bust cache after 120 days public cacheObsoleteAfterSeconds = 60 * 60 * 24 * 120 @@ -63,6 +65,10 @@ export default class EnrichmentServiceClearbit extends LoggerBase implements IEn return enriched } + async hasRemainingCredits(): Promise { + return true + } + private async getDataUsingEmail(email: string): Promise { let response: IMemberEnrichmentClearbitAPIResponse diff --git a/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts index 0ac91d758f..e630110b2f 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/progai/service.ts @@ -124,6 +124,10 @@ export default class EnrichmentServiceProgAI extends LoggerBase implements IEnri return enrichableUsingGithubHandle || enrichableUsingEmail } + async hasRemainingCredits(): Promise { + return true + } + async getData(input: IEnrichmentSourceInput): Promise { let enriched: IMemberEnrichmentDataProgAI = null diff --git a/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts index 2d5c350355..47a476c31e 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts @@ -9,7 +9,11 @@ import { IMemberEnrichmentDataNormalized, } from '../../types' -import { IMemberEnrichmentDataSerp, IMemberEnrichmentSerpApiResponse } from './types' +import { + IMemberEnrichmentDataSerp, + IMemberEnrichmentSerpApiResponse, + ISerpApiAccountUsageData, +} from './types' export default class EnrichmentServiceSerpApi extends LoggerBase implements IEnrichmentService { public source: MemberEnrichmentSource = MemberEnrichmentSource.SERP @@ -45,6 +49,20 @@ export default class EnrichmentServiceSerpApi extends LoggerBase implements IEnr ) } + async hasRemainingCredits(): Promise { + const config = { + method: 'get', + url: `https://serpapi.com/account`, + params: { + api_key: process.env['CROWD_ENRICHMENT_SERP_API_KEY'], + }, + } + + const response: ISerpApiAccountUsageData = (await axios(config)).data + + return response.total_searches_left > 0 + } + async getData(input: IEnrichmentSourceInput): Promise { let enriched: IMemberEnrichmentDataSerp = null diff --git a/services/apps/premium/members_enrichment_worker/src/sources/serp/types.ts b/services/apps/premium/members_enrichment_worker/src/sources/serp/types.ts index 87484aca0e..f9c3f64e30 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/serp/types.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/serp/types.ts @@ -7,6 +7,24 @@ export interface IMemberEnrichmentSerpApiResponse { search_information: IMemberEnrichmentSerpApiResponseSearchInformation } +export interface ISerpApiAccountUsageData { + account_id: string + api_key: string + account_email: string + account_status: string + plan_id: string + plan_name: string + plan_monthly_price: number + searches_per_month: number + plan_searches_left: number + extra_credits: number + total_searches_left: number + this_month_usage: number + this_hour_searches: number + last_hour_searches: number + account_rate_limit_per_hour: number +} + export interface IMemberEnrichmentSerpApiResponseSearchInformation { query_displayed: string total_results: number diff --git a/services/apps/premium/members_enrichment_worker/src/types.ts b/services/apps/premium/members_enrichment_worker/src/types.ts index 0aff644cbc..0cab324638 100644 --- a/services/apps/premium/members_enrichment_worker/src/types.ts +++ b/services/apps/premium/members_enrichment_worker/src/types.ts @@ -37,6 +37,10 @@ export interface IEnrichmentService { // can the source enrich using this input isEnrichableBySource(input: IEnrichmentSourceInput): boolean + // does the source have credits to enrich members, if returned false the source will be skipped + // response will be saved to redis for 60 seconds and will be used for subsequent calls + hasRemainingCredits(): Promise + // SQL filter to get enrichable members for a source // members table is available as "members" alias // memberIdentities table is available as "mi" alias diff --git a/services/libs/redis/src/cache.ts b/services/libs/redis/src/cache.ts index 210db84386..b3393ae784 100644 --- a/services/libs/redis/src/cache.ts +++ b/services/libs/redis/src/cache.ts @@ -43,6 +43,12 @@ export class RedisCache extends LoggerBase implements ICache { } } + async exists(key: string): Promise { + const actualKey = this.prefixer(key) + const value = await this.client.exists(actualKey) + return value === 1 + } + async increment(key: string, incrementBy = 1, ttlSeconds?: number): Promise { const actualKey = this.prefixer(key) From 66f54f07755a4be52313c4c858b65533c8fe8f2b Mon Sep 17 00:00:00 2001 From: anilb Date: Mon, 4 Nov 2024 11:26:41 +0100 Subject: [PATCH 4/6] handle axios errors when getting remaining credits --- .../src/sources/serp/service.ts | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts index 47a476c31e..d89bb5d470 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/serp/service.ts @@ -50,17 +50,22 @@ export default class EnrichmentServiceSerpApi extends LoggerBase implements IEnr } async hasRemainingCredits(): Promise { - const config = { - method: 'get', - url: `https://serpapi.com/account`, - params: { - api_key: process.env['CROWD_ENRICHMENT_SERP_API_KEY'], - }, - } + try { + const config = { + method: 'get', + url: `https://serpapi.com/account`, + params: { + api_key: process.env['CROWD_ENRICHMENT_SERP_API_KEY'], + }, + } - const response: ISerpApiAccountUsageData = (await axios(config)).data + const response: ISerpApiAccountUsageData = (await axios(config)).data - return response.total_searches_left > 0 + return response.total_searches_left > 0 + } catch (error) { + this.log.error('Error while checking serpapi account usage', error) + return false + } } async getData(input: IEnrichmentSourceInput): Promise { From 5e47a4571cd694dbad31212c311858aeba62eadf Mon Sep 17 00:00:00 2001 From: anilb Date: Tue, 5 Nov 2024 09:42:49 +0100 Subject: [PATCH 5/6] selecting among multiple gh identities when enriching, prioritizing members with most activities --- .../src/activities.ts | 2 + .../src/activities/enrichment.ts | 14 ++++- .../src/activities/getMembers.ts | 4 +- .../members_enrichment_worker/src/types.ts | 2 +- .../src/workflows/enrichMember.ts | 31 ++++++++-- .../src/workflows/getMembersToEnrich.ts | 9 ++- .../data-access-layer/src/activities/sql.ts | 22 +++++++ .../members_enrichment_worker/index.ts | 59 ++++++++++--------- services/libs/types/src/premium/enrichment.ts | 6 ++ 9 files changed, 109 insertions(+), 40 deletions(-) diff --git a/services/apps/premium/members_enrichment_worker/src/activities.ts b/services/apps/premium/members_enrichment_worker/src/activities.ts index cd22e736ea..98d7c43e9d 100644 --- a/services/apps/premium/members_enrichment_worker/src/activities.ts +++ b/services/apps/premium/members_enrichment_worker/src/activities.ts @@ -1,5 +1,6 @@ import { findMemberEnrichmentCache, + findMemberIdentityWithTheMostActivityInPlatform, getEnrichmentData, insertMemberEnrichmentCache, isCacheObsolete, @@ -50,4 +51,5 @@ export { touchMemberEnrichmentCacheUpdatedAt, updateMemberEnrichmentCache, isEnrichableBySource, + findMemberIdentityWithTheMostActivityInPlatform, } diff --git a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts index 4fe5a8e82f..845e963628 100644 --- a/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts +++ b/services/apps/premium/members_enrichment_worker/src/activities/enrichment.ts @@ -1,3 +1,4 @@ +import { findMemberIdentityWithTheMostActivityInPlatform as findMemberIdentityWithTheMostActivityInPlatformQuestDb } from '@crowd/data-access-layer/src/activities' import { findMemberEnrichmentCacheDb, insertMemberEnrichmentCacheDb, @@ -5,7 +6,11 @@ import { updateMemberEnrichmentCacheDb, } from '@crowd/data-access-layer/src/old/apps/premium/members_enrichment_worker' import { RedisCache } from '@crowd/redis' -import { IMemberEnrichmentCache, MemberEnrichmentSource } from '@crowd/types' +import { + IEnrichableMemberIdentityActivityAggregate, + IMemberEnrichmentCache, + MemberEnrichmentSource, +} from '@crowd/types' import { EnrichmentSourceServiceFactory } from '../factory' import { svc } from '../main' @@ -117,3 +122,10 @@ export async function touchMemberEnrichmentCacheUpdatedAt( ): Promise { await touchMemberEnrichmentCacheUpdatedAtDb(svc.postgres.writer.connection(), memberId, source) } + +export async function findMemberIdentityWithTheMostActivityInPlatform( + memberId: string, + platform: string, +): Promise { + return findMemberIdentityWithTheMostActivityInPlatformQuestDb(svc.questdbSQL, memberId, platform) +} diff --git a/services/apps/premium/members_enrichment_worker/src/activities/getMembers.ts b/services/apps/premium/members_enrichment_worker/src/activities/getMembers.ts index 6b99f3c42e..09698a6512 100644 --- a/services/apps/premium/members_enrichment_worker/src/activities/getMembers.ts +++ b/services/apps/premium/members_enrichment_worker/src/activities/getMembers.ts @@ -11,7 +11,7 @@ import { svc } from '../main' export async function getEnrichableMembers( limit: number, sources: MemberEnrichmentSource[], - afterId: string, + afterCursor: { activityCount: number; memberId: string } | null, ): Promise { let rows: IEnrichableMember[] = [] const sourceInputs: IMemberEnrichmentSourceQueryInput[] = sources.map((s) => { @@ -23,7 +23,7 @@ export async function getEnrichableMembers( } }) const db = svc.postgres.reader - rows = await fetchMembersForEnrichment(db, limit, sourceInputs, afterId) + rows = await fetchMembersForEnrichment(db, limit, sourceInputs, afterCursor) return rows } diff --git a/services/apps/premium/members_enrichment_worker/src/types.ts b/services/apps/premium/members_enrichment_worker/src/types.ts index 0cab324638..4bffd0555e 100644 --- a/services/apps/premium/members_enrichment_worker/src/types.ts +++ b/services/apps/premium/members_enrichment_worker/src/types.ts @@ -70,7 +70,7 @@ export interface IMemberEnrichmentDataNormalizedOrganization { } export interface IGetMembersForEnrichmentArgs { - afterId?: string + afterCursor: { activityCount: number; memberId: string } | null } export interface IMemberEnrichmentSocialData { diff --git a/services/apps/premium/members_enrichment_worker/src/workflows/enrichMember.ts b/services/apps/premium/members_enrichment_worker/src/workflows/enrichMember.ts index 4f9c243b7c..de65524a62 100644 --- a/services/apps/premium/members_enrichment_worker/src/workflows/enrichMember.ts +++ b/services/apps/premium/members_enrichment_worker/src/workflows/enrichMember.ts @@ -19,6 +19,7 @@ const { updateMemberEnrichmentCache, isCacheObsolete, normalizeEnrichmentData, + findMemberIdentityWithTheMostActivityInPlatform, } = proxyActivities({ startToCloseTimeout: '20 seconds', retry: { @@ -42,12 +43,6 @@ export async function enrichMember( // cache is obsolete when it's not found or cache.updatedAt is older than cacheObsoleteAfterSeconds if (await isCacheObsolete(source, cache)) { const enrichmentInput: IEnrichmentSourceInput = { - github: input.identities.find( - (i) => - i.verified && - i.platform === PlatformType.GITHUB && - i.type === MemberIdentityType.USERNAME, - ), email: input.identities.find((i) => i.verified && i.type === MemberIdentityType.EMAIL), linkedin: input.identities.find( (i) => @@ -61,6 +56,30 @@ export async function enrichMember( activityCount: input.activityCount || 0, } + // there can be multiple verified identities in github, we select the one with the most activities + const verifiedGithubIdentities = input.identities.filter( + (i) => + i.verified && + i.platform === PlatformType.GITHUB && + i.type === MemberIdentityType.USERNAME, + ) + + if (verifiedGithubIdentities.length > 1) { + const ghIdentityWithTheMostActivities = + await findMemberIdentityWithTheMostActivityInPlatform(input.id, PlatformType.GITHUB) + if (ghIdentityWithTheMostActivities) { + enrichmentInput.github = input.identities.find( + (i) => + i.verified && + i.platform === PlatformType.GITHUB && + i.type === MemberIdentityType.USERNAME && + i.value === ghIdentityWithTheMostActivities.username, + ) + } + } else { + enrichmentInput.github = verifiedGithubIdentities?.[0] || undefined + } + const data = await getEnrichmentData(source, enrichmentInput) if (!cache) { diff --git a/services/apps/premium/members_enrichment_worker/src/workflows/getMembersToEnrich.ts b/services/apps/premium/members_enrichment_worker/src/workflows/getMembersToEnrich.ts index 72898e92e8..fc68598cac 100644 --- a/services/apps/premium/members_enrichment_worker/src/workflows/getMembersToEnrich.ts +++ b/services/apps/premium/members_enrichment_worker/src/workflows/getMembersToEnrich.ts @@ -19,14 +19,14 @@ const { getEnrichableMembers } = proxyActivities({ export async function getMembersToEnrich(args: IGetMembersForEnrichmentArgs): Promise { const MEMBER_ENRICHMENT_PER_RUN = 100 - const afterId = args?.afterId || null + const afterCursor = args?.afterCursor || null const sources = [ MemberEnrichmentSource.PROGAI, MemberEnrichmentSource.CLEARBIT, MemberEnrichmentSource.SERP, ] - const members = await getEnrichableMembers(MEMBER_ENRICHMENT_PER_RUN, sources, afterId) + const members = await getEnrichableMembers(MEMBER_ENRICHMENT_PER_RUN, sources, afterCursor) if (members.length === 0) { return @@ -54,6 +54,9 @@ export async function getMembersToEnrich(args: IGetMembersForEnrichmentArgs): Pr ) await continueAsNew({ - afterId: members[members.length - 1].id, + afterCursor: { + memberId: members[members.length - 1].id, + activityCount: members[members.length - 1].activityCount, + }, }) } diff --git a/services/libs/data-access-layer/src/activities/sql.ts b/services/libs/data-access-layer/src/activities/sql.ts index 6967f15156..f33e6517bb 100644 --- a/services/libs/data-access-layer/src/activities/sql.ts +++ b/services/libs/data-access-layer/src/activities/sql.ts @@ -8,6 +8,7 @@ import { ActivityDisplayVariant, IActivityBySentimentMoodResult, IActivityByTypeAndPlatformResult, + IEnrichableMemberIdentityActivityAggregate, IMemberIdentity, ITimeseriesDatapoint, MemberIdentityType, @@ -1391,3 +1392,24 @@ export async function getLastActivitiesForMembers( segmentIds, ) } + +export async function findMemberIdentityWithTheMostActivityInPlatform( + qdbConn: DbConnOrTx, + platform: string, + memberId: string, +): Promise { + const query = ` + SELECT count(a.id) AS "activityCount", a.platform, a.username + FROM activities a + WHERE a."memberId" = $(memberId) + AND a.platform = $(platform) + GROUP BY a.platform, a.username + ORDER BY activity_count DESC + LIMIT 1; + ` + + return qdbConn.oneOrNone(query, { + memberId, + platform, + }) +} diff --git a/services/libs/data-access-layer/src/old/apps/premium/members_enrichment_worker/index.ts b/services/libs/data-access-layer/src/old/apps/premium/members_enrichment_worker/index.ts index 497472754f..1f7d8b6915 100644 --- a/services/libs/data-access-layer/src/old/apps/premium/members_enrichment_worker/index.ts +++ b/services/libs/data-access-layer/src/old/apps/premium/members_enrichment_worker/index.ts @@ -16,9 +16,11 @@ export async function fetchMembersForEnrichment( db: DbStore, limit: number, sourceInputs: IMemberEnrichmentSourceQueryInput[], - afterId: string, + afterCursor: { activityCount: number; memberId: string } | null, ): Promise { - const idFilter = afterId ? ' and members.id < $2 ' : '' + const cursorFilter = afterCursor + ? `AND ((coalesce("activitySummary".total_count, 0) < $2) OR (coalesce("activitySummary".total_count, 0) = $2 AND members.id < $3))` + : '' const sourceInnerQueryItems = [] const enrichableBySqlConditions = [] @@ -48,42 +50,45 @@ export async function fetchMembersForEnrichment( WITH "activitySummary" AS ( SELECT msa."memberId", - sum(msa."activityCount") AS total_count + SUM(msa."activityCount") AS total_count FROM "memberSegmentsAgg" msa WHERE msa."segmentId" IN ( SELECT id FROM segments WHERE "grandparentId" IS NOT NULL AND "parentId" IS NOT NULL ) - group by msa."memberId" + GROUP BY msa."memberId" ) SELECT members."id", members."tenantId", members."displayName", - members.attributes->'location'->>'default' as location, - members.attributes->'websiteUrl'->>'default' as website, - json_agg(json_build_object( - 'platform', mi.platform, - 'value', mi.value, - 'type', mi.type, - 'verified', mi.verified - )) as identities, - MAX("activitySummary".total_count) as "activityCount" - FROM members - INNER JOIN tenants ON tenants.id = members."tenantId" - INNER JOIN "memberIdentities" mi ON mi."memberId" = members.id - INNER JOIN "activitySummary" on "activitySummary"."memberId" = members.id - WHERE - ${enrichableBySqlJoined} - AND tenants."deletedAt" IS NULL - AND members."deletedAt" IS NULL - AND (${sourceInnerQueryItems.join(' OR ')}) - ${idFilter} - GROUP BY members.id - ORDER BY members.id desc - LIMIT $1;`, - [limit, afterId], + members.attributes->'location'->>'default' AS location, + members.attributes->'websiteUrl'->>'default' AS website, + JSON_AGG( + JSON_BUILD_OBJECT( + 'platform', mi.platform, + 'value', mi.value, + 'type', mi.type, + 'verified', mi.verified + ) + ) AS identities, + MAX(coalesce("activitySummary".total_count, 0)) AS "activityCount" + FROM members + INNER JOIN tenants ON tenants.id = members."tenantId" + INNER JOIN "memberIdentities" mi ON mi."memberId" = members.id + LEFT JOIN "activitySummary" ON "activitySummary"."memberId" = members.id + WHERE + ${enrichableBySqlJoined} + AND tenants."deletedAt" IS NULL + AND members."deletedAt" IS NULL + AND (${sourceInnerQueryItems.join(' OR ')}) + ${cursorFilter} + GROUP BY members.id + ORDER BY "activityCount" DESC, members.id DESC + LIMIT $1; + `, + afterCursor ? [limit, afterCursor.activityCount, afterCursor.memberId] : [limit], ) } diff --git a/services/libs/types/src/premium/enrichment.ts b/services/libs/types/src/premium/enrichment.ts index defb3b8e2d..e6754a3ee7 100644 --- a/services/libs/types/src/premium/enrichment.ts +++ b/services/libs/types/src/premium/enrichment.ts @@ -24,3 +24,9 @@ export interface IEnrichableMember { identities: IMemberIdentity[] activityCount: number } + +export interface IEnrichableMemberIdentityActivityAggregate { + activityCount: number + username: string + platform: string +} From abb976dd1f2ec3eca5aadd202844c6153acaf491 Mon Sep 17 00:00:00 2001 From: anilb Date: Tue, 5 Nov 2024 09:50:25 +0100 Subject: [PATCH 6/6] use the new afterCursor when scheduling member enrichment --- .../src/schedules/getMembersToEnrich.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/apps/premium/members_enrichment_worker/src/schedules/getMembersToEnrich.ts b/services/apps/premium/members_enrichment_worker/src/schedules/getMembersToEnrich.ts index 5a69c80633..e318fce93a 100644 --- a/services/apps/premium/members_enrichment_worker/src/schedules/getMembersToEnrich.ts +++ b/services/apps/premium/members_enrichment_worker/src/schedules/getMembersToEnrich.ts @@ -27,7 +27,7 @@ export const scheduleMembersEnrichment = async () => { }, args: [ { - afterId: null, + afterCursor: null, }, ], },