diff --git a/app/[locale]/developers/apps/types.ts b/app/[locale]/developers/apps/types.ts index e894c6b11c3..c11bc9ae332 100644 --- a/app/[locale]/developers/apps/types.ts +++ b/app/[locale]/developers/apps/types.ts @@ -112,29 +112,3 @@ export type DeveloperAppsByCategory = Record< DeveloperAppCategorySlug, DeveloperApp[] > - -/** - * Pre-computed randomized selections for developer apps. - * Computed daily in the trigger.dev task to ensure all users see the same selections. - */ -export interface DeveloperAppsComputedSelections { - /** Main page highlights - top app from 3 random categories (3 IDs) */ - mainPageHighlights: string[] - /** Category page highlights - top 3 apps per category (7 categories × 3 = 21 IDs) */ - categoryHighlights: Record - /** Category preview apps - 5 random apps per category for main page cards (7 × 5 = 35 IDs) */ - categoryPreviews: Record - /** ISO date when selections were computed (for debugging) */ - computedAt: string -} - -/** - * Envelope type for developer tools data. - * Contains both the app data and pre-computed selections. - */ -export interface DeveloperToolsDataEnvelope { - /** All apps indexed by ID for quick lookup (used by app modal) */ - appsById: Record - /** Pre-computed randomized selections (refreshed daily) */ - selections: DeveloperAppsComputedSelections -} diff --git a/app/[locale]/developers/apps/utils.ts b/app/[locale]/developers/apps/utils.ts index 194a5380ff5..ecf9061ee05 100644 --- a/app/[locale]/developers/apps/utils.ts +++ b/app/[locale]/developers/apps/utils.ts @@ -1,220 +1,7 @@ import type { TagProps } from "@/components/ui/tag" -import { getDayOfYear, getWeekNumber } from "@/lib/utils/date" - -import { DEV_APP_CATEGORIES, DEV_APP_CATEGORY_SLUGS } from "./constants" -import type { - DeveloperApp, - DeveloperAppCategorySlug, - DeveloperAppsByCategory, -} from "./types" - -// Number of top apps to show in highlights section -const HIGHLIGHTS_PER_CATEGORY = 9 -// Number of preview apps to show in category cards -const PREVIEWS_PER_CATEGORY = 5 - -/** - * Transform flat array of apps into an object grouped by category slug - */ -export const transformDeveloperAppsData = ( - data: DeveloperApp[] -): DeveloperAppsByCategory => { - const initialAcc = Object.values(DEV_APP_CATEGORY_SLUGS).reduce( - (acc, slug) => { - acc[slug] = [] - return acc - }, - {} as DeveloperAppsByCategory - ) - - return data.reduce((acc, app) => { - const slug = DEV_APP_CATEGORY_SLUGS[app.category] - acc[slug].push(app) - return acc - }, initialAcc) -} - -/** - * Seeded random number generator for deterministic randomization - * Uses Linear Congruential Generator algorithm - * - * Why not Math.random() or timestamp? - * - Math.random(): Non-deterministic, every user sees different highlights - * - Timestamp: Different on every page load, causes jarring UX - * - Seeded: Same seed = same "random" sequence = consistent highlights for all users - */ -function seededRandom(seed: number) { - let value = seed - return () => { - value = (value * 9301 + 49297) % 233280 - return value / 233280 - } -} - -/** - * Shuffle array using Fisher-Yates algorithm with seeded randomization - * Ensures deterministic shuffle: same seed always produces same order - */ -function seededShuffle(array: T[], seed: number): T[] { - const shuffled = [...array] - const random = seededRandom(seed) - - for (let i = shuffled.length - 1; i > 0; i--) { - const j = Math.floor(random() * (i + 1)) - ;[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]] - } - - return shuffled -} - -/** - * Get maximum star count across all repos for an app - */ -function getMaxStarCount(app: DeveloperApp): number { - return Math.max(...app.repos.map((repo) => repo.stargazers || 0), 0) -} - -/** - * Generate seed offset based on category name for variety across categories - */ -function getCategorySeedOffset(category: string): number { - return category.length -} - -/** - * Get highlighted apps grouped by category - * - * Algorithm: - * 1. Filter to apps with GitHub repos updated in last 6 months + have banner/thumbnail images - * 2. Group by category slug - * 3. Sort each category by highest GitHub star count - * 4. Take top 9 apps per category - * 5. Shuffle each category's top 9 using weekly seed for deterministic rotation - * - * @returns Object mapping category slugs to arrays of up to 9 highlighted apps - */ -export function getHighlightsByCategory( - apps: DeveloperApp[], - now: Date = new Date() -): Record { - const sixMonthsAgo = new Date(now) - sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6) - - // Filter apps with GitHub repos updated in last 6 months and have required images - const recentApps = apps.filter((app) => { - // Must have both banner and thumbnail images for highlights section - if (!app.banner_url || !app.thumbnail_url) return false - - const hasGitHubRepo = app.repos.some((repo) => - repo.href.includes("github.com") - ) - if (!hasGitHubRepo) return false - - const latestUpdate = app.repos.reduce((latest, repo) => { - if (!repo.lastUpdated) return latest - const date = new Date(repo.lastUpdated) - return date > latest ? date : latest - }, new Date(0)) - - return latestUpdate > sixMonthsAgo - }) - - // Group by category slug - const byCategory: Record = {} - for (const app of recentApps) { - const categorySlug = DEV_APP_CATEGORY_SLUGS[app.category] - if (!categorySlug) continue - if (!byCategory[categorySlug]) { - byCategory[categorySlug] = [] - } - byCategory[categorySlug].push(app) - } - - // Get weekly seed for deterministic randomization - const weekSeed = getWeekNumber(now) - - // Sort by stars, take top N, randomize - const result: Record = {} - for (const [category, categoryApps] of Object.entries(byCategory)) { - // Sort by highest star count - const sorted = [...categoryApps].sort((a, b) => { - return getMaxStarCount(b) - getMaxStarCount(a) - }) - - // Take top N and randomize - const topN = sorted.slice(0, HIGHLIGHTS_PER_CATEGORY) - const randomized = seededShuffle( - topN, - weekSeed + getCategorySeedOffset(category) - ) - - result[category] = randomized - } - - return result as Record -} - -/** - * Get highlights for main /developers/apps page - * Returns top app from top 3 randomly-ordered categories - * - * @returns Array of 3 apps (one from each of top 3 categories) - */ -export function getMainPageHighlights( - highlightsByCategory: Record, - now: Date = new Date() -): DeveloperApp[] { - const weekSeed = getWeekNumber(now) - - // Get categories with highlights - const categoriesWithHighlights = Object.entries(highlightsByCategory).filter( - ([, apps]) => apps.length > 0 - ) - - // Randomize category order - const randomizedCategories = seededShuffle(categoriesWithHighlights, weekSeed) - - // Take top app from top 3 categories - return randomizedCategories - .slice(0, 3) - .map(([, apps]) => apps[0]) - .filter(Boolean) -} - - -/** - * Get randomized preview apps per category for main page cards - * - * Simpler than highlights: no filtering, no star sorting - just shuffle and take N. - * Uses daily seed for rotation instead of weekly. - * - * @param dataByCategory - Apps already grouped by category - * @returns Same structure but with max N randomized apps per category - */ -export function getRandomPreviewsByCategory( - dataByCategory: DeveloperAppsByCategory, - now: Date = new Date() -): DeveloperAppsByCategory { - const daySeed = getDayOfYear(now) - - const result = {} as DeveloperAppsByCategory - - for (const [category, apps] of Object.entries(dataByCategory)) { - // Shuffle with daily seed + category offset for variety - const shuffled = seededShuffle( - apps, - daySeed + getCategorySeedOffset(category) - ) - // Take first N after shuffle - result[category as DeveloperAppCategorySlug] = shuffled.slice( - 0, - PREVIEWS_PER_CATEGORY - ) - } - - return result -} +import { DEV_APP_CATEGORIES } from "./constants" +import type { DeveloperAppCategorySlug } from "./types" /** * Gets the tag style for a developer app category based on its slug. diff --git a/src/data-layer/fetchers/fetchDeveloperToolsBuidlGuidl.ts b/src/data-layer/fetchers/developer-tools/fetchBuidlGuidl.ts similarity index 87% rename from src/data-layer/fetchers/fetchDeveloperToolsBuidlGuidl.ts rename to src/data-layer/fetchers/developer-tools/fetchBuidlGuidl.ts index a722d8173aa..b47b16b939a 100644 --- a/src/data-layer/fetchers/fetchDeveloperToolsBuidlGuidl.ts +++ b/src/data-layer/fetchers/developer-tools/fetchBuidlGuidl.ts @@ -1,8 +1,6 @@ import { DeveloperAppsResponse } from "@/lib/types" -export async function fetchDeveloperToolsBuidlGuidl(): Promise< - DeveloperAppsResponse[] -> { +export async function fetchBuidlGuidl(): Promise { const url = "https://raw.githubusercontent.com/BuidlGuidl/Developer-Tooling/refs/heads/main/output/results.json" diff --git a/src/data-layer/fetchers/fetchDeveloperToolsGitHub.ts b/src/data-layer/fetchers/developer-tools/fetchGitHub.ts similarity index 96% rename from src/data-layer/fetchers/fetchDeveloperToolsGitHub.ts rename to src/data-layer/fetchers/developer-tools/fetchGitHub.ts index fc47a03c9a2..b4b916464a4 100644 --- a/src/data-layer/fetchers/fetchDeveloperToolsGitHub.ts +++ b/src/data-layer/fetchers/developer-tools/fetchGitHub.ts @@ -2,7 +2,7 @@ import type { DeveloperAppsResponse } from "@/lib/types" import { retry, sleep } from "@/lib/utils/fetch" -import type { DeveloperApp } from "../../../app/[locale]/developers/apps/types" +import type { DeveloperApp } from "./utils" type RepoInfo = { owner: string @@ -100,7 +100,7 @@ async function fetchReposBatch( return results } -export async function fetchDeveloperToolsGitHub( +export async function fetchGitHub( appData: DeveloperAppsResponse[] ): Promise { // Collect all unique repo URLs diff --git a/src/data-layer/fetchers/fetchDeveloperToolsNpmJs.ts b/src/data-layer/fetchers/developer-tools/fetchNpmJs.ts similarity index 97% rename from src/data-layer/fetchers/fetchDeveloperToolsNpmJs.ts rename to src/data-layer/fetchers/developer-tools/fetchNpmJs.ts index 05e720037d9..e75a913bce3 100644 --- a/src/data-layer/fetchers/fetchDeveloperToolsNpmJs.ts +++ b/src/data-layer/fetchers/developer-tools/fetchNpmJs.ts @@ -1,6 +1,6 @@ import { retry, sleep } from "@/lib/utils/fetch" -import type { DeveloperApp } from "../../../app/[locale]/developers/apps/types" +import type { DeveloperApp } from "./utils" type ParsedNpmUrl = { packageName: string @@ -137,7 +137,7 @@ async function fetchBulkDownloads( return results } -export async function fetchDeveloperToolsNpmJs( +export async function fetchNpmJs( appData: DeveloperApp[] ): Promise { // Collect all unique npm URLs and their package names diff --git a/src/data-layer/fetchers/fetchDeveloperTools.ts b/src/data-layer/fetchers/developer-tools/index.ts similarity index 83% rename from src/data-layer/fetchers/fetchDeveloperTools.ts rename to src/data-layer/fetchers/developer-tools/index.ts index 0f7e7bef87f..40583ce5f1b 100644 --- a/src/data-layer/fetchers/fetchDeveloperTools.ts +++ b/src/data-layer/fetchers/developer-tools/index.ts @@ -1,18 +1,20 @@ +import { fetchBuidlGuidl } from "./fetchBuidlGuidl" +import { fetchGitHub } from "./fetchGitHub" +import { fetchNpmJs } from "./fetchNpmJs" import type { DeveloperApp, DeveloperAppsComputedSelections, DeveloperToolsDataEnvelope, -} from "../../../app/[locale]/developers/apps/types" +} from "./utils" import { getHighlightsByCategory, getMainPageHighlights, getRandomPreviewsByCategory, transformDeveloperAppsData, -} from "../../../app/[locale]/developers/apps/utils" +} from "./utils" -import { fetchDeveloperToolsBuidlGuidl } from "./fetchDeveloperToolsBuidlGuidl" -import { fetchDeveloperToolsGitHub } from "./fetchDeveloperToolsGitHub" -import { fetchDeveloperToolsNpmJs } from "./fetchDeveloperToolsNpmJs" +// Re-export types for consumers +export type { DeveloperToolsDataEnvelope } from "./utils" /** * Fetches and enriches developer tools data. @@ -31,15 +33,15 @@ export async function fetchDeveloperTools(): Promise console.log("Starting developer tools data enrichment pipeline") // Step 1: Fetch base data from BuidlGuidl - const rawData = await fetchDeveloperToolsBuidlGuidl() + const rawData = await fetchBuidlGuidl() console.log(`Fetched ${rawData.length} developer tools from BuidlGuidl`) // Step 2: Enrich with GitHub data (stars, last commit) - const withGitHub = await fetchDeveloperToolsGitHub(rawData) + const withGitHub = await fetchGitHub(rawData) console.log("Enriched with GitHub data") // Step 3: Enrich with npm data (download counts) - const enrichedData = await fetchDeveloperToolsNpmJs(withGitHub) + const enrichedData = await fetchNpmJs(withGitHub) console.log("Enriched with npm data") // Step 4: Build lookup map diff --git a/src/data-layer/fetchers/developer-tools/utils.ts b/src/data-layer/fetchers/developer-tools/utils.ts new file mode 100644 index 00000000000..28ddb652237 --- /dev/null +++ b/src/data-layer/fetchers/developer-tools/utils.ts @@ -0,0 +1,304 @@ +import { getDayOfYear, getWeekNumber } from "@/lib/utils/date" + +// Import the base DeveloperApp type from app code (type-only import) +// This is acceptable as it's a shared data contract, not a presentation dependency +import type { DeveloperApp } from "../../../../app/[locale]/developers/apps/types" + +// Re-export for convenience +export type { DeveloperApp } + +// ============================================================================= +// Types +// ============================================================================= + +/** + * Category slug type derived from the category mapping. + * These are URL-friendly identifiers for developer app categories. + */ +export type DeveloperAppCategorySlug = + | "interoperability" + | "transactions" + | "analytics" + | "education" + | "sdks" + | "contracts" + | "security" + +/** + * Apps grouped by category slug. + */ +export type DeveloperAppsByCategory = Record< + DeveloperAppCategorySlug, + DeveloperApp[] +> + +/** + * Pre-computed randomized selections for developer apps. + * Computed daily in the trigger.dev task to ensure all users see the same selections. + */ +export interface DeveloperAppsComputedSelections { + /** Main page highlights - top app from 3 random categories (3 IDs) */ + mainPageHighlights: string[] + /** Category page highlights - top 3 apps per category (7 categories × 3 = 21 IDs) */ + categoryHighlights: Record + /** Category preview apps - 5 random apps per category for main page cards (7 × 5 = 35 IDs) */ + categoryPreviews: Record + /** ISO date when selections were computed (for debugging) */ + computedAt: string +} + +/** + * Envelope type for developer tools data. + * Contains both the app data and pre-computed selections. + */ +export interface DeveloperToolsDataEnvelope { + /** All apps indexed by ID for quick lookup (used by app modal) */ + appsById: Record + /** Pre-computed randomized selections (refreshed daily) */ + selections: DeveloperAppsComputedSelections +} + +// ============================================================================= +// Constants +// ============================================================================= + +/** + * Maps human-readable category names to URL-friendly slugs. + * This is the data-layer copy of the constant - no UI dependencies. + */ +export const DEV_APP_CATEGORY_SLUGS: Record = + { + "Cross-Chain & Interoperability": "interoperability", + "Transaction & Wallet Infrastructure": "transactions", + "Data, Analytics & Tracing": "analytics", + "Education & Community Resources": "education", + "Client Libraries & SDKs (Front-End)": "sdks", + "Smart Contract Development & Toolchains": "contracts", + "Security, Testing & Formal Verification": "security", + } + +/** + * List of all category slugs for iteration. + */ +export const DEV_APP_CATEGORY_SLUG_LIST: DeveloperAppCategorySlug[] = [ + "interoperability", + "transactions", + "analytics", + "education", + "sdks", + "contracts", + "security", +] + +// Number of top apps to show in highlights section +const HIGHLIGHTS_PER_CATEGORY = 9 +// Number of preview apps to show in category cards +const PREVIEWS_PER_CATEGORY = 5 + +// ============================================================================= +// Seeded Randomization Utilities +// ============================================================================= + +/** + * Seeded random number generator for deterministic randomization. + * Uses Linear Congruential Generator algorithm. + * + * Why not Math.random() or timestamp? + * - Math.random(): Non-deterministic, every user sees different highlights + * - Timestamp: Different on every page load, causes jarring UX + * - Seeded: Same seed = same "random" sequence = consistent highlights for all users + */ +function seededRandom(seed: number) { + let value = seed + return () => { + value = (value * 9301 + 49297) % 233280 + return value / 233280 + } +} + +/** + * Shuffle array using Fisher-Yates algorithm with seeded randomization. + * Ensures deterministic shuffle: same seed always produces same order. + */ +function seededShuffle(array: T[], seed: number): T[] { + const shuffled = [...array] + const random = seededRandom(seed) + + for (let i = shuffled.length - 1; i > 0; i--) { + const j = Math.floor(random() * (i + 1)) + ;[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]] + } + + return shuffled +} + +/** + * Get maximum star count across all repos for an app. + */ +function getMaxStarCount(app: DeveloperApp): number { + return Math.max(...app.repos.map((repo) => repo.stargazers || 0), 0) +} + +/** + * Generate seed offset based on category name for variety across categories. + */ +function getCategorySeedOffset(category: string): number { + return category.length +} + +// ============================================================================= +// Data Transformation Functions +// ============================================================================= + +/** + * Transform flat array of apps into an object grouped by category slug. + */ +export function transformDeveloperAppsData( + data: DeveloperApp[] +): DeveloperAppsByCategory { + const initialAcc = DEV_APP_CATEGORY_SLUG_LIST.reduce((acc, slug) => { + acc[slug] = [] + return acc + }, {} as DeveloperAppsByCategory) + + return data.reduce((acc, app) => { + const slug = DEV_APP_CATEGORY_SLUGS[app.category] + if (slug) { + acc[slug].push(app) + } + return acc + }, initialAcc) +} + +/** + * Get highlighted apps grouped by category. + * + * Algorithm: + * 1. Filter to apps with GitHub repos updated in last 6 months + have banner/thumbnail images + * 2. Group by category slug + * 3. Sort each category by highest GitHub star count + * 4. Take top 9 apps per category + * 5. Shuffle each category's top 9 using weekly seed for deterministic rotation + * + * @returns Object mapping category slugs to arrays of up to 9 highlighted apps + */ +export function getHighlightsByCategory( + apps: DeveloperApp[], + now: Date = new Date() +): Record { + const sixMonthsAgo = new Date(now) + sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6) + + // Filter apps with GitHub repos updated in last 6 months and have required images + const recentApps = apps.filter((app) => { + // Must have both banner and thumbnail images for highlights section + if (!app.banner_url || !app.thumbnail_url) return false + + const hasGitHubRepo = app.repos.some((repo) => + repo.href.includes("github.com") + ) + if (!hasGitHubRepo) return false + + const latestUpdate = app.repos.reduce((latest, repo) => { + if (!repo.lastUpdated) return latest + const date = new Date(repo.lastUpdated) + return date > latest ? date : latest + }, new Date(0)) + + return latestUpdate > sixMonthsAgo + }) + + // Group by category slug + const byCategory: Record = {} + for (const app of recentApps) { + const categorySlug = DEV_APP_CATEGORY_SLUGS[app.category] + if (!categorySlug) continue + if (!byCategory[categorySlug]) { + byCategory[categorySlug] = [] + } + byCategory[categorySlug].push(app) + } + + // Get weekly seed for deterministic randomization + const weekSeed = getWeekNumber(now) + + // Sort by stars, take top N, randomize + const result: Record = {} + for (const [category, categoryApps] of Object.entries(byCategory)) { + // Sort by highest star count + const sorted = [...categoryApps].sort((a, b) => { + return getMaxStarCount(b) - getMaxStarCount(a) + }) + + // Take top N and randomize + const topN = sorted.slice(0, HIGHLIGHTS_PER_CATEGORY) + const randomized = seededShuffle( + topN, + weekSeed + getCategorySeedOffset(category) + ) + + result[category] = randomized + } + + return result as Record +} + +/** + * Get highlights for main /developers/apps page. + * Returns top app from top 3 randomly-ordered categories. + * + * @returns Array of 3 apps (one from each of top 3 categories) + */ +export function getMainPageHighlights( + highlightsByCategory: Record, + now: Date = new Date() +): DeveloperApp[] { + const weekSeed = getWeekNumber(now) + + // Get categories with highlights + const categoriesWithHighlights = Object.entries(highlightsByCategory).filter( + ([, apps]) => apps.length > 0 + ) + + // Randomize category order + const randomizedCategories = seededShuffle(categoriesWithHighlights, weekSeed) + + // Take top app from top 3 categories + return randomizedCategories + .slice(0, 3) + .map(([, apps]) => apps[0]) + .filter(Boolean) +} + +/** + * Get randomized preview apps per category for main page cards. + * + * Simpler than highlights: no filtering, no star sorting - just shuffle and take N. + * Uses daily seed for rotation instead of weekly. + * + * @param dataByCategory - Apps already grouped by category + * @returns Same structure but with max N randomized apps per category + */ +export function getRandomPreviewsByCategory( + dataByCategory: DeveloperAppsByCategory, + now: Date = new Date() +): DeveloperAppsByCategory { + const daySeed = getDayOfYear(now) + + const result = {} as DeveloperAppsByCategory + + for (const [category, apps] of Object.entries(dataByCategory)) { + // Shuffle with daily seed + category offset for variety + const shuffled = seededShuffle( + apps, + daySeed + getCategorySeedOffset(category) + ) + // Take first N after shuffle + result[category as DeveloperAppCategorySlug] = shuffled.slice( + 0, + PREVIEWS_PER_CATEGORY + ) + } + + return result +} diff --git a/src/data-layer/index.ts b/src/data-layer/index.ts index 9759d6f8b41..a140fd9b392 100644 --- a/src/data-layer/index.ts +++ b/src/data-layer/index.ts @@ -15,8 +15,7 @@ import type { } from "@/lib/types" import type { CommunityEventsReturnType } from "@/lib/interfaces" -import type { DeveloperToolsDataEnvelope } from "../../app/[locale]/developers/apps/types" - +import type { DeveloperToolsDataEnvelope } from "./fetchers/developer-tools/utils" import type { BeaconChainData } from "./fetchers/fetchBeaconChain" import type { CoinGeckoCoinMarketResponse } from "./fetchers/fetchStablecoinsData" import { get } from "./storage" @@ -45,5 +44,4 @@ export const getStablecoinsData = () => get(KEYS.ST export const getTotalEthStakedData = () => get(KEYS.TOTAL_ETH_STAKED) export const getTotalValueLockedData = () => get(KEYS.TOTAL_VALUE_LOCKED) export const getEventsData = () => get(KEYS.EVENTS) -export const getDeveloperToolsData = () => - get(KEYS.DEVELOPER_APPS) +export const getDeveloperToolsData = () => get(KEYS.DEVELOPER_APPS) diff --git a/src/data-layer/tasks.ts b/src/data-layer/tasks.ts index 71a3d2b1d8e..50b5bb895cf 100644 --- a/src/data-layer/tasks.ts +++ b/src/data-layer/tasks.ts @@ -7,12 +7,12 @@ import { schedules } from "@trigger.dev/sdk/v3" +import { fetchDeveloperTools } from "./fetchers/developer-tools" import { fetchApps } from "./fetchers/fetchApps" import { fetchBeaconChain } from "./fetchers/fetchBeaconChain" import { fetchBlobscanStats } from "./fetchers/fetchBlobscanStats" import { fetchCalendarEvents } from "./fetchers/fetchCalendarEvents" import { fetchCommunityPicks } from "./fetchers/fetchCommunityPicks" -import { fetchDeveloperTools } from "./fetchers/fetchDeveloperTools" import { fetchEthereumMarketcap } from "./fetchers/fetchEthereumMarketcap" import { fetchEthereumStablecoinsMcap } from "./fetchers/fetchEthereumStablecoinsMcap" import { fetchEthPrice } from "./fetchers/fetchEthPrice"