diff --git a/.claude/skills/data-layer/SKILL.md b/.claude/skills/data-layer/SKILL.md index 5feb0bc4826..ba672d601e6 100644 --- a/.claude/skills/data-layer/SKILL.md +++ b/.claude/skills/data-layer/SKILL.md @@ -8,39 +8,87 @@ description: This skill provides patterns for working with the data-layer module ## Architecture ``` -src/data-layer/ # Isolated, framework-agnostic module -├── api/ # Fetch functions (one per data source) -├── index.ts # Getter functions (pure passthrough) -└── registry.ts # Task registry (hourly/daily) +src/data-layer/ +├── fetchers/ # Fetch functions (one per data source) +├── index.ts # Public API - typed getter functions +├── tasks.ts # KEYS constant + Trigger.dev scheduled tasks +├── storage.ts # get/set abstraction (Netlify Blobs or mock files) +└── mocks/ # Mock data files for local development + +src/lib/data/ +└── index.ts # Next.js caching adapter (createCachedGetter) +``` + +## Key Files + +### tasks.ts - Single Source of Truth + +Defines all task keys and scheduled jobs: + +```typescript +export const KEYS = { + ETH_PRICE: "fetch-eth-price", + L2BEAT: "fetch-l2beat", + // ... +} as const + +const DAILY: Task[] = [ + [KEYS.APPS, fetchApps], + [KEYS.EVENTS, fetchEvents], +] + +const HOURLY: Task[] = [ + [KEYS.ETH_PRICE, fetchEthPrice], + [KEYS.BEACONCHAIN_EPOCH, fetchBeaconChainEpoch], +] +``` -src/lib/data/ # Next.js caching adapter -└── index.ts # Cached wrappers via createCachedGetter() +### index.ts - Simple Getters + +One-liner passthrough functions: + +```typescript +export const getEthPrice = () => get(KEYS.ETH_PRICE) +export const getL2beatData = () => get(KEYS.L2BEAT) ``` +### storage.ts - Storage Abstraction + +Simple get/set that switches between Netlify Blobs (prod) and local JSON files (dev): + +```typescript +export async function get(key: string): Promise +export async function set(key: string, data: unknown): Promise +``` + +Uses `USE_MOCK_DATA=true` env var for local development. + ## Rules ### 1. Getters must be pure passthrough -In `src/data-layer/index.ts`, getter functions must only call `getData(TASK_ID)` with no transformations: +No transformations in `index.ts` - just `get(KEYS.X)`: ```typescript // Correct -export async function getEventsData(): Promise { - return getData(FETCH_EVENTS_TASK_ID) -} +export const getEventsData = () => get(KEYS.EVENTS) // Wrong - no transformations in getters -export async function getEventsData(): Promise { - const data = await getData(FETCH_EVENTS_TASK_ID) - return data?.map((e) => ({ ...e, computed: derive(e) })) ?? null +export const getEventsData = () => { + const data = await get(KEYS.EVENTS) + return data?.map(transform) ?? null } ``` -All transformations belong in the fetch task (`src/data-layer/api/`), not in the getter. +All transformations belong in the fetcher (`src/data-layer/fetchers/`). + +### 2. KEYS is the single source of truth -### 2. Expose via lib/data for caching +All task IDs are defined in `KEYS` in `tasks.ts`. The getter in `index.ts` and the task tuple in `DAILY`/`HOURLY` must use the same key. -When a data function needs caching/revalidation, add a cached wrapper in `src/lib/data/index.ts`: +### 3. Expose via lib/data for caching + +Add cached wrapper in `src/lib/data/index.ts`: ```typescript export const getEventsData = createCachedGetter( @@ -52,25 +100,37 @@ export const getEventsData = createCachedGetter( ## Adding a New Data Source -1. Create fetch function in `src/data-layer/api/fetchNewData.ts`: +1. **Create fetcher** in `src/data-layer/fetchers/fetchNewData.ts`: ```typescript - export const FETCH_NEW_DATA_TASK_ID = "fetch-new-data" - export async function fetchNewData(): Promise { // Fetch and transform data here } ``` -2. Add getter in `src/data-layer/index.ts`: +2. **Add key** to `KEYS` in `src/data-layer/tasks.ts`: ```typescript - export async function getNewData(): Promise { - return getData(FETCH_NEW_DATA_TASK_ID) - } + export const KEYS = { + // ...existing keys + NEW_DATA: "fetch-new-data", + } as const + ``` + +3. **Add task tuple** to `DAILY` or `HOURLY` in `tasks.ts`: + ```typescript + const DAILY: Task[] = [ + // ...existing tasks + [KEYS.NEW_DATA, fetchNewData], + ] + ``` + +4. **Add getter** in `src/data-layer/index.ts`: + ```typescript + export const getNewData = () => get(KEYS.NEW_DATA) ``` -3. Register in `src/data-layer/registry.ts` (hourlyTasks or dailyTasks) +5. **Add mock file** at `src/data-layer/mocks/fetch-new-data.json` for local development -4. Add cached wrapper in `src/lib/data/index.ts`: +6. **Add cached wrapper** in `src/lib/data/index.ts`: ```typescript export const getNewData = createCachedGetter( dataLayer.getNewData, diff --git a/.prettierrc b/.prettierrc index dd48c13bb21..aabb3458d6b 100644 --- a/.prettierrc +++ b/.prettierrc @@ -4,7 +4,13 @@ "singleQuote": false, "tabWidth": 2, "trailingComma": "es5", - "plugins": [ - "prettier-plugin-tailwindcss" + "plugins": ["prettier-plugin-tailwindcss"], + "overrides": [ + { + "files": "src/data-layer/index.ts", + "options": { + "printWidth": 120 + } + } ] } \ No newline at end of file diff --git a/app/[locale]/community/events/utils.ts b/app/[locale]/community/events/utils.ts index 9faa472e46e..2d686fd6169 100644 --- a/app/[locale]/community/events/utils.ts +++ b/app/[locale]/community/events/utils.ts @@ -8,7 +8,7 @@ import { parseLocationToContinent } from "@/lib/utils/geography" import { slugify } from "@/lib/utils/url" import communityMeetups from "@/data/community-meetups.json" -import { getEventTypes } from "@/data-layer/api/fetchEvents" +import { getEventTypes } from "@/data-layer/fetchers/fetchEvents" // Map EventType to Tag component status colors export const TAG_STATUS_MAPPING: Record = { diff --git a/package.json b/package.json index 50eeda9576d..8b5d1711bed 100644 --- a/package.json +++ b/package.json @@ -28,8 +28,8 @@ "test:e2e:ui": "playwright test --project=e2e --ui", "test:e2e:debug": "playwright test --project=e2e --debug", "test:e2e:report": "playwright show-report tests/__report__", - "trigger:dev": "dotenv -e .env -- npx trigger.dev@latest dev", - "trigger:deploy": "dotenv -e .env -- npx trigger.dev@latest deploy" + "trigger:dev": "npx trigger.dev@latest dev --env-file .env.local", + "trigger:deploy": "npx trigger.dev@latest deploy --env-file .env.local" }, "dependencies": { "@aws-sdk/client-ses": "^3.859.0", @@ -140,7 +140,6 @@ "chromatic": "12.0.0", "decompress": "^4.2.1", "dotenv": "^16.5.0", - "dotenv-cli": "^11.0.0", "eslint": "^8.57.1", "eslint-config-next": "^14.2.2", "eslint-config-prettier": "^9", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9ad55f968d8..6aacdbc2ab9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -332,9 +332,6 @@ importers: dotenv: specifier: ^16.5.0 version: 16.5.0 - dotenv-cli: - specifier: ^11.0.0 - version: 11.0.0 eslint: specifier: ^8.57.1 version: 8.57.1 @@ -5882,22 +5879,10 @@ packages: resolution: {integrity: sha512-1gxPBJpI/pcjQhKgIU91II6Wkay+dLcN3M6rf2uwP8hRur3HtQXjVrdAK3sjC0piaEuxzMwjXChcETiJl47lAQ==} engines: {node: '>=18'} - dotenv-cli@11.0.0: - resolution: {integrity: sha512-r5pA8idbk7GFWuHEU7trSTflWcdBpQEK+Aw17UrSHjS6CReuhrrPcyC3zcQBPQvhArRHnBo/h6eLH1fkCvNlww==} - hasBin: true - - dotenv-expand@12.0.3: - resolution: {integrity: sha512-uc47g4b+4k/M/SeaW1y4OApx+mtLWl92l5LMPP0GNXctZqELk+YGgOPIIC5elYmUH4OuoK3JLhuRUYegeySiFA==} - engines: {node: '>=12'} - dotenv@16.5.0: resolution: {integrity: sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==} engines: {node: '>=12'} - dotenv@17.2.3: - resolution: {integrity: sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==} - engines: {node: '>=12'} - dset@3.1.4: resolution: {integrity: sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==} engines: {node: '>=4'} @@ -17202,21 +17187,8 @@ snapshots: dependencies: type-fest: 4.41.0 - dotenv-cli@11.0.0: - dependencies: - cross-spawn: 7.0.6 - dotenv: 17.2.3 - dotenv-expand: 12.0.3 - minimist: 1.2.8 - - dotenv-expand@12.0.3: - dependencies: - dotenv: 16.5.0 - dotenv@16.5.0: {} - dotenv@17.2.3: {} - dset@3.1.4: {} dunder-proto@1.0.1: diff --git a/src/data-layer/docs.md b/src/data-layer/docs.md index fa04f063f3b..03210a2f921 100644 --- a/src/data-layer/docs.md +++ b/src/data-layer/docs.md @@ -1,54 +1,43 @@ -# Data Layer Documentation +# Data Layer -The data layer is a centralized system for fetching, storing, and retrieving external data for the Ethereum.org website. It uses Trigger.dev for scheduled background jobs and Netlify Blobs for persistent storage. - -## Overview - -The data layer provides: -- **Scheduled data fetching** - Automated background jobs that fetch data from external APIs -- **Persistent storage** - Centralized storage for fetched data using Netlify Blobs -- **Type-safe access** - Clean public API with automatic type inference -- **Framework-agnostic design** - Core data-layer has no framework dependencies -- **Mock data support** - Local development without external dependencies +Centralized system for fetching and storing external data using Trigger.dev scheduled jobs and Netlify Blobs storage. ## Architecture ``` src/data-layer/ -├── api/ # Data fetching functions (one per external data source) -├── storage/ # Storage abstraction layer (unified Storage interface) -├── trigger/ # Trigger.dev scheduled tasks (parallelized) -├── mocks/ # Mock data files for local development +├── fetchers/ # Fetch functions (one per data source) +├── mocks/ # Mock JSON files for local development ├── index.ts # Public API - typed getter functions -├── registry.ts # Central registry of all tasks -└── types.ts # Shared type definitions (including Storage interface) +├── tasks.ts # KEYS constant + Trigger.dev scheduled tasks +└── storage.ts # get/set abstraction (Netlify Blobs or mock files) src/lib/data/ -└── index.ts # Next.js adapter - adds caching layer +└── index.ts # Next.js caching adapter ``` ## Key Rules -### 1. Data-layer getters must be pure passthrough +### 1. Getters must be pure passthrough ```typescript -// ✅ Correct -export async function getEventsData(): Promise { - return getData(FETCH_EVENTS_TASK_ID) -} +// Correct +export const getEventsData = () => get(KEYS.EVENTS) -// ❌ Wrong - no transformations in getters -export async function getEventsData(): Promise { - const data = await getData(FETCH_EVENTS_TASK_ID) - return data?.map((e) => ({ ...e, computed: derive(e) })) ?? null +// Wrong - no transformations in getters +export const getEventsData = async () => { + const data = await get(KEYS.EVENTS) + return data?.map(transform) ?? null } ``` -Put all transformations in the fetch task (`/api`), not in the getter. +All transformations belong in the fetcher (`/fetchers`). -### 2. Expose via `@/lib/data` for caching +### 2. KEYS is the single source of truth -If a data function needs caching/revalidation, expose it through `@/lib/data`: +All storage keys are defined in `KEYS` in `tasks.ts`. The getter in `index.ts` and the task tuple in `DAILY`/`HOURLY` must use the same key. + +### 3. Use `@/lib/data` for caching ```typescript // src/lib/data/index.ts @@ -61,309 +50,23 @@ export const getEventsData = createCachedGetter( Direct `@/data-layer` imports work but have no caching. -## Components - -### 1. Public API (`src/data-layer/index.ts`) - -The data-layer exports a clean, framework-agnostic public API with typed getter functions: - -```typescript -import { getEthPrice, getL2beatData } from "@/data-layer" - -// Types flow automatically - no generics needed! -const price = await getEthPrice() // Returns MetricReturnData | null -const l2beat = await getL2beatData() // Returns L2beatData | null -``` - -**Available Functions:** -- `getEthPrice()` - Ethereum price data -- `getL2beatData()` - L2BEAT scaling summary -- `getAppsData()` - Apps organized by category -- `getGrowThePieData()` - GrowThePie fundamentals -- `getGrowThePieBlockspaceData()` - GrowThePie blockspace data -- `getGrowThePieMasterData()` - GrowThePie master data -- `getCommunityPicks()` - Community picks -- `getCalendarEvents()` - Community calendar events -- `getRSSData()` - RSS feeds from community blogs -- `getAttestantPosts()` - Attestant blog posts -- `getBeaconchainEpochData()` - Beaconchain epoch data -- `getBeaconchainEthstoreData()` - Beaconchain ETH store data -- `getBlobscanStats()` - Blobscan statistics -- `getEthereumMarketcapData()` - Ethereum market cap -- `getEthereumStablecoinsMcapData()` - Ethereum stablecoins market cap -- `getGFIs()` - GitHub good first issues -- `getGitHistory()` - GitHub commit history -- `getGithubRepoData()` - GitHub repository data -- `getStablecoinsData()` - Ethereum stablecoins data -- `getTotalEthStakedData()` - Total ETH staked -- `getTotalValueLockedData()` - Total value locked - -**Benefits:** -- ✅ No internal details exposed (no task IDs or storage paths) -- ✅ Automatic type inference -- ✅ Framework-agnostic (can be extracted to separate service) - -### 2. Next.js Adapter (`src/lib/data/index.ts`) - -The adapter provides Next.js-specific caching using `unstable_cache`: - -```typescript -import { getEthPrice } from "@/lib/data" - -// Automatically cached + typed ✨ -const price = await getEthPrice() -``` - -**Cache Durations:** -- **1 hour** (`BASE_TIME_UNIT`) - Most frequently updated data (prices, metrics, feeds) -- **24 hours** (`BASE_TIME_UNIT * 24`) - Less frequently changing data (apps, community picks, GitHub repo data) - -**Why separate adapter?** -- Caching is the app's concern, not the data-layer's -- Data-layer stays framework-agnostic -- Easy to adjust cache settings per data source -- Easy to extract data-layer to its own service later - -### 3. API Functions (`/api`) - -Each API function: -- Fetches data from an external source -- Exports a unique `TASK_ID` constant -- Returns typed data -- Handles errors gracefully -- Includes console logging - -Example: -```typescript -export const FETCH_ETH_PRICE_TASK_ID = "fetch-eth-price" - -export async function fetchEthPrice(): Promise { - // Fetch logic... - return { value: price, timestamp: Date.now() } -} -``` - -### 4. Storage Layer (`/storage`) - -**Unified Storage Interface** (`types.ts`): -```typescript -export interface Storage { - get(taskId: TaskId): Promise | null> - set(taskId: TaskId, data: unknown, metadata?: StorageMetadata): Promise -} -``` - -**Implementations:** -- `netlifyBlobsStorage.ts` - Production storage using Netlify Blobs -- `mockStorage.ts` - Local development using JSON files - -**Public API** (`getter.ts` / `setter.ts`): -- `getData(taskId)` - Retrieve data without metadata -- `getData(taskId, { withMetadata: true })` - Retrieve data with metadata -- `setData(taskId, data)` - Store data with automatic metadata - -**Storage Configuration:** -- Requires `SITE_ID` (auto-provided by Netlify) and `NETLIFY_BLOBS_TOKEN` environment variables -- Throws error if credentials are missing (no silent fallback) -- Use `USE_MOCK_DATA=true` for local development - -### 5. Registry (`registry.ts`) - -Central registry organizing tasks by schedule: -- `dailyTasks` - Tasks that run daily at midnight UTC -- `hourlyTasks` - Tasks that run hourly -- `tasks` - Combined array of all tasks - -Each task entry: -```typescript -{ - id: "fetch-eth-price", - fetchFunction: fetchEthPrice -} -``` - -### 6. Trigger.dev Tasks (`/trigger/tasks`) - -**Daily Task** (`daily.ts`): -- Runs at midnight UTC (`0 0 * * *`) -- Executes all tasks in `dailyTasks` registry **in parallel** using `Promise.allSettled` -- Stores results using `setData` -- Individual task failures don't stop other tasks - -**Hourly Task** (`hourly.ts`): -- Runs every hour (`0 * * * *`) -- Executes all tasks in `hourlyTasks` registry **in parallel** using `Promise.allSettled` -- Stores results using `setData` -- Individual task failures don't stop other tasks - -**Why consolidated tasks?** -Trigger.dev free tier limits to 10 schedules. By consolidating into 2 tasks (daily/hourly), we stay within limits while supporting many data sources. - -**Why parallelized?** -Tasks now run concurrently instead of sequentially, dramatically reducing execution time. - -## Usage - -### In App Code (Recommended) - -Use the Next.js adapter for automatic caching: - -```typescript -import { getEthPrice, getL2beatData } from "@/lib/data" - -// In a Next.js page or API route -export default async function Page() { - const price = await getEthPrice() // Cached + typed ✨ - const l2beat = await getL2beatData() - - return
Price: {price?.value}
-} -``` - -### Direct Data-Layer Access - -For non-Next.js contexts or when you don't need caching: - -```typescript -import { getEthPrice } from "@/data-layer" - -const price = await getEthPrice() // Returns MetricReturnData | null -``` - -### Type Imports - -Import types from their canonical locations: - -```typescript -import type { MetricReturnData, L2beatData } from "@/lib/types" -import { getEthPrice, getL2beatData } from "@/lib/data" -``` - -## Testing - -Unit tests are available in `tests/unit/data-layer/`: - -```bash -# Run all unit tests -npm run test:unit -``` - -Tests validate: -- ✅ Functions execute without errors -- ✅ Return types match expected structures -- ✅ Handle null cases gracefully -- ✅ Data structure validation when data is present - -See `tests/unit/data-layer/getters.spec.ts` for test examples. - ## Adding a New Data Source -1. **Create API function** in `/api`: - ```typescript - // src/data-layer/api/fetchNewData.ts - export const FETCH_NEW_DATA_TASK_ID = "fetch-new-data" - - export async function fetchNewData(): Promise { - // Fetch logic... - } - ``` - -2. **Add type to `src/lib/types.ts`** (if not already defined): - ```typescript - export type YourDataType = { - // type definition - } - ``` - -3. **Add getter function** to `src/data-layer/index.ts`: - ```typescript - import { FETCH_NEW_DATA_TASK_ID } from "./api/fetchNewData" - import type { YourDataType } from "@/lib/types" - - export async function getNewData(): Promise { - return getData(FETCH_NEW_DATA_TASK_ID) - } - ``` - -4. **Add to registry** in `registry.ts`: - ```typescript - import { FETCH_NEW_DATA_TASK_ID, fetchNewData } from "./api/fetchNewData" - - // Add to dailyTasks or hourlyTasks array - { - id: FETCH_NEW_DATA_TASK_ID, - fetchFunction: fetchNewData, - } - ``` - -5. **Add adapter function** to `src/lib/data/index.ts`: - ```typescript - export const getNewData = unstable_cache( - () => dataLayer.getNewData(), - ["new-data"], - { revalidate: CACHE_REVALIDATE_HOUR } - ) - ``` - -6. **Task is automatically discovered** by Trigger.dev and will run on the appropriate schedule +1. **Create fetcher** in `src/data-layer/fetchers/fetchNewData.ts` +2. **Add type** to `src/lib/types.ts` (if needed) +3. **Add key** to `KEYS` in `tasks.ts` +4. **Add task tuple** to `DAILY` or `HOURLY` in `tasks.ts` +5. **Add getter** in `src/data-layer/index.ts` +6. **Add mock file** at `src/data-layer/mocks/{key}.json` for local development +7. **Add cached wrapper** in `src/lib/data/index.ts` ## Environment Variables -**Required for production:** -- `SITE_ID` - Netlify site ID (auto-provided by Netlify during builds) -- `NETLIFY_BLOBS_TOKEN` - Netlify Blobs access token (required, throws error if missing) +**Production:** +- `SITE_ID` - Netlify site ID (auto-provided) +- `NETLIFY_BLOBS_TOKEN` - Netlify Blobs access token - `TRIGGER_PROJECT_ID` - Trigger.dev project ID -**Optional for local development:** -- `USE_MOCK_DATA=true` - Use mock storage instead of Netlify Blobs - -## Error Handling - -- **Parallel Execution** - Tasks run concurrently using `Promise.allSettled` -- **Graceful Degradation** - Individual task failures don't stop other tasks from running -- **Error Logging** - All errors are logged with task context -- **Retry Logic** - Trigger.dev handles retries (configured in `trigger.config.ts`) - -## Storage Metadata - -Each stored item includes metadata: -```typescript -{ - storedAt: "2024-01-01T00:00:00.000Z" // ISO timestamp -} -``` - -This allows tracking when data was last updated. - -## Mock Data - -Mock data files are stored in `src/data-layer/mocks/` and can be regenerated: - -```bash -npx dotenv-cli -e .env -- npx ts-node -r tsconfig-paths/register -O '{"module":"commonjs"}' src/data-layer/mocks/generate-mocks.ts -``` - -This pulls data from Netlify Blobs storage and saves it as JSON files for local development. - -## Troubleshooting - -### Mock storage not working -- Ensure `USE_MOCK_DATA=true` is set in `.env` -- Verify mock files exist in `/data-layer/mocks/` -- Regenerate mocks if needed: `npm run generate-mocks` (if script exists) - -### Netlify Blobs errors -- Verify `SITE_ID` (auto-provided by Netlify) and `NETLIFY_BLOBS_TOKEN` are set -- Check error message for specific configuration issues -- Storage will throw clear error if credentials are missing - -### Trigger.dev tasks not running -- Check `TRIGGER_PROJECT_ID` is set -- Verify task is registered in `registry.ts` -- Check Trigger.dev dashboard for errors -- Tasks run in parallel - check logs for individual task failures +**Local development:** +- `USE_MOCK_DATA=true` - Use mock storage -### Type errors -- Ensure types are defined in `src/lib/types.ts` -- Import types from `@/lib/types`, not from data-layer -- Import functions from `@/lib/data` (adapter) or `@/data-layer` (direct) diff --git a/src/data-layer/api/fetchApps.ts b/src/data-layer/fetchers/fetchApps.ts similarity index 100% rename from src/data-layer/api/fetchApps.ts rename to src/data-layer/fetchers/fetchApps.ts diff --git a/src/data-layer/api/fetchBeaconChainEpoch.ts b/src/data-layer/fetchers/fetchBeaconChainEpoch.ts similarity index 100% rename from src/data-layer/api/fetchBeaconChainEpoch.ts rename to src/data-layer/fetchers/fetchBeaconChainEpoch.ts diff --git a/src/data-layer/api/fetchBeaconChainEthstore.ts b/src/data-layer/fetchers/fetchBeaconChainEthstore.ts similarity index 100% rename from src/data-layer/api/fetchBeaconChainEthstore.ts rename to src/data-layer/fetchers/fetchBeaconChainEthstore.ts diff --git a/src/data-layer/api/fetchBlobscanStats.ts b/src/data-layer/fetchers/fetchBlobscanStats.ts similarity index 100% rename from src/data-layer/api/fetchBlobscanStats.ts rename to src/data-layer/fetchers/fetchBlobscanStats.ts diff --git a/src/data-layer/api/fetchCalendarEvents.ts b/src/data-layer/fetchers/fetchCalendarEvents.ts similarity index 100% rename from src/data-layer/api/fetchCalendarEvents.ts rename to src/data-layer/fetchers/fetchCalendarEvents.ts diff --git a/src/data-layer/api/fetchCommunityPicks.ts b/src/data-layer/fetchers/fetchCommunityPicks.ts similarity index 100% rename from src/data-layer/api/fetchCommunityPicks.ts rename to src/data-layer/fetchers/fetchCommunityPicks.ts diff --git a/src/data-layer/api/fetchEthPrice.ts b/src/data-layer/fetchers/fetchEthPrice.ts similarity index 100% rename from src/data-layer/api/fetchEthPrice.ts rename to src/data-layer/fetchers/fetchEthPrice.ts diff --git a/src/data-layer/api/fetchEthereumMarketcap.ts b/src/data-layer/fetchers/fetchEthereumMarketcap.ts similarity index 100% rename from src/data-layer/api/fetchEthereumMarketcap.ts rename to src/data-layer/fetchers/fetchEthereumMarketcap.ts diff --git a/src/data-layer/api/fetchEthereumStablecoinsMcap.ts b/src/data-layer/fetchers/fetchEthereumStablecoinsMcap.ts similarity index 100% rename from src/data-layer/api/fetchEthereumStablecoinsMcap.ts rename to src/data-layer/fetchers/fetchEthereumStablecoinsMcap.ts diff --git a/src/data-layer/api/fetchEvents.ts b/src/data-layer/fetchers/fetchEvents.ts similarity index 100% rename from src/data-layer/api/fetchEvents.ts rename to src/data-layer/fetchers/fetchEvents.ts diff --git a/src/data-layer/api/fetchGFIs.ts b/src/data-layer/fetchers/fetchGFIs.ts similarity index 100% rename from src/data-layer/api/fetchGFIs.ts rename to src/data-layer/fetchers/fetchGFIs.ts diff --git a/src/data-layer/api/fetchGitHistory.ts b/src/data-layer/fetchers/fetchGitHistory.ts similarity index 100% rename from src/data-layer/api/fetchGitHistory.ts rename to src/data-layer/fetchers/fetchGitHistory.ts diff --git a/src/data-layer/api/fetchGithubRepoData.ts b/src/data-layer/fetchers/fetchGithubRepoData.ts similarity index 100% rename from src/data-layer/api/fetchGithubRepoData.ts rename to src/data-layer/fetchers/fetchGithubRepoData.ts diff --git a/src/data-layer/api/fetchGrowThePie.ts b/src/data-layer/fetchers/fetchGrowThePie.ts similarity index 100% rename from src/data-layer/api/fetchGrowThePie.ts rename to src/data-layer/fetchers/fetchGrowThePie.ts diff --git a/src/data-layer/api/fetchGrowThePieBlockspace.ts b/src/data-layer/fetchers/fetchGrowThePieBlockspace.ts similarity index 100% rename from src/data-layer/api/fetchGrowThePieBlockspace.ts rename to src/data-layer/fetchers/fetchGrowThePieBlockspace.ts diff --git a/src/data-layer/api/fetchGrowThePieMaster.ts b/src/data-layer/fetchers/fetchGrowThePieMaster.ts similarity index 100% rename from src/data-layer/api/fetchGrowThePieMaster.ts rename to src/data-layer/fetchers/fetchGrowThePieMaster.ts diff --git a/src/data-layer/api/fetchL2beat.ts b/src/data-layer/fetchers/fetchL2beat.ts similarity index 100% rename from src/data-layer/api/fetchL2beat.ts rename to src/data-layer/fetchers/fetchL2beat.ts diff --git a/src/data-layer/api/fetchPosts.ts b/src/data-layer/fetchers/fetchPosts.ts similarity index 100% rename from src/data-layer/api/fetchPosts.ts rename to src/data-layer/fetchers/fetchPosts.ts diff --git a/src/data-layer/api/fetchRSS.ts b/src/data-layer/fetchers/fetchRSS.ts similarity index 100% rename from src/data-layer/api/fetchRSS.ts rename to src/data-layer/fetchers/fetchRSS.ts diff --git a/src/data-layer/api/fetchStablecoinsData.ts b/src/data-layer/fetchers/fetchStablecoinsData.ts similarity index 100% rename from src/data-layer/api/fetchStablecoinsData.ts rename to src/data-layer/fetchers/fetchStablecoinsData.ts diff --git a/src/data-layer/api/fetchTotalEthStaked.ts b/src/data-layer/fetchers/fetchTotalEthStaked.ts similarity index 100% rename from src/data-layer/api/fetchTotalEthStaked.ts rename to src/data-layer/fetchers/fetchTotalEthStaked.ts diff --git a/src/data-layer/api/fetchTotalValueLocked.ts b/src/data-layer/fetchers/fetchTotalValueLocked.ts similarity index 100% rename from src/data-layer/api/fetchTotalValueLocked.ts rename to src/data-layer/fetchers/fetchTotalValueLocked.ts diff --git a/src/data-layer/index.ts b/src/data-layer/index.ts index f8a1efb3445..97125507141 100644 --- a/src/data-layer/index.ts +++ b/src/data-layer/index.ts @@ -16,213 +16,31 @@ import type { } from "@/lib/types" import type { CommunityEventsReturnType } from "@/lib/interfaces" -import { FETCH_APPS_TASK_ID } from "./api/fetchApps" -import { FETCH_BEACONCHAIN_EPOCH_TASK_ID } from "./api/fetchBeaconChainEpoch" -import { FETCH_BEACONCHAIN_ETHSTORE_TASK_ID } from "./api/fetchBeaconChainEthstore" -import { FETCH_BLOBSCAN_STATS_TASK_ID } from "./api/fetchBlobscanStats" -import { FETCH_CALENDAR_EVENTS_TASK_ID } from "./api/fetchCalendarEvents" -import { FETCH_COMMUNITY_PICKS_TASK_ID } from "./api/fetchCommunityPicks" -import { FETCH_ETHEREUM_MARKETCAP_TASK_ID } from "./api/fetchEthereumMarketcap" -import { FETCH_ETHEREUM_STABLECOINS_MCAP_TASK_ID } from "./api/fetchEthereumStablecoinsMcap" -import { FETCH_ETH_PRICE_TASK_ID } from "./api/fetchEthPrice" -import { FETCH_EVENTS_TASK_ID } from "./api/fetchEvents" -import { FETCH_GFIS_TASK_ID } from "./api/fetchGFIs" -import { FETCH_GIT_HISTORY_TASK_ID } from "./api/fetchGitHistory" -import { FETCH_GITHUB_REPO_DATA_TASK_ID } from "./api/fetchGithubRepoData" -import { FETCH_GROW_THE_PIE_TASK_ID } from "./api/fetchGrowThePie" -import { FETCH_GROW_THE_PIE_BLOCKSPACE_TASK_ID } from "./api/fetchGrowThePieBlockspace" -import { FETCH_GROW_THE_PIE_MASTER_TASK_ID } from "./api/fetchGrowThePieMaster" -import { FETCH_L2BEAT_TASK_ID } from "./api/fetchL2beat" -import { FETCH_POSTS_TASK_ID } from "./api/fetchPosts" -import { FETCH_RSS_TASK_ID } from "./api/fetchRSS" -import { - CoinGeckoCoinMarketResponse, - FETCH_STABLECOINS_DATA_TASK_ID, -} from "./api/fetchStablecoinsData" -import { FETCH_TOTAL_ETH_STAKED_TASK_ID } from "./api/fetchTotalEthStaked" -import { FETCH_TOTAL_VALUE_LOCKED_TASK_ID } from "./api/fetchTotalValueLocked" -import { getData } from "./storage/getter" - -/** - * Get Ethereum price data. - * @returns The latest ETH price in USD, or null if not available - */ -export async function getEthPrice(): Promise { - return getData(FETCH_ETH_PRICE_TASK_ID) -} - -/** - * Get L2BEAT scaling summary data. - * @returns L2BEAT project data including TVL, maturity, and other metrics, or null if not available - */ -export async function getL2beatData(): Promise { - return getData(FETCH_L2BEAT_TASK_ID) -} - -/** - * Get apps data organized by category. - * @returns Apps data grouped by category, or null if not available - */ -export async function getAppsData(): Promise | null> { - return getData>(FETCH_APPS_TASK_ID) -} - -/** - * Get GrowThePie fundamentals data. - * @returns Transaction counts, costs, and active addresses for Layer 2 networks, or null if not available - */ -export async function getGrowThePieData(): Promise { - return getData(FETCH_GROW_THE_PIE_TASK_ID) -} - -/** - * Get GrowThePie blockspace data. - * @returns Blockspace usage data (NFT, DeFi, social, token transfers, unlabeled) for each network, or null if not available - */ -export async function getGrowThePieBlockspaceData(): Promise | null> { - return getData>( - FETCH_GROW_THE_PIE_BLOCKSPACE_TASK_ID - ) -} - -/** - * Get GrowThePie master data containing chain launch dates. - * @returns Launch dates for all chains indexed by their URL key, or null if not available - */ -export async function getGrowThePieMasterData(): Promise { - return getData(FETCH_GROW_THE_PIE_MASTER_TASK_ID) -} - -/** - * Get community picks data. - * @returns Community picks data, or null if not available - */ -export async function getCommunityPicks(): Promise { - return getData(FETCH_COMMUNITY_PICKS_TASK_ID) -} - -/** - * Get community calendar events. - * @returns Past and upcoming community events, or null if not available - */ -export async function getCalendarEvents(): Promise { - return getData(FETCH_CALENDAR_EVENTS_TASK_ID) -} - -/** - * Get RSS feeds from community blogs. - * @returns Array of RSS items grouped by source, or null if not available - */ -export async function getRSSData(): Promise { - return getData(FETCH_RSS_TASK_ID) -} - -/** - * Get Attestant blog posts. - * @returns Array of RSS items from the Attestant blog, or null if not available - */ -export async function getAttestantPosts(): Promise { - return getData(FETCH_POSTS_TASK_ID) -} - -/** - * Get beaconchain epoch data. - * @returns Latest epoch data including total ETH staked and validator count, or null if not available - */ -export async function getBeaconchainEpochData(): Promise { - return getData(FETCH_BEACONCHAIN_EPOCH_TASK_ID) -} - -/** - * Get beaconchain ETH store data (APR). - * @returns ETH store APR data, or null if not available - */ -export async function getBeaconchainEthstoreData(): Promise { - return getData(FETCH_BEACONCHAIN_ETHSTORE_TASK_ID) -} - -/** - * Get blobscan overall stats. - * @returns Blobscan statistics including blob fees, gas usage, and transaction counts, or null if not available - */ -export async function getBlobscanStats(): Promise { - return getData(FETCH_BLOBSCAN_STATS_TASK_ID) -} - -/** - * Get Ethereum market cap data. - * @returns Ethereum market cap metric data, or null if not available - */ -export async function getEthereumMarketcapData(): Promise { - return getData(FETCH_ETHEREUM_MARKETCAP_TASK_ID) -} - -/** - * Get Ethereum stablecoins market cap data. - * @returns Ethereum stablecoins market cap metric data, or null if not available - */ -export async function getEthereumStablecoinsMcapData(): Promise { - return getData(FETCH_ETHEREUM_STABLECOINS_MCAP_TASK_ID) -} - -/** - * Get GitHub good first issues. - * @returns Array of GitHub issues labeled as "good first issue", or null if not available - */ -export async function getGFIs(): Promise { - return getData(FETCH_GFIS_TASK_ID) -} - -/** - * Get GitHub commit history. - * @returns Recent commit history from the repository, or null if not available - */ -export async function getGitHistory(): Promise { - return getData(FETCH_GIT_HISTORY_TASK_ID) -} - -/** - * Get GitHub repository data for local environment frameworks. - * @returns Star counts and languages for each repository, or null if not available - */ -export async function getGithubRepoData(): Promise | null> { - return getData>(FETCH_GITHUB_REPO_DATA_TASK_ID) -} - -/** - * Get Ethereum stablecoins data from CoinGecko. - * @returns Market data for stablecoins on Ethereum, or null if not available - */ -export async function getStablecoinsData(): Promise { - return getData(FETCH_STABLECOINS_DATA_TASK_ID) -} - -/** - * Get total ETH staked data. - * @returns Total ETH staked metric data, or null if not available - */ -export async function getTotalEthStakedData(): Promise { - return getData(FETCH_TOTAL_ETH_STAKED_TASK_ID) -} - -/** - * Get total value locked (TVL) data. - * @returns TVL metric data, or null if not available - */ -export async function getTotalValueLockedData(): Promise { - return getData(FETCH_TOTAL_VALUE_LOCKED_TASK_ID) -} - -/** - * Get events data from Geode Labs API. - * @returns Array of upcoming events sorted by start time, or null if not available - */ -export async function getEventsData(): Promise { - return getData(FETCH_EVENTS_TASK_ID) -} +import type { CoinGeckoCoinMarketResponse } from "./fetchers/fetchStablecoinsData" +import { get } from "./storage" +import { KEYS } from "./tasks" + +export { KEYS } + +export const getEthPrice = () => get(KEYS.ETH_PRICE) +export const getL2beatData = () => get(KEYS.L2BEAT) +export const getAppsData = () => get>(KEYS.APPS) +export const getGrowThePieData = () => get(KEYS.GROW_THE_PIE) +export const getGrowThePieBlockspaceData = () => get>(KEYS.GROW_THE_PIE_BLOCKSPACE) +export const getGrowThePieMasterData = () => get(KEYS.GROW_THE_PIE_MASTER) +export const getCommunityPicks = () => get(KEYS.COMMUNITY_PICKS) +export const getCalendarEvents = () => get(KEYS.CALENDAR_EVENTS) +export const getRSSData = () => get(KEYS.RSS) +export const getAttestantPosts = () => get(KEYS.POSTS) +export const getBeaconchainEpochData = () => get(KEYS.BEACONCHAIN_EPOCH) +export const getBeaconchainEthstoreData = () => get(KEYS.BEACONCHAIN_ETHSTORE) +export const getBlobscanStats = () => get(KEYS.BLOBSCAN_STATS) +export const getEthereumMarketcapData = () => get(KEYS.ETHEREUM_MARKETCAP) +export const getEthereumStablecoinsMcapData = () => get(KEYS.ETHEREUM_STABLECOINS_MCAP) +export const getGFIs = () => get(KEYS.GFIS) +export const getGitHistory = () => get(KEYS.GIT_HISTORY) +export const getGithubRepoData = () => get>(KEYS.GITHUB_REPO_DATA) +export const getStablecoinsData = () => get(KEYS.STABLECOINS_DATA) +export const getTotalEthStakedData = () => get(KEYS.TOTAL_ETH_STAKED) +export const getTotalValueLockedData = () => get(KEYS.TOTAL_VALUE_LOCKED) +export const getEventsData = () => get(KEYS.EVENTS) diff --git a/src/data-layer/mocks/README.md b/src/data-layer/mocks/README.md index 788895cee76..a04300aaf6b 100644 --- a/src/data-layer/mocks/README.md +++ b/src/data-layer/mocks/README.md @@ -1,15 +1,5 @@ # Mock Data Files -These mock data files are generated from Netlify Blobs storage for local development. +JSON files for local development. Used when `USE_MOCK_DATA=true`. -## Usage - -These files can be used to mock the data-layer storage in local development environments without needing to connect to Netlify Blobs. - -## Generation - -To regenerate these files, run: - -```bash -npx dotenv-cli -e .env -- npx ts-node -r tsconfig-paths/register -O '{"module":"commonjs"}' src/data-layer/mocks/generate-mocks.ts -``` +Each file is named after its storage key (e.g., `fetch-eth-price.json` for `KEYS.ETH_PRICE`). diff --git a/src/data-layer/mocks/generate-mocks.ts b/src/data-layer/mocks/generate-mocks.ts deleted file mode 100644 index 7055002bf9c..00000000000 --- a/src/data-layer/mocks/generate-mocks.ts +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Script to generate mock data files from storage for local development. - * - * This script pulls data from Netlify Blobs storage and saves it as JSON files - * in the /data-layer/mocks directory. These mock files can be used for local - * development without needing to connect to Netlify Blobs. - * - * Run with: - * - npx dotenv-cli -e .env -- npx ts-node -r tsconfig-paths/register -O '{"module":"commonjs"}' src/data-layer/mocks/generate-mocks.ts - */ - -import * as fs from "fs" -import * as path from "path" - -import { getData } from "../storage/getter" - -// Define all task IDs to avoid importing registry (which imports API files with path aliases) -const TASK_IDS = [ - "fetch-apps", - "fetch-beaconchain-epoch", - "fetch-beaconchain-ethstore", - "fetch-blobscan-stats", - "fetch-calendar-events", - "fetch-community-picks", - "fetch-ethereum-marketcap", - "fetch-ethereum-stablecoins-mcap", - "fetch-eth-price", - "fetch-gfis", - "fetch-git-history", - "fetch-github-repo-data", - "fetch-grow-the-pie", - "fetch-grow-the-pie-blockspace", - "fetch-grow-the-pie-master", - "fetch-l2beat", - "fetch-posts", - "fetch-rss", - "fetch-stablecoins-data", - "fetch-total-eth-staked", - "fetch-total-value-locked", -] as const - -const MOCKS_DIR = __dirname - -async function generateMocks() { - console.log("=".repeat(60)) - console.log("Generating mock data files from storage") - console.log("=".repeat(60)) - console.log("") - - // Ensure mocks directory exists - if (!fs.existsSync(MOCKS_DIR)) { - fs.mkdirSync(MOCKS_DIR, { recursive: true }) - console.log(`📁 Created mocks directory: ${MOCKS_DIR}`) - } - - let successCount = 0 - let notFoundCount = 0 - const errors: Array<{ taskId: string; error: string }> = [] - - console.log(`📦 Fetching data for ${TASK_IDS.length} tasks...\n`) - - for (const taskId of TASK_IDS) { - try { - console.log(` Fetching: ${taskId}...`) - const data = await getData(taskId) - - if (data === null) { - console.log(` ⚠️ No data found, skipping`) - notFoundCount++ - continue - } - - // Save data as JSON file - const filePath = path.join(MOCKS_DIR, `${taskId}.json`) - const jsonContent = JSON.stringify(data, null, 2) - fs.writeFileSync(filePath, jsonContent, "utf-8") - - const fileSize = (jsonContent.length / 1024).toFixed(2) - console.log(` ✅ Saved (${fileSize} KB)`) - successCount++ - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error) - console.log(` ❌ Error: ${errorMessage}`) - errors.push({ taskId, error: errorMessage }) - } - } - - // Generate index file - const indexContent = `/** - * Mock data files for local development. - * - * These files are generated from Netlify Blobs storage and can be used - * for local development without needing to connect to Netlify Blobs. - * - * Generated: ${new Date().toISOString()} - * Total files: ${successCount} - */ - -export const mockTaskIds = ${JSON.stringify( - TASK_IDS.filter((id) => { - const filePath = path.join(MOCKS_DIR, `${id}.json`) - return fs.existsSync(filePath) - }), - null, - 2 - )} as const - -export type MockTaskId = (typeof mockTaskIds)[number] -` - - const indexPath = path.join(MOCKS_DIR, "index.ts") - fs.writeFileSync(indexPath, indexContent, "utf-8") - console.log(`\n📄 Generated index file: ${indexPath}`) - - // Generate README - const readmeContent = `# Mock Data Files - -These mock data files are generated from Netlify Blobs storage for local development. - -## Usage - -These files can be used to mock the data-layer storage in local development environments without needing to connect to Netlify Blobs. - -## Generation - -To regenerate these files, run: - -\`\`\`bash -npx dotenv-cli -e .env -- npx ts-node -r tsconfig-paths/register -O '{"module":"commonjs"}' src/data-layer/mocks/generate-mocks.ts -\`\`\` -` - - const readmePath = path.join(MOCKS_DIR, "README.md") - fs.writeFileSync(readmePath, readmeContent, "utf-8") - console.log(`📄 Generated README: ${readmePath}`) - - // Summary - console.log("\n" + "=".repeat(60)) - console.log("Summary") - console.log("=".repeat(60)) - console.log(` ✅ Successfully generated: ${successCount} files`) - console.log(` ℹ️ Not found: ${notFoundCount} tasks`) - if (errors.length > 0) { - console.log(` ❌ Errors: ${errors.length}`) - console.log("\n Error details:") - errors.forEach(({ taskId, error }) => { - console.log(` - ${taskId}: ${error}`) - }) - } - console.log(` 📁 Output directory: ${MOCKS_DIR}`) - console.log("") - console.log("✅ Mock generation completed!") -} - -// Run the script -generateMocks().catch((error) => { - console.error("❌ Failed to generate mocks:", error) - if (error instanceof Error) { - console.error(" Error message:", error.message) - console.error(" Stack:", error.stack) - } - process.exit(1) -}) diff --git a/src/data-layer/registry.ts b/src/data-layer/registry.ts deleted file mode 100644 index e3b9069bc6c..00000000000 --- a/src/data-layer/registry.ts +++ /dev/null @@ -1,173 +0,0 @@ -/** - * Registry of all data-layer components. - * - * This includes Trigger.dev scheduled tasks - * Tasks are automatically discovered by Trigger.dev, but this registry - * provides reference. - * - * Task IDs can be used as keys for storage. - */ - -import { FETCH_APPS_TASK_ID, fetchApps } from "./api/fetchApps" -import { - FETCH_BEACONCHAIN_EPOCH_TASK_ID, - fetchBeaconChainEpoch, -} from "./api/fetchBeaconChainEpoch" -import { - FETCH_BEACONCHAIN_ETHSTORE_TASK_ID, - fetchBeaconChainEthstore, -} from "./api/fetchBeaconChainEthstore" -import { - FETCH_BLOBSCAN_STATS_TASK_ID, - fetchBlobscanStats, -} from "./api/fetchBlobscanStats" -import { - FETCH_CALENDAR_EVENTS_TASK_ID, - fetchCalendarEvents, -} from "./api/fetchCalendarEvents" -import { - FETCH_COMMUNITY_PICKS_TASK_ID, - fetchCommunityPicks, -} from "./api/fetchCommunityPicks" -import { - FETCH_ETHEREUM_MARKETCAP_TASK_ID, - fetchEthereumMarketcap, -} from "./api/fetchEthereumMarketcap" -import { - FETCH_ETHEREUM_STABLECOINS_MCAP_TASK_ID, - fetchEthereumStablecoinsMcap, -} from "./api/fetchEthereumStablecoinsMcap" -import { FETCH_ETH_PRICE_TASK_ID, fetchEthPrice } from "./api/fetchEthPrice" -import { FETCH_EVENTS_TASK_ID, fetchEvents } from "./api/fetchEvents" -import { FETCH_GFIS_TASK_ID, fetchGFIs } from "./api/fetchGFIs" -import { - FETCH_GIT_HISTORY_TASK_ID, - fetchGitHistory, -} from "./api/fetchGitHistory" -import { - FETCH_GITHUB_REPO_DATA_TASK_ID, - fetchGithubRepoData, -} from "./api/fetchGithubRepoData" -import { - FETCH_GROW_THE_PIE_TASK_ID, - fetchGrowThePie, -} from "./api/fetchGrowThePie" -import { - FETCH_GROW_THE_PIE_BLOCKSPACE_TASK_ID, - fetchGrowThePieBlockspace, -} from "./api/fetchGrowThePieBlockspace" -import { - FETCH_GROW_THE_PIE_MASTER_TASK_ID, - fetchGrowThePieMaster, -} from "./api/fetchGrowThePieMaster" -import { FETCH_L2BEAT_TASK_ID, fetchL2beat } from "./api/fetchL2beat" -import { FETCH_POSTS_TASK_ID, fetchAttestantPosts } from "./api/fetchPosts" -import { FETCH_RSS_TASK_ID, fetchRSS } from "./api/fetchRSS" -import { - FETCH_STABLECOINS_DATA_TASK_ID, - fetchStablecoinsData, -} from "./api/fetchStablecoinsData" -import { - FETCH_TOTAL_ETH_STAKED_TASK_ID, - fetchTotalEthStaked, -} from "./api/fetchTotalEthStaked" -import { - FETCH_TOTAL_VALUE_LOCKED_TASK_ID, - fetchTotalValueLocked, -} from "./api/fetchTotalValueLocked" - -export const dailyTasks = [ - { - id: FETCH_APPS_TASK_ID, - fetchFunction: fetchApps, - }, - { - id: FETCH_CALENDAR_EVENTS_TASK_ID, - fetchFunction: fetchCalendarEvents, - }, - { - id: FETCH_EVENTS_TASK_ID, - fetchFunction: fetchEvents, - }, - { - id: FETCH_COMMUNITY_PICKS_TASK_ID, - fetchFunction: fetchCommunityPicks, - }, - { - id: FETCH_GFIS_TASK_ID, - fetchFunction: fetchGFIs, - }, - { - id: FETCH_GIT_HISTORY_TASK_ID, - fetchFunction: fetchGitHistory, - }, - { - id: FETCH_GROW_THE_PIE_TASK_ID, - fetchFunction: fetchGrowThePie, - }, - { - id: FETCH_GROW_THE_PIE_BLOCKSPACE_TASK_ID, - fetchFunction: fetchGrowThePieBlockspace, - }, - { - id: FETCH_GROW_THE_PIE_MASTER_TASK_ID, - fetchFunction: fetchGrowThePieMaster, - }, - { - id: FETCH_L2BEAT_TASK_ID, - fetchFunction: fetchL2beat, - }, - { - id: FETCH_POSTS_TASK_ID, - fetchFunction: fetchAttestantPosts, - }, - { - id: FETCH_RSS_TASK_ID, - fetchFunction: fetchRSS, - }, - { - id: FETCH_GITHUB_REPO_DATA_TASK_ID, - fetchFunction: fetchGithubRepoData, - }, -] as const - -export const hourlyTasks = [ - { - id: FETCH_BEACONCHAIN_EPOCH_TASK_ID, - fetchFunction: fetchBeaconChainEpoch, - }, - { - id: FETCH_BEACONCHAIN_ETHSTORE_TASK_ID, - fetchFunction: fetchBeaconChainEthstore, - }, - { - id: FETCH_BLOBSCAN_STATS_TASK_ID, - fetchFunction: fetchBlobscanStats, - }, - { - id: FETCH_ETHEREUM_MARKETCAP_TASK_ID, - fetchFunction: fetchEthereumMarketcap, - }, - { - id: FETCH_ETHEREUM_STABLECOINS_MCAP_TASK_ID, - fetchFunction: fetchEthereumStablecoinsMcap, - }, - { - id: FETCH_ETH_PRICE_TASK_ID, - fetchFunction: fetchEthPrice, - }, - { - id: FETCH_TOTAL_ETH_STAKED_TASK_ID, - fetchFunction: fetchTotalEthStaked, - }, - { - id: FETCH_TOTAL_VALUE_LOCKED_TASK_ID, - fetchFunction: fetchTotalValueLocked, - }, - { - id: FETCH_STABLECOINS_DATA_TASK_ID, - fetchFunction: fetchStablecoinsData, - }, -] as const - -export const tasks = [...dailyTasks, ...hourlyTasks] as const diff --git a/src/data-layer/storage.ts b/src/data-layer/storage.ts new file mode 100644 index 00000000000..1e8b792e96f --- /dev/null +++ b/src/data-layer/storage.ts @@ -0,0 +1,61 @@ +/** + * Data storage - reads/writes to Netlify Blobs (prod) or local JSON files (dev). + */ + +import * as fs from "fs" +import * as path from "path" + +import { getStore } from "@netlify/blobs" + +const USE_MOCK = process.env.USE_MOCK_DATA === "true" + +// Netlify Blobs store (lazy init) +let blobStore: ReturnType | null = null + +function getBlobs() { + if (blobStore) return blobStore + + const siteID = process.env.SITE_ID + const token = process.env.NETLIFY_BLOBS_TOKEN + + if (!siteID || !token) { + throw new Error("Missing SITE_ID or NETLIFY_BLOBS_TOKEN") + } + + blobStore = getStore({ + name: "data-layer", + siteID, + token, + } as Parameters[0]) + + return blobStore +} + +// Mock file path +function mockPath(key: string): string { + return path.resolve(process.cwd(), `src/data-layer/mocks/${key}.json`) +} + +/** Get data by key */ +export async function get(key: string): Promise { + if (USE_MOCK) { + const filePath = mockPath(key) + if (!fs.existsSync(filePath)) return null + return JSON.parse(fs.readFileSync(filePath, "utf-8")) as T + } + + const blob = await getBlobs().get(key, { type: "text" }) + return blob ? (JSON.parse(blob) as T) : null +} + +/** Store data by key */ +export async function set(key: string, data: unknown): Promise { + if (USE_MOCK) { + fs.writeFileSync(mockPath(key), JSON.stringify(data, null, 2), "utf-8") + return + } + + await getBlobs().set(key, JSON.stringify(data), { + metadata: { storedAt: new Date().toISOString() }, + }) +} diff --git a/src/data-layer/storage/getter.ts b/src/data-layer/storage/getter.ts deleted file mode 100644 index 6b48d4b71d4..00000000000 --- a/src/data-layer/storage/getter.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { Storage, StorageMetadata, TaskId } from "../types" - -import { mockStorage } from "./mockStorage" -import { netlifyBlobsStorage } from "./netlifyBlobsStorage" - -const defaultStorage: Storage = - process.env.USE_MOCK_DATA === "true" ? mockStorage : netlifyBlobsStorage - -/** Get data from storage. Pass `{ withMetadata: true }` to include metadata. */ -export async function getData(taskId: TaskId): Promise -export async function getData( - taskId: TaskId, - options: { withMetadata: true } -): Promise<{ data: T; metadata: StorageMetadata } | null> -export async function getData( - taskId: TaskId, - options?: { withMetadata?: boolean } -): Promise { - try { - const result = await defaultStorage.get(taskId) - - if (!result) { - console.warn(`[Data Layer] No data found for task: ${taskId}`) - return null - } - - return options?.withMetadata ? result : result.data - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error) - console.error(`[Data Layer] Error for "${taskId}": ${errorMessage}`) - throw error - } -} diff --git a/src/data-layer/storage/mockStorage.ts b/src/data-layer/storage/mockStorage.ts deleted file mode 100644 index 4c251617b97..00000000000 --- a/src/data-layer/storage/mockStorage.ts +++ /dev/null @@ -1,48 +0,0 @@ -import * as fs from "fs" -import * as fsPromises from "fs/promises" -import * as path from "path" - -import type { Storage, StorageMetadata, StorageResult, TaskId } from "../types" - -function getMocksDir(): string { - const possiblePaths = [ - path.resolve(process.cwd(), "src/data-layer/mocks"), - path.resolve(process.cwd(), "src", "data-layer", "mocks"), - path.resolve(__dirname, "../mocks"), - ] - - for (const dirPath of possiblePaths) { - if (fs.existsSync(dirPath)) return dirPath - } - - return possiblePaths[0] -} - -/** Mock storage that reads from local JSON files for development. */ -export const mockStorage: Storage = { - async get(taskId: TaskId): Promise | null> { - const mocksDir = getMocksDir() - const filePath = path.join(mocksDir, `${taskId}.json`) - - if (!fs.existsSync(mocksDir)) { - throw new Error(`[Mock Storage] Directory not found: ${mocksDir}`) - } - - if (!fs.existsSync(filePath)) { - throw new Error(`[Mock Storage] File not found: ${filePath}`) - } - - const fileContent = await fsPromises.readFile(filePath, "utf-8") - const data = JSON.parse(fileContent) as T - const stats = await fsPromises.stat(filePath) - const metadata: StorageMetadata = { storedAt: stats.mtime.toISOString() } - - return { data, metadata } - }, - - async set(taskId: TaskId, data: unknown): Promise { - const mocksDir = getMocksDir() - const filePath = path.join(mocksDir, `${taskId}.json`) - fs.writeFileSync(filePath, JSON.stringify(data, null, 2), "utf-8") - }, -} diff --git a/src/data-layer/storage/netlifyBlobsStorage.ts b/src/data-layer/storage/netlifyBlobsStorage.ts deleted file mode 100644 index dc4642abf20..00000000000 --- a/src/data-layer/storage/netlifyBlobsStorage.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { getStore } from "@netlify/blobs" - -import type { Storage, StorageMetadata, StorageResult, TaskId } from "../types" - -let store: ReturnType | null = null - -function getBlobStore() { - if (store) return store - - // SITE_ID is automatically provided by Netlify during builds - const siteID = process.env.SITE_ID - const token = process.env.NETLIFY_BLOBS_TOKEN - - if (!siteID || !token) { - throw new Error("Missing SITE_ID or NETLIFY_BLOBS_TOKEN env vars") - } - - store = getStore({ - name: "data-layer", - siteID, - token, - } as Parameters[0]) - - return store -} - -export const netlifyBlobsStorage: Storage = { - async get(taskId: TaskId): Promise | null> { - const blobStore = getBlobStore() - const blob = await blobStore.get(taskId, { type: "text" }) - - if (!blob) return null - - const data = JSON.parse(blob) as T - const blobMetadataResult = await blobStore.getMetadata(taskId) - const storedAtValue = blobMetadataResult?.metadata?.storedAt - const metadata: StorageMetadata = { - storedAt: - typeof storedAtValue === "string" - ? storedAtValue - : new Date().toISOString(), - } - - return { data, metadata } - }, - - async set( - taskId: TaskId, - data: unknown, - metadata?: StorageMetadata - ): Promise { - const blobStore = getBlobStore() - await blobStore.set(taskId, JSON.stringify(data), { - metadata: metadata ? { storedAt: metadata.storedAt } : {}, - }) - }, -} diff --git a/src/data-layer/storage/setter.ts b/src/data-layer/storage/setter.ts deleted file mode 100644 index 33eebd982a7..00000000000 --- a/src/data-layer/storage/setter.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { Storage, StorageMetadata, TaskId } from "../types" - -import { mockStorage } from "./mockStorage" -import { netlifyBlobsStorage } from "./netlifyBlobsStorage" - -const defaultStorage: Storage = - process.env.USE_MOCK_DATA === "true" ? mockStorage : netlifyBlobsStorage - -export async function setData(taskId: TaskId, data: T): Promise { - const metadata: StorageMetadata = { storedAt: new Date().toISOString() } - await defaultStorage.set(taskId, data, metadata) -} diff --git a/src/data-layer/tasks.ts b/src/data-layer/tasks.ts new file mode 100644 index 00000000000..3acb86f292f --- /dev/null +++ b/src/data-layer/tasks.ts @@ -0,0 +1,122 @@ +/** + * Trigger.dev scheduled tasks for data fetching. + * + * Daily tasks run at midnight UTC. + * Hourly tasks run every hour. + */ + +import { schedules } from "@trigger.dev/sdk/v3" + +import { fetchApps } from "./fetchers/fetchApps" +import { fetchBeaconChainEpoch } from "./fetchers/fetchBeaconChainEpoch" +import { fetchBeaconChainEthstore } from "./fetchers/fetchBeaconChainEthstore" +import { fetchBlobscanStats } from "./fetchers/fetchBlobscanStats" +import { fetchCalendarEvents } from "./fetchers/fetchCalendarEvents" +import { fetchCommunityPicks } from "./fetchers/fetchCommunityPicks" +import { fetchEthereumMarketcap } from "./fetchers/fetchEthereumMarketcap" +import { fetchEthereumStablecoinsMcap } from "./fetchers/fetchEthereumStablecoinsMcap" +import { fetchEthPrice } from "./fetchers/fetchEthPrice" +import { fetchEvents } from "./fetchers/fetchEvents" +import { fetchGFIs } from "./fetchers/fetchGFIs" +import { fetchGitHistory } from "./fetchers/fetchGitHistory" +import { fetchGithubRepoData } from "./fetchers/fetchGithubRepoData" +import { fetchGrowThePie } from "./fetchers/fetchGrowThePie" +import { fetchGrowThePieBlockspace } from "./fetchers/fetchGrowThePieBlockspace" +import { fetchGrowThePieMaster } from "./fetchers/fetchGrowThePieMaster" +import { fetchL2beat } from "./fetchers/fetchL2beat" +import { fetchAttestantPosts } from "./fetchers/fetchPosts" +import { fetchRSS } from "./fetchers/fetchRSS" +import { fetchStablecoinsData } from "./fetchers/fetchStablecoinsData" +import { fetchTotalEthStaked } from "./fetchers/fetchTotalEthStaked" +import { fetchTotalValueLocked } from "./fetchers/fetchTotalValueLocked" +import { set } from "./storage" + +export const KEYS = { + APPS: "fetch-apps", + CALENDAR_EVENTS: "fetch-calendar-events", + COMMUNITY_PICKS: "fetch-community-picks", + GFIS: "fetch-gfis", + GIT_HISTORY: "fetch-git-history", + GROW_THE_PIE: "fetch-grow-the-pie", + GROW_THE_PIE_BLOCKSPACE: "fetch-grow-the-pie-blockspace", + GROW_THE_PIE_MASTER: "fetch-grow-the-pie-master", + L2BEAT: "fetch-l2beat", + POSTS: "fetch-posts", + RSS: "fetch-rss", + GITHUB_REPO_DATA: "fetch-github-repo-data", + EVENTS: "fetch-events", + BEACONCHAIN_EPOCH: "fetch-beaconchain-epoch", + BEACONCHAIN_ETHSTORE: "fetch-beaconchain-ethstore", + BLOBSCAN_STATS: "fetch-blobscan-stats", + ETHEREUM_MARKETCAP: "fetch-ethereum-marketcap", + ETHEREUM_STABLECOINS_MCAP: "fetch-ethereum-stablecoins-mcap", + ETH_PRICE: "fetch-eth-price", + TOTAL_ETH_STAKED: "fetch-total-eth-staked", + TOTAL_VALUE_LOCKED: "fetch-total-value-locked", + STABLECOINS_DATA: "fetch-stablecoins-data", +} as const + +// Task definition: storage key + fetch function +type Task = [string, () => Promise] + +const DAILY: Task[] = [ + [KEYS.APPS, fetchApps], + [KEYS.CALENDAR_EVENTS, fetchCalendarEvents], + [KEYS.COMMUNITY_PICKS, fetchCommunityPicks], + [KEYS.GFIS, fetchGFIs], + [KEYS.GIT_HISTORY, fetchGitHistory], + [KEYS.GROW_THE_PIE, fetchGrowThePie], + [KEYS.GROW_THE_PIE_BLOCKSPACE, fetchGrowThePieBlockspace], + [KEYS.GROW_THE_PIE_MASTER, fetchGrowThePieMaster], + [KEYS.L2BEAT, fetchL2beat], + [KEYS.POSTS, fetchAttestantPosts], + [KEYS.RSS, fetchRSS], + [KEYS.GITHUB_REPO_DATA, fetchGithubRepoData], + [KEYS.EVENTS, fetchEvents], +] + +const HOURLY: Task[] = [ + [KEYS.BEACONCHAIN_EPOCH, fetchBeaconChainEpoch], + [KEYS.BEACONCHAIN_ETHSTORE, fetchBeaconChainEthstore], + [KEYS.BLOBSCAN_STATS, fetchBlobscanStats], + [KEYS.ETHEREUM_MARKETCAP, fetchEthereumMarketcap], + [KEYS.ETHEREUM_STABLECOINS_MCAP, fetchEthereumStablecoinsMcap], + [KEYS.ETH_PRICE, fetchEthPrice], + [KEYS.TOTAL_ETH_STAKED, fetchTotalEthStaked], + [KEYS.TOTAL_VALUE_LOCKED, fetchTotalValueLocked], + [KEYS.STABLECOINS_DATA, fetchStablecoinsData], +] + +async function runTasks(tasks: Task[]) { + const results = await Promise.allSettled( + tasks.map(async ([key, fetch]) => { + const data = await fetch() + await set(key, data) + console.log(`✓ ${key}`) + return key + }) + ) + + const summary = results.map((r, i) => ({ + key: tasks[i][0], + ok: r.status === "fulfilled", + error: r.status === "rejected" ? String(r.reason) : undefined, + })) + + const failed = summary.filter((s) => !s.ok) + failed.forEach((f) => console.error(`✗ ${f.key}: ${f.error}`)) + + return summary +} + +export const dailyTask = schedules.task({ + id: "daily-data-fetch", + cron: "0 0 * * *", + run: () => runTasks(DAILY), +}) + +export const hourlyTask = schedules.task({ + id: "hourly-data-fetch", + cron: "0 * * * *", + run: () => runTasks(HOURLY), +}) diff --git a/src/data-layer/trigger/old-tasks/revalidate-apps.ts b/src/data-layer/trigger/old-tasks/revalidate-apps.ts index cd08ce2f81f..31683d95656 100644 --- a/src/data-layer/trigger/old-tasks/revalidate-apps.ts +++ b/src/data-layer/trigger/old-tasks/revalidate-apps.ts @@ -2,7 +2,7 @@ import { schedules } from "@trigger.dev/sdk/v3" import { slugify } from "@/lib/utils/url" -import { fetchApps } from "@/data-layer/api/fetchApps" +import { fetchApps } from "@/data-layer/fetchers/fetchApps" import { revalidatePaths } from "./utils" diff --git a/src/data-layer/trigger/tasks/daily.ts b/src/data-layer/trigger/tasks/daily.ts deleted file mode 100644 index 6ef4d661a76..00000000000 --- a/src/data-layer/trigger/tasks/daily.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { schedules } from "@trigger.dev/sdk/v3" - -import { dailyTasks } from "../../registry" -import { setData } from "../../storage/setter" - -/** - * Single Trigger.dev scheduled task for all daily tasks - * Runs daily at midnight UTC and executes all daily fetch functions - * This keeps us under the free tier limit of 10 schedules - */ -export const dailyTask = schedules.task({ - id: "daily-data-fetch", - // Run daily at midnight UTC - cron: "0 0 * * *", - run: async () => { - const results: Record = {} - - // Execute all daily tasks in parallel - const taskPromises = dailyTasks.map(async (task) => { - try { - console.log(`Fetching data for task: ${task.id}`) - const data = await task.fetchFunction() - await setData(task.id, data) - return { - taskId: task.id, - success: true, - data, - } - } catch (error) { - console.error(`Error fetching data for task ${task.id}:`, error) - return { - taskId: task.id, - success: false, - error: error instanceof Error ? error.message : String(error), - } - } - }) - - // Wait for all tasks to complete (success or failure) - const settledResults = await Promise.allSettled(taskPromises) - - // Process results - for (const result of settledResults) { - if (result.status === "fulfilled") { - const { taskId, success, ...rest } = result.value - results[taskId] = { success, ...rest } - } else { - // This shouldn't happen since we catch errors in the map, but handle it just in case - console.error("Unexpected error in Promise.allSettled:", result.reason) - } - } - - return results - }, -}) diff --git a/src/data-layer/trigger/tasks/hourly.ts b/src/data-layer/trigger/tasks/hourly.ts deleted file mode 100644 index 6611fa5f448..00000000000 --- a/src/data-layer/trigger/tasks/hourly.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { schedules } from "@trigger.dev/sdk/v3" - -import { hourlyTasks } from "../../registry" -import { setData } from "../../storage/setter" - -/** - * Single Trigger.dev scheduled task for all hourly tasks - * Runs every hour and executes all hourly fetch functions - * This keeps us under the free tier limit of 10 schedules - */ -export const hourlyTask = schedules.task({ - id: "hourly-data-fetch", - // Run every hour to keep data current - cron: "0 * * * *", - run: async () => { - const results: Record = {} - - // Execute all hourly tasks in parallel - const taskPromises = hourlyTasks.map(async (task) => { - try { - console.log(`Fetching data for task: ${task.id}`) - const data = await task.fetchFunction() - await setData(task.id, data) - return { - taskId: task.id, - success: true, - data, - } - } catch (error) { - console.error(`Error fetching data for task ${task.id}:`, error) - return { - taskId: task.id, - success: false, - error: error instanceof Error ? error.message : String(error), - } - } - }) - - // Wait for all tasks to complete (success or failure) - const settledResults = await Promise.allSettled(taskPromises) - - // Process results - for (const result of settledResults) { - if (result.status === "fulfilled") { - const { taskId, success, ...rest } = result.value - results[taskId] = { success, ...rest } - } else { - // This shouldn't happen since we catch errors in the map, but handle it just in case - console.error("Unexpected error in Promise.allSettled:", result.reason) - } - } - - return results - }, -}) diff --git a/src/data-layer/trigger/tasks/index.ts b/src/data-layer/trigger/tasks/index.ts deleted file mode 100644 index 82a09060476..00000000000 --- a/src/data-layer/trigger/tasks/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Import all Trigger.dev scheduled tasks - * - * Trigger.dev automatically discovers tasks from this directory. - * Importing these files registers the tasks. - */ - -import "./daily" -import "./hourly" diff --git a/src/data-layer/types.ts b/src/data-layer/types.ts deleted file mode 100644 index 91651742294..00000000000 --- a/src/data-layer/types.ts +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Type definitions for the data-layer module. - */ - -import type { tasks } from "./registry" - -export type TaskId = (typeof tasks)[number]["id"] - -/** - * Metadata stored alongside data. - */ -export interface StorageMetadata { - storedAt: string -} - -/** - * Result shape when retrieving data with metadata. - */ -export interface StorageResult { - data: T - metadata: StorageMetadata -} - -/** - * Unified storage interface. - * Implementations must provide both get and set methods. - */ -export interface Storage { - /** - * Retrieve data for a task. - * @param taskId - The task ID to use as the storage key - * @returns The stored data with metadata, or null if not found - */ - get(taskId: TaskId): Promise | null> - - /** - * Store data for a task with optional metadata. - * @param taskId - The task ID to use as the storage key - * @param data - Data to store (will be serialized) - * @param metadata - Optional metadata about the stored data - */ - set(taskId: TaskId, data: unknown, metadata?: StorageMetadata): Promise -} diff --git a/trigger.config.ts b/trigger.config.ts index 06550145d50..e0ceb0b86a1 100644 --- a/trigger.config.ts +++ b/trigger.config.ts @@ -22,11 +22,8 @@ export default defineConfig({ randomize: true, }, }, - // Directories containing Trigger.dev task definitions - dirs: [ - "./src/data-layer/trigger/tasks", - "./src/data-layer/trigger/old-tasks", - ], + // Task definitions file + dirs: ["./src/data-layer"], // Initialize Sentry for error tracking in Trigger.dev tasks // Uses the same Sentry configuration as the Next.js app // Note: Trigger.dev already initializes OpenTelemetry, so we skip Sentry's OpenTelemetry setup