Repository navigation
feat(caretaker): implement Cloud Run webhook ingestion service #28015
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 7 commits
8d23977
c9cbd86
2d09842
a6ef7a9
40c59c1
a8a0fe8
5650049
7c2a01b
afa05a5
812b530
05334cc
ca0de47
177e670
e3aabc4
799aefb
df24dbb
02edc10
ed4870d
503e006
d1ce311
c210f2d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| node_modules | ||
| dist | ||
| npm-debug.log | ||
| .git | ||
| .gitignore | ||
| *.py | ||
| *.pyc | ||
| __pycache__ | ||
| requirements.txt | ||
| project.toml | ||
| **/*.test.ts | ||
|
|
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| FROM node:20-slim | ||
| WORKDIR /app | ||
| COPY package*.json ./ | ||
| RUN npm ci | ||
| COPY . . | ||
| EXPOSE 8080 | ||
| CMD ["npx", "tsx", "server.ts"] | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| /** | ||
| * @license | ||
| * Copyright 2026 Google LLC | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| import { describe, it, expect } from 'vitest'; | ||
| import { verifyGithubSignature } from './github.js'; | ||
| import * as crypto from 'node:crypto'; | ||
|
|
||
| describe('verifyGithubSignature', () => { | ||
| const secret = 'my-secret'; | ||
| const payload = '{"test":true}'; | ||
|
|
||
| it('should return true for a valid signature', () => { | ||
| const hmac = crypto.createHmac('sha256', secret); | ||
| hmac.update(payload); | ||
| const validSignature = 'sha256=' + hmac.digest('hex'); | ||
|
|
||
| const result = verifyGithubSignature(payload, validSignature, secret); | ||
| expect(result).toBe(true); | ||
| }); | ||
|
|
||
| it('should return false if signatureHeader is missing', () => { | ||
| const result = verifyGithubSignature(payload, undefined, secret); | ||
| expect(result).toBe(false); | ||
| }); | ||
|
|
||
| it('should return false for an invalid signature', () => { | ||
| const result = verifyGithubSignature( | ||
| payload, | ||
| 'sha256=invalid-signature', | ||
| secret, | ||
| ); | ||
| expect(result).toBe(false); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| /** | ||
| * @license | ||
| * Copyright 2026 Google LLC | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| import * as crypto from 'node:crypto'; | ||
|
|
||
| /** | ||
| * Verify that the payload was sent from GitHub using HMAC SHA256. | ||
| * | ||
| * @param payloadBody - The raw body of the request (Buffer or string). | ||
| * @param signatureHeader - The value of the X-Hub-Signature-256 header. | ||
|
gundermanc marked this conversation as resolved.
|
||
| * @param secret - The GitHub Webhook secret. | ||
| * @returns True if the signature is valid, false otherwise. | ||
| */ | ||
| export function verifyGithubSignature( | ||
| payloadBody: Buffer | string, | ||
| signatureHeader: string | undefined, | ||
| secret: string, | ||
| ): boolean { | ||
| if (!signatureHeader || signatureHeader.length !== 71) { | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: magic number. can we define a named const indicating what 71 represents here? |
||
| return false; | ||
| } | ||
|
|
||
| if (!Buffer.isBuffer(payloadBody) && typeof payloadBody !== 'string') { | ||
| return false; | ||
| } | ||
|
|
||
| const hmac = crypto.createHmac('sha256', secret); | ||
| hmac.update(payloadBody); | ||
| const expectedSignature = 'sha256=' + hmac.digest('hex'); | ||
|
|
||
| try { | ||
| return crypto.timingSafeEqual( | ||
| Buffer.from(expectedSignature), | ||
| Buffer.from(signatureHeader), | ||
| ); | ||
| } catch { | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we need to ... or plan to have any sort of logging for errors like this? |
||
| return false; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| /** | ||
| * @license | ||
| * Copyright 2026 Google LLC | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| import { describe, it, expect, vi, beforeEach } from 'vitest'; | ||
| import type { Mock } from 'vitest'; | ||
| import { IssuesStore } from './issuesStore.js'; | ||
| import { Firestore, Transaction } from '@google-cloud/firestore'; | ||
|
|
||
| describe('IssuesStore', () => { | ||
| let mockTransaction: { | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Using mocks is ok but I would be deliberate about what you mock vs. what uses the real object or a realistic stand-in. The risk of mocks is they can allow you to test modules in isolation and miss gaps when they are combined. They can also mislead you into locking in the wrong behavior if the mock behaves differently than the real code. When possible I recommend doing integration style tests and choosing a layer that's the 'cutoff' and mocking everything below that. Firestore is an external dependency, so perhaps it makes sense to have a single local mock firestore implementation that behaves somewhat realistically and is used in all your tests. Alternatively you might be able to use a non-prod instance, but that has tradeoffs. |
||
| get: Mock; | ||
| set: Mock; | ||
| }; | ||
| let mockDb: Firestore; | ||
| let store: IssuesStore; | ||
|
|
||
| beforeEach(() => { | ||
| // Assign mock read/write methods for transaction | ||
| mockTransaction = { | ||
| get: vi.fn(), | ||
| set: vi.fn(), | ||
| }; | ||
|
|
||
| // Mock Firestore client | ||
| mockDb = { | ||
| collection: vi.fn().mockReturnThis(), | ||
| doc: vi.fn().mockReturnValue({}), | ||
| runTransaction: vi | ||
| .fn() | ||
| .mockImplementation( | ||
| (callback: (tx: Transaction) => Promise<unknown>) => { | ||
| return callback(mockTransaction as unknown as Transaction); | ||
| }, | ||
| ), | ||
| } as unknown as Firestore; | ||
|
|
||
| store = new IssuesStore(mockDb, 'issues-collection'); | ||
| }); | ||
|
|
||
| it('should initialize a new issue if it does not exist', async () => { | ||
| // The transaction should mock that the document does not exist | ||
| mockTransaction.get.mockResolvedValue({ exists: false }); | ||
|
|
||
| const result = await store.createIssue( | ||
| 'google', | ||
| 'gemini-cli', | ||
| 123, | ||
| 'Test Title', | ||
| ); | ||
|
|
||
| expect(result).toBe(true); | ||
| expect(mockTransaction.get).toHaveBeenCalled(); | ||
| expect(mockTransaction.set).toHaveBeenCalledWith( | ||
| expect.anything(), | ||
| expect.objectContaining({ | ||
| status: 'UNTRIAGED', | ||
| github_metadata: expect.objectContaining({ | ||
| owner: 'google', | ||
| repo: 'gemini-cli', | ||
| issue_number: 123, | ||
| title: 'Test Title', | ||
| }), | ||
| }), | ||
| ); | ||
| }); | ||
|
|
||
| it('should return false and skip creation if the issue already exists', async () => { | ||
| // The transaction should mock that the document already exists | ||
| mockTransaction.get.mockResolvedValue({ exists: true }); | ||
|
|
||
| const result = await store.createIssue( | ||
| 'google', | ||
| 'gemini-cli', | ||
| 123, | ||
| 'Test Title', | ||
| ); | ||
|
|
||
| expect(result).toBe(false); | ||
| expect(mockTransaction.get).toHaveBeenCalled(); | ||
| expect(mockTransaction.set).not.toHaveBeenCalled(); | ||
| }); | ||
| }); | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| /** | ||
| * @license | ||
| * Copyright 2026 Google LLC | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| import { | ||
| Firestore, | ||
| FieldValue, | ||
| DocumentReference, | ||
| Transaction, | ||
| } from '@google-cloud/firestore'; | ||
|
|
||
| export class IssuesStore { | ||
| private db: Firestore; | ||
|
gundermanc marked this conversation as resolved.
Outdated
|
||
| private collectionName: string; | ||
|
|
||
| constructor(db: Firestore, collectionName: string) { | ||
| this.db = db; | ||
| this.collectionName = collectionName; | ||
| } | ||
|
|
||
| // Generates the standardized Firestore document reference for an issue | ||
| getIssueRef( | ||
| owner: string, | ||
| repo: string, | ||
| issueNumber: number, | ||
| ): DocumentReference { | ||
| const docId = `github_${owner}_${repo}_${issueNumber}`; | ||
| return this.db.collection(this.collectionName).doc(docId); | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We are storing issues right in the top level Firestore collection passed in via the environment variable. This means that any new data types we store will be in this same collection as well, or perhaps nested, leaving the top level collection containing nested collections and individual issues docs. Should we proactively create a nested collection for issues to keep the door open to extend this? |
||
| } | ||
|
|
||
| // Initializes a new issue document in a transaction | ||
| async createIssue( | ||
| owner: string, | ||
| repo: string, | ||
| issueNumber: number, | ||
| title: string, | ||
| ): Promise<boolean> { | ||
| const docRef = this.getIssueRef(owner, repo, issueNumber); | ||
|
|
||
| return this.db.runTransaction(async (transaction: Transaction) => { | ||
|
gundermanc marked this conversation as resolved.
Outdated
|
||
| const snapshot = await transaction.get(docRef); | ||
|
|
||
| if (!snapshot.exists) { | ||
| transaction.set(docRef, { | ||
| status: 'UNTRIAGED', | ||
| triage_attempts: 0, | ||
| workable_spec: {}, | ||
| lock: { | ||
| holder: null, | ||
| expires_at: null, | ||
| }, | ||
| created_at: FieldValue.serverTimestamp(), | ||
| updated_at: FieldValue.serverTimestamp(), | ||
| github_metadata: { | ||
| owner: owner, | ||
| repo: repo, | ||
| issue_number: issueNumber, | ||
| title: title, | ||
| }, | ||
| }); | ||
| return true; | ||
| } | ||
| return false; | ||
| }); | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| { | ||
| "name": "ingestion-service", | ||
| "version": "1.0.0", | ||
| "description": "Ingestion service for triage worker", | ||
| "main": "server.ts", | ||
| "scripts": { | ||
| "dev": "tsx watch server.ts", | ||
| "start": "tsx server.ts", | ||
| "test": "vitest run" | ||
| }, | ||
| "dependencies": { | ||
| "@google-cloud/firestore": "^7.7.0", | ||
| "@google-cloud/pubsub": "^4.4.0", | ||
| "dotenv": "^16.4.5", | ||
| "express": "^4.19.2" | ||
| }, | ||
| "devDependencies": { | ||
| "@types/express": "^4.17.21", | ||
| "@types/node": "^20.12.12", | ||
| "tsx": "^4.9.3", | ||
| "typescript": "^5.4.5", | ||
| "vitest": "^1.6.0" | ||
| } | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[P1] Production anti-pattern.
Running
tsx(orts-node) in a production container introduces significant memory overhead and startup latency. Add a"build": "tsc"script topackage.jsonand run the compiled JavaScript here instead.