From 710f54d24c0662bc1eaf1ff5874aeea7e5f1073f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 08:10:22 +0900 Subject: [PATCH 01/58] test(integration): add status load authority generator --- ...erate-plugin-delivery-status-authority.mjs | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs diff --git a/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs b/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs new file mode 100644 index 000000000..3a16d6452 --- /dev/null +++ b/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs @@ -0,0 +1,63 @@ +import { createHmac, randomUUID } from 'node:crypto'; +import { writeFile } from 'node:fs/promises'; + +const DEFAULT_ITERATIONS = 1000; +const MAX_ITERATIONS = 10000; +const WORKSPACE_ID = '11111111-1111-4111-8111-111111111111'; +const USER_ID = '22222222-2222-4222-8222-222222222222'; +const DELIVERY_ID = '33333333-3333-4333-8333-333333333333'; +const PATH = `/v1/plugins/delivery-attempts/${DELIVERY_ID}`; + +function fail(message) { + throw new Error(message); +} + +function iterations(value) { + const parsed = value === undefined ? DEFAULT_ITERATIONS : Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > MAX_ITERATIONS) { + return fail('K6_ITERATIONS must be an integer between 1 and 10000'); + } + return parsed; +} + +function secret(value) { + if (typeof value !== 'string' || Buffer.byteLength(value, 'utf8') < 32) { + return fail('INTEGRATION_OPERATOR_CONTEXT_SECRET must contain at least 32 UTF-8 bytes'); + } + return value; +} + +const outputPath = process.argv[2]; +if (!outputPath) { + fail('authority output path is required'); +} + +const count = iterations(process.env.K6_ITERATIONS); +const contextSecret = secret(process.env.INTEGRATION_OPERATOR_CONTEXT_SECRET); +const issuedAt = String(Math.floor(Date.now() / 1000)); +const authorities = Array.from({ length: count }, () => { + const evidenceId = randomUUID(); + const signature = createHmac('sha256', contextSecret) + .update( + `life-os.integration-operator-context.v1\n${WORKSPACE_ID}\n${USER_ID}\n${evidenceId}\n${issuedAt}\nGET\n${PATH}`, + 'utf8', + ) + .digest('base64url'); + + return Object.freeze({ + path: PATH, + deliveryId: DELIVERY_ID, + headers: Object.freeze({ + 'x-life-os-workspace-id': WORKSPACE_ID, + 'x-life-os-user-id': USER_ID, + 'x-life-os-context-evidence-id': evidenceId, + 'x-life-os-context-issued-at': issuedAt, + 'x-life-os-context-signature': signature, + }), + }); +}); + +await writeFile(outputPath, `${JSON.stringify(authorities)}\n`, { + encoding: 'utf8', + mode: 0o600, +}); From ab1c5bad3e286a2e059a5e86305186636cae0a39 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 08:10:41 +0900 Subject: [PATCH 02/58] test(integration): add real status k6 contract --- .../perf/plugin-delivery-attempt-status.k6.js | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 apps/integration-service/perf/plugin-delivery-attempt-status.k6.js diff --git a/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js b/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js new file mode 100644 index 000000000..5f43fed46 --- /dev/null +++ b/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js @@ -0,0 +1,75 @@ +import http from 'k6/http'; +import { check } from 'k6'; +import { SharedArray } from 'k6/data'; +import exec from 'k6/execution'; + +function boundedInteger(name, fallback, minimum, maximum) { + const raw = __ENV[name]; + const value = raw === undefined || raw === '' ? fallback : Number(raw); + if (!Number.isInteger(value) || value < minimum || value > maximum) { + throw new Error(`${name} must be an integer between ${minimum} and ${maximum}`); + } + return value; +} + +const iterations = boundedInteger('K6_ITERATIONS', 1000, 1, 10000); +const vus = boundedInteger('K6_VUS', 10, 1, 100); +const authorityFile = __ENV.K6_AUTHORITY_FILE; +const baseUrl = __ENV.K6_BASE_URL; + +if (!authorityFile) { + throw new Error('K6_AUTHORITY_FILE is required'); +} +if (!baseUrl || !/^http:\/\/127\.0\.0\.1:\d+$/u.test(baseUrl)) { + throw new Error('K6_BASE_URL must be an explicit loopback HTTP endpoint'); +} + +const authorities = new SharedArray('plugin-delivery-status-authority', () => { + const parsed = JSON.parse(open(authorityFile)); + if (!Array.isArray(parsed) || parsed.length !== iterations) { + throw new Error('authority bundle must contain exactly K6_ITERATIONS entries'); + } + return parsed; +}); + +export const options = { + scenarios: { + plugin_delivery_status: { + executor: 'shared-iterations', + vus, + iterations, + maxDuration: '45s', + }, + }, + thresholds: { + 'http_req_duration{endpoint:plugin_delivery_status}': ['p(95)<20'], + 'http_req_failed{endpoint:plugin_delivery_status}': ['rate==0'], + checks: ['rate==1'], + }, + summaryTrendStats: ['avg', 'med', 'p(90)', 'p(95)', 'p(99)', 'max'], +}; + +export default function () { + const authority = authorities[exec.scenario.iterationInTest]; + if (!authority) { + throw new Error('missing unique authority for iteration'); + } + + const response = http.get(`${baseUrl}${authority.path}`, { + headers: authority.headers, + redirects: 0, + timeout: '2s', + tags: { endpoint: 'plugin_delivery_status' }, + }); + + check(response, { + 'status is 200': (result) => result.status === 200, + 'status evidence matches delivery': (result) => + result.status === 200 && result.json('deliveryId') === authority.deliveryId, + 'status evidence remains credential-free': (result) => + result.status === 200 && + !Object.prototype.hasOwnProperty.call(result.json(), 'claimToken') && + !Object.prototype.hasOwnProperty.call(result.json(), 'claimTokenDigest') && + !Object.prototype.hasOwnProperty.call(result.json(), 'credential'), + }); +} From ff2c56dc911b0652435e95ebc62625aa665fac22 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 08:11:12 +0900 Subject: [PATCH 03/58] test(integration): seed durable status performance fixture --- .../seed-plugin-delivery-attempt-status.sql | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql diff --git a/apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql b/apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql new file mode 100644 index 000000000..a8c052b0b --- /dev/null +++ b/apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql @@ -0,0 +1,81 @@ +BEGIN; + +INSERT INTO plugin_integration.plugin_installation_record ( + installation_id, + workspace_id, + installed_by_user_id, + plugin_id, + plugin_contract_version, + manifest_sha256, + granted_capabilities, + installation_status, + installed_at, + revoked_at +) VALUES ( + '77777777-7777-4777-8777-777777777777', + '11111111-1111-4111-8111-111111111111', + '22222222-2222-4222-8222-222222222222', + 'life-os.performance.status', + 'life-os.plugin.v1', + repeat('0', 64), + ARRAY[]::text[], + 'active', + clock_timestamp() - INTERVAL '10 minutes', + NULL +); + +INSERT INTO plugin_integration.plugin_delivery_origin_grant_record ( + authority_version, + grant_id, + installation_id, + workspace_id, + granted_by_user_id, + origin_uri, + grant_status, + granted_at, + revoked_at +) VALUES ( + 'life-os.plugin-delivery-origin.v1', + '66666666-6666-4666-8666-666666666666', + '77777777-7777-4777-8777-777777777777', + '11111111-1111-4111-8111-111111111111', + '22222222-2222-4222-8222-222222222222', + 'https://performance.example.test', + 'active', + clock_timestamp() - INTERVAL '9 minutes', + NULL +); + +INSERT INTO plugin_integration.plugin_delivery_attempt_record ( + authority_version, + delivery_id, + grant_id, + installation_id, + workspace_id, + requested_by_user_id, + delivery_status, + attempt_count, + max_attempts, + requested_at, + updated_at, + next_attempt_at, + terminal_at, + last_outcome_code +) VALUES ( + 'life-os.plugin-delivery-attempt.v1', + '33333333-3333-4333-8333-333333333333', + '66666666-6666-4666-8666-666666666666', + '77777777-7777-4777-8777-777777777777', + '11111111-1111-4111-8111-111111111111', + '22222222-2222-4222-8222-222222222222', + 'pending', + 0, + 3, + clock_timestamp() - INTERVAL '1 minute', + clock_timestamp() - INTERVAL '1 minute', + clock_timestamp() + INTERVAL '5 minutes', + NULL, + NULL +); + +COMMIT; From e252d947fc08b2e3cca29d24affcdd499eb0abd8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 08:11:54 +0900 Subject: [PATCH 04/58] test(integration): verify delivery status p95 on PostgreSQL --- .../verify-plugin-delivery-status-k6.yml | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 .github/workflows/verify-plugin-delivery-status-k6.yml diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml new file mode 100644 index 000000000..7bf40e769 --- /dev/null +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -0,0 +1,118 @@ +name: Verify plugin delivery status k6 + +on: + push: + branches: + - perf/plugin-delivery-status-k6-v1 + +permissions: + contents: read + +concurrency: + group: verify-plugin-delivery-status-k6-${{ github.ref }} + cancel-in-progress: true + +jobs: + status_p95: + runs-on: ubuntu-24.04 + timeout-minutes: 12 + env: + INTEGRATION_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/life_os_test + INTEGRATION_OPERATOR_CONTEXT_SECRET: performance-context-secret-value-32-bytes-minimum + INTEGRATION_PLUGIN_VAULT_ORIGIN: https://vault.performance.example.test + INTEGRATION_PLUGIN_VAULT_TOKEN: performance-vault-fixture-token + INTEGRATION_PLUGIN_VAULT_MOUNT: secret + INTEGRATION_SERVICE_PORT: '4107' + K6_BASE_URL: http://127.0.0.1:4107 + K6_ITERATIONS: '1000' + K6_VUS: '10' + K6_AUTHORITY_FILE: /tmp/plugin-delivery-status-authority.json + services: + postgres: + image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_DB: life_os_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d life_os_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - name: Checkout exact verifier head + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + ref: ${{ github.sha }} + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + + - name: Set up k6 + uses: grafana/setup-k6-action@db07bd9765aac508ef18982e52ab937fe633a065 # v1.2.1 + with: + k6-version: '2.1.0' + + - name: Install frozen workspace + shell: bash + run: | + set -Eeuo pipefail + corepack enable + pnpm install --frozen-lockfile + pnpm --filter @life-os/plugin-sdk build + pnpm --filter @life-os/integration-service build + + - name: Migrate and seed Integration-owned PostgreSQL + shell: bash + run: | + set -Eeuo pipefail + for migration in apps/integration-service/migrations/*.sql; do + psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -f "$migration" + done + psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 \ + -f apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql + + - name: Measure authenticated status path without warm-up exclusion + shell: bash + run: | + set -Eeuo pipefail + service_log=/tmp/integration-service.log + cleanup() { + status=$? + trap - EXIT + if [ -n "${service_pid:-}" ]; then + kill "$service_pid" 2>/dev/null || true + wait "$service_pid" 2>/dev/null || true + fi + if [ "$status" -ne 0 ]; then + cat "$service_log" || true + fi + rm -f "$K6_AUTHORITY_FILE" + exit "$status" + } + trap cleanup EXIT + + node apps/integration-service/dist/server.js >"$service_log" 2>&1 & + service_pid=$! + + ready=0 + for _ in $(seq 1 30); do + code=$(curl --silent --output /dev/null --write-out '%{http_code}' \ + --max-time 1 \ + "$K6_BASE_URL/v1/plugins/delivery-attempts/33333333-3333-4333-8333-333333333333" || true) + if [ "$code" = '401' ]; then + ready=1 + break + fi + sleep 1 + done + test "$ready" -eq 1 + + node apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ + "$K6_AUTHORITY_FILE" + k6 run apps/integration-service/perf/plugin-delivery-attempt-status.k6.js From f7802a856aeccc59bc3f29b95a02411a56a60027 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 08:14:08 +0900 Subject: [PATCH 05/58] test(integration): verify status with current k6 stable --- .github/workflows/verify-plugin-delivery-status-k6.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml index 7bf40e769..c7d24cd1d 100644 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -56,7 +56,7 @@ jobs: - name: Set up k6 uses: grafana/setup-k6-action@db07bd9765aac508ef18982e52ab937fe633a065 # v1.2.1 with: - k6-version: '2.1.0' + k6-version: '2.2.0' - name: Install frozen workspace shell: bash From 0f6562b46449ba157066ade03ac2cab281afe47d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:10:37 +0900 Subject: [PATCH 06/58] fix(integration): honor verified TLS in status benchmark --- .../verify-plugin-delivery-status-k6.yml | 81 ++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml index c7d24cd1d..b01f8cb37 100644 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 12 env: - INTEGRATION_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/life_os_test + INTEGRATION_DATABASE_URL: postgresql://postgres:postgres@postgres.performance.test:5432/life_os_test INTEGRATION_OPERATOR_CONTEXT_SECRET: performance-context-secret-value-32-bytes-minimum INTEGRATION_PLUGIN_VAULT_ORIGIN: https://vault.performance.example.test INTEGRATION_PLUGIN_VAULT_TOKEN: performance-vault-fixture-token @@ -27,6 +27,8 @@ jobs: K6_ITERATIONS: '1000' K6_VUS: '10' K6_AUTHORITY_FILE: /tmp/plugin-delivery-status-authority.json + PGSSLMODE: verify-full + PGSSLROOTCERT: /tmp/integration-postgres-ca.crt services: postgres: image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 @@ -67,6 +69,77 @@ jobs: pnpm --filter @life-os/plugin-sdk build pnpm --filter @life-os/integration-service build + - name: Configure verified-TLS PostgreSQL fixture + shell: bash + run: | + set -Eeuo pipefail + hostname=postgres.performance.test + ca_key=/tmp/integration-postgres-ca.key + ca_cert="$PGSSLROOTCERT" + server_key=/tmp/integration-postgres-server.key + server_csr=/tmp/integration-postgres-server.csr + server_cert=/tmp/integration-postgres-server.crt + server_ext=/tmp/integration-postgres-server.ext + + mapfile -t postgres_containers < <( + docker ps --filter publish=5432 --format '{{.ID}}' + ) + test "${#postgres_containers[@]}" -eq 1 + postgres_container="${postgres_containers[0]}" + + openssl genrsa -out "$ca_key" 2048 + openssl req -x509 -new -sha256 -days 1 \ + -key "$ca_key" \ + -subj '/CN=LifeOS Integration CI PostgreSQL CA' \ + -out "$ca_cert" + openssl genrsa -out "$server_key" 2048 + openssl req -new -sha256 \ + -key "$server_key" \ + -subj "/CN=$hostname" \ + -out "$server_csr" + printf 'subjectAltName=DNS:%s\nextendedKeyUsage=serverAuth\n' "$hostname" >"$server_ext" + openssl x509 -req -sha256 -days 1 \ + -in "$server_csr" \ + -CA "$ca_cert" \ + -CAkey "$ca_key" \ + -CAcreateserial \ + -extfile "$server_ext" \ + -out "$server_cert" + chmod 0600 "$server_key" + + docker cp "$server_cert" "$postgres_container:/var/lib/postgresql/server.crt" + docker cp "$server_key" "$postgres_container:/var/lib/postgresql/server.key" + docker exec --user root "$postgres_container" \ + chown postgres:postgres /var/lib/postgresql/server.crt /var/lib/postgresql/server.key + docker exec --user root "$postgres_container" \ + chmod 0644 /var/lib/postgresql/server.crt + docker exec --user root "$postgres_container" \ + chmod 0600 /var/lib/postgresql/server.key + docker exec "$postgres_container" psql -U postgres -d life_os_test -v ON_ERROR_STOP=1 \ + -c "ALTER SYSTEM SET ssl = 'on'" \ + -c "ALTER SYSTEM SET ssl_cert_file = '/var/lib/postgresql/server.crt'" \ + -c "ALTER SYSTEM SET ssl_key_file = '/var/lib/postgresql/server.key'" + docker restart "$postgres_container" >/dev/null + + ready=0 + for _ in $(seq 1 30); do + if docker exec "$postgres_container" pg_isready -U postgres -d life_os_test >/dev/null 2>&1; then + ready=1 + break + fi + sleep 1 + done + test "$ready" -eq 1 + printf '127.0.0.1 %s\n' "$hostname" | sudo tee -a /etc/hosts >/dev/null + printf 'NODE_EXTRA_CA_CERTS=%s\n' "$ca_cert" >>"$GITHUB_ENV" + + rm -f \ + "$ca_key" \ + "$server_key" \ + "$server_csr" \ + "$server_ext" \ + "${ca_cert}.srl" + - name: Migrate and seed Integration-owned PostgreSQL shell: bash run: | @@ -76,6 +149,9 @@ jobs: done psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 \ -f apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql + psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -tAc \ + 'SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()' | + grep -Fx t - name: Measure authenticated status path without warm-up exclusion shell: bash @@ -109,6 +185,9 @@ jobs: ready=1 break fi + if ! kill -0 "$service_pid" 2>/dev/null; then + break + fi sleep 1 done test "$ready" -eq 1 From 388d78b9855b527ff6906d4fb72f819b12759761 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:15:42 +0900 Subject: [PATCH 07/58] perf(integration): make replay consume one round trip --- ...plugin_operator_context_replay_consume.sql | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 apps/integration-service/migrations/0012_plugin_operator_context_replay_consume.sql diff --git a/apps/integration-service/migrations/0012_plugin_operator_context_replay_consume.sql b/apps/integration-service/migrations/0012_plugin_operator_context_replay_consume.sql new file mode 100644 index 000000000..d02c5a0e0 --- /dev/null +++ b/apps/integration-service/migrations/0012_plugin_operator_context_replay_consume.sql @@ -0,0 +1,47 @@ +CREATE OR REPLACE FUNCTION plugin_integration.consume_plugin_operator_context_replay( + p_evidence_id uuid, + p_consumed_at timestamptz, + p_expires_at timestamptz +) +RETURNS boolean +LANGUAGE plpgsql +AS $$ +DECLARE + consumed_evidence_id uuid; +BEGIN + INSERT INTO plugin_integration.plugin_operator_context_replay_record ( + evidence_id, + consumed_at, + expires_at + ) VALUES ( + p_evidence_id, + p_consumed_at, + p_expires_at + ) + ON CONFLICT (evidence_id) DO UPDATE + SET consumed_at = EXCLUDED.consumed_at, + expires_at = EXCLUDED.expires_at + WHERE plugin_integration.plugin_operator_context_replay_record.expires_at < now() + RETURNING evidence_id INTO consumed_evidence_id; + + IF consumed_evidence_id IS NULL THEN + RETURN false; + END IF; + + DELETE FROM plugin_integration.plugin_operator_context_replay_record AS replay_record + WHERE replay_record.evidence_id IN ( + SELECT expired_record.evidence_id + FROM plugin_integration.plugin_operator_context_replay_record AS expired_record + WHERE expired_record.expires_at < now() + AND expired_record.evidence_id <> p_evidence_id + ORDER BY expired_record.expires_at + FOR UPDATE SKIP LOCKED + LIMIT 32 + ); + + RETURN true; +END; +$$; + +COMMENT ON FUNCTION plugin_integration.consume_plugin_operator_context_replay(uuid, timestamptz, timestamptz) IS + 'Atomically consumes one Plugin operator evidence identity and performs bounded post-consume expiry cleanup in the same PostgreSQL round trip.'; From 295332d6b592cb1e2af866dfd6d2952b84b1db09 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:16:01 +0900 Subject: [PATCH 08/58] perf(integration): collapse replay consume round trip --- .../src/plugin-operator-replay.ts | 45 +++++++------------ 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/apps/integration-service/src/plugin-operator-replay.ts b/apps/integration-service/src/plugin-operator-replay.ts index 84904e2ee..837007fe4 100644 --- a/apps/integration-service/src/plugin-operator-replay.ts +++ b/apps/integration-service/src/plugin-operator-replay.ts @@ -39,8 +39,8 @@ export class PluginOperatorReplayValidationError extends Error { } } -interface ReplayEvidenceRow { - evidence_id: unknown; +interface ReplayConsumeRow { + consumed: unknown; } /** Fails closed for malformed replay evidence without reflecting the value. */ @@ -87,10 +87,13 @@ function replayEvidence( /** * PostgreSQL implementation of the one-time operator evidence guard. * - * A primary-key insert is the distributed compare-and-set boundary: exactly one - * service instance can consume a signed evidence UUID. Expired rows are pruned - * against the database clock, while still-valid rows remain durable across - * processes so horizontal replicas cannot replay the same authority independently. + * The Integration-owned consume function is the distributed compare-and-set boundary: + * exactly one service instance can consume a still-valid signed evidence UUID. It performs + * a conflict-safe insert/expired replacement first, then bounded expiry cleanup with + * `SKIP LOCKED`, so cleanup cannot turn concurrent current evidence into a lock cycle. + * Keeping consume and cleanup inside one PostgreSQL invocation removes an otherwise + * unconditional network round trip from every authenticated operator request while the + * expiry index continues to bound cleanup work. */ export class PostgresPluginOperatorReplayGuard implements PluginOperatorReplayGuardPort @@ -101,33 +104,19 @@ export class PostgresPluginOperatorReplayGuard /** Atomically consumes one evidence UUID and returns false for an existing winner. */ async consume(evidence: PluginOperatorReplayEvidence): Promise { const safe = replayEvidence(evidence); - await this.client.query( - `DELETE FROM plugin_integration.plugin_operator_context_replay_record - WHERE expires_at < now()`, - ); - const inserted = await this.client.query( - `INSERT INTO plugin_integration.plugin_operator_context_replay_record ( - evidence_id, consumed_at, expires_at - ) VALUES ($1::uuid, $2::timestamptz, $3::timestamptz) - ON CONFLICT (evidence_id) DO NOTHING - RETURNING evidence_id`, + const result = await this.client.query( + `SELECT plugin_integration.consume_plugin_operator_context_replay( + $1::uuid, $2::timestamptz, $3::timestamptz + ) AS consumed`, [safe.evidenceId, safe.consumedAt, safe.expiresAt], ); - if (inserted.rows.length > 1 || inserted.rowCount === null) { - return invalid(); - } - if (inserted.rows.length === 0) { - if (inserted.rowCount !== 0) { - return invalid(); - } - return false; - } if ( - inserted.rowCount !== 1 || - evidenceId(inserted.rows[0]?.evidence_id) !== safe.evidenceId + result.rowCount !== 1 || + result.rows.length !== 1 || + typeof result.rows[0]?.consumed !== 'boolean' ) { return invalid(); } - return true; + return result.rows[0].consumed; } } From e05637a241095df1c169002a3d1c75c9e9e2b939 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:16:21 +0900 Subject: [PATCH 09/58] test(integration): cover single-call replay consume --- .../src/plugin-operator-replay.test.ts | 54 +++++++------------ 1 file changed, 18 insertions(+), 36 deletions(-) diff --git a/apps/integration-service/src/plugin-operator-replay.test.ts b/apps/integration-service/src/plugin-operator-replay.test.ts index 803433db6..6fd7bee50 100644 --- a/apps/integration-service/src/plugin-operator-replay.test.ts +++ b/apps/integration-service/src/plugin-operator-replay.test.ts @@ -52,28 +52,19 @@ function evidence( } describe('PostgresPluginOperatorReplayGuard', () => { - it('atomically consumes one evidence UUID after pruning only rows expired by the database clock', async () => { + it('delegates atomic consume and bounded expiry cleanup through one PostgreSQL round trip', async () => { const client = new ScriptedSqlClient([ - { rows: [], rowCount: 0 }, - { rows: [{ evidence_id: EVIDENCE_ID }], rowCount: 1 }, + { rows: [{ consumed: true }], rowCount: 1 }, ]); const guard = new PostgresPluginOperatorReplayGuard(client); await expect(guard.consume(evidence())).resolves.toBe(true); - expect(client.queries).toHaveLength(2); + expect(client.queries).toHaveLength(1); expect(client.queries[0]?.text).toContain( - 'DELETE FROM plugin_integration.plugin_operator_context_replay_record', - ); - expect(client.queries[0]?.text).toContain('expires_at < now()'); - expect(client.queries[0]?.values).toBeUndefined(); - expect(client.queries[1]?.text).toContain( - 'INSERT INTO plugin_integration.plugin_operator_context_replay_record', - ); - expect(client.queries[1]?.text).toContain( - 'ON CONFLICT (evidence_id) DO NOTHING', + 'plugin_integration.consume_plugin_operator_context_replay', ); - expect(client.queries[1]?.values).toEqual([ + expect(client.queries[0]?.values).toEqual([ EVIDENCE_ID, CONSUMED_AT, EXPIRES_AT, @@ -82,8 +73,7 @@ describe('PostgresPluginOperatorReplayGuard', () => { it('normalizes accepted UUID evidence to lowercase before persistence', async () => { const client = new ScriptedSqlClient([ - { rows: [], rowCount: 0 }, - { rows: [{ evidence_id: LOWERCASE_EVIDENCE_ID }], rowCount: 1 }, + { rows: [{ consumed: true }], rowCount: 1 }, ]); const guard = new PostgresPluginOperatorReplayGuard(client); @@ -93,21 +83,21 @@ describe('PostgresPluginOperatorReplayGuard', () => { ), ).resolves.toBe(true); - expect(client.queries[1]?.values).toEqual([ + expect(client.queries[0]?.values).toEqual([ LOWERCASE_EVIDENCE_ID, CONSUMED_AT, EXPIRES_AT, ]); }); - it('returns false when another service instance already consumed the evidence UUID', async () => { + it('returns false when another service instance already consumed still-valid evidence', async () => { const client = new ScriptedSqlClient([ - { rows: [], rowCount: 0 }, - { rows: [], rowCount: 0 }, + { rows: [{ consumed: false }], rowCount: 1 }, ]); const guard = new PostgresPluginOperatorReplayGuard(client); await expect(guard.consume(evidence())).resolves.toBe(false); + expect(client.queries).toHaveLength(1); }); it('rejects malformed or contradictory evidence before issuing SQL', async () => { @@ -126,31 +116,23 @@ describe('PostgresPluginOperatorReplayGuard', () => { } }); - it('rejects ambiguous or corrupted INSERT evidence instead of granting authority', async () => { - for (const inserted of [ - { rows: [{ evidence_id: EVIDENCE_ID }], rowCount: null }, - { rows: [], rowCount: 1 }, + it('rejects ambiguous or corrupted consume evidence instead of granting authority', async () => { + for (const result of [ + { rows: [{ consumed: true }], rowCount: null }, + { rows: [], rowCount: 0 }, { - rows: [ - { evidence_id: EVIDENCE_ID }, - { evidence_id: EVIDENCE_ID }, - ], + rows: [{ consumed: true }, { consumed: true }], rowCount: 2, }, - { - rows: [{ evidence_id: '88888888-8888-4888-8888-888888888888' }], - rowCount: 1, - }, + { rows: [{ consumed: 'true' }], rowCount: 1 }, ] satisfies readonly PluginOperatorReplaySqlResult[]) { - const client = new ScriptedSqlClient([ - { rows: [], rowCount: 0 }, - inserted, - ]); + const client = new ScriptedSqlClient([result]); const guard = new PostgresPluginOperatorReplayGuard(client); await expect(guard.consume(evidence())).rejects.toBeInstanceOf( PluginOperatorReplayValidationError, ); + expect(client.queries).toHaveLength(1); } }); }); From 19489da369fe9bed065bca297b3b3e63e79a8b60 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:17:16 +0900 Subject: [PATCH 10/58] test(integration): gate replay hot-path repair --- .../workflows/verify-plugin-delivery-status-k6.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml index b01f8cb37..72398c5d5 100644 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -60,14 +60,23 @@ jobs: with: k6-version: '2.2.0' - - name: Install frozen workspace + - name: Install and verify Integration source shell: bash run: | set -Eeuo pipefail corepack enable pnpm install --frozen-lockfile pnpm --filter @life-os/plugin-sdk build + pnpm --filter @life-os/integration-service typecheck pnpm --filter @life-os/integration-service build + env -u INTEGRATION_DATABASE_URL \ + pnpm --filter @life-os/integration-service test + pnpm exec prettier --single-quote --check \ + .github/workflows/verify-plugin-delivery-status-k6.yml \ + apps/integration-service/src/plugin-operator-replay.ts \ + apps/integration-service/src/plugin-operator-replay.test.ts \ + apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ + apps/integration-service/perf/plugin-delivery-attempt-status.k6.js - name: Configure verified-TLS PostgreSQL fixture shell: bash From 6adfd90b9bf10248be9da230339881b706d14b25 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:21:30 +0900 Subject: [PATCH 11/58] chore(integration): expose canonical formatter delta --- .github/workflows/verify-plugin-delivery-status-k6.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml index 72398c5d5..7fc5b5732 100644 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -71,12 +71,19 @@ jobs: pnpm --filter @life-os/integration-service build env -u INTEGRATION_DATABASE_URL \ pnpm --filter @life-os/integration-service test - pnpm exec prettier --single-quote --check \ + pnpm exec prettier --single-quote --write \ .github/workflows/verify-plugin-delivery-status-k6.yml \ apps/integration-service/src/plugin-operator-replay.ts \ apps/integration-service/src/plugin-operator-replay.test.ts \ apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ apps/integration-service/perf/plugin-delivery-attempt-status.k6.js + git diff -- \ + .github/workflows/verify-plugin-delivery-status-k6.yml \ + apps/integration-service/src/plugin-operator-replay.ts \ + apps/integration-service/src/plugin-operator-replay.test.ts \ + apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ + apps/integration-service/perf/plugin-delivery-attempt-status.k6.js + exit 1 - name: Configure verified-TLS PostgreSQL fixture shell: bash From fd494cd4d1905f3b238dea6fce68882d56e9a1f3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:24:04 +0900 Subject: [PATCH 12/58] style(integration): apply canonical replay formatting --- apps/integration-service/src/plugin-operator-replay.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/apps/integration-service/src/plugin-operator-replay.ts b/apps/integration-service/src/plugin-operator-replay.ts index 837007fe4..9514171d6 100644 --- a/apps/integration-service/src/plugin-operator-replay.ts +++ b/apps/integration-service/src/plugin-operator-replay.ts @@ -1,7 +1,6 @@ const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu; -const ISO_INSTANT_PATTERN = - /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u; +const ISO_INSTANT_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u; /** One verified one-time operator evidence identity and its bounded lifetime. */ export interface PluginOperatorReplayEvidence { @@ -95,9 +94,7 @@ function replayEvidence( * unconditional network round trip from every authenticated operator request while the * expiry index continues to bound cleanup work. */ -export class PostgresPluginOperatorReplayGuard - implements PluginOperatorReplayGuardPort -{ +export class PostgresPluginOperatorReplayGuard implements PluginOperatorReplayGuardPort { /** Creates the guard over a bounded parameterized SQL client. */ constructor(private readonly client: PluginOperatorReplaySqlClient) {} From 547ea778dbd070d6e92616e0beb0b7a264f69541 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:24:15 +0900 Subject: [PATCH 13/58] style(integration): format status authority generator --- .../perf/generate-plugin-delivery-status-authority.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs b/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs index 3a16d6452..54c592ccb 100644 --- a/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs +++ b/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs @@ -22,7 +22,9 @@ function iterations(value) { function secret(value) { if (typeof value !== 'string' || Buffer.byteLength(value, 'utf8') < 32) { - return fail('INTEGRATION_OPERATOR_CONTEXT_SECRET must contain at least 32 UTF-8 bytes'); + return fail( + 'INTEGRATION_OPERATOR_CONTEXT_SECRET must contain at least 32 UTF-8 bytes', + ); } return value; } From 850d007edc757dd400b4839510e1efbc7ddc349d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:24:27 +0900 Subject: [PATCH 14/58] style(integration): format delivery status k6 --- .../perf/plugin-delivery-attempt-status.k6.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js b/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js index 5f43fed46..603306483 100644 --- a/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js +++ b/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js @@ -7,7 +7,9 @@ function boundedInteger(name, fallback, minimum, maximum) { const raw = __ENV[name]; const value = raw === undefined || raw === '' ? fallback : Number(raw); if (!Number.isInteger(value) || value < minimum || value > maximum) { - throw new Error(`${name} must be an integer between ${minimum} and ${maximum}`); + throw new Error( + `${name} must be an integer between ${minimum} and ${maximum}`, + ); } return value; } @@ -27,7 +29,9 @@ if (!baseUrl || !/^http:\/\/127\.0\.0\.1:\d+$/u.test(baseUrl)) { const authorities = new SharedArray('plugin-delivery-status-authority', () => { const parsed = JSON.parse(open(authorityFile)); if (!Array.isArray(parsed) || parsed.length !== iterations) { - throw new Error('authority bundle must contain exactly K6_ITERATIONS entries'); + throw new Error( + 'authority bundle must contain exactly K6_ITERATIONS entries', + ); } return parsed; }); @@ -65,11 +69,15 @@ export default function () { check(response, { 'status is 200': (result) => result.status === 200, 'status evidence matches delivery': (result) => - result.status === 200 && result.json('deliveryId') === authority.deliveryId, + result.status === 200 && + result.json('deliveryId') === authority.deliveryId, 'status evidence remains credential-free': (result) => result.status === 200 && !Object.prototype.hasOwnProperty.call(result.json(), 'claimToken') && - !Object.prototype.hasOwnProperty.call(result.json(), 'claimTokenDigest') && + !Object.prototype.hasOwnProperty.call( + result.json(), + 'claimTokenDigest', + ) && !Object.prototype.hasOwnProperty.call(result.json(), 'credential'), }); } From 1053da2b4fd35fce7b925253ac69552bdb19aeb3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:25:12 +0900 Subject: [PATCH 15/58] test(integration): prove replay consume concurrency in PostgreSQL --- ...erator-replay-postgres.integration.test.ts | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts diff --git a/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts b/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts new file mode 100644 index 000000000..579b15bfa --- /dev/null +++ b/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts @@ -0,0 +1,120 @@ +import { randomUUID } from 'node:crypto'; +import { Pool, type QueryResultRow } from 'pg'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { parsePluginDeliveryAttemptTestDatabaseTarget } from './plugin-delivery-attempt-test-database'; + +const DATABASE_URL = process.env.INTEGRATION_DATABASE_URL; +const TEST_DATABASE_TARGET = DATABASE_URL + ? parsePluginDeliveryAttemptTestDatabaseTarget(DATABASE_URL) + : undefined; +const describeWithPostgres = TEST_DATABASE_TARGET ? describe : describe.skip; + +interface ConsumedRow extends QueryResultRow { + readonly consumed: boolean; +} + +let pool: Pool; + +beforeAll(() => { + if (TEST_DATABASE_TARGET && DATABASE_URL) { + pool = new Pool({ connectionString: DATABASE_URL, max: 10 }); + } +}); + +afterAll(async () => { + if (pool) { + await pool.end(); + } +}); + +function evidenceLifetime() { + const consumedAt = new Date(); + const expiresAt = new Date(consumedAt.getTime() + 60_000); + return [consumedAt.toISOString(), expiresAt.toISOString()] as const; +} + +async function consume(evidenceId: string): Promise { + const [consumedAt, expiresAt] = evidenceLifetime(); + const result = await pool.query( + `SELECT plugin_integration.consume_plugin_operator_context_replay( + $1::uuid, $2::timestamptz, $3::timestamptz + ) AS consumed`, + [evidenceId, consumedAt, expiresAt], + ); + if (result.rowCount !== 1 || result.rows.length !== 1) { + throw new Error('Replay consume acceptance returned ambiguous evidence'); + } + return result.rows[0]?.consumed === true; +} + +async function deleteEvidence(evidenceIds: readonly string[]): Promise { + if (evidenceIds.length === 0) { + return; + } + await pool.query( + `DELETE FROM plugin_integration.plugin_operator_context_replay_record + WHERE evidence_id = ANY($1::uuid[])`, + [[...evidenceIds]], + ); +} + +describeWithPostgres('Plugin operator replay PostgreSQL consume authority', () => { + it('admits exactly one winner when ten replicas race the same evidence identity', async () => { + const evidenceId = randomUUID(); + try { + const winners = await Promise.all( + Array.from({ length: 10 }, () => consume(evidenceId)), + ); + + expect(winners.filter(Boolean)).toHaveLength(1); + expect(winners.filter((winner) => !winner)).toHaveLength(9); + await expect(consume(evidenceId)).resolves.toBe(false); + } finally { + await deleteEvidence([evidenceId]); + } + }); + + it('replaces expired evidence once without allowing a second current winner', async () => { + const evidenceId = randomUUID(); + try { + await pool.query( + `INSERT INTO plugin_integration.plugin_operator_context_replay_record ( + evidence_id, consumed_at, expires_at + ) VALUES ($1::uuid, now() - interval '2 seconds', now() - interval '1 second')`, + [evidenceId], + ); + + await expect(consume(evidenceId)).resolves.toBe(true); + await expect(consume(evidenceId)).resolves.toBe(false); + } finally { + await deleteEvidence([evidenceId]); + } + }); + + it('bounds opportunistic expiry cleanup to thirty-two other rows per winner', async () => { + const winnerId = randomUUID(); + const expiredIds = Array.from({ length: 40 }, () => randomUUID()); + try { + await pool.query( + `INSERT INTO plugin_integration.plugin_operator_context_replay_record ( + evidence_id, consumed_at, expires_at + ) + SELECT evidence_id, now() - interval '2 seconds', now() - interval '1 second' + FROM unnest($1::uuid[]) AS evidence_id`, + [expiredIds], + ); + + await expect(consume(winnerId)).resolves.toBe(true); + const remaining = await pool.query<{ count: string }>( + `SELECT count(*)::text AS count + FROM plugin_integration.plugin_operator_context_replay_record + WHERE evidence_id = ANY($1::uuid[])`, + [expiredIds], + ); + + expect(remaining.rows[0]?.count).toBe('8'); + } finally { + await deleteEvidence([winnerId, ...expiredIds]); + } + }); +}); From a1942042d747f0328c307c52ae425805d302ff23 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:25:39 +0900 Subject: [PATCH 16/58] test(integration): verify replay repair and status p95 --- .../verify-plugin-delivery-status-k6.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml index 7fc5b5732..f5040c766 100644 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -71,19 +71,13 @@ jobs: pnpm --filter @life-os/integration-service build env -u INTEGRATION_DATABASE_URL \ pnpm --filter @life-os/integration-service test - pnpm exec prettier --single-quote --write \ + pnpm exec prettier --single-quote --check \ .github/workflows/verify-plugin-delivery-status-k6.yml \ apps/integration-service/src/plugin-operator-replay.ts \ apps/integration-service/src/plugin-operator-replay.test.ts \ + apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts \ apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ apps/integration-service/perf/plugin-delivery-attempt-status.k6.js - git diff -- \ - .github/workflows/verify-plugin-delivery-status-k6.yml \ - apps/integration-service/src/plugin-operator-replay.ts \ - apps/integration-service/src/plugin-operator-replay.test.ts \ - apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ - apps/integration-service/perf/plugin-delivery-attempt-status.k6.js - exit 1 - name: Configure verified-TLS PostgreSQL fixture shell: bash @@ -156,18 +150,21 @@ jobs: "$server_ext" \ "${ca_cert}.srl" - - name: Migrate and seed Integration-owned PostgreSQL + - name: Migrate and verify Integration-owned PostgreSQL shell: bash run: | set -Eeuo pipefail for migration in apps/integration-service/migrations/*.sql; do psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -f "$migration" done - psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 \ - -f apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -tAc \ 'SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()' | grep -Fx t + pnpm --filter @life-os/integration-service exec vitest run \ + src/plugin-operator-replay-postgres.integration.test.ts \ + --no-file-parallelism + psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 \ + -f apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql - name: Measure authenticated status path without warm-up exclusion shell: bash From b15b0dc82218bf9f1099bb17937b44119f8ce094 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:28:36 +0900 Subject: [PATCH 17/58] style(integration): format replay PostgreSQL acceptance --- ...erator-replay-postgres.integration.test.ts | 115 +++++++++--------- 1 file changed, 60 insertions(+), 55 deletions(-) diff --git a/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts b/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts index 579b15bfa..13a67cef9 100644 --- a/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts +++ b/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts @@ -1,7 +1,9 @@ import { randomUUID } from 'node:crypto'; import { Pool, type QueryResultRow } from 'pg'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { parsePluginDeliveryAttemptTestDatabaseTarget } from './plugin-delivery-attempt-test-database'; +import { + parsePluginDeliveryAttemptTestDatabaseTarget, +} from './plugin-delivery-attempt-test-database'; const DATABASE_URL = process.env.INTEGRATION_DATABASE_URL; const TEST_DATABASE_TARGET = DATABASE_URL @@ -58,63 +60,66 @@ async function deleteEvidence(evidenceIds: readonly string[]): Promise { ); } -describeWithPostgres('Plugin operator replay PostgreSQL consume authority', () => { - it('admits exactly one winner when ten replicas race the same evidence identity', async () => { - const evidenceId = randomUUID(); - try { - const winners = await Promise.all( - Array.from({ length: 10 }, () => consume(evidenceId)), - ); +describeWithPostgres( + 'Plugin operator replay PostgreSQL consume authority', + () => { + it('admits exactly one winner when ten replicas race the same evidence identity', async () => { + const evidenceId = randomUUID(); + try { + const winners = await Promise.all( + Array.from({ length: 10 }, () => consume(evidenceId)), + ); - expect(winners.filter(Boolean)).toHaveLength(1); - expect(winners.filter((winner) => !winner)).toHaveLength(9); - await expect(consume(evidenceId)).resolves.toBe(false); - } finally { - await deleteEvidence([evidenceId]); - } - }); + expect(winners.filter(Boolean)).toHaveLength(1); + expect(winners.filter((winner) => !winner)).toHaveLength(9); + await expect(consume(evidenceId)).resolves.toBe(false); + } finally { + await deleteEvidence([evidenceId]); + } + }); - it('replaces expired evidence once without allowing a second current winner', async () => { - const evidenceId = randomUUID(); - try { - await pool.query( - `INSERT INTO plugin_integration.plugin_operator_context_replay_record ( - evidence_id, consumed_at, expires_at - ) VALUES ($1::uuid, now() - interval '2 seconds', now() - interval '1 second')`, - [evidenceId], - ); + it('replaces expired evidence once without allowing a second current winner', async () => { + const evidenceId = randomUUID(); + try { + await pool.query( + `INSERT INTO plugin_integration.plugin_operator_context_replay_record ( + evidence_id, consumed_at, expires_at + ) VALUES ($1::uuid, now() - interval '2 seconds', now() - interval '1 second')`, + [evidenceId], + ); - await expect(consume(evidenceId)).resolves.toBe(true); - await expect(consume(evidenceId)).resolves.toBe(false); - } finally { - await deleteEvidence([evidenceId]); - } - }); + await expect(consume(evidenceId)).resolves.toBe(true); + await expect(consume(evidenceId)).resolves.toBe(false); + } finally { + await deleteEvidence([evidenceId]); + } + }); - it('bounds opportunistic expiry cleanup to thirty-two other rows per winner', async () => { - const winnerId = randomUUID(); - const expiredIds = Array.from({ length: 40 }, () => randomUUID()); - try { - await pool.query( - `INSERT INTO plugin_integration.plugin_operator_context_replay_record ( - evidence_id, consumed_at, expires_at - ) - SELECT evidence_id, now() - interval '2 seconds', now() - interval '1 second' - FROM unnest($1::uuid[]) AS evidence_id`, - [expiredIds], - ); + it('bounds opportunistic expiry cleanup to thirty-two other rows per winner', async () => { + const winnerId = randomUUID(); + const expiredIds = Array.from({ length: 40 }, () => randomUUID()); + try { + await pool.query( + `INSERT INTO plugin_integration.plugin_operator_context_replay_record ( + evidence_id, consumed_at, expires_at + ) + SELECT evidence_id, now() - interval '2 seconds', now() - interval '1 second' + FROM unnest($1::uuid[]) AS evidence_id`, + [expiredIds], + ); - await expect(consume(winnerId)).resolves.toBe(true); - const remaining = await pool.query<{ count: string }>( - `SELECT count(*)::text AS count - FROM plugin_integration.plugin_operator_context_replay_record - WHERE evidence_id = ANY($1::uuid[])`, - [expiredIds], - ); + await expect(consume(winnerId)).resolves.toBe(true); + const remaining = await pool.query<{ count: string }>( + `SELECT count(*)::text AS count + FROM plugin_integration.plugin_operator_context_replay_record + WHERE evidence_id = ANY($1::uuid[])`, + [expiredIds], + ); - expect(remaining.rows[0]?.count).toBe('8'); - } finally { - await deleteEvidence([winnerId, ...expiredIds]); - } - }); -}); + expect(remaining.rows[0]?.count).toBe('8'); + } finally { + await deleteEvidence([winnerId, ...expiredIds]); + } + }); + }, +); From 5c0a8e2e96378699cbfab0dc8dd73da7d855e977 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:39:08 +0900 Subject: [PATCH 18/58] test(integration): expose canonical replay acceptance format --- .../workflows/verify-plugin-delivery-status-k6.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml index f5040c766..7edee061b 100644 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -60,7 +60,7 @@ jobs: with: k6-version: '2.2.0' - - name: Install and verify Integration source + - name: Install and expose canonical replay acceptance format shell: bash run: | set -Eeuo pipefail @@ -71,13 +71,11 @@ jobs: pnpm --filter @life-os/integration-service build env -u INTEGRATION_DATABASE_URL \ pnpm --filter @life-os/integration-service test - pnpm exec prettier --single-quote --check \ - .github/workflows/verify-plugin-delivery-status-k6.yml \ - apps/integration-service/src/plugin-operator-replay.ts \ - apps/integration-service/src/plugin-operator-replay.test.ts \ - apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts \ - apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ - apps/integration-service/perf/plugin-delivery-attempt-status.k6.js + pnpm exec prettier --single-quote --write \ + apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts + git diff -- \ + apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts + exit 1 - name: Configure verified-TLS PostgreSQL fixture shell: bash From cfb3f5aa62dca5e7e119ab7e28fee1c2ba0a7ee0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:40:37 +0900 Subject: [PATCH 19/58] style(integration): apply canonical replay acceptance format --- .../src/plugin-operator-replay-postgres.integration.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts b/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts index 13a67cef9..de76c705e 100644 --- a/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts +++ b/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts @@ -1,9 +1,7 @@ import { randomUUID } from 'node:crypto'; import { Pool, type QueryResultRow } from 'pg'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { - parsePluginDeliveryAttemptTestDatabaseTarget, -} from './plugin-delivery-attempt-test-database'; +import { parsePluginDeliveryAttemptTestDatabaseTarget } from './plugin-delivery-attempt-test-database'; const DATABASE_URL = process.env.INTEGRATION_DATABASE_URL; const TEST_DATABASE_TARGET = DATABASE_URL From 94529316d51bde83e4fc4d26d7c4f471692272d5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:41:03 +0900 Subject: [PATCH 20/58] test(integration): restore final status p95 verifier --- .../workflows/verify-plugin-delivery-status-k6.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml index 7edee061b..f5040c766 100644 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -60,7 +60,7 @@ jobs: with: k6-version: '2.2.0' - - name: Install and expose canonical replay acceptance format + - name: Install and verify Integration source shell: bash run: | set -Eeuo pipefail @@ -71,11 +71,13 @@ jobs: pnpm --filter @life-os/integration-service build env -u INTEGRATION_DATABASE_URL \ pnpm --filter @life-os/integration-service test - pnpm exec prettier --single-quote --write \ - apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts - git diff -- \ - apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts - exit 1 + pnpm exec prettier --single-quote --check \ + .github/workflows/verify-plugin-delivery-status-k6.yml \ + apps/integration-service/src/plugin-operator-replay.ts \ + apps/integration-service/src/plugin-operator-replay.test.ts \ + apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts \ + apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ + apps/integration-service/perf/plugin-delivery-attempt-status.k6.js - name: Configure verified-TLS PostgreSQL fixture shell: bash From f4ddeeef3781a45b2ef4a2b585195e52c2245b3c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:44:22 +0900 Subject: [PATCH 21/58] test(integration): use dedicated replay acceptance database --- .github/workflows/verify-plugin-delivery-status-k6.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml index f5040c766..eca719cfb 100644 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 12 env: - INTEGRATION_DATABASE_URL: postgresql://postgres:postgres@postgres.performance.test:5432/life_os_test + INTEGRATION_DATABASE_URL: postgresql://life_os:life_os@postgres.performance.test:5432/life_os_integration?sslmode=verify-full INTEGRATION_OPERATOR_CONTEXT_SECRET: performance-context-secret-value-32-bytes-minimum INTEGRATION_PLUGIN_VAULT_ORIGIN: https://vault.performance.example.test INTEGRATION_PLUGIN_VAULT_TOKEN: performance-vault-fixture-token @@ -125,6 +125,9 @@ jobs: chmod 0644 /var/lib/postgresql/server.crt docker exec --user root "$postgres_container" \ chmod 0600 /var/lib/postgresql/server.key + docker exec "$postgres_container" psql -U postgres -d postgres -v ON_ERROR_STOP=1 \ + -c "CREATE ROLE life_os LOGIN PASSWORD 'life_os'" \ + -c "CREATE DATABASE life_os_integration OWNER life_os" docker exec "$postgres_container" psql -U postgres -d life_os_test -v ON_ERROR_STOP=1 \ -c "ALTER SYSTEM SET ssl = 'on'" \ -c "ALTER SYSTEM SET ssl_cert_file = '/var/lib/postgresql/server.crt'" \ From a24862ee07b6522f37f60f17013b09f546c923f2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:46:43 +0900 Subject: [PATCH 22/58] test(integration): separate runtime and harness database authority --- .github/workflows/verify-plugin-delivery-status-k6.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml index eca719cfb..97f7434d6 100644 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 12 env: - INTEGRATION_DATABASE_URL: postgresql://life_os:life_os@postgres.performance.test:5432/life_os_integration?sslmode=verify-full + INTEGRATION_DATABASE_URL: postgresql://life_os:life_os@postgres.performance.test:5432/life_os_integration INTEGRATION_OPERATOR_CONTEXT_SECRET: performance-context-secret-value-32-bytes-minimum INTEGRATION_PLUGIN_VAULT_ORIGIN: https://vault.performance.example.test INTEGRATION_PLUGIN_VAULT_TOKEN: performance-vault-fixture-token @@ -163,9 +163,10 @@ jobs: psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -tAc \ 'SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()' | grep -Fx t - pnpm --filter @life-os/integration-service exec vitest run \ - src/plugin-operator-replay-postgres.integration.test.ts \ - --no-file-parallelism + INTEGRATION_DATABASE_URL="${INTEGRATION_DATABASE_URL}?sslmode=verify-full" \ + pnpm --filter @life-os/integration-service exec vitest run \ + src/plugin-operator-replay-postgres.integration.test.ts \ + --no-file-parallelism psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 \ -f apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql From a81c8f243eaedb5f7efddc8a6c208083c8c256e3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:48:51 +0900 Subject: [PATCH 23/58] test(integration): retire completed status p95 verifier --- .../verify-plugin-delivery-status-k6.yml | 214 ------------------ 1 file changed, 214 deletions(-) delete mode 100644 .github/workflows/verify-plugin-delivery-status-k6.yml diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml deleted file mode 100644 index 97f7434d6..000000000 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ /dev/null @@ -1,214 +0,0 @@ -name: Verify plugin delivery status k6 - -on: - push: - branches: - - perf/plugin-delivery-status-k6-v1 - -permissions: - contents: read - -concurrency: - group: verify-plugin-delivery-status-k6-${{ github.ref }} - cancel-in-progress: true - -jobs: - status_p95: - runs-on: ubuntu-24.04 - timeout-minutes: 12 - env: - INTEGRATION_DATABASE_URL: postgresql://life_os:life_os@postgres.performance.test:5432/life_os_integration - INTEGRATION_OPERATOR_CONTEXT_SECRET: performance-context-secret-value-32-bytes-minimum - INTEGRATION_PLUGIN_VAULT_ORIGIN: https://vault.performance.example.test - INTEGRATION_PLUGIN_VAULT_TOKEN: performance-vault-fixture-token - INTEGRATION_PLUGIN_VAULT_MOUNT: secret - INTEGRATION_SERVICE_PORT: '4107' - K6_BASE_URL: http://127.0.0.1:4107 - K6_ITERATIONS: '1000' - K6_VUS: '10' - K6_AUTHORITY_FILE: /tmp/plugin-delivery-status-authority.json - PGSSLMODE: verify-full - PGSSLROOTCERT: /tmp/integration-postgres-ca.crt - services: - postgres: - image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 - env: - POSTGRES_DB: life_os_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U postgres -d life_os_test" - --health-interval 5s - --health-timeout 5s - --health-retries 10 - steps: - - name: Checkout exact verifier head - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - with: - persist-credentials: false - ref: ${{ github.sha }} - - - name: Set up Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 - with: - node-version: 22 - - - name: Set up k6 - uses: grafana/setup-k6-action@db07bd9765aac508ef18982e52ab937fe633a065 # v1.2.1 - with: - k6-version: '2.2.0' - - - name: Install and verify Integration source - shell: bash - run: | - set -Eeuo pipefail - corepack enable - pnpm install --frozen-lockfile - pnpm --filter @life-os/plugin-sdk build - pnpm --filter @life-os/integration-service typecheck - pnpm --filter @life-os/integration-service build - env -u INTEGRATION_DATABASE_URL \ - pnpm --filter @life-os/integration-service test - pnpm exec prettier --single-quote --check \ - .github/workflows/verify-plugin-delivery-status-k6.yml \ - apps/integration-service/src/plugin-operator-replay.ts \ - apps/integration-service/src/plugin-operator-replay.test.ts \ - apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts \ - apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ - apps/integration-service/perf/plugin-delivery-attempt-status.k6.js - - - name: Configure verified-TLS PostgreSQL fixture - shell: bash - run: | - set -Eeuo pipefail - hostname=postgres.performance.test - ca_key=/tmp/integration-postgres-ca.key - ca_cert="$PGSSLROOTCERT" - server_key=/tmp/integration-postgres-server.key - server_csr=/tmp/integration-postgres-server.csr - server_cert=/tmp/integration-postgres-server.crt - server_ext=/tmp/integration-postgres-server.ext - - mapfile -t postgres_containers < <( - docker ps --filter publish=5432 --format '{{.ID}}' - ) - test "${#postgres_containers[@]}" -eq 1 - postgres_container="${postgres_containers[0]}" - - openssl genrsa -out "$ca_key" 2048 - openssl req -x509 -new -sha256 -days 1 \ - -key "$ca_key" \ - -subj '/CN=LifeOS Integration CI PostgreSQL CA' \ - -out "$ca_cert" - openssl genrsa -out "$server_key" 2048 - openssl req -new -sha256 \ - -key "$server_key" \ - -subj "/CN=$hostname" \ - -out "$server_csr" - printf 'subjectAltName=DNS:%s\nextendedKeyUsage=serverAuth\n' "$hostname" >"$server_ext" - openssl x509 -req -sha256 -days 1 \ - -in "$server_csr" \ - -CA "$ca_cert" \ - -CAkey "$ca_key" \ - -CAcreateserial \ - -extfile "$server_ext" \ - -out "$server_cert" - chmod 0600 "$server_key" - - docker cp "$server_cert" "$postgres_container:/var/lib/postgresql/server.crt" - docker cp "$server_key" "$postgres_container:/var/lib/postgresql/server.key" - docker exec --user root "$postgres_container" \ - chown postgres:postgres /var/lib/postgresql/server.crt /var/lib/postgresql/server.key - docker exec --user root "$postgres_container" \ - chmod 0644 /var/lib/postgresql/server.crt - docker exec --user root "$postgres_container" \ - chmod 0600 /var/lib/postgresql/server.key - docker exec "$postgres_container" psql -U postgres -d postgres -v ON_ERROR_STOP=1 \ - -c "CREATE ROLE life_os LOGIN PASSWORD 'life_os'" \ - -c "CREATE DATABASE life_os_integration OWNER life_os" - docker exec "$postgres_container" psql -U postgres -d life_os_test -v ON_ERROR_STOP=1 \ - -c "ALTER SYSTEM SET ssl = 'on'" \ - -c "ALTER SYSTEM SET ssl_cert_file = '/var/lib/postgresql/server.crt'" \ - -c "ALTER SYSTEM SET ssl_key_file = '/var/lib/postgresql/server.key'" - docker restart "$postgres_container" >/dev/null - - ready=0 - for _ in $(seq 1 30); do - if docker exec "$postgres_container" pg_isready -U postgres -d life_os_test >/dev/null 2>&1; then - ready=1 - break - fi - sleep 1 - done - test "$ready" -eq 1 - printf '127.0.0.1 %s\n' "$hostname" | sudo tee -a /etc/hosts >/dev/null - printf 'NODE_EXTRA_CA_CERTS=%s\n' "$ca_cert" >>"$GITHUB_ENV" - - rm -f \ - "$ca_key" \ - "$server_key" \ - "$server_csr" \ - "$server_ext" \ - "${ca_cert}.srl" - - - name: Migrate and verify Integration-owned PostgreSQL - shell: bash - run: | - set -Eeuo pipefail - for migration in apps/integration-service/migrations/*.sql; do - psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -f "$migration" - done - psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -tAc \ - 'SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()' | - grep -Fx t - INTEGRATION_DATABASE_URL="${INTEGRATION_DATABASE_URL}?sslmode=verify-full" \ - pnpm --filter @life-os/integration-service exec vitest run \ - src/plugin-operator-replay-postgres.integration.test.ts \ - --no-file-parallelism - psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 \ - -f apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql - - - name: Measure authenticated status path without warm-up exclusion - shell: bash - run: | - set -Eeuo pipefail - service_log=/tmp/integration-service.log - cleanup() { - status=$? - trap - EXIT - if [ -n "${service_pid:-}" ]; then - kill "$service_pid" 2>/dev/null || true - wait "$service_pid" 2>/dev/null || true - fi - if [ "$status" -ne 0 ]; then - cat "$service_log" || true - fi - rm -f "$K6_AUTHORITY_FILE" - exit "$status" - } - trap cleanup EXIT - - node apps/integration-service/dist/server.js >"$service_log" 2>&1 & - service_pid=$! - - ready=0 - for _ in $(seq 1 30); do - code=$(curl --silent --output /dev/null --write-out '%{http_code}' \ - --max-time 1 \ - "$K6_BASE_URL/v1/plugins/delivery-attempts/33333333-3333-4333-8333-333333333333" || true) - if [ "$code" = '401' ]; then - ready=1 - break - fi - if ! kill -0 "$service_pid" 2>/dev/null; then - break - fi - sleep 1 - done - test "$ready" -eq 1 - - node apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ - "$K6_AUTHORITY_FILE" - k6 run apps/integration-service/perf/plugin-delivery-attempt-status.k6.js From e1e22e3ad9f0ab59974dce5912ae150dcfc3c6ae Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 10:12:03 +0900 Subject: [PATCH 24/58] test(integration): reject mutable replay SQL evidence --- .../src/plugin-operator-replay.test.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/apps/integration-service/src/plugin-operator-replay.test.ts b/apps/integration-service/src/plugin-operator-replay.test.ts index 6fd7bee50..e2a346454 100644 --- a/apps/integration-service/src/plugin-operator-replay.test.ts +++ b/apps/integration-service/src/plugin-operator-replay.test.ts @@ -135,4 +135,25 @@ describe('PostgresPluginOperatorReplayGuard', () => { expect(client.queries).toHaveLength(1); } }); + + it('observes hostile replay SQL rows once before granting consume authority', async () => { + let rowsReads = 0; + const result = { + rowCount: 1, + get rows(): readonly unknown[] { + rowsReads += 1; + return rowsReads === 1 + ? [{ consumed: 'not-a-boolean' }] + : [{ consumed: true }]; + }, + } satisfies PluginOperatorReplaySqlResult; + const client = new ScriptedSqlClient([result]); + const guard = new PostgresPluginOperatorReplayGuard(client); + + await expect(guard.consume(evidence())).rejects.toBeInstanceOf( + PluginOperatorReplayValidationError, + ); + expect(rowsReads).toBe(1); + expect(client.queries).toHaveLength(1); + }); }); From b9c4594f58c9075b6599e7230b2e7ccdf40460ad Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 10:12:29 +0900 Subject: [PATCH 25/58] ci(integration): verify replay SQL evidence snapshot --- ...fy-plugin-operator-replay-sql-snapshot.yml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/verify-plugin-operator-replay-sql-snapshot.yml diff --git a/.github/workflows/verify-plugin-operator-replay-sql-snapshot.yml b/.github/workflows/verify-plugin-operator-replay-sql-snapshot.yml new file mode 100644 index 000000000..92a4e4514 --- /dev/null +++ b/.github/workflows/verify-plugin-operator-replay-sql-snapshot.yml @@ -0,0 +1,33 @@ +name: Verify Plugin Operator Replay SQL Snapshot + +on: + push: + branches: + - perf/plugin-delivery-status-k6-v1 + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Checkout exact branch head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 1 + - name: Set up Node 24 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: '24' + - name: Enable repository pnpm + run: corepack enable + - name: Install frozen workspace + run: pnpm install --frozen-lockfile + - name: Check focused formatting + run: pnpm exec prettier --check apps/integration-service/src/plugin-operator-replay.ts apps/integration-service/src/plugin-operator-replay.test.ts + - name: Run hostile replay SQL evidence regression + run: pnpm --filter @life-os/integration-service test -- plugin-operator-replay.test.ts + - name: Typecheck Integration + run: pnpm --filter @life-os/integration-service typecheck From b563f701102732eebfb75d4f3ddfebaa4df9d648 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 10:14:50 +0900 Subject: [PATCH 26/58] fix(integration): snapshot replay SQL evidence --- .../src/plugin-operator-replay.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/integration-service/src/plugin-operator-replay.ts b/apps/integration-service/src/plugin-operator-replay.ts index 9514171d6..94c7610b1 100644 --- a/apps/integration-service/src/plugin-operator-replay.ts +++ b/apps/integration-service/src/plugin-operator-replay.ts @@ -107,13 +107,14 @@ export class PostgresPluginOperatorReplayGuard implements PluginOperatorReplayGu ) AS consumed`, [safe.evidenceId, safe.consumedAt, safe.expiresAt], ); - if ( - result.rowCount !== 1 || - result.rows.length !== 1 || - typeof result.rows[0]?.consumed !== 'boolean' - ) { + const rows = result.rows; + if (result.rowCount !== 1 || !Array.isArray(rows) || rows.length !== 1) { return invalid(); } - return result.rows[0].consumed; + const consumed = rows[0]?.consumed; + if (typeof consumed !== 'boolean') { + return invalid(); + } + return consumed; } } From 8b19326a5ccac58fee73c7126c6a7aa3eab012f0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 11:04:26 +0900 Subject: [PATCH 27/58] test(integration): reject hostile replay SQL access --- .../src/plugin-operator-replay.test.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/apps/integration-service/src/plugin-operator-replay.test.ts b/apps/integration-service/src/plugin-operator-replay.test.ts index e2a346454..812bde65b 100644 --- a/apps/integration-service/src/plugin-operator-replay.test.ts +++ b/apps/integration-service/src/plugin-operator-replay.test.ts @@ -156,4 +156,37 @@ describe('PostgresPluginOperatorReplayGuard', () => { expect(rowsReads).toBe(1); expect(client.queries).toHaveLength(1); }); + + it('fails closed when replay SQL collection or row access is revoked', async () => { + const revokedRows = Proxy.revocable([{ consumed: true }], {}); + revokedRows.revoke(); + const revokedRow = Proxy.revocable({ consumed: true }, {}); + revokedRow.revoke(); + + for (const result of [ + { + rows: revokedRows.proxy as unknown as readonly unknown[], + rowCount: 1, + }, + { + rows: [revokedRow.proxy], + rowCount: 1, + }, + { + rowCount: 1, + get rows(): readonly unknown[] { + throw new Error('password=must-not-escape-replay-sql-evidence'); + }, + }, + ] satisfies readonly PluginOperatorReplaySqlResult[]) { + const client = new ScriptedSqlClient([result]); + const guard = new PostgresPluginOperatorReplayGuard(client); + + await expect(guard.consume(evidence())).rejects.toMatchObject({ + name: 'PluginOperatorReplayValidationError', + message: 'Plugin operator replay evidence is invalid', + }); + expect(client.queries).toHaveLength(1); + } + }); }); From c8282a94cfc3c83579686b13c7816f1c3ca6c7c8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 11:04:58 +0900 Subject: [PATCH 28/58] fix(integration): bound hostile replay SQL evidence --- .../src/plugin-operator-replay.ts | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/apps/integration-service/src/plugin-operator-replay.ts b/apps/integration-service/src/plugin-operator-replay.ts index 94c7610b1..4128e1b95 100644 --- a/apps/integration-service/src/plugin-operator-replay.ts +++ b/apps/integration-service/src/plugin-operator-replay.ts @@ -47,6 +47,15 @@ function invalid(): never { throw new PluginOperatorReplayValidationError(); } +/** Collapses hostile SQL-result access to the fixed credential-free evidence error. */ +function boundedReplaySqlRead(read: () => T): T { + try { + return read(); + } catch { + return invalid(); + } +} + /** Requires a canonical UUIDv4 evidence identity. */ function evidenceId(value: unknown): string { if (typeof value !== 'string' || !UUID_V4_PATTERN.test(value)) { @@ -107,11 +116,18 @@ export class PostgresPluginOperatorReplayGuard implements PluginOperatorReplayGu ) AS consumed`, [safe.evidenceId, safe.consumedAt, safe.expiresAt], ); - const rows = result.rows; - if (result.rowCount !== 1 || !Array.isArray(rows) || rows.length !== 1) { + const rows = boundedReplaySqlRead(() => result.rows); + const rowCount = boundedReplaySqlRead(() => result.rowCount); + const rowsAreArray = boundedReplaySqlRead(() => Array.isArray(rows)); + if (!rowsAreArray) { + return invalid(); + } + const rowsLength = boundedReplaySqlRead(() => rows.length); + if (rowCount !== 1 || rowsLength !== 1) { return invalid(); } - const consumed = rows[0]?.consumed; + const row = boundedReplaySqlRead(() => rows[0]); + const consumed = boundedReplaySqlRead(() => row?.consumed); if (typeof consumed !== 'boolean') { return invalid(); } From 8f540c5765c0632caaedef9730b1f2a4c146416e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 13:04:25 +0900 Subject: [PATCH 29/58] test(integration): reject hostile replay request evidence --- .../src/plugin-operator-replay.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/apps/integration-service/src/plugin-operator-replay.test.ts b/apps/integration-service/src/plugin-operator-replay.test.ts index 812bde65b..d57aa69b7 100644 --- a/apps/integration-service/src/plugin-operator-replay.test.ts +++ b/apps/integration-service/src/plugin-operator-replay.test.ts @@ -116,6 +116,29 @@ describe('PostgresPluginOperatorReplayGuard', () => { } }); + it('fails closed before SQL when replay request evidence access is hostile', async () => { + const revokedEvidence = Proxy.revocable(evidence(), {}); + revokedEvidence.revoke(); + const throwingEvidence = { + get evidenceId(): string { + throw new Error('password=must-not-escape-replay-request-evidence'); + }, + consumedAt: CONSUMED_AT, + expiresAt: EXPIRES_AT, + }; + + for (const candidate of [revokedEvidence.proxy, throwingEvidence]) { + const client = new ScriptedSqlClient([]); + const guard = new PostgresPluginOperatorReplayGuard(client); + + await expect(guard.consume(candidate)).rejects.toMatchObject({ + name: 'PluginOperatorReplayValidationError', + message: 'Plugin operator replay evidence is invalid', + }); + expect(client.queries).toHaveLength(0); + } + }); + it('rejects ambiguous or corrupted consume evidence instead of granting authority', async () => { for (const result of [ { rows: [{ consumed: true }], rowCount: null }, From a200c24203083e4694f32e61cf04df9497c89c3a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 13:04:47 +0900 Subject: [PATCH 30/58] fix(integration): bound replay request evidence access --- .../src/plugin-operator-replay.ts | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/apps/integration-service/src/plugin-operator-replay.ts b/apps/integration-service/src/plugin-operator-replay.ts index 4128e1b95..562e8c49f 100644 --- a/apps/integration-service/src/plugin-operator-replay.ts +++ b/apps/integration-service/src/plugin-operator-replay.ts @@ -47,8 +47,8 @@ function invalid(): never { throw new PluginOperatorReplayValidationError(); } -/** Collapses hostile SQL-result access to the fixed credential-free evidence error. */ -function boundedReplaySqlRead(read: () => T): T { +/** Collapses hostile caller or SQL evidence access to the fixed credential-free error. */ +function boundedReplayEvidenceRead(read: () => T): T { try { return read(); } catch { @@ -80,13 +80,16 @@ function instant(value: unknown): string { function replayEvidence( value: PluginOperatorReplayEvidence, ): PluginOperatorReplayEvidence { - const consumedAt = instant(value.consumedAt); - const expiresAt = instant(value.expiresAt); + const rawEvidenceId = boundedReplayEvidenceRead(() => value.evidenceId); + const rawConsumedAt = boundedReplayEvidenceRead(() => value.consumedAt); + const rawExpiresAt = boundedReplayEvidenceRead(() => value.expiresAt); + const consumedAt = instant(rawConsumedAt); + const expiresAt = instant(rawExpiresAt); if (new Date(expiresAt).getTime() < new Date(consumedAt).getTime()) { return invalid(); } return Object.freeze({ - evidenceId: evidenceId(value.evidenceId), + evidenceId: evidenceId(rawEvidenceId), consumedAt, expiresAt, }); @@ -116,18 +119,18 @@ export class PostgresPluginOperatorReplayGuard implements PluginOperatorReplayGu ) AS consumed`, [safe.evidenceId, safe.consumedAt, safe.expiresAt], ); - const rows = boundedReplaySqlRead(() => result.rows); - const rowCount = boundedReplaySqlRead(() => result.rowCount); - const rowsAreArray = boundedReplaySqlRead(() => Array.isArray(rows)); + const rows = boundedReplayEvidenceRead(() => result.rows); + const rowCount = boundedReplayEvidenceRead(() => result.rowCount); + const rowsAreArray = boundedReplayEvidenceRead(() => Array.isArray(rows)); if (!rowsAreArray) { return invalid(); } - const rowsLength = boundedReplaySqlRead(() => rows.length); + const rowsLength = boundedReplayEvidenceRead(() => rows.length); if (rowCount !== 1 || rowsLength !== 1) { return invalid(); } - const row = boundedReplaySqlRead(() => rows[0]); - const consumed = boundedReplaySqlRead(() => row?.consumed); + const row = boundedReplayEvidenceRead(() => rows[0]); + const consumed = boundedReplayEvidenceRead(() => row?.consumed); if (typeof consumed !== 'boolean') { return invalid(); } From 003dfd0e4b57837417625b7611e48a3172b47284 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 13:07:01 +0900 Subject: [PATCH 31/58] test(integration): reject expired replay evidence --- ...erator-replay-postgres.integration.test.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts b/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts index de76c705e..dbfd748f7 100644 --- a/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts +++ b/apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts @@ -76,6 +76,35 @@ describeWithPostgres( } }); + it('rejects evidence that is already expired at the PostgreSQL authority boundary', async () => { + const evidenceId = randomUUID(); + try { + const consumeExpired = async () => { + const result = await pool.query( + `SELECT plugin_integration.consume_plugin_operator_context_replay( + $1::uuid, + now() - interval '2 seconds', + now() - interval '1 second' + ) AS consumed`, + [evidenceId], + ); + return result.rows[0]?.consumed === true; + }; + + await expect(consumeExpired()).resolves.toBe(false); + await expect(consumeExpired()).resolves.toBe(false); + const persisted = await pool.query<{ count: string }>( + `SELECT count(*)::text AS count + FROM plugin_integration.plugin_operator_context_replay_record + WHERE evidence_id = $1::uuid`, + [evidenceId], + ); + expect(persisted.rows[0]?.count).toBe('0'); + } finally { + await deleteEvidence([evidenceId]); + } + }); + it('replaces expired evidence once without allowing a second current winner', async () => { const evidenceId = randomUUID(); try { @@ -97,6 +126,10 @@ describeWithPostgres( const winnerId = randomUUID(); const expiredIds = Array.from({ length: 40 }, () => randomUUID()); try { + await pool.query( + `DELETE FROM plugin_integration.plugin_operator_context_replay_record + WHERE expires_at < now()`, + ); await pool.query( `INSERT INTO plugin_integration.plugin_operator_context_replay_record ( evidence_id, consumed_at, expires_at From 214e273cf3c0685e62f0adf1bc6984a628ba48c6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 13:07:11 +0900 Subject: [PATCH 32/58] fix(integration): reject expired replay evidence in database --- .../0012_plugin_operator_context_replay_consume.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/integration-service/migrations/0012_plugin_operator_context_replay_consume.sql b/apps/integration-service/migrations/0012_plugin_operator_context_replay_consume.sql index d02c5a0e0..95ecd7f6b 100644 --- a/apps/integration-service/migrations/0012_plugin_operator_context_replay_consume.sql +++ b/apps/integration-service/migrations/0012_plugin_operator_context_replay_consume.sql @@ -9,6 +9,10 @@ AS $$ DECLARE consumed_evidence_id uuid; BEGIN + IF p_expires_at <= now() THEN + RETURN false; + END IF; + INSERT INTO plugin_integration.plugin_operator_context_replay_record ( evidence_id, consumed_at, @@ -44,4 +48,4 @@ END; $$; COMMENT ON FUNCTION plugin_integration.consume_plugin_operator_context_replay(uuid, timestamptz, timestamptz) IS - 'Atomically consumes one Plugin operator evidence identity and performs bounded post-consume expiry cleanup in the same PostgreSQL round trip.'; + 'Atomically consumes one unexpired Plugin operator evidence identity and performs bounded post-consume expiry cleanup in the same PostgreSQL round trip.'; From f15c0940c7c015b6cb8b44f509da0eec546c4fda Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 13:07:48 +0900 Subject: [PATCH 33/58] fix(integration): exclusively create k6 authority bundle --- .../perf/generate-plugin-delivery-status-authority.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs b/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs index 54c592ccb..4666cf642 100644 --- a/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs +++ b/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs @@ -61,5 +61,6 @@ const authorities = Array.from({ length: count }, () => { await writeFile(outputPath, `${JSON.stringify(authorities)}\n`, { encoding: 'utf8', + flag: 'wx', mode: 0o600, }); From 4fc85bf0bb1001a9fc48f09e16a6eef1f140c72e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 13:08:18 +0900 Subject: [PATCH 34/58] test(integration): require exact status evidence keys --- .../perf/plugin-delivery-attempt-status.k6.js | 48 +++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js b/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js index 603306483..e1d987d2d 100644 --- a/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js +++ b/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js @@ -3,6 +3,28 @@ import { check } from 'k6'; import { SharedArray } from 'k6/data'; import exec from 'k6/execution'; +const STATUS_EVIDENCE_KEYS = Object.freeze( + [ + 'attemptCount', + 'authorityVersion', + 'checkedAt', + 'claimState', + 'controlSequence', + 'deliveryId', + 'deliveryStatus', + 'grantId', + 'installationId', + 'lastOutcomeCode', + 'maxAttempts', + 'nextAttemptAt', + 'requestedAt', + 'requestedByUserId', + 'terminalAt', + 'updatedAt', + 'workspaceId', + ].sort(), +); + function boundedInteger(name, fallback, minimum, maximum) { const raw = __ENV[name]; const value = raw === undefined || raw === '' ? fallback : Number(raw); @@ -14,6 +36,22 @@ function boundedInteger(name, fallback, minimum, maximum) { return value; } +function hasExactStatusEvidenceKeys(result) { + try { + const body = result.json(); + if (body === null || typeof body !== 'object' || Array.isArray(body)) { + return false; + } + const keys = Object.keys(body).sort(); + return ( + keys.length === STATUS_EVIDENCE_KEYS.length && + keys.every((key, index) => key === STATUS_EVIDENCE_KEYS[index]) + ); + } catch { + return false; + } +} + const iterations = boundedInteger('K6_ITERATIONS', 1000, 1, 10000); const vus = boundedInteger('K6_VUS', 10, 1, 100); const authorityFile = __ENV.K6_AUTHORITY_FILE; @@ -71,13 +109,7 @@ export default function () { 'status evidence matches delivery': (result) => result.status === 200 && result.json('deliveryId') === authority.deliveryId, - 'status evidence remains credential-free': (result) => - result.status === 200 && - !Object.prototype.hasOwnProperty.call(result.json(), 'claimToken') && - !Object.prototype.hasOwnProperty.call( - result.json(), - 'claimTokenDigest', - ) && - !Object.prototype.hasOwnProperty.call(result.json(), 'credential'), + 'status evidence uses the exact credential-free contract': (result) => + result.status === 200 && hasExactStatusEvidenceKeys(result), }); } From bab0e71620e33143cd949b26cd073642c28bea0e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 13:09:55 +0900 Subject: [PATCH 35/58] test(integration): require TLS for signed k6 authority --- .../perf/plugin-delivery-attempt-status.k6.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js b/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js index e1d987d2d..e21dc9a33 100644 --- a/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js +++ b/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js @@ -60,8 +60,8 @@ const baseUrl = __ENV.K6_BASE_URL; if (!authorityFile) { throw new Error('K6_AUTHORITY_FILE is required'); } -if (!baseUrl || !/^http:\/\/127\.0\.0\.1:\d+$/u.test(baseUrl)) { - throw new Error('K6_BASE_URL must be an explicit loopback HTTP endpoint'); +if (!baseUrl || !/^https:\/\/127\.0\.0\.1:\d+$/u.test(baseUrl)) { + throw new Error('K6_BASE_URL must be an explicit loopback HTTPS endpoint'); } const authorities = new SharedArray('plugin-delivery-status-authority', () => { From c0eee740f948f9d24820c5750b94650650a5cf8a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 13:10:30 +0900 Subject: [PATCH 36/58] test(integration): restore TLS k6 current-head acceptance --- .../verify-plugin-delivery-status-k6.yml | 306 ++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100644 .github/workflows/verify-plugin-delivery-status-k6.yml diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml new file mode 100644 index 000000000..e384094c2 --- /dev/null +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -0,0 +1,306 @@ +name: Verify plugin delivery status k6 + +on: + push: + branches: + - perf/plugin-delivery-status-k6-v1 + +permissions: + contents: read + +concurrency: + group: verify-plugin-delivery-status-k6-${{ github.ref }} + cancel-in-progress: true + +jobs: + status_p95: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + INTEGRATION_DATABASE_URL: postgresql://life_os:life_os@postgres.performance.test:5432/life_os_integration + INTEGRATION_OPERATOR_CONTEXT_SECRET: performance-context-secret-value-32-bytes-minimum + INTEGRATION_PLUGIN_VAULT_ORIGIN: https://vault.performance.example.test + INTEGRATION_PLUGIN_VAULT_TOKEN: performance-vault-fixture-token + INTEGRATION_PLUGIN_VAULT_MOUNT: secret + INTEGRATION_SERVICE_PORT: '4107' + K6_TLS_PROXY_PORT: '4443' + K6_BASE_URL: https://127.0.0.1:4443 + K6_ITERATIONS: '1000' + K6_VUS: '10' + K6_AUTHORITY_FILE: /tmp/plugin-delivery-status-authority.json + PGSSLMODE: verify-full + PGSSLROOTCERT: /tmp/integration-postgres-ca.crt + services: + postgres: + image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_DB: life_os_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d life_os_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - name: Checkout exact verifier head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 1 + persist-credentials: false + ref: ${{ github.sha }} + + - name: Set up Node 24 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: '24' + + - name: Set up k6 + uses: grafana/setup-k6-action@db07bd9765aac508ef18982e52ab937fe633a065 + with: + k6-version: '2.2.0' + + - name: Install and verify Integration source + shell: bash + run: | + set -Eeuo pipefail + corepack enable + pnpm install --frozen-lockfile + pnpm --filter @life-os/plugin-sdk build + pnpm --filter @life-os/integration-service typecheck + pnpm --filter @life-os/integration-service build + env -u INTEGRATION_DATABASE_URL \ + pnpm --filter @life-os/integration-service test + pnpm exec prettier --single-quote --check \ + .github/workflows/verify-plugin-delivery-status-k6.yml \ + apps/integration-service/migrations/0012_plugin_operator_context_replay_consume.sql \ + apps/integration-service/src/plugin-operator-replay.ts \ + apps/integration-service/src/plugin-operator-replay.test.ts \ + apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts \ + apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ + apps/integration-service/perf/plugin-delivery-attempt-status.k6.js + + - name: Configure verified-TLS PostgreSQL fixture + shell: bash + run: | + set -Eeuo pipefail + hostname=postgres.performance.test + ca_key=/tmp/integration-postgres-ca.key + ca_cert="$PGSSLROOTCERT" + server_key=/tmp/integration-postgres-server.key + server_csr=/tmp/integration-postgres-server.csr + server_cert=/tmp/integration-postgres-server.crt + server_ext=/tmp/integration-postgres-server.ext + + mapfile -t postgres_containers < <( + docker ps --filter publish=5432 --format '{{.ID}}' + ) + test "${#postgres_containers[@]}" -eq 1 + postgres_container="${postgres_containers[0]}" + + openssl genrsa -out "$ca_key" 2048 + openssl req -x509 -new -sha256 -days 1 \ + -key "$ca_key" \ + -subj '/CN=LifeOS Integration CI PostgreSQL CA' \ + -out "$ca_cert" + openssl genrsa -out "$server_key" 2048 + openssl req -new -sha256 \ + -key "$server_key" \ + -subj "/CN=$hostname" \ + -out "$server_csr" + printf 'subjectAltName=DNS:%s\nextendedKeyUsage=serverAuth\n' "$hostname" >"$server_ext" + openssl x509 -req -sha256 -days 1 \ + -in "$server_csr" \ + -CA "$ca_cert" \ + -CAkey "$ca_key" \ + -CAcreateserial \ + -extfile "$server_ext" \ + -out "$server_cert" + chmod 0600 "$server_key" + + docker cp "$server_cert" "$postgres_container:/var/lib/postgresql/server.crt" + docker cp "$server_key" "$postgres_container:/var/lib/postgresql/server.key" + docker exec --user root "$postgres_container" \ + chown postgres:postgres /var/lib/postgresql/server.crt /var/lib/postgresql/server.key + docker exec --user root "$postgres_container" \ + chmod 0644 /var/lib/postgresql/server.crt + docker exec --user root "$postgres_container" \ + chmod 0600 /var/lib/postgresql/server.key + docker exec "$postgres_container" psql -U postgres -d postgres -v ON_ERROR_STOP=1 \ + -c "CREATE ROLE life_os LOGIN PASSWORD 'life_os'" \ + -c "CREATE DATABASE life_os_integration OWNER life_os" + docker exec "$postgres_container" psql -U postgres -d life_os_test -v ON_ERROR_STOP=1 \ + -c "ALTER SYSTEM SET ssl = 'on'" \ + -c "ALTER SYSTEM SET ssl_cert_file = '/var/lib/postgresql/server.crt'" \ + -c "ALTER SYSTEM SET ssl_key_file = '/var/lib/postgresql/server.key'" + docker restart "$postgres_container" >/dev/null + + ready=0 + for _ in $(seq 1 30); do + if docker exec "$postgres_container" pg_isready -U postgres -d life_os_test >/dev/null 2>&1; then + ready=1 + break + fi + sleep 1 + done + test "$ready" -eq 1 + printf '127.0.0.1 %s\n' "$hostname" | sudo tee -a /etc/hosts >/dev/null + printf 'NODE_EXTRA_CA_CERTS=%s\n' "$ca_cert" >>"$GITHUB_ENV" + + rm -f \ + "$ca_key" \ + "$server_key" \ + "$server_csr" \ + "$server_ext" \ + "${ca_cert}.srl" + + - name: Migrate and verify Integration-owned PostgreSQL + shell: bash + run: | + set -Eeuo pipefail + for migration in apps/integration-service/migrations/*.sql; do + psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -f "$migration" + done + psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -tAc \ + 'SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()' | + grep -Fx t + INTEGRATION_DATABASE_URL="${INTEGRATION_DATABASE_URL}?sslmode=verify-full" \ + pnpm --filter @life-os/integration-service exec vitest run \ + src/plugin-operator-replay-postgres.integration.test.ts \ + --no-file-parallelism + psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 \ + -f apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql + + - name: Measure authenticated HTTPS status path without warm-up exclusion + shell: bash + run: | + set -Eeuo pipefail + service_log=/tmp/integration-service.log + proxy_log=/tmp/integration-https-proxy.log + http_ca_key=/tmp/integration-http-ca.key + http_ca_cert=/tmp/integration-http-ca.crt + http_server_key=/tmp/integration-http-server.key + http_server_csr=/tmp/integration-http-server.csr + http_server_cert=/tmp/integration-http-server.crt + http_server_ext=/tmp/integration-http-server.ext + + cleanup() { + status=$? + trap - EXIT + if [ -n "${proxy_pid:-}" ]; then + kill "$proxy_pid" 2>/dev/null || true + wait "$proxy_pid" 2>/dev/null || true + fi + if [ -n "${service_pid:-}" ]; then + kill "$service_pid" 2>/dev/null || true + wait "$service_pid" 2>/dev/null || true + fi + if [ "$status" -ne 0 ]; then + cat "$service_log" || true + cat "$proxy_log" || true + fi + rm -f \ + "$K6_AUTHORITY_FILE" \ + "$http_ca_key" \ + "$http_ca_cert" \ + "$http_server_key" \ + "$http_server_csr" \ + "$http_server_cert" \ + "$http_server_ext" \ + "${http_ca_cert}.srl" + exit "$status" + } + trap cleanup EXIT + + openssl genrsa -out "$http_ca_key" 2048 + openssl req -x509 -new -sha256 -days 1 \ + -key "$http_ca_key" \ + -subj '/CN=LifeOS Integration CI HTTPS CA' \ + -out "$http_ca_cert" + openssl genrsa -out "$http_server_key" 2048 + openssl req -new -sha256 \ + -key "$http_server_key" \ + -subj '/CN=127.0.0.1' \ + -out "$http_server_csr" + printf 'subjectAltName=IP:127.0.0.1\nextendedKeyUsage=serverAuth\n' >"$http_server_ext" + openssl x509 -req -sha256 -days 1 \ + -in "$http_server_csr" \ + -CA "$http_ca_cert" \ + -CAkey "$http_ca_key" \ + -CAcreateserial \ + -extfile "$http_server_ext" \ + -out "$http_server_cert" + chmod 0600 "$http_server_key" + + node apps/integration-service/dist/server.js >"$service_log" 2>&1 & + service_pid=$! + + HTTP_SERVER_KEY="$http_server_key" \ + HTTP_SERVER_CERT="$http_server_cert" \ + node --input-type=module >"$proxy_log" 2>&1 <<'NODE' & + import { readFileSync } from 'node:fs'; + import { request } from 'node:http'; + import { createServer } from 'node:https'; + + const targetPort = Number(process.env.INTEGRATION_SERVICE_PORT); + const listenPort = Number(process.env.K6_TLS_PROXY_PORT); + const server = createServer( + { + key: readFileSync(process.env.HTTP_SERVER_KEY), + cert: readFileSync(process.env.HTTP_SERVER_CERT), + }, + (requestFromClient, responseToClient) => { + const upstream = request( + { + hostname: '127.0.0.1', + port: targetPort, + method: requestFromClient.method, + path: requestFromClient.url, + headers: requestFromClient.headers, + }, + (upstreamResponse) => { + responseToClient.writeHead( + upstreamResponse.statusCode ?? 502, + upstreamResponse.headers, + ); + upstreamResponse.pipe(responseToClient); + }, + ); + upstream.on('error', () => { + if (!responseToClient.headersSent) { + responseToClient.writeHead(502); + } + responseToClient.end(); + }); + requestFromClient.pipe(upstream); + }, + ); + + server.listen(listenPort, '127.0.0.1'); + process.once('SIGTERM', () => server.close(() => process.exit(0))); + NODE + proxy_pid=$! + + ready=0 + for _ in $(seq 1 30); do + code=$(curl --silent --output /dev/null --write-out '%{http_code}' \ + --cacert "$http_ca_cert" \ + --max-time 1 \ + "$K6_BASE_URL/v1/plugins/delivery-attempts/33333333-3333-4333-8333-333333333333" || true) + if [ "$code" = '401' ]; then + ready=1 + break + fi + if ! kill -0 "$service_pid" 2>/dev/null || ! kill -0 "$proxy_pid" 2>/dev/null; then + break + fi + sleep 1 + done + test "$ready" -eq 1 + + node apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ + "$K6_AUTHORITY_FILE" + SSL_CERT_FILE="$http_ca_cert" \ + k6 run apps/integration-service/perf/plugin-delivery-attempt-status.k6.js From 8020d0f9dd7de7d8524d49f7eca1829cf6b9efa5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 13:12:39 +0900 Subject: [PATCH 37/58] test(ci): avoid unsupported SQL prettier check --- .github/workflows/verify-plugin-delivery-status-k6.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml index e384094c2..df7b47038 100644 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -75,7 +75,6 @@ jobs: pnpm --filter @life-os/integration-service test pnpm exec prettier --single-quote --check \ .github/workflows/verify-plugin-delivery-status-k6.yml \ - apps/integration-service/migrations/0012_plugin_operator_context_replay_consume.sql \ apps/integration-service/src/plugin-operator-replay.ts \ apps/integration-service/src/plugin-operator-replay.test.ts \ apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts \ From 3806f4ad3802a082755f93a773011d06613f70a8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 19:16:44 +0900 Subject: [PATCH 38/58] fix(ci): focus replay verifier and build sdk dependency --- .../verify-plugin-operator-replay-sql-snapshot.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify-plugin-operator-replay-sql-snapshot.yml b/.github/workflows/verify-plugin-operator-replay-sql-snapshot.yml index 92a4e4514..17e066e24 100644 --- a/.github/workflows/verify-plugin-operator-replay-sql-snapshot.yml +++ b/.github/workflows/verify-plugin-operator-replay-sql-snapshot.yml @@ -25,9 +25,14 @@ jobs: run: corepack enable - name: Install frozen workspace run: pnpm install --frozen-lockfile + - name: Build Integration workspace dependency + run: pnpm --filter @life-os/plugin-sdk build - name: Check focused formatting run: pnpm exec prettier --check apps/integration-service/src/plugin-operator-replay.ts apps/integration-service/src/plugin-operator-replay.test.ts - name: Run hostile replay SQL evidence regression - run: pnpm --filter @life-os/integration-service test -- plugin-operator-replay.test.ts + run: >- + pnpm --filter @life-os/integration-service exec vitest run + --no-file-parallelism + src/plugin-operator-replay.test.ts - name: Typecheck Integration run: pnpm --filter @life-os/integration-service typecheck From 5c6e05118050f61a6d56457c2863aa8c8b8cd871 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 19:18:15 +0900 Subject: [PATCH 39/58] fix(perf): keep scenario controls out of reserved k6 env --- .../perf/plugin-delivery-attempt-status.k6.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js b/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js index e21dc9a33..fd2760381 100644 --- a/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js +++ b/apps/integration-service/perf/plugin-delivery-attempt-status.k6.js @@ -52,23 +52,25 @@ function hasExactStatusEvidenceKeys(result) { } } -const iterations = boundedInteger('K6_ITERATIONS', 1000, 1, 10000); -const vus = boundedInteger('K6_VUS', 10, 1, 100); -const authorityFile = __ENV.K6_AUTHORITY_FILE; -const baseUrl = __ENV.K6_BASE_URL; +const iterations = boundedInteger('LIFEOS_PERF_ITERATIONS', 1000, 1, 10000); +const vus = boundedInteger('LIFEOS_PERF_VUS', 10, 1, 100); +const authorityFile = __ENV.LIFEOS_PERF_AUTHORITY_FILE; +const baseUrl = __ENV.LIFEOS_PERF_BASE_URL; if (!authorityFile) { - throw new Error('K6_AUTHORITY_FILE is required'); + throw new Error('LIFEOS_PERF_AUTHORITY_FILE is required'); } if (!baseUrl || !/^https:\/\/127\.0\.0\.1:\d+$/u.test(baseUrl)) { - throw new Error('K6_BASE_URL must be an explicit loopback HTTPS endpoint'); + throw new Error( + 'LIFEOS_PERF_BASE_URL must be an explicit loopback HTTPS endpoint', + ); } const authorities = new SharedArray('plugin-delivery-status-authority', () => { const parsed = JSON.parse(open(authorityFile)); if (!Array.isArray(parsed) || parsed.length !== iterations) { throw new Error( - 'authority bundle must contain exactly K6_ITERATIONS entries', + 'authority bundle must contain exactly LIFEOS_PERF_ITERATIONS entries', ); } return parsed; From e3da325f3e0067db39c4e8e1824c1060db051f34 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 19:18:26 +0900 Subject: [PATCH 40/58] fix(perf): align authority fixture with custom controls --- .../perf/generate-plugin-delivery-status-authority.mjs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs b/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs index 4666cf642..0f9950e14 100644 --- a/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs +++ b/apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs @@ -15,7 +15,9 @@ function fail(message) { function iterations(value) { const parsed = value === undefined ? DEFAULT_ITERATIONS : Number(value); if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > MAX_ITERATIONS) { - return fail('K6_ITERATIONS must be an integer between 1 and 10000'); + return fail( + 'LIFEOS_PERF_ITERATIONS must be an integer between 1 and 10000', + ); } return parsed; } @@ -34,7 +36,7 @@ if (!outputPath) { fail('authority output path is required'); } -const count = iterations(process.env.K6_ITERATIONS); +const count = iterations(process.env.LIFEOS_PERF_ITERATIONS); const contextSecret = secret(process.env.INTEGRATION_OPERATOR_CONTEXT_SECRET); const issuedAt = String(Math.floor(Date.now() / 1000)); const authorities = Array.from({ length: count }, () => { From ad75adc421a8df46575f9f0442dc3dcb7acf4f39 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 16 Sep 2026 19:18:59 +0900 Subject: [PATCH 41/58] fix(perf): remove k6 option env collision --- .../verify-plugin-delivery-status-k6.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml index df7b47038..d218b5b05 100644 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -23,11 +23,11 @@ jobs: INTEGRATION_PLUGIN_VAULT_TOKEN: performance-vault-fixture-token INTEGRATION_PLUGIN_VAULT_MOUNT: secret INTEGRATION_SERVICE_PORT: '4107' - K6_TLS_PROXY_PORT: '4443' - K6_BASE_URL: https://127.0.0.1:4443 - K6_ITERATIONS: '1000' - K6_VUS: '10' - K6_AUTHORITY_FILE: /tmp/plugin-delivery-status-authority.json + LIFEOS_PERF_TLS_PROXY_PORT: '4443' + LIFEOS_PERF_BASE_URL: https://127.0.0.1:4443 + LIFEOS_PERF_ITERATIONS: '1000' + LIFEOS_PERF_VUS: '10' + LIFEOS_PERF_AUTHORITY_FILE: /tmp/plugin-delivery-status-authority.json PGSSLMODE: verify-full PGSSLROOTCERT: /tmp/integration-postgres-ca.crt services: @@ -201,7 +201,7 @@ jobs: cat "$proxy_log" || true fi rm -f \ - "$K6_AUTHORITY_FILE" \ + "$LIFEOS_PERF_AUTHORITY_FILE" \ "$http_ca_key" \ "$http_ca_cert" \ "$http_server_key" \ @@ -244,7 +244,7 @@ jobs: import { createServer } from 'node:https'; const targetPort = Number(process.env.INTEGRATION_SERVICE_PORT); - const listenPort = Number(process.env.K6_TLS_PROXY_PORT); + const listenPort = Number(process.env.LIFEOS_PERF_TLS_PROXY_PORT); const server = createServer( { key: readFileSync(process.env.HTTP_SERVER_KEY), @@ -287,7 +287,7 @@ jobs: code=$(curl --silent --output /dev/null --write-out '%{http_code}' \ --cacert "$http_ca_cert" \ --max-time 1 \ - "$K6_BASE_URL/v1/plugins/delivery-attempts/33333333-3333-4333-8333-333333333333" || true) + "$LIFEOS_PERF_BASE_URL/v1/plugins/delivery-attempts/33333333-3333-4333-8333-333333333333" || true) if [ "$code" = '401' ]; then ready=1 break @@ -300,6 +300,6 @@ jobs: test "$ready" -eq 1 node apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ - "$K6_AUTHORITY_FILE" + "$LIFEOS_PERF_AUTHORITY_FILE" SSL_CERT_FILE="$http_ca_cert" \ k6 run apps/integration-service/perf/plugin-delivery-attempt-status.k6.js From bb1c8647475b1d5c7f5417320d72dece2b92f7e1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 00:19:06 +0900 Subject: [PATCH 42/58] test(perf): profile delivery status latency stages --- .../perf/profile-plugin-delivery-status.mjs | 237 ++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 apps/integration-service/perf/profile-plugin-delivery-status.mjs diff --git a/apps/integration-service/perf/profile-plugin-delivery-status.mjs b/apps/integration-service/perf/profile-plugin-delivery-status.mjs new file mode 100644 index 000000000..f4ea644f4 --- /dev/null +++ b/apps/integration-service/perf/profile-plugin-delivery-status.mjs @@ -0,0 +1,237 @@ +import { createHmac, randomUUID } from 'node:crypto'; +import { request } from 'node:http'; +import { performance } from 'node:perf_hooks'; +import pg from 'pg'; + +const { Pool } = pg; +const DEFAULT_ITERATIONS = 1000; +const DEFAULT_VUS = 10; +const MAX_ITERATIONS = 10000; +const MAX_VUS = 100; +const WORKSPACE_ID = '11111111-1111-4111-8111-111111111111'; +const USER_ID = '22222222-2222-4222-8222-222222222222'; +const DELIVERY_ID = '33333333-3333-4333-8333-333333333333'; +const PATH = `/v1/plugins/delivery-attempts/${DELIVERY_ID}`; + +function fail(message) { + throw new Error(message); +} + +function boundedInteger(value, fallback, maximum, name) { + const parsed = value === undefined ? fallback : Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > maximum) { + return fail(`${name} is outside the profiling bound`); + } + return parsed; +} + +function requireSecret(value) { + if (typeof value !== 'string' || Buffer.byteLength(value, 'utf8') < 32) { + return fail('Integration profiling context secret is unavailable'); + } + return value; +} + +function requireDatabaseUrl(value) { + if (typeof value !== 'string' || value.length === 0) { + return fail('Integration profiling database authority is unavailable'); + } + return value; +} + +function percentile(sorted, fraction) { + const index = Math.max(0, Math.ceil(sorted.length * fraction) - 1); + return sorted[index]; +} + +function report(profile, samples) { + const sorted = [...samples].sort((left, right) => left - right); + const average = sorted.reduce((sum, value) => sum + value, 0) / sorted.length; + process.stdout.write( + `${JSON.stringify({ + profile, + count: sorted.length, + avgMs: Number(average.toFixed(2)), + p50Ms: Number(percentile(sorted, 0.5).toFixed(2)), + p90Ms: Number(percentile(sorted, 0.9).toFixed(2)), + p95Ms: Number(percentile(sorted, 0.95).toFixed(2)), + p99Ms: Number(percentile(sorted, 0.99).toFixed(2)), + maxMs: Number(sorted.at(-1).toFixed(2)), + })}\n`, + ); +} + +async function runConcurrent(count, concurrency, operation) { + const samples = new Array(count); + let nextIndex = 0; + + async function worker() { + while (true) { + const index = nextIndex; + nextIndex += 1; + if (index >= count) { + return; + } + const startedAt = performance.now(); + await operation(index); + samples[index] = performance.now() - startedAt; + } + } + + await Promise.all(Array.from({ length: concurrency }, () => worker())); + return samples; +} + +function signedHeaders(secret) { + const evidenceId = randomUUID(); + const issuedAt = String(Math.floor(Date.now() / 1000)); + const signature = createHmac('sha256', secret) + .update( + `life-os.integration-operator-context.v1\n${WORKSPACE_ID}\n${USER_ID}\n${evidenceId}\n${issuedAt}\nGET\n${PATH}`, + 'utf8', + ) + .digest('base64url'); + return { + 'x-life-os-workspace-id': WORKSPACE_ID, + 'x-life-os-user-id': USER_ID, + 'x-life-os-context-evidence-id': evidenceId, + 'x-life-os-context-issued-at': issuedAt, + 'x-life-os-context-signature': signature, + }; +} + +async function requestDirectStatus(port, secret, agent) { + await new Promise((resolve, reject) => { + const outbound = request( + { + hostname: '127.0.0.1', + port, + method: 'GET', + path: PATH, + headers: signedHeaders(secret), + agent, + }, + (response) => { + const chunks = []; + response.on('data', (chunk) => chunks.push(chunk)); + response.once('error', reject); + response.on('end', () => { + if (response.statusCode !== 200) { + reject(new Error('Direct Integration status profile received non-200')); + return; + } + try { + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')); + if (body.deliveryId !== DELIVERY_ID) { + reject(new Error('Direct Integration status profile received wrong delivery')); + return; + } + } catch { + reject(new Error('Direct Integration status profile received invalid JSON')); + return; + } + resolve(); + }); + }, + ); + outbound.once('error', reject); + outbound.end(); + }); +} + +const iterations = boundedInteger( + process.env.LIFEOS_PERF_ITERATIONS, + DEFAULT_ITERATIONS, + MAX_ITERATIONS, + 'LIFEOS_PERF_ITERATIONS', +); +const vus = boundedInteger( + process.env.LIFEOS_PERF_VUS, + DEFAULT_VUS, + MAX_VUS, + 'LIFEOS_PERF_VUS', +); +const contextSecret = requireSecret(process.env.INTEGRATION_OPERATOR_CONTEXT_SECRET); +const databaseUrl = requireDatabaseUrl(process.env.INTEGRATION_DATABASE_URL); +const servicePort = boundedInteger( + process.env.INTEGRATION_SERVICE_PORT, + 4107, + 65535, + 'INTEGRATION_SERVICE_PORT', +); + +const { Agent } = await import('node:http'); +const agent = new Agent({ keepAlive: true, maxSockets: vus }); +try { + const directHttpSamples = await runConcurrent(iterations, vus, () => + requestDirectStatus(servicePort, contextSecret, agent), + ); + report('direct_http_application', directHttpSamples); +} finally { + agent.destroy(); +} + +const pool = new Pool({ + connectionString: databaseUrl, + ssl: { rejectUnauthorized: true }, + max: vus, + connectionTimeoutMillis: 5000, + statement_timeout: 5000, + query_timeout: 6000, + idleTimeoutMillis: 30000, +}); + +try { + await pool.query('SELECT 1'); + + const replaySamples = await runConcurrent(iterations, vus, async () => { + const consumedAt = new Date(); + const expiresAt = new Date(consumedAt.getTime() + 60_000); + const result = await pool.query( + `SELECT plugin_integration.consume_plugin_operator_context_replay( + $1::uuid, $2::timestamptz, $3::timestamptz + ) AS consumed`, + [randomUUID(), consumedAt.toISOString(), expiresAt.toISOString()], + ); + if (result.rowCount !== 1 || result.rows[0]?.consumed !== true) { + return fail('Replay persistence profile did not consume evidence'); + } + }); + report('postgres_replay_consume', replaySamples); + + const statusSamples = await runConcurrent(iterations, vus, async () => { + const result = await pool.query( + `SELECT authority_version, delivery_id, grant_id, installation_id, + workspace_id, requested_by_user_id, delivery_status, attempt_count, + max_attempts, requested_at, updated_at, next_attempt_at, terminal_at, + last_outcome_code, control_sequence, + (claim_token_digest IS NOT NULL) AS has_claim_token_digest, + claim_started_at, claim_expires_at + FROM plugin_integration.plugin_delivery_attempt_record + WHERE delivery_id = $1::uuid + AND workspace_id = $2::uuid + AND requested_by_user_id = $3::uuid + LIMIT 2`, + [DELIVERY_ID, WORKSPACE_ID, USER_ID], + ); + if (result.rowCount !== 1) { + return fail('Delivery status persistence profile did not find exact evidence'); + } + }); + report('postgres_status_read', statusSamples); + + const durability = await pool.query( + 'SELECT current_setting($1) AS synchronous_commit, current_setting($2) AS fsync', + ['synchronous_commit', 'fsync'], + ); + const row = durability.rows[0]; + process.stdout.write( + `${JSON.stringify({ + profile: 'postgres_durability', + synchronousCommit: row?.synchronous_commit, + fsync: row?.fsync, + })}\n`, + ); +} finally { + await pool.end(); +} From 50ed9207ff1f640e831805454163270b8c6e1874 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 00:20:14 +0900 Subject: [PATCH 43/58] test(perf): retain post-failure latency profile --- .../workflows/verify-plugin-delivery-status-k6.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml index d218b5b05..f709ce348 100644 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -79,7 +79,8 @@ jobs: apps/integration-service/src/plugin-operator-replay.test.ts \ apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts \ apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ - apps/integration-service/perf/plugin-delivery-attempt-status.k6.js + apps/integration-service/perf/plugin-delivery-attempt-status.k6.js \ + apps/integration-service/perf/profile-plugin-delivery-status.mjs - name: Configure verified-TLS PostgreSQL fixture shell: bash @@ -301,5 +302,13 @@ jobs: node apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ "$LIFEOS_PERF_AUTHORITY_FILE" + + acceptance_status=0 + set +e SSL_CERT_FILE="$http_ca_cert" \ k6 run apps/integration-service/perf/plugin-delivery-attempt-status.k6.js + acceptance_status=$? + set -e + + node apps/integration-service/perf/profile-plugin-delivery-status.mjs + exit "$acceptance_status" From dc35367f75258363af281dcf2f0b62188de79c70 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 00:41:25 +0900 Subject: [PATCH 44/58] test(perf): isolate server latency profile --- .../perf/profile-plugin-delivery-status.mjs | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/apps/integration-service/perf/profile-plugin-delivery-status.mjs b/apps/integration-service/perf/profile-plugin-delivery-status.mjs index f4ea644f4..c3cffcfef 100644 --- a/apps/integration-service/perf/profile-plugin-delivery-status.mjs +++ b/apps/integration-service/perf/profile-plugin-delivery-status.mjs @@ -100,7 +100,7 @@ function signedHeaders(secret) { }; } -async function requestDirectStatus(port, secret, agent) { +async function requestDirectStatus(port, headers, agent) { await new Promise((resolve, reject) => { const outbound = request( { @@ -108,7 +108,7 @@ async function requestDirectStatus(port, secret, agent) { port, method: 'GET', path: PATH, - headers: signedHeaders(secret), + headers, agent, }, (response) => { @@ -160,11 +160,14 @@ const servicePort = boundedInteger( 'INTEGRATION_SERVICE_PORT', ); +const directHeaders = Array.from({ length: iterations }, () => + signedHeaders(contextSecret), +); const { Agent } = await import('node:http'); const agent = new Agent({ keepAlive: true, maxSockets: vus }); try { - const directHttpSamples = await runConcurrent(iterations, vus, () => - requestDirectStatus(servicePort, contextSecret, agent), + const directHttpSamples = await runConcurrent(iterations, vus, (index) => + requestDirectStatus(servicePort, directHeaders[index], agent), ); report('direct_http_application', directHttpSamples); } finally { @@ -184,14 +187,20 @@ const pool = new Pool({ try { await pool.query('SELECT 1'); - const replaySamples = await runConcurrent(iterations, vus, async () => { - const consumedAt = new Date(); - const expiresAt = new Date(consumedAt.getTime() + 60_000); + const replayConsumedAt = new Date(); + const replayExpiresAt = new Date(replayConsumedAt.getTime() + 60_000); + const replayEvidence = Array.from({ length: iterations }, () => ({ + evidenceId: randomUUID(), + consumedAt: replayConsumedAt.toISOString(), + expiresAt: replayExpiresAt.toISOString(), + })); + const replaySamples = await runConcurrent(iterations, vus, async (index) => { + const evidence = replayEvidence[index]; const result = await pool.query( `SELECT plugin_integration.consume_plugin_operator_context_replay( $1::uuid, $2::timestamptz, $3::timestamptz ) AS consumed`, - [randomUUID(), consumedAt.toISOString(), expiresAt.toISOString()], + [evidence.evidenceId, evidence.consumedAt, evidence.expiresAt], ); if (result.rowCount !== 1 || result.rows[0]?.consumed !== true) { return fail('Replay persistence profile did not consume evidence'); From 448f35bb3a6bb76255b0b5af76681382d2a81e02 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 01:08:31 +0900 Subject: [PATCH 45/58] test(perf): align direct profile with k6 request timing --- .../perf/profile-plugin-delivery-status.mjs | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/apps/integration-service/perf/profile-plugin-delivery-status.mjs b/apps/integration-service/perf/profile-plugin-delivery-status.mjs index c3cffcfef..124e80eb8 100644 --- a/apps/integration-service/perf/profile-plugin-delivery-status.mjs +++ b/apps/integration-service/perf/profile-plugin-delivery-status.mjs @@ -82,6 +82,29 @@ async function runConcurrent(count, concurrency, operation) { return samples; } +async function runConcurrentDurations(count, concurrency, operation) { + const samples = new Array(count); + let nextIndex = 0; + + async function worker() { + while (true) { + const index = nextIndex; + nextIndex += 1; + if (index >= count) { + return; + } + const durationMs = await operation(index); + if (!Number.isFinite(durationMs) || durationMs < 0) { + return fail('Direct Integration status profile returned invalid timing'); + } + samples[index] = durationMs; + } + } + + await Promise.all(Array.from({ length: concurrency }, () => worker())); + return samples; +} + function signedHeaders(secret) { const evidenceId = randomUUID(); const issuedAt = String(Math.floor(Date.now() / 1000)); @@ -101,7 +124,8 @@ function signedHeaders(secret) { } async function requestDirectStatus(port, headers, agent) { - await new Promise((resolve, reject) => { + return await new Promise((resolve, reject) => { + const startedAt = performance.now(); const outbound = request( { hostname: '127.0.0.1', @@ -116,6 +140,7 @@ async function requestDirectStatus(port, headers, agent) { response.on('data', (chunk) => chunks.push(chunk)); response.once('error', reject); response.on('end', () => { + const durationMs = performance.now() - startedAt; if (response.statusCode !== 200) { reject(new Error('Direct Integration status profile received non-200')); return; @@ -130,7 +155,7 @@ async function requestDirectStatus(port, headers, agent) { reject(new Error('Direct Integration status profile received invalid JSON')); return; } - resolve(); + resolve(durationMs); }); }, ); @@ -166,8 +191,10 @@ const directHeaders = Array.from({ length: iterations }, () => const { Agent } = await import('node:http'); const agent = new Agent({ keepAlive: true, maxSockets: vus }); try { - const directHttpSamples = await runConcurrent(iterations, vus, (index) => - requestDirectStatus(servicePort, directHeaders[index], agent), + const directHttpSamples = await runConcurrentDurations( + iterations, + vus, + (index) => requestDirectStatus(servicePort, directHeaders[index], agent), ); report('direct_http_application', directHttpSamples); } finally { From a149cc8908690ac0095fec636a0daf7f475b22e2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 06:08:04 +0900 Subject: [PATCH 46/58] style(perf): restore canonical profiler formatting --- .../perf/profile-plugin-delivery-status.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/integration-service/perf/profile-plugin-delivery-status.mjs b/apps/integration-service/perf/profile-plugin-delivery-status.mjs index 124e80eb8..b15e5aedd 100644 --- a/apps/integration-service/perf/profile-plugin-delivery-status.mjs +++ b/apps/integration-service/perf/profile-plugin-delivery-status.mjs @@ -95,7 +95,9 @@ async function runConcurrentDurations(count, concurrency, operation) { } const durationMs = await operation(index); if (!Number.isFinite(durationMs) || durationMs < 0) { - return fail('Direct Integration status profile returned invalid timing'); + return fail( + 'Direct Integration status profile returned invalid timing', + ); } samples[index] = durationMs; } From 4794e341b34f0bce068f04821fd636960302df17 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 10:03:47 +0900 Subject: [PATCH 47/58] style(perf): finish canonical profiler formatting --- .../perf/profile-plugin-delivery-status.mjs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/integration-service/perf/profile-plugin-delivery-status.mjs b/apps/integration-service/perf/profile-plugin-delivery-status.mjs index b15e5aedd..6c7eb4d03 100644 --- a/apps/integration-service/perf/profile-plugin-delivery-status.mjs +++ b/apps/integration-service/perf/profile-plugin-delivery-status.mjs @@ -144,17 +144,25 @@ async function requestDirectStatus(port, headers, agent) { response.on('end', () => { const durationMs = performance.now() - startedAt; if (response.statusCode !== 200) { - reject(new Error('Direct Integration status profile received non-200')); + reject( + new Error('Direct Integration status profile received non-200'), + ); return; } try { const body = JSON.parse(Buffer.concat(chunks).toString('utf8')); if (body.deliveryId !== DELIVERY_ID) { - reject(new Error('Direct Integration status profile received wrong delivery')); + reject( + new Error( + 'Direct Integration status profile received wrong delivery', + ), + ); return; } } catch { - reject(new Error('Direct Integration status profile received invalid JSON')); + reject( + new Error('Direct Integration status profile received invalid JSON'), + ); return; } resolve(durationMs); From 4fd46c88671633b6adc8d609c901dcfa9c548e61 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 15:09:40 +0900 Subject: [PATCH 48/58] style(perf): wrap profiler context secret --- .../perf/profile-plugin-delivery-status.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/integration-service/perf/profile-plugin-delivery-status.mjs b/apps/integration-service/perf/profile-plugin-delivery-status.mjs index 6c7eb4d03..c644ba5af 100644 --- a/apps/integration-service/perf/profile-plugin-delivery-status.mjs +++ b/apps/integration-service/perf/profile-plugin-delivery-status.mjs @@ -186,7 +186,9 @@ const vus = boundedInteger( MAX_VUS, 'LIFEOS_PERF_VUS', ); -const contextSecret = requireSecret(process.env.INTEGRATION_OPERATOR_CONTEXT_SECRET); +const contextSecret = requireSecret( + process.env.INTEGRATION_OPERATOR_CONTEXT_SECRET, +); const databaseUrl = requireDatabaseUrl(process.env.INTEGRATION_DATABASE_URL); const servicePort = boundedInteger( process.env.INTEGRATION_SERVICE_PORT, From 26f6cadad676da3faffe65da11dbe9ee77679761 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 21:08:18 +0900 Subject: [PATCH 49/58] ci(perf): expose exact profiler formatting delta --- ...ose-plugin-delivery-profile-formatting.yml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/diagnose-plugin-delivery-profile-formatting.yml diff --git a/.github/workflows/diagnose-plugin-delivery-profile-formatting.yml b/.github/workflows/diagnose-plugin-delivery-profile-formatting.yml new file mode 100644 index 000000000..1993d2c0c --- /dev/null +++ b/.github/workflows/diagnose-plugin-delivery-profile-formatting.yml @@ -0,0 +1,42 @@ +name: Diagnose plugin delivery profile formatting + +on: + push: + branches: + - perf/plugin-delivery-status-k6-v1 + paths: + - .github/workflows/diagnose-plugin-delivery-profile-formatting.yml + +permissions: + contents: read + +jobs: + diagnose: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout exact branch head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 1 + persist-credentials: false + ref: ${{ github.sha }} + - name: Set up Node 24 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: '24' + - name: Install frozen workspace + shell: bash + run: | + set -Eeuo pipefail + corepack enable + pnpm install --frozen-lockfile + - name: Print canonical profiler formatting patch + shell: bash + run: | + set -Eeuo pipefail + pnpm exec prettier --single-quote --write \ + apps/integration-service/perf/profile-plugin-delivery-status.mjs + git --no-pager diff --check + git --no-pager diff -- \ + apps/integration-service/perf/profile-plugin-delivery-status.mjs From 86ace33b5f03820e4b243c750a6c2b060f217e0e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 02:38:07 +0900 Subject: [PATCH 50/58] style(perf): apply profiler formatting --- .../perf/profile-plugin-delivery-status.mjs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/integration-service/perf/profile-plugin-delivery-status.mjs b/apps/integration-service/perf/profile-plugin-delivery-status.mjs index c644ba5af..0d6572566 100644 --- a/apps/integration-service/perf/profile-plugin-delivery-status.mjs +++ b/apps/integration-service/perf/profile-plugin-delivery-status.mjs @@ -161,7 +161,9 @@ async function requestDirectStatus(port, headers, agent) { } } catch { reject( - new Error('Direct Integration status profile received invalid JSON'), + new Error( + 'Direct Integration status profile received invalid JSON', + ), ); return; } @@ -263,7 +265,9 @@ try { [DELIVERY_ID, WORKSPACE_ID, USER_ID], ); if (result.rowCount !== 1) { - return fail('Delivery status persistence profile did not find exact evidence'); + return fail( + 'Delivery status persistence profile did not find exact evidence', + ); } }); report('postgres_status_read', statusSamples); From fc44d0d16b9b331aae4f1e02b41597ea49d56c74 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 02:38:12 +0900 Subject: [PATCH 51/58] ci(perf): retire profiler formatter diagnostic --- ...ose-plugin-delivery-profile-formatting.yml | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 .github/workflows/diagnose-plugin-delivery-profile-formatting.yml diff --git a/.github/workflows/diagnose-plugin-delivery-profile-formatting.yml b/.github/workflows/diagnose-plugin-delivery-profile-formatting.yml deleted file mode 100644 index 1993d2c0c..000000000 --- a/.github/workflows/diagnose-plugin-delivery-profile-formatting.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Diagnose plugin delivery profile formatting - -on: - push: - branches: - - perf/plugin-delivery-status-k6-v1 - paths: - - .github/workflows/diagnose-plugin-delivery-profile-formatting.yml - -permissions: - contents: read - -jobs: - diagnose: - runs-on: ubuntu-24.04 - timeout-minutes: 15 - steps: - - name: Checkout exact branch head - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - fetch-depth: 1 - persist-credentials: false - ref: ${{ github.sha }} - - name: Set up Node 24 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 - with: - node-version: '24' - - name: Install frozen workspace - shell: bash - run: | - set -Eeuo pipefail - corepack enable - pnpm install --frozen-lockfile - - name: Print canonical profiler formatting patch - shell: bash - run: | - set -Eeuo pipefail - pnpm exec prettier --single-quote --write \ - apps/integration-service/perf/profile-plugin-delivery-status.mjs - git --no-pager diff --check - git --no-pager diff -- \ - apps/integration-service/perf/profile-plugin-delivery-status.mjs From cf8361bdd5419acf6c5d2a38df44b66fb942fa66 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 08:02:36 +0900 Subject: [PATCH 52/58] ci(perf): retire replay verifier after exact green --- ...fy-plugin-operator-replay-sql-snapshot.yml | 38 ------------------- 1 file changed, 38 deletions(-) delete mode 100644 .github/workflows/verify-plugin-operator-replay-sql-snapshot.yml diff --git a/.github/workflows/verify-plugin-operator-replay-sql-snapshot.yml b/.github/workflows/verify-plugin-operator-replay-sql-snapshot.yml deleted file mode 100644 index 17e066e24..000000000 --- a/.github/workflows/verify-plugin-operator-replay-sql-snapshot.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Verify Plugin Operator Replay SQL Snapshot - -on: - push: - branches: - - perf/plugin-delivery-status-k6-v1 - -permissions: - contents: read - -jobs: - verify: - runs-on: ubuntu-24.04 - timeout-minutes: 20 - steps: - - name: Checkout exact branch head - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - fetch-depth: 1 - - name: Set up Node 24 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 - with: - node-version: '24' - - name: Enable repository pnpm - run: corepack enable - - name: Install frozen workspace - run: pnpm install --frozen-lockfile - - name: Build Integration workspace dependency - run: pnpm --filter @life-os/plugin-sdk build - - name: Check focused formatting - run: pnpm exec prettier --check apps/integration-service/src/plugin-operator-replay.ts apps/integration-service/src/plugin-operator-replay.test.ts - - name: Run hostile replay SQL evidence regression - run: >- - pnpm --filter @life-os/integration-service exec vitest run - --no-file-parallelism - src/plugin-operator-replay.test.ts - - name: Typecheck Integration - run: pnpm --filter @life-os/integration-service typecheck From 840c9c2bcaa2de9a1655e7be639d91dbaa4636fe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 08:02:41 +0900 Subject: [PATCH 53/58] ci(perf): retire k6 verifier after exact green --- .../verify-plugin-delivery-status-k6.yml | 314 ------------------ 1 file changed, 314 deletions(-) delete mode 100644 .github/workflows/verify-plugin-delivery-status-k6.yml diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml deleted file mode 100644 index f709ce348..000000000 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ /dev/null @@ -1,314 +0,0 @@ -name: Verify plugin delivery status k6 - -on: - push: - branches: - - perf/plugin-delivery-status-k6-v1 - -permissions: - contents: read - -concurrency: - group: verify-plugin-delivery-status-k6-${{ github.ref }} - cancel-in-progress: true - -jobs: - status_p95: - runs-on: ubuntu-24.04 - timeout-minutes: 15 - env: - INTEGRATION_DATABASE_URL: postgresql://life_os:life_os@postgres.performance.test:5432/life_os_integration - INTEGRATION_OPERATOR_CONTEXT_SECRET: performance-context-secret-value-32-bytes-minimum - INTEGRATION_PLUGIN_VAULT_ORIGIN: https://vault.performance.example.test - INTEGRATION_PLUGIN_VAULT_TOKEN: performance-vault-fixture-token - INTEGRATION_PLUGIN_VAULT_MOUNT: secret - INTEGRATION_SERVICE_PORT: '4107' - LIFEOS_PERF_TLS_PROXY_PORT: '4443' - LIFEOS_PERF_BASE_URL: https://127.0.0.1:4443 - LIFEOS_PERF_ITERATIONS: '1000' - LIFEOS_PERF_VUS: '10' - LIFEOS_PERF_AUTHORITY_FILE: /tmp/plugin-delivery-status-authority.json - PGSSLMODE: verify-full - PGSSLROOTCERT: /tmp/integration-postgres-ca.crt - services: - postgres: - image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 - env: - POSTGRES_DB: life_os_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U postgres -d life_os_test" - --health-interval 5s - --health-timeout 5s - --health-retries 10 - steps: - - name: Checkout exact verifier head - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - fetch-depth: 1 - persist-credentials: false - ref: ${{ github.sha }} - - - name: Set up Node 24 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 - with: - node-version: '24' - - - name: Set up k6 - uses: grafana/setup-k6-action@db07bd9765aac508ef18982e52ab937fe633a065 - with: - k6-version: '2.2.0' - - - name: Install and verify Integration source - shell: bash - run: | - set -Eeuo pipefail - corepack enable - pnpm install --frozen-lockfile - pnpm --filter @life-os/plugin-sdk build - pnpm --filter @life-os/integration-service typecheck - pnpm --filter @life-os/integration-service build - env -u INTEGRATION_DATABASE_URL \ - pnpm --filter @life-os/integration-service test - pnpm exec prettier --single-quote --check \ - .github/workflows/verify-plugin-delivery-status-k6.yml \ - apps/integration-service/src/plugin-operator-replay.ts \ - apps/integration-service/src/plugin-operator-replay.test.ts \ - apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts \ - apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ - apps/integration-service/perf/plugin-delivery-attempt-status.k6.js \ - apps/integration-service/perf/profile-plugin-delivery-status.mjs - - - name: Configure verified-TLS PostgreSQL fixture - shell: bash - run: | - set -Eeuo pipefail - hostname=postgres.performance.test - ca_key=/tmp/integration-postgres-ca.key - ca_cert="$PGSSLROOTCERT" - server_key=/tmp/integration-postgres-server.key - server_csr=/tmp/integration-postgres-server.csr - server_cert=/tmp/integration-postgres-server.crt - server_ext=/tmp/integration-postgres-server.ext - - mapfile -t postgres_containers < <( - docker ps --filter publish=5432 --format '{{.ID}}' - ) - test "${#postgres_containers[@]}" -eq 1 - postgres_container="${postgres_containers[0]}" - - openssl genrsa -out "$ca_key" 2048 - openssl req -x509 -new -sha256 -days 1 \ - -key "$ca_key" \ - -subj '/CN=LifeOS Integration CI PostgreSQL CA' \ - -out "$ca_cert" - openssl genrsa -out "$server_key" 2048 - openssl req -new -sha256 \ - -key "$server_key" \ - -subj "/CN=$hostname" \ - -out "$server_csr" - printf 'subjectAltName=DNS:%s\nextendedKeyUsage=serverAuth\n' "$hostname" >"$server_ext" - openssl x509 -req -sha256 -days 1 \ - -in "$server_csr" \ - -CA "$ca_cert" \ - -CAkey "$ca_key" \ - -CAcreateserial \ - -extfile "$server_ext" \ - -out "$server_cert" - chmod 0600 "$server_key" - - docker cp "$server_cert" "$postgres_container:/var/lib/postgresql/server.crt" - docker cp "$server_key" "$postgres_container:/var/lib/postgresql/server.key" - docker exec --user root "$postgres_container" \ - chown postgres:postgres /var/lib/postgresql/server.crt /var/lib/postgresql/server.key - docker exec --user root "$postgres_container" \ - chmod 0644 /var/lib/postgresql/server.crt - docker exec --user root "$postgres_container" \ - chmod 0600 /var/lib/postgresql/server.key - docker exec "$postgres_container" psql -U postgres -d postgres -v ON_ERROR_STOP=1 \ - -c "CREATE ROLE life_os LOGIN PASSWORD 'life_os'" \ - -c "CREATE DATABASE life_os_integration OWNER life_os" - docker exec "$postgres_container" psql -U postgres -d life_os_test -v ON_ERROR_STOP=1 \ - -c "ALTER SYSTEM SET ssl = 'on'" \ - -c "ALTER SYSTEM SET ssl_cert_file = '/var/lib/postgresql/server.crt'" \ - -c "ALTER SYSTEM SET ssl_key_file = '/var/lib/postgresql/server.key'" - docker restart "$postgres_container" >/dev/null - - ready=0 - for _ in $(seq 1 30); do - if docker exec "$postgres_container" pg_isready -U postgres -d life_os_test >/dev/null 2>&1; then - ready=1 - break - fi - sleep 1 - done - test "$ready" -eq 1 - printf '127.0.0.1 %s\n' "$hostname" | sudo tee -a /etc/hosts >/dev/null - printf 'NODE_EXTRA_CA_CERTS=%s\n' "$ca_cert" >>"$GITHUB_ENV" - - rm -f \ - "$ca_key" \ - "$server_key" \ - "$server_csr" \ - "$server_ext" \ - "${ca_cert}.srl" - - - name: Migrate and verify Integration-owned PostgreSQL - shell: bash - run: | - set -Eeuo pipefail - for migration in apps/integration-service/migrations/*.sql; do - psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -f "$migration" - done - psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -tAc \ - 'SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()' | - grep -Fx t - INTEGRATION_DATABASE_URL="${INTEGRATION_DATABASE_URL}?sslmode=verify-full" \ - pnpm --filter @life-os/integration-service exec vitest run \ - src/plugin-operator-replay-postgres.integration.test.ts \ - --no-file-parallelism - psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 \ - -f apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql - - - name: Measure authenticated HTTPS status path without warm-up exclusion - shell: bash - run: | - set -Eeuo pipefail - service_log=/tmp/integration-service.log - proxy_log=/tmp/integration-https-proxy.log - http_ca_key=/tmp/integration-http-ca.key - http_ca_cert=/tmp/integration-http-ca.crt - http_server_key=/tmp/integration-http-server.key - http_server_csr=/tmp/integration-http-server.csr - http_server_cert=/tmp/integration-http-server.crt - http_server_ext=/tmp/integration-http-server.ext - - cleanup() { - status=$? - trap - EXIT - if [ -n "${proxy_pid:-}" ]; then - kill "$proxy_pid" 2>/dev/null || true - wait "$proxy_pid" 2>/dev/null || true - fi - if [ -n "${service_pid:-}" ]; then - kill "$service_pid" 2>/dev/null || true - wait "$service_pid" 2>/dev/null || true - fi - if [ "$status" -ne 0 ]; then - cat "$service_log" || true - cat "$proxy_log" || true - fi - rm -f \ - "$LIFEOS_PERF_AUTHORITY_FILE" \ - "$http_ca_key" \ - "$http_ca_cert" \ - "$http_server_key" \ - "$http_server_csr" \ - "$http_server_cert" \ - "$http_server_ext" \ - "${http_ca_cert}.srl" - exit "$status" - } - trap cleanup EXIT - - openssl genrsa -out "$http_ca_key" 2048 - openssl req -x509 -new -sha256 -days 1 \ - -key "$http_ca_key" \ - -subj '/CN=LifeOS Integration CI HTTPS CA' \ - -out "$http_ca_cert" - openssl genrsa -out "$http_server_key" 2048 - openssl req -new -sha256 \ - -key "$http_server_key" \ - -subj '/CN=127.0.0.1' \ - -out "$http_server_csr" - printf 'subjectAltName=IP:127.0.0.1\nextendedKeyUsage=serverAuth\n' >"$http_server_ext" - openssl x509 -req -sha256 -days 1 \ - -in "$http_server_csr" \ - -CA "$http_ca_cert" \ - -CAkey "$http_ca_key" \ - -CAcreateserial \ - -extfile "$http_server_ext" \ - -out "$http_server_cert" - chmod 0600 "$http_server_key" - - node apps/integration-service/dist/server.js >"$service_log" 2>&1 & - service_pid=$! - - HTTP_SERVER_KEY="$http_server_key" \ - HTTP_SERVER_CERT="$http_server_cert" \ - node --input-type=module >"$proxy_log" 2>&1 <<'NODE' & - import { readFileSync } from 'node:fs'; - import { request } from 'node:http'; - import { createServer } from 'node:https'; - - const targetPort = Number(process.env.INTEGRATION_SERVICE_PORT); - const listenPort = Number(process.env.LIFEOS_PERF_TLS_PROXY_PORT); - const server = createServer( - { - key: readFileSync(process.env.HTTP_SERVER_KEY), - cert: readFileSync(process.env.HTTP_SERVER_CERT), - }, - (requestFromClient, responseToClient) => { - const upstream = request( - { - hostname: '127.0.0.1', - port: targetPort, - method: requestFromClient.method, - path: requestFromClient.url, - headers: requestFromClient.headers, - }, - (upstreamResponse) => { - responseToClient.writeHead( - upstreamResponse.statusCode ?? 502, - upstreamResponse.headers, - ); - upstreamResponse.pipe(responseToClient); - }, - ); - upstream.on('error', () => { - if (!responseToClient.headersSent) { - responseToClient.writeHead(502); - } - responseToClient.end(); - }); - requestFromClient.pipe(upstream); - }, - ); - - server.listen(listenPort, '127.0.0.1'); - process.once('SIGTERM', () => server.close(() => process.exit(0))); - NODE - proxy_pid=$! - - ready=0 - for _ in $(seq 1 30); do - code=$(curl --silent --output /dev/null --write-out '%{http_code}' \ - --cacert "$http_ca_cert" \ - --max-time 1 \ - "$LIFEOS_PERF_BASE_URL/v1/plugins/delivery-attempts/33333333-3333-4333-8333-333333333333" || true) - if [ "$code" = '401' ]; then - ready=1 - break - fi - if ! kill -0 "$service_pid" 2>/dev/null || ! kill -0 "$proxy_pid" 2>/dev/null; then - break - fi - sleep 1 - done - test "$ready" -eq 1 - - node apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ - "$LIFEOS_PERF_AUTHORITY_FILE" - - acceptance_status=0 - set +e - SSL_CERT_FILE="$http_ca_cert" \ - k6 run apps/integration-service/perf/plugin-delivery-attempt-status.k6.js - acceptance_status=$? - set -e - - node apps/integration-service/perf/profile-plugin-delivery-status.mjs - exit "$acceptance_status" From a12888bfa51c20be79fca2a2bf58a221a6e559ae Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 19 Sep 2026 12:07:13 +0900 Subject: [PATCH 54/58] ci(perf): restore k6 verifier after failed exact head --- .../verify-plugin-delivery-status-k6.yml | 314 ++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 .github/workflows/verify-plugin-delivery-status-k6.yml diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml new file mode 100644 index 000000000..f709ce348 --- /dev/null +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -0,0 +1,314 @@ +name: Verify plugin delivery status k6 + +on: + push: + branches: + - perf/plugin-delivery-status-k6-v1 + +permissions: + contents: read + +concurrency: + group: verify-plugin-delivery-status-k6-${{ github.ref }} + cancel-in-progress: true + +jobs: + status_p95: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + INTEGRATION_DATABASE_URL: postgresql://life_os:life_os@postgres.performance.test:5432/life_os_integration + INTEGRATION_OPERATOR_CONTEXT_SECRET: performance-context-secret-value-32-bytes-minimum + INTEGRATION_PLUGIN_VAULT_ORIGIN: https://vault.performance.example.test + INTEGRATION_PLUGIN_VAULT_TOKEN: performance-vault-fixture-token + INTEGRATION_PLUGIN_VAULT_MOUNT: secret + INTEGRATION_SERVICE_PORT: '4107' + LIFEOS_PERF_TLS_PROXY_PORT: '4443' + LIFEOS_PERF_BASE_URL: https://127.0.0.1:4443 + LIFEOS_PERF_ITERATIONS: '1000' + LIFEOS_PERF_VUS: '10' + LIFEOS_PERF_AUTHORITY_FILE: /tmp/plugin-delivery-status-authority.json + PGSSLMODE: verify-full + PGSSLROOTCERT: /tmp/integration-postgres-ca.crt + services: + postgres: + image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_DB: life_os_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d life_os_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - name: Checkout exact verifier head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 1 + persist-credentials: false + ref: ${{ github.sha }} + + - name: Set up Node 24 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: '24' + + - name: Set up k6 + uses: grafana/setup-k6-action@db07bd9765aac508ef18982e52ab937fe633a065 + with: + k6-version: '2.2.0' + + - name: Install and verify Integration source + shell: bash + run: | + set -Eeuo pipefail + corepack enable + pnpm install --frozen-lockfile + pnpm --filter @life-os/plugin-sdk build + pnpm --filter @life-os/integration-service typecheck + pnpm --filter @life-os/integration-service build + env -u INTEGRATION_DATABASE_URL \ + pnpm --filter @life-os/integration-service test + pnpm exec prettier --single-quote --check \ + .github/workflows/verify-plugin-delivery-status-k6.yml \ + apps/integration-service/src/plugin-operator-replay.ts \ + apps/integration-service/src/plugin-operator-replay.test.ts \ + apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts \ + apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ + apps/integration-service/perf/plugin-delivery-attempt-status.k6.js \ + apps/integration-service/perf/profile-plugin-delivery-status.mjs + + - name: Configure verified-TLS PostgreSQL fixture + shell: bash + run: | + set -Eeuo pipefail + hostname=postgres.performance.test + ca_key=/tmp/integration-postgres-ca.key + ca_cert="$PGSSLROOTCERT" + server_key=/tmp/integration-postgres-server.key + server_csr=/tmp/integration-postgres-server.csr + server_cert=/tmp/integration-postgres-server.crt + server_ext=/tmp/integration-postgres-server.ext + + mapfile -t postgres_containers < <( + docker ps --filter publish=5432 --format '{{.ID}}' + ) + test "${#postgres_containers[@]}" -eq 1 + postgres_container="${postgres_containers[0]}" + + openssl genrsa -out "$ca_key" 2048 + openssl req -x509 -new -sha256 -days 1 \ + -key "$ca_key" \ + -subj '/CN=LifeOS Integration CI PostgreSQL CA' \ + -out "$ca_cert" + openssl genrsa -out "$server_key" 2048 + openssl req -new -sha256 \ + -key "$server_key" \ + -subj "/CN=$hostname" \ + -out "$server_csr" + printf 'subjectAltName=DNS:%s\nextendedKeyUsage=serverAuth\n' "$hostname" >"$server_ext" + openssl x509 -req -sha256 -days 1 \ + -in "$server_csr" \ + -CA "$ca_cert" \ + -CAkey "$ca_key" \ + -CAcreateserial \ + -extfile "$server_ext" \ + -out "$server_cert" + chmod 0600 "$server_key" + + docker cp "$server_cert" "$postgres_container:/var/lib/postgresql/server.crt" + docker cp "$server_key" "$postgres_container:/var/lib/postgresql/server.key" + docker exec --user root "$postgres_container" \ + chown postgres:postgres /var/lib/postgresql/server.crt /var/lib/postgresql/server.key + docker exec --user root "$postgres_container" \ + chmod 0644 /var/lib/postgresql/server.crt + docker exec --user root "$postgres_container" \ + chmod 0600 /var/lib/postgresql/server.key + docker exec "$postgres_container" psql -U postgres -d postgres -v ON_ERROR_STOP=1 \ + -c "CREATE ROLE life_os LOGIN PASSWORD 'life_os'" \ + -c "CREATE DATABASE life_os_integration OWNER life_os" + docker exec "$postgres_container" psql -U postgres -d life_os_test -v ON_ERROR_STOP=1 \ + -c "ALTER SYSTEM SET ssl = 'on'" \ + -c "ALTER SYSTEM SET ssl_cert_file = '/var/lib/postgresql/server.crt'" \ + -c "ALTER SYSTEM SET ssl_key_file = '/var/lib/postgresql/server.key'" + docker restart "$postgres_container" >/dev/null + + ready=0 + for _ in $(seq 1 30); do + if docker exec "$postgres_container" pg_isready -U postgres -d life_os_test >/dev/null 2>&1; then + ready=1 + break + fi + sleep 1 + done + test "$ready" -eq 1 + printf '127.0.0.1 %s\n' "$hostname" | sudo tee -a /etc/hosts >/dev/null + printf 'NODE_EXTRA_CA_CERTS=%s\n' "$ca_cert" >>"$GITHUB_ENV" + + rm -f \ + "$ca_key" \ + "$server_key" \ + "$server_csr" \ + "$server_ext" \ + "${ca_cert}.srl" + + - name: Migrate and verify Integration-owned PostgreSQL + shell: bash + run: | + set -Eeuo pipefail + for migration in apps/integration-service/migrations/*.sql; do + psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -f "$migration" + done + psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -tAc \ + 'SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()' | + grep -Fx t + INTEGRATION_DATABASE_URL="${INTEGRATION_DATABASE_URL}?sslmode=verify-full" \ + pnpm --filter @life-os/integration-service exec vitest run \ + src/plugin-operator-replay-postgres.integration.test.ts \ + --no-file-parallelism + psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 \ + -f apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql + + - name: Measure authenticated HTTPS status path without warm-up exclusion + shell: bash + run: | + set -Eeuo pipefail + service_log=/tmp/integration-service.log + proxy_log=/tmp/integration-https-proxy.log + http_ca_key=/tmp/integration-http-ca.key + http_ca_cert=/tmp/integration-http-ca.crt + http_server_key=/tmp/integration-http-server.key + http_server_csr=/tmp/integration-http-server.csr + http_server_cert=/tmp/integration-http-server.crt + http_server_ext=/tmp/integration-http-server.ext + + cleanup() { + status=$? + trap - EXIT + if [ -n "${proxy_pid:-}" ]; then + kill "$proxy_pid" 2>/dev/null || true + wait "$proxy_pid" 2>/dev/null || true + fi + if [ -n "${service_pid:-}" ]; then + kill "$service_pid" 2>/dev/null || true + wait "$service_pid" 2>/dev/null || true + fi + if [ "$status" -ne 0 ]; then + cat "$service_log" || true + cat "$proxy_log" || true + fi + rm -f \ + "$LIFEOS_PERF_AUTHORITY_FILE" \ + "$http_ca_key" \ + "$http_ca_cert" \ + "$http_server_key" \ + "$http_server_csr" \ + "$http_server_cert" \ + "$http_server_ext" \ + "${http_ca_cert}.srl" + exit "$status" + } + trap cleanup EXIT + + openssl genrsa -out "$http_ca_key" 2048 + openssl req -x509 -new -sha256 -days 1 \ + -key "$http_ca_key" \ + -subj '/CN=LifeOS Integration CI HTTPS CA' \ + -out "$http_ca_cert" + openssl genrsa -out "$http_server_key" 2048 + openssl req -new -sha256 \ + -key "$http_server_key" \ + -subj '/CN=127.0.0.1' \ + -out "$http_server_csr" + printf 'subjectAltName=IP:127.0.0.1\nextendedKeyUsage=serverAuth\n' >"$http_server_ext" + openssl x509 -req -sha256 -days 1 \ + -in "$http_server_csr" \ + -CA "$http_ca_cert" \ + -CAkey "$http_ca_key" \ + -CAcreateserial \ + -extfile "$http_server_ext" \ + -out "$http_server_cert" + chmod 0600 "$http_server_key" + + node apps/integration-service/dist/server.js >"$service_log" 2>&1 & + service_pid=$! + + HTTP_SERVER_KEY="$http_server_key" \ + HTTP_SERVER_CERT="$http_server_cert" \ + node --input-type=module >"$proxy_log" 2>&1 <<'NODE' & + import { readFileSync } from 'node:fs'; + import { request } from 'node:http'; + import { createServer } from 'node:https'; + + const targetPort = Number(process.env.INTEGRATION_SERVICE_PORT); + const listenPort = Number(process.env.LIFEOS_PERF_TLS_PROXY_PORT); + const server = createServer( + { + key: readFileSync(process.env.HTTP_SERVER_KEY), + cert: readFileSync(process.env.HTTP_SERVER_CERT), + }, + (requestFromClient, responseToClient) => { + const upstream = request( + { + hostname: '127.0.0.1', + port: targetPort, + method: requestFromClient.method, + path: requestFromClient.url, + headers: requestFromClient.headers, + }, + (upstreamResponse) => { + responseToClient.writeHead( + upstreamResponse.statusCode ?? 502, + upstreamResponse.headers, + ); + upstreamResponse.pipe(responseToClient); + }, + ); + upstream.on('error', () => { + if (!responseToClient.headersSent) { + responseToClient.writeHead(502); + } + responseToClient.end(); + }); + requestFromClient.pipe(upstream); + }, + ); + + server.listen(listenPort, '127.0.0.1'); + process.once('SIGTERM', () => server.close(() => process.exit(0))); + NODE + proxy_pid=$! + + ready=0 + for _ in $(seq 1 30); do + code=$(curl --silent --output /dev/null --write-out '%{http_code}' \ + --cacert "$http_ca_cert" \ + --max-time 1 \ + "$LIFEOS_PERF_BASE_URL/v1/plugins/delivery-attempts/33333333-3333-4333-8333-333333333333" || true) + if [ "$code" = '401' ]; then + ready=1 + break + fi + if ! kill -0 "$service_pid" 2>/dev/null || ! kill -0 "$proxy_pid" 2>/dev/null; then + break + fi + sleep 1 + done + test "$ready" -eq 1 + + node apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ + "$LIFEOS_PERF_AUTHORITY_FILE" + + acceptance_status=0 + set +e + SSL_CERT_FILE="$http_ca_cert" \ + k6 run apps/integration-service/perf/plugin-delivery-attempt-status.k6.js + acceptance_status=$? + set -e + + node apps/integration-service/perf/profile-plugin-delivery-status.mjs + exit "$acceptance_status" From 9ce9f89e6f94ca38e6107c768f0cb4da64b34db4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 19 Sep 2026 19:06:00 +0900 Subject: [PATCH 55/58] chore(perf): retire exact-green k6 verifier --- .../verify-plugin-delivery-status-k6.yml | 314 ------------------ 1 file changed, 314 deletions(-) delete mode 100644 .github/workflows/verify-plugin-delivery-status-k6.yml diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml deleted file mode 100644 index f709ce348..000000000 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ /dev/null @@ -1,314 +0,0 @@ -name: Verify plugin delivery status k6 - -on: - push: - branches: - - perf/plugin-delivery-status-k6-v1 - -permissions: - contents: read - -concurrency: - group: verify-plugin-delivery-status-k6-${{ github.ref }} - cancel-in-progress: true - -jobs: - status_p95: - runs-on: ubuntu-24.04 - timeout-minutes: 15 - env: - INTEGRATION_DATABASE_URL: postgresql://life_os:life_os@postgres.performance.test:5432/life_os_integration - INTEGRATION_OPERATOR_CONTEXT_SECRET: performance-context-secret-value-32-bytes-minimum - INTEGRATION_PLUGIN_VAULT_ORIGIN: https://vault.performance.example.test - INTEGRATION_PLUGIN_VAULT_TOKEN: performance-vault-fixture-token - INTEGRATION_PLUGIN_VAULT_MOUNT: secret - INTEGRATION_SERVICE_PORT: '4107' - LIFEOS_PERF_TLS_PROXY_PORT: '4443' - LIFEOS_PERF_BASE_URL: https://127.0.0.1:4443 - LIFEOS_PERF_ITERATIONS: '1000' - LIFEOS_PERF_VUS: '10' - LIFEOS_PERF_AUTHORITY_FILE: /tmp/plugin-delivery-status-authority.json - PGSSLMODE: verify-full - PGSSLROOTCERT: /tmp/integration-postgres-ca.crt - services: - postgres: - image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 - env: - POSTGRES_DB: life_os_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U postgres -d life_os_test" - --health-interval 5s - --health-timeout 5s - --health-retries 10 - steps: - - name: Checkout exact verifier head - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - fetch-depth: 1 - persist-credentials: false - ref: ${{ github.sha }} - - - name: Set up Node 24 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 - with: - node-version: '24' - - - name: Set up k6 - uses: grafana/setup-k6-action@db07bd9765aac508ef18982e52ab937fe633a065 - with: - k6-version: '2.2.0' - - - name: Install and verify Integration source - shell: bash - run: | - set -Eeuo pipefail - corepack enable - pnpm install --frozen-lockfile - pnpm --filter @life-os/plugin-sdk build - pnpm --filter @life-os/integration-service typecheck - pnpm --filter @life-os/integration-service build - env -u INTEGRATION_DATABASE_URL \ - pnpm --filter @life-os/integration-service test - pnpm exec prettier --single-quote --check \ - .github/workflows/verify-plugin-delivery-status-k6.yml \ - apps/integration-service/src/plugin-operator-replay.ts \ - apps/integration-service/src/plugin-operator-replay.test.ts \ - apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts \ - apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ - apps/integration-service/perf/plugin-delivery-attempt-status.k6.js \ - apps/integration-service/perf/profile-plugin-delivery-status.mjs - - - name: Configure verified-TLS PostgreSQL fixture - shell: bash - run: | - set -Eeuo pipefail - hostname=postgres.performance.test - ca_key=/tmp/integration-postgres-ca.key - ca_cert="$PGSSLROOTCERT" - server_key=/tmp/integration-postgres-server.key - server_csr=/tmp/integration-postgres-server.csr - server_cert=/tmp/integration-postgres-server.crt - server_ext=/tmp/integration-postgres-server.ext - - mapfile -t postgres_containers < <( - docker ps --filter publish=5432 --format '{{.ID}}' - ) - test "${#postgres_containers[@]}" -eq 1 - postgres_container="${postgres_containers[0]}" - - openssl genrsa -out "$ca_key" 2048 - openssl req -x509 -new -sha256 -days 1 \ - -key "$ca_key" \ - -subj '/CN=LifeOS Integration CI PostgreSQL CA' \ - -out "$ca_cert" - openssl genrsa -out "$server_key" 2048 - openssl req -new -sha256 \ - -key "$server_key" \ - -subj "/CN=$hostname" \ - -out "$server_csr" - printf 'subjectAltName=DNS:%s\nextendedKeyUsage=serverAuth\n' "$hostname" >"$server_ext" - openssl x509 -req -sha256 -days 1 \ - -in "$server_csr" \ - -CA "$ca_cert" \ - -CAkey "$ca_key" \ - -CAcreateserial \ - -extfile "$server_ext" \ - -out "$server_cert" - chmod 0600 "$server_key" - - docker cp "$server_cert" "$postgres_container:/var/lib/postgresql/server.crt" - docker cp "$server_key" "$postgres_container:/var/lib/postgresql/server.key" - docker exec --user root "$postgres_container" \ - chown postgres:postgres /var/lib/postgresql/server.crt /var/lib/postgresql/server.key - docker exec --user root "$postgres_container" \ - chmod 0644 /var/lib/postgresql/server.crt - docker exec --user root "$postgres_container" \ - chmod 0600 /var/lib/postgresql/server.key - docker exec "$postgres_container" psql -U postgres -d postgres -v ON_ERROR_STOP=1 \ - -c "CREATE ROLE life_os LOGIN PASSWORD 'life_os'" \ - -c "CREATE DATABASE life_os_integration OWNER life_os" - docker exec "$postgres_container" psql -U postgres -d life_os_test -v ON_ERROR_STOP=1 \ - -c "ALTER SYSTEM SET ssl = 'on'" \ - -c "ALTER SYSTEM SET ssl_cert_file = '/var/lib/postgresql/server.crt'" \ - -c "ALTER SYSTEM SET ssl_key_file = '/var/lib/postgresql/server.key'" - docker restart "$postgres_container" >/dev/null - - ready=0 - for _ in $(seq 1 30); do - if docker exec "$postgres_container" pg_isready -U postgres -d life_os_test >/dev/null 2>&1; then - ready=1 - break - fi - sleep 1 - done - test "$ready" -eq 1 - printf '127.0.0.1 %s\n' "$hostname" | sudo tee -a /etc/hosts >/dev/null - printf 'NODE_EXTRA_CA_CERTS=%s\n' "$ca_cert" >>"$GITHUB_ENV" - - rm -f \ - "$ca_key" \ - "$server_key" \ - "$server_csr" \ - "$server_ext" \ - "${ca_cert}.srl" - - - name: Migrate and verify Integration-owned PostgreSQL - shell: bash - run: | - set -Eeuo pipefail - for migration in apps/integration-service/migrations/*.sql; do - psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -f "$migration" - done - psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -tAc \ - 'SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()' | - grep -Fx t - INTEGRATION_DATABASE_URL="${INTEGRATION_DATABASE_URL}?sslmode=verify-full" \ - pnpm --filter @life-os/integration-service exec vitest run \ - src/plugin-operator-replay-postgres.integration.test.ts \ - --no-file-parallelism - psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 \ - -f apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql - - - name: Measure authenticated HTTPS status path without warm-up exclusion - shell: bash - run: | - set -Eeuo pipefail - service_log=/tmp/integration-service.log - proxy_log=/tmp/integration-https-proxy.log - http_ca_key=/tmp/integration-http-ca.key - http_ca_cert=/tmp/integration-http-ca.crt - http_server_key=/tmp/integration-http-server.key - http_server_csr=/tmp/integration-http-server.csr - http_server_cert=/tmp/integration-http-server.crt - http_server_ext=/tmp/integration-http-server.ext - - cleanup() { - status=$? - trap - EXIT - if [ -n "${proxy_pid:-}" ]; then - kill "$proxy_pid" 2>/dev/null || true - wait "$proxy_pid" 2>/dev/null || true - fi - if [ -n "${service_pid:-}" ]; then - kill "$service_pid" 2>/dev/null || true - wait "$service_pid" 2>/dev/null || true - fi - if [ "$status" -ne 0 ]; then - cat "$service_log" || true - cat "$proxy_log" || true - fi - rm -f \ - "$LIFEOS_PERF_AUTHORITY_FILE" \ - "$http_ca_key" \ - "$http_ca_cert" \ - "$http_server_key" \ - "$http_server_csr" \ - "$http_server_cert" \ - "$http_server_ext" \ - "${http_ca_cert}.srl" - exit "$status" - } - trap cleanup EXIT - - openssl genrsa -out "$http_ca_key" 2048 - openssl req -x509 -new -sha256 -days 1 \ - -key "$http_ca_key" \ - -subj '/CN=LifeOS Integration CI HTTPS CA' \ - -out "$http_ca_cert" - openssl genrsa -out "$http_server_key" 2048 - openssl req -new -sha256 \ - -key "$http_server_key" \ - -subj '/CN=127.0.0.1' \ - -out "$http_server_csr" - printf 'subjectAltName=IP:127.0.0.1\nextendedKeyUsage=serverAuth\n' >"$http_server_ext" - openssl x509 -req -sha256 -days 1 \ - -in "$http_server_csr" \ - -CA "$http_ca_cert" \ - -CAkey "$http_ca_key" \ - -CAcreateserial \ - -extfile "$http_server_ext" \ - -out "$http_server_cert" - chmod 0600 "$http_server_key" - - node apps/integration-service/dist/server.js >"$service_log" 2>&1 & - service_pid=$! - - HTTP_SERVER_KEY="$http_server_key" \ - HTTP_SERVER_CERT="$http_server_cert" \ - node --input-type=module >"$proxy_log" 2>&1 <<'NODE' & - import { readFileSync } from 'node:fs'; - import { request } from 'node:http'; - import { createServer } from 'node:https'; - - const targetPort = Number(process.env.INTEGRATION_SERVICE_PORT); - const listenPort = Number(process.env.LIFEOS_PERF_TLS_PROXY_PORT); - const server = createServer( - { - key: readFileSync(process.env.HTTP_SERVER_KEY), - cert: readFileSync(process.env.HTTP_SERVER_CERT), - }, - (requestFromClient, responseToClient) => { - const upstream = request( - { - hostname: '127.0.0.1', - port: targetPort, - method: requestFromClient.method, - path: requestFromClient.url, - headers: requestFromClient.headers, - }, - (upstreamResponse) => { - responseToClient.writeHead( - upstreamResponse.statusCode ?? 502, - upstreamResponse.headers, - ); - upstreamResponse.pipe(responseToClient); - }, - ); - upstream.on('error', () => { - if (!responseToClient.headersSent) { - responseToClient.writeHead(502); - } - responseToClient.end(); - }); - requestFromClient.pipe(upstream); - }, - ); - - server.listen(listenPort, '127.0.0.1'); - process.once('SIGTERM', () => server.close(() => process.exit(0))); - NODE - proxy_pid=$! - - ready=0 - for _ in $(seq 1 30); do - code=$(curl --silent --output /dev/null --write-out '%{http_code}' \ - --cacert "$http_ca_cert" \ - --max-time 1 \ - "$LIFEOS_PERF_BASE_URL/v1/plugins/delivery-attempts/33333333-3333-4333-8333-333333333333" || true) - if [ "$code" = '401' ]; then - ready=1 - break - fi - if ! kill -0 "$service_pid" 2>/dev/null || ! kill -0 "$proxy_pid" 2>/dev/null; then - break - fi - sleep 1 - done - test "$ready" -eq 1 - - node apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ - "$LIFEOS_PERF_AUTHORITY_FILE" - - acceptance_status=0 - set +e - SSL_CERT_FILE="$http_ca_cert" \ - k6 run apps/integration-service/perf/plugin-delivery-attempt-status.k6.js - acceptance_status=$? - set -e - - node apps/integration-service/perf/profile-plugin-delivery-status.mjs - exit "$acceptance_status" From a5de82ba0de4647b0cff766da24a2282b0fccff0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 19 Sep 2026 19:07:02 +0900 Subject: [PATCH 56/58] ci(perf): require replicated exact-head p95 evidence --- .../verify-plugin-delivery-status-k6.yml | 319 ++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 .github/workflows/verify-plugin-delivery-status-k6.yml diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml new file mode 100644 index 000000000..8eb4b47e0 --- /dev/null +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -0,0 +1,319 @@ +name: Verify plugin delivery status k6 + +on: + push: + branches: + - perf/plugin-delivery-status-k6-v1 + +permissions: + contents: read + +concurrency: + group: verify-plugin-delivery-status-k6-${{ github.ref }} + cancel-in-progress: true + +jobs: + status_p95: + name: status_p95_replica_${{ matrix.replica }} + strategy: + fail-fast: false + matrix: + replica: [1, 2, 3] + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + INTEGRATION_DATABASE_URL: postgresql://life_os:life_os@postgres.performance.test:5432/life_os_integration + INTEGRATION_OPERATOR_CONTEXT_SECRET: performance-context-secret-value-32-bytes-minimum + INTEGRATION_PLUGIN_VAULT_ORIGIN: https://vault.performance.example.test + INTEGRATION_PLUGIN_VAULT_TOKEN: performance-vault-fixture-token + INTEGRATION_PLUGIN_VAULT_MOUNT: secret + INTEGRATION_SERVICE_PORT: '4107' + LIFEOS_PERF_TLS_PROXY_PORT: '4443' + LIFEOS_PERF_BASE_URL: https://127.0.0.1:4443 + LIFEOS_PERF_ITERATIONS: '1000' + LIFEOS_PERF_VUS: '10' + LIFEOS_PERF_AUTHORITY_FILE: /tmp/plugin-delivery-status-authority.json + PGSSLMODE: verify-full + PGSSLROOTCERT: /tmp/integration-postgres-ca.crt + services: + postgres: + image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_DB: life_os_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d life_os_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - name: Checkout exact verifier head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 1 + persist-credentials: false + ref: ${{ github.sha }} + + - name: Set up Node 24 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: '24' + + - name: Set up k6 + uses: grafana/setup-k6-action@db07bd9765aac508ef18982e52ab937fe633a065 + with: + k6-version: '2.2.0' + + - name: Install and verify Integration source + shell: bash + run: | + set -Eeuo pipefail + corepack enable + pnpm install --frozen-lockfile + pnpm --filter @life-os/plugin-sdk build + pnpm --filter @life-os/integration-service typecheck + pnpm --filter @life-os/integration-service build + env -u INTEGRATION_DATABASE_URL \ + pnpm --filter @life-os/integration-service test + pnpm exec prettier --single-quote --check \ + .github/workflows/verify-plugin-delivery-status-k6.yml \ + apps/integration-service/src/plugin-operator-replay.ts \ + apps/integration-service/src/plugin-operator-replay.test.ts \ + apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts \ + apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ + apps/integration-service/perf/plugin-delivery-attempt-status.k6.js \ + apps/integration-service/perf/profile-plugin-delivery-status.mjs + + - name: Configure verified-TLS PostgreSQL fixture + shell: bash + run: | + set -Eeuo pipefail + hostname=postgres.performance.test + ca_key=/tmp/integration-postgres-ca.key + ca_cert="$PGSSLROOTCERT" + server_key=/tmp/integration-postgres-server.key + server_csr=/tmp/integration-postgres-server.csr + server_cert=/tmp/integration-postgres-server.crt + server_ext=/tmp/integration-postgres-server.ext + + mapfile -t postgres_containers < <( + docker ps --filter publish=5432 --format '{{.ID}}' + ) + test "${#postgres_containers[@]}" -eq 1 + postgres_container="${postgres_containers[0]}" + + openssl genrsa -out "$ca_key" 2048 + openssl req -x509 -new -sha256 -days 1 \ + -key "$ca_key" \ + -subj '/CN=LifeOS Integration CI PostgreSQL CA' \ + -out "$ca_cert" + openssl genrsa -out "$server_key" 2048 + openssl req -new -sha256 \ + -key "$server_key" \ + -subj "/CN=$hostname" \ + -out "$server_csr" + printf 'subjectAltName=DNS:%s\nextendedKeyUsage=serverAuth\n' "$hostname" >"$server_ext" + openssl x509 -req -sha256 -days 1 \ + -in "$server_csr" \ + -CA "$ca_cert" \ + -CAkey "$ca_key" \ + -CAcreateserial \ + -extfile "$server_ext" \ + -out "$server_cert" + chmod 0600 "$server_key" + + docker cp "$server_cert" "$postgres_container:/var/lib/postgresql/server.crt" + docker cp "$server_key" "$postgres_container:/var/lib/postgresql/server.key" + docker exec --user root "$postgres_container" \ + chown postgres:postgres /var/lib/postgresql/server.crt /var/lib/postgresql/server.key + docker exec --user root "$postgres_container" \ + chmod 0644 /var/lib/postgresql/server.crt + docker exec --user root "$postgres_container" \ + chmod 0600 /var/lib/postgresql/server.key + docker exec "$postgres_container" psql -U postgres -d postgres -v ON_ERROR_STOP=1 \ + -c "CREATE ROLE life_os LOGIN PASSWORD 'life_os'" \ + -c "CREATE DATABASE life_os_integration OWNER life_os" + docker exec "$postgres_container" psql -U postgres -d life_os_test -v ON_ERROR_STOP=1 \ + -c "ALTER SYSTEM SET ssl = 'on'" \ + -c "ALTER SYSTEM SET ssl_cert_file = '/var/lib/postgresql/server.crt'" \ + -c "ALTER SYSTEM SET ssl_key_file = '/var/lib/postgresql/server.key'" + docker restart "$postgres_container" >/dev/null + + ready=0 + for _ in $(seq 1 30); do + if docker exec "$postgres_container" pg_isready -U postgres -d life_os_test >/dev/null 2>&1; then + ready=1 + break + fi + sleep 1 + done + test "$ready" -eq 1 + printf '127.0.0.1 %s\n' "$hostname" | sudo tee -a /etc/hosts >/dev/null + printf 'NODE_EXTRA_CA_CERTS=%s\n' "$ca_cert" >>"$GITHUB_ENV" + + rm -f \ + "$ca_key" \ + "$server_key" \ + "$server_csr" \ + "$server_ext" \ + "${ca_cert}.srl" + + - name: Migrate and verify Integration-owned PostgreSQL + shell: bash + run: | + set -Eeuo pipefail + for migration in apps/integration-service/migrations/*.sql; do + psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -f "$migration" + done + psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -tAc \ + 'SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()' | + grep -Fx t + INTEGRATION_DATABASE_URL="${INTEGRATION_DATABASE_URL}?sslmode=verify-full" \ + pnpm --filter @life-os/integration-service exec vitest run \ + src/plugin-operator-replay-postgres.integration.test.ts \ + --no-file-parallelism + psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 \ + -f apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql + + - name: Measure authenticated HTTPS status path without warm-up exclusion + shell: bash + run: | + set -Eeuo pipefail + service_log=/tmp/integration-service.log + proxy_log=/tmp/integration-https-proxy.log + http_ca_key=/tmp/integration-http-ca.key + http_ca_cert=/tmp/integration-http-ca.crt + http_server_key=/tmp/integration-http-server.key + http_server_csr=/tmp/integration-http-server.csr + http_server_cert=/tmp/integration-http-server.crt + http_server_ext=/tmp/integration-http-server.ext + + cleanup() { + status=$? + trap - EXIT + if [ -n "${proxy_pid:-}" ]; then + kill "$proxy_pid" 2>/dev/null || true + wait "$proxy_pid" 2>/dev/null || true + fi + if [ -n "${service_pid:-}" ]; then + kill "$service_pid" 2>/dev/null || true + wait "$service_pid" 2>/dev/null || true + fi + if [ "$status" -ne 0 ]; then + cat "$service_log" || true + cat "$proxy_log" || true + fi + rm -f \ + "$LIFEOS_PERF_AUTHORITY_FILE" \ + "$http_ca_key" \ + "$http_ca_cert" \ + "$http_server_key" \ + "$http_server_csr" \ + "$http_server_cert" \ + "$http_server_ext" \ + "${http_ca_cert}.srl" + exit "$status" + } + trap cleanup EXIT + + openssl genrsa -out "$http_ca_key" 2048 + openssl req -x509 -new -sha256 -days 1 \ + -key "$http_ca_key" \ + -subj '/CN=LifeOS Integration CI HTTPS CA' \ + -out "$http_ca_cert" + openssl genrsa -out "$http_server_key" 2048 + openssl req -new -sha256 \ + -key "$http_server_key" \ + -subj '/CN=127.0.0.1' \ + -out "$http_server_csr" + printf 'subjectAltName=IP:127.0.0.1\nextendedKeyUsage=serverAuth\n' >"$http_server_ext" + openssl x509 -req -sha256 -days 1 \ + -in "$http_server_csr" \ + -CA "$http_ca_cert" \ + -CAkey "$http_ca_key" \ + -CAcreateserial \ + -extfile "$http_server_ext" \ + -out "$http_server_cert" + chmod 0600 "$http_server_key" + + node apps/integration-service/dist/server.js >"$service_log" 2>&1 & + service_pid=$! + + HTTP_SERVER_KEY="$http_server_key" \ + HTTP_SERVER_CERT="$http_server_cert" \ + node --input-type=module >"$proxy_log" 2>&1 <<'NODE' & + import { readFileSync } from 'node:fs'; + import { request } from 'node:http'; + import { createServer } from 'node:https'; + + const targetPort = Number(process.env.INTEGRATION_SERVICE_PORT); + const listenPort = Number(process.env.LIFEOS_PERF_TLS_PROXY_PORT); + const server = createServer( + { + key: readFileSync(process.env.HTTP_SERVER_KEY), + cert: readFileSync(process.env.HTTP_SERVER_CERT), + }, + (requestFromClient, responseToClient) => { + const upstream = request( + { + hostname: '127.0.0.1', + port: targetPort, + method: requestFromClient.method, + path: requestFromClient.url, + headers: requestFromClient.headers, + }, + (upstreamResponse) => { + responseToClient.writeHead( + upstreamResponse.statusCode ?? 502, + upstreamResponse.headers, + ); + upstreamResponse.pipe(responseToClient); + }, + ); + upstream.on('error', () => { + if (!responseToClient.headersSent) { + responseToClient.writeHead(502); + } + responseToClient.end(); + }); + requestFromClient.pipe(upstream); + }, + ); + + server.listen(listenPort, '127.0.0.1'); + process.once('SIGTERM', () => server.close(() => process.exit(0))); + NODE + proxy_pid=$! + + ready=0 + for _ in $(seq 1 30); do + code=$(curl --silent --output /dev/null --write-out '%{http_code}' \ + --cacert "$http_ca_cert" \ + --max-time 1 \ + "$LIFEOS_PERF_BASE_URL/v1/plugins/delivery-attempts/33333333-3333-4333-8333-333333333333" || true) + if [ "$code" = '401' ]; then + ready=1 + break + fi + if ! kill -0 "$service_pid" 2>/dev/null || ! kill -0 "$proxy_pid" 2>/dev/null; then + break + fi + sleep 1 + done + test "$ready" -eq 1 + + node apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ + "$LIFEOS_PERF_AUTHORITY_FILE" + + acceptance_status=0 + set +e + SSL_CERT_FILE="$http_ca_cert" \ + k6 run apps/integration-service/perf/plugin-delivery-attempt-status.k6.js + acceptance_status=$? + set -e + + node apps/integration-service/perf/profile-plugin-delivery-status.mjs + exit "$acceptance_status" From 24d8079642054f8fc470851d5dab8317fb2ee9bb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 19 Sep 2026 23:07:37 +0900 Subject: [PATCH 57/58] ci(perf): retain replicated p95 evidence --- .../verify-plugin-delivery-status-k6.yml | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml index 8eb4b47e0..65f9417b1 100644 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ b/.github/workflows/verify-plugin-delivery-status-k6.yml @@ -33,6 +33,8 @@ jobs: LIFEOS_PERF_ITERATIONS: '1000' LIFEOS_PERF_VUS: '10' LIFEOS_PERF_AUTHORITY_FILE: /tmp/plugin-delivery-status-authority.json + LIFEOS_PERF_K6_LOG: /tmp/plugin-delivery-status-k6.log + LIFEOS_PERF_PROFILE_LOG: /tmp/plugin-delivery-status-profile.ndjson PGSSLMODE: verify-full PGSSLROOTCERT: /tmp/integration-postgres-ca.crt services: @@ -179,6 +181,7 @@ jobs: -f apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql - name: Measure authenticated HTTPS status path without warm-up exclusion + id: performance shell: bash run: | set -Eeuo pipefail @@ -311,9 +314,22 @@ jobs: acceptance_status=0 set +e SSL_CERT_FILE="$http_ca_cert" \ - k6 run apps/integration-service/perf/plugin-delivery-attempt-status.k6.js - acceptance_status=$? + k6 run apps/integration-service/perf/plugin-delivery-attempt-status.k6.js \ + 2>&1 | tee "$LIFEOS_PERF_K6_LOG" + acceptance_status=${PIPESTATUS[0]} set -e - node apps/integration-service/perf/profile-plugin-delivery-status.mjs + node apps/integration-service/perf/profile-plugin-delivery-status.mjs | + tee "$LIFEOS_PERF_PROFILE_LOG" exit "$acceptance_status" + + - name: Retain per-replica performance evidence + if: ${{ always() && steps.performance.outcome != 'skipped' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: plugin-delivery-status-p95-replica-${{ matrix.replica }} + path: | + /tmp/plugin-delivery-status-k6.log + /tmp/plugin-delivery-status-profile.ndjson + if-no-files-found: error + retention-days: 7 From 7967976ffe11fa0afd71250ce751abe63456adf2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 05:05:20 +0900 Subject: [PATCH 58/58] ci(perf): retire proven delivery-status verifier --- .../verify-plugin-delivery-status-k6.yml | 335 ------------------ 1 file changed, 335 deletions(-) delete mode 100644 .github/workflows/verify-plugin-delivery-status-k6.yml diff --git a/.github/workflows/verify-plugin-delivery-status-k6.yml b/.github/workflows/verify-plugin-delivery-status-k6.yml deleted file mode 100644 index 65f9417b1..000000000 --- a/.github/workflows/verify-plugin-delivery-status-k6.yml +++ /dev/null @@ -1,335 +0,0 @@ -name: Verify plugin delivery status k6 - -on: - push: - branches: - - perf/plugin-delivery-status-k6-v1 - -permissions: - contents: read - -concurrency: - group: verify-plugin-delivery-status-k6-${{ github.ref }} - cancel-in-progress: true - -jobs: - status_p95: - name: status_p95_replica_${{ matrix.replica }} - strategy: - fail-fast: false - matrix: - replica: [1, 2, 3] - runs-on: ubuntu-24.04 - timeout-minutes: 15 - env: - INTEGRATION_DATABASE_URL: postgresql://life_os:life_os@postgres.performance.test:5432/life_os_integration - INTEGRATION_OPERATOR_CONTEXT_SECRET: performance-context-secret-value-32-bytes-minimum - INTEGRATION_PLUGIN_VAULT_ORIGIN: https://vault.performance.example.test - INTEGRATION_PLUGIN_VAULT_TOKEN: performance-vault-fixture-token - INTEGRATION_PLUGIN_VAULT_MOUNT: secret - INTEGRATION_SERVICE_PORT: '4107' - LIFEOS_PERF_TLS_PROXY_PORT: '4443' - LIFEOS_PERF_BASE_URL: https://127.0.0.1:4443 - LIFEOS_PERF_ITERATIONS: '1000' - LIFEOS_PERF_VUS: '10' - LIFEOS_PERF_AUTHORITY_FILE: /tmp/plugin-delivery-status-authority.json - LIFEOS_PERF_K6_LOG: /tmp/plugin-delivery-status-k6.log - LIFEOS_PERF_PROFILE_LOG: /tmp/plugin-delivery-status-profile.ndjson - PGSSLMODE: verify-full - PGSSLROOTCERT: /tmp/integration-postgres-ca.crt - services: - postgres: - image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 - env: - POSTGRES_DB: life_os_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U postgres -d life_os_test" - --health-interval 5s - --health-timeout 5s - --health-retries 10 - steps: - - name: Checkout exact verifier head - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - fetch-depth: 1 - persist-credentials: false - ref: ${{ github.sha }} - - - name: Set up Node 24 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 - with: - node-version: '24' - - - name: Set up k6 - uses: grafana/setup-k6-action@db07bd9765aac508ef18982e52ab937fe633a065 - with: - k6-version: '2.2.0' - - - name: Install and verify Integration source - shell: bash - run: | - set -Eeuo pipefail - corepack enable - pnpm install --frozen-lockfile - pnpm --filter @life-os/plugin-sdk build - pnpm --filter @life-os/integration-service typecheck - pnpm --filter @life-os/integration-service build - env -u INTEGRATION_DATABASE_URL \ - pnpm --filter @life-os/integration-service test - pnpm exec prettier --single-quote --check \ - .github/workflows/verify-plugin-delivery-status-k6.yml \ - apps/integration-service/src/plugin-operator-replay.ts \ - apps/integration-service/src/plugin-operator-replay.test.ts \ - apps/integration-service/src/plugin-operator-replay-postgres.integration.test.ts \ - apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ - apps/integration-service/perf/plugin-delivery-attempt-status.k6.js \ - apps/integration-service/perf/profile-plugin-delivery-status.mjs - - - name: Configure verified-TLS PostgreSQL fixture - shell: bash - run: | - set -Eeuo pipefail - hostname=postgres.performance.test - ca_key=/tmp/integration-postgres-ca.key - ca_cert="$PGSSLROOTCERT" - server_key=/tmp/integration-postgres-server.key - server_csr=/tmp/integration-postgres-server.csr - server_cert=/tmp/integration-postgres-server.crt - server_ext=/tmp/integration-postgres-server.ext - - mapfile -t postgres_containers < <( - docker ps --filter publish=5432 --format '{{.ID}}' - ) - test "${#postgres_containers[@]}" -eq 1 - postgres_container="${postgres_containers[0]}" - - openssl genrsa -out "$ca_key" 2048 - openssl req -x509 -new -sha256 -days 1 \ - -key "$ca_key" \ - -subj '/CN=LifeOS Integration CI PostgreSQL CA' \ - -out "$ca_cert" - openssl genrsa -out "$server_key" 2048 - openssl req -new -sha256 \ - -key "$server_key" \ - -subj "/CN=$hostname" \ - -out "$server_csr" - printf 'subjectAltName=DNS:%s\nextendedKeyUsage=serverAuth\n' "$hostname" >"$server_ext" - openssl x509 -req -sha256 -days 1 \ - -in "$server_csr" \ - -CA "$ca_cert" \ - -CAkey "$ca_key" \ - -CAcreateserial \ - -extfile "$server_ext" \ - -out "$server_cert" - chmod 0600 "$server_key" - - docker cp "$server_cert" "$postgres_container:/var/lib/postgresql/server.crt" - docker cp "$server_key" "$postgres_container:/var/lib/postgresql/server.key" - docker exec --user root "$postgres_container" \ - chown postgres:postgres /var/lib/postgresql/server.crt /var/lib/postgresql/server.key - docker exec --user root "$postgres_container" \ - chmod 0644 /var/lib/postgresql/server.crt - docker exec --user root "$postgres_container" \ - chmod 0600 /var/lib/postgresql/server.key - docker exec "$postgres_container" psql -U postgres -d postgres -v ON_ERROR_STOP=1 \ - -c "CREATE ROLE life_os LOGIN PASSWORD 'life_os'" \ - -c "CREATE DATABASE life_os_integration OWNER life_os" - docker exec "$postgres_container" psql -U postgres -d life_os_test -v ON_ERROR_STOP=1 \ - -c "ALTER SYSTEM SET ssl = 'on'" \ - -c "ALTER SYSTEM SET ssl_cert_file = '/var/lib/postgresql/server.crt'" \ - -c "ALTER SYSTEM SET ssl_key_file = '/var/lib/postgresql/server.key'" - docker restart "$postgres_container" >/dev/null - - ready=0 - for _ in $(seq 1 30); do - if docker exec "$postgres_container" pg_isready -U postgres -d life_os_test >/dev/null 2>&1; then - ready=1 - break - fi - sleep 1 - done - test "$ready" -eq 1 - printf '127.0.0.1 %s\n' "$hostname" | sudo tee -a /etc/hosts >/dev/null - printf 'NODE_EXTRA_CA_CERTS=%s\n' "$ca_cert" >>"$GITHUB_ENV" - - rm -f \ - "$ca_key" \ - "$server_key" \ - "$server_csr" \ - "$server_ext" \ - "${ca_cert}.srl" - - - name: Migrate and verify Integration-owned PostgreSQL - shell: bash - run: | - set -Eeuo pipefail - for migration in apps/integration-service/migrations/*.sql; do - psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -f "$migration" - done - psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 -tAc \ - 'SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()' | - grep -Fx t - INTEGRATION_DATABASE_URL="${INTEGRATION_DATABASE_URL}?sslmode=verify-full" \ - pnpm --filter @life-os/integration-service exec vitest run \ - src/plugin-operator-replay-postgres.integration.test.ts \ - --no-file-parallelism - psql "$INTEGRATION_DATABASE_URL" -v ON_ERROR_STOP=1 \ - -f apps/integration-service/perf/seed-plugin-delivery-attempt-status.sql - - - name: Measure authenticated HTTPS status path without warm-up exclusion - id: performance - shell: bash - run: | - set -Eeuo pipefail - service_log=/tmp/integration-service.log - proxy_log=/tmp/integration-https-proxy.log - http_ca_key=/tmp/integration-http-ca.key - http_ca_cert=/tmp/integration-http-ca.crt - http_server_key=/tmp/integration-http-server.key - http_server_csr=/tmp/integration-http-server.csr - http_server_cert=/tmp/integration-http-server.crt - http_server_ext=/tmp/integration-http-server.ext - - cleanup() { - status=$? - trap - EXIT - if [ -n "${proxy_pid:-}" ]; then - kill "$proxy_pid" 2>/dev/null || true - wait "$proxy_pid" 2>/dev/null || true - fi - if [ -n "${service_pid:-}" ]; then - kill "$service_pid" 2>/dev/null || true - wait "$service_pid" 2>/dev/null || true - fi - if [ "$status" -ne 0 ]; then - cat "$service_log" || true - cat "$proxy_log" || true - fi - rm -f \ - "$LIFEOS_PERF_AUTHORITY_FILE" \ - "$http_ca_key" \ - "$http_ca_cert" \ - "$http_server_key" \ - "$http_server_csr" \ - "$http_server_cert" \ - "$http_server_ext" \ - "${http_ca_cert}.srl" - exit "$status" - } - trap cleanup EXIT - - openssl genrsa -out "$http_ca_key" 2048 - openssl req -x509 -new -sha256 -days 1 \ - -key "$http_ca_key" \ - -subj '/CN=LifeOS Integration CI HTTPS CA' \ - -out "$http_ca_cert" - openssl genrsa -out "$http_server_key" 2048 - openssl req -new -sha256 \ - -key "$http_server_key" \ - -subj '/CN=127.0.0.1' \ - -out "$http_server_csr" - printf 'subjectAltName=IP:127.0.0.1\nextendedKeyUsage=serverAuth\n' >"$http_server_ext" - openssl x509 -req -sha256 -days 1 \ - -in "$http_server_csr" \ - -CA "$http_ca_cert" \ - -CAkey "$http_ca_key" \ - -CAcreateserial \ - -extfile "$http_server_ext" \ - -out "$http_server_cert" - chmod 0600 "$http_server_key" - - node apps/integration-service/dist/server.js >"$service_log" 2>&1 & - service_pid=$! - - HTTP_SERVER_KEY="$http_server_key" \ - HTTP_SERVER_CERT="$http_server_cert" \ - node --input-type=module >"$proxy_log" 2>&1 <<'NODE' & - import { readFileSync } from 'node:fs'; - import { request } from 'node:http'; - import { createServer } from 'node:https'; - - const targetPort = Number(process.env.INTEGRATION_SERVICE_PORT); - const listenPort = Number(process.env.LIFEOS_PERF_TLS_PROXY_PORT); - const server = createServer( - { - key: readFileSync(process.env.HTTP_SERVER_KEY), - cert: readFileSync(process.env.HTTP_SERVER_CERT), - }, - (requestFromClient, responseToClient) => { - const upstream = request( - { - hostname: '127.0.0.1', - port: targetPort, - method: requestFromClient.method, - path: requestFromClient.url, - headers: requestFromClient.headers, - }, - (upstreamResponse) => { - responseToClient.writeHead( - upstreamResponse.statusCode ?? 502, - upstreamResponse.headers, - ); - upstreamResponse.pipe(responseToClient); - }, - ); - upstream.on('error', () => { - if (!responseToClient.headersSent) { - responseToClient.writeHead(502); - } - responseToClient.end(); - }); - requestFromClient.pipe(upstream); - }, - ); - - server.listen(listenPort, '127.0.0.1'); - process.once('SIGTERM', () => server.close(() => process.exit(0))); - NODE - proxy_pid=$! - - ready=0 - for _ in $(seq 1 30); do - code=$(curl --silent --output /dev/null --write-out '%{http_code}' \ - --cacert "$http_ca_cert" \ - --max-time 1 \ - "$LIFEOS_PERF_BASE_URL/v1/plugins/delivery-attempts/33333333-3333-4333-8333-333333333333" || true) - if [ "$code" = '401' ]; then - ready=1 - break - fi - if ! kill -0 "$service_pid" 2>/dev/null || ! kill -0 "$proxy_pid" 2>/dev/null; then - break - fi - sleep 1 - done - test "$ready" -eq 1 - - node apps/integration-service/perf/generate-plugin-delivery-status-authority.mjs \ - "$LIFEOS_PERF_AUTHORITY_FILE" - - acceptance_status=0 - set +e - SSL_CERT_FILE="$http_ca_cert" \ - k6 run apps/integration-service/perf/plugin-delivery-attempt-status.k6.js \ - 2>&1 | tee "$LIFEOS_PERF_K6_LOG" - acceptance_status=${PIPESTATUS[0]} - set -e - - node apps/integration-service/perf/profile-plugin-delivery-status.mjs | - tee "$LIFEOS_PERF_PROFILE_LOG" - exit "$acceptance_status" - - - name: Retain per-replica performance evidence - if: ${{ always() && steps.performance.outcome != 'skipped' }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: plugin-delivery-status-p95-replica-${{ matrix.replica }} - path: | - /tmp/plugin-delivery-status-k6.log - /tmp/plugin-delivery-status-profile.ndjson - if-no-files-found: error - retention-days: 7