From a3c92b3cb5ae005f0c47a9d7d70b7e5749ed40dc Mon Sep 17 00:00:00 2001 From: susumutomita Date: Tue, 2 Jun 2026 14:07:53 +0900 Subject: [PATCH] feat(disruption): cross-account executor + CDK construct (ADR-031) [#1419] --- .../handler-no-direct-sdk-import.json | 42 +++++ bun.lock | 43 +++++ .../disruption-executor-lambda.ts | 176 ++++++++++++++++++ .../handlers/describe-stack-handler/index.ts | 149 +-------------- .../dispatch-command.ts | 126 +++++++++++++ .../disruption-executor-handler/execute.ts | 122 ++++++++++++ .../executor-store.ts | 105 +++++++++++ .../disruption-executor-handler/index.ts | 108 +++++++++++ .../disruption-executor-handler/route.ts | 90 +++++++++ .../schedule-revert.ts | 72 +++++++ .../send-dispatch.ts | 107 +++++++++++ .../handlers/shared/assume-competitor-role.ts | 169 +++++++++++++++++ .../problem-deploy-backend-stack.ts | 12 ++ infrastructure/package.json | 1 + ...roblem-deploy-backend-stack-events.test.ts | 5 +- .../assume-competitor-role.test.ts | 133 +++++++++++++ .../disruption-dispatch-command.test.ts | 150 +++++++++++++++ .../problem-deploy/disruption-execute.test.ts | 135 ++++++++++++++ .../disruption-executor-lambda.test.ts | 136 ++++++++++++++ .../disruption-executor-store.test.ts | 125 +++++++++++++ .../problem-deploy/disruption-route.test.ts | 126 +++++++++++++ .../disruption-schedule-revert.test.ts | 103 ++++++++++ .../disruption-send-dispatch.test.ts | 102 ++++++++++ 23 files changed, 2191 insertions(+), 146 deletions(-) create mode 100644 infrastructure/lib/problem-deploy/disruption-executor-lambda.ts create mode 100644 infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/dispatch-command.ts create mode 100644 infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/execute.ts create mode 100644 infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/executor-store.ts create mode 100644 infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/index.ts create mode 100644 infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/route.ts create mode 100644 infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/schedule-revert.ts create mode 100644 infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/send-dispatch.ts create mode 100644 infrastructure/lib/problem-deploy/handlers/shared/assume-competitor-role.ts create mode 100644 infrastructure/test/problem-deploy/assume-competitor-role.test.ts create mode 100644 infrastructure/test/problem-deploy/disruption-dispatch-command.test.ts create mode 100644 infrastructure/test/problem-deploy/disruption-execute.test.ts create mode 100644 infrastructure/test/problem-deploy/disruption-executor-lambda.test.ts create mode 100644 infrastructure/test/problem-deploy/disruption-executor-store.test.ts create mode 100644 infrastructure/test/problem-deploy/disruption-route.test.ts create mode 100644 infrastructure/test/problem-deploy/disruption-schedule-revert.test.ts create mode 100644 infrastructure/test/problem-deploy/disruption-send-dispatch.test.ts diff --git a/.claude/harness/baselines/handler-no-direct-sdk-import.json b/.claude/harness/baselines/handler-no-direct-sdk-import.json index 56eec0411..d54da9090 100644 --- a/.claude/harness/baselines/handler-no-direct-sdk-import.json +++ b/.claude/harness/baselines/handler-no-direct-sdk-import.json @@ -29,6 +29,48 @@ "filePath": "infrastructure/lib/problem-deploy/handlers/generic-scoring-handler/index.ts", "line": 7, "match": "@aws-sdk/lib-dynamodb" + }, + { + "ruleId": "handler-no-direct-sdk-import", + "filePath": "infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/index.ts", + "line": 15, + "match": "@aws-sdk/client-cloudformation" + }, + { + "ruleId": "handler-no-direct-sdk-import", + "filePath": "infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/index.ts", + "line": 16, + "match": "@aws-sdk/client-dynamodb" + }, + { + "ruleId": "handler-no-direct-sdk-import", + "filePath": "infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/index.ts", + "line": 17, + "match": "@aws-sdk/client-lambda" + }, + { + "ruleId": "handler-no-direct-sdk-import", + "filePath": "infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/index.ts", + "line": 18, + "match": "@aws-sdk/client-scheduler" + }, + { + "ruleId": "handler-no-direct-sdk-import", + "filePath": "infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/index.ts", + "line": 19, + "match": "@aws-sdk/client-ssm" + }, + { + "ruleId": "handler-no-direct-sdk-import", + "filePath": "infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/index.ts", + "line": 20, + "match": "@aws-sdk/client-sts" + }, + { + "ruleId": "handler-no-direct-sdk-import", + "filePath": "infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/index.ts", + "line": 21, + "match": "@aws-sdk/lib-dynamodb" } ] } diff --git a/bun.lock b/bun.lock index 6658df7db..5fd96b371 100644 --- a/bun.lock +++ b/bun.lock @@ -132,6 +132,7 @@ "@aws-sdk/client-cloudwatch-logs": "^3.1053.0", "@aws-sdk/client-codebuild": "^3.1048.0", "@aws-sdk/client-codepipeline": "^3.1048.0", + "@aws-sdk/client-scheduler": "^3.1048.0", "@aws-sdk/client-sfn": "^3.1048.0", "@cdklabs/sbt-aws": "0.3.9", "@tenkacloud/coordination-plugin-sdk": "workspace:*", @@ -333,6 +334,8 @@ "@aws-sdk/client-s3": ["@aws-sdk/client-s3@3.1053.0", "", { "dependencies": { "@aws-crypto/sha1-browser": "5.2.0", "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.13", "@aws-sdk/credential-provider-node": "^3.972.44", "@aws-sdk/middleware-bucket-endpoint": "^3.972.15", "@aws-sdk/middleware-expect-continue": "^3.972.13", "@aws-sdk/middleware-flexible-checksums": "^3.974.21", "@aws-sdk/middleware-location-constraint": "^3.972.11", "@aws-sdk/middleware-sdk-s3": "^3.972.42", "@aws-sdk/middleware-ssec": "^3.972.11", "@aws-sdk/signature-v4-multi-region": "^3.996.28", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.3", "@smithy/fetch-http-handler": "^5.4.3", "@smithy/node-http-handler": "^4.7.3", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-/oGxoB6p1Nqs935Blt+v1o+anSCEf2n3RjIrcLz84i4cn2Gr+Z7JpDdUkG5+74r5ctqEPG7k/phTGbJ9fNKnHg=="], + "@aws-sdk/client-scheduler": ["@aws-sdk/client-scheduler@3.1058.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.15", "@aws-sdk/credential-provider-node": "^3.972.48", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/fetch-http-handler": "^5.4.5", "@smithy/node-http-handler": "^4.7.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-yBY1eFXWv7SC6ny436XBicdBUBZh4gvXlybi76VBftRX+FyZ1NKbVSvuxjSldqf2bwiJ+f3GvcYqCoOEUnCQSA=="], + "@aws-sdk/client-sfn": ["@aws-sdk/client-sfn@3.1048.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.11", "@aws-sdk/credential-provider-node": "^3.972.42", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/fetch-http-handler": "^5.4.2", "@smithy/node-http-handler": "^4.7.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-f+F2WkFX9HOqd7Wq7phc45olVkd8jpTgbZmxPvj2ItixrgT/XiobZn6vLmUF+b/2f0wO074yFT3p2go8If7K9Q=="], "@aws-sdk/client-ssm": ["@aws-sdk/client-ssm@3.1048.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.11", "@aws-sdk/credential-provider-node": "^3.972.42", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/fetch-http-handler": "^5.4.2", "@smithy/node-http-handler": "^4.7.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-Q/9t+BeHQnbsYCmvNwh7FDs5JQATuqoSRPdhYZ5hij/mG431VTDWVxqwD4Gze+4AMDVPcOTi9Jv0fXYc1BJVRg=="], @@ -2045,6 +2048,18 @@ "@aws-sdk/client-s3/@aws-sdk/types": ["@aws-sdk/types@3.973.9", "", { "dependencies": { "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-kuBfgQVdcz5Bmapc4A13YbpVw/pXkesfhetcFYwbntqas8sF41OHyd4o28+/TG2ZQdHBsv90Lsu5y6oitvYCdg=="], + "@aws-sdk/client-scheduler/@aws-sdk/core": ["@aws-sdk/core@3.974.15", "", { "dependencies": { "@aws-sdk/types": "^3.973.9", "@aws-sdk/xml-builder": "^3.972.26", "@aws/lambda-invoke-store": "^0.2.2", "@smithy/core": "^3.24.5", "@smithy/signature-v4": "^5.4.5", "@smithy/types": "^4.14.2", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-UpA0rTGW/tHGITcCqHisbuuEPraYg9GG+mWmXjY5+RxZBMLGe6aL9oe0ix50LztwAcPIkGZLH0yWdMIkCM10hw=="], + + "@aws-sdk/client-scheduler/@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.48", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.41", "@aws-sdk/credential-provider-http": "^3.972.43", "@aws-sdk/credential-provider-ini": "^3.972.46", "@aws-sdk/credential-provider-process": "^3.972.41", "@aws-sdk/credential-provider-sso": "^3.972.45", "@aws-sdk/credential-provider-web-identity": "^3.972.45", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/credential-provider-imds": "^4.3.6", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-QIbtJP0olSLZ2ImEu636pP+7JJbPfaL3xSJIFXhu472CWuondCc4bGOa8OeyhOFet8z4H1D/ZFKXc39FboWwYA=="], + + "@aws-sdk/client-scheduler/@aws-sdk/types": ["@aws-sdk/types@3.973.9", "", { "dependencies": { "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-kuBfgQVdcz5Bmapc4A13YbpVw/pXkesfhetcFYwbntqas8sF41OHyd4o28+/TG2ZQdHBsv90Lsu5y6oitvYCdg=="], + + "@aws-sdk/client-scheduler/@smithy/core": ["@smithy/core@3.24.5", "", { "dependencies": { "@aws-crypto/crc32": "5.2.0", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-Kt8phUg45M15EjhYAbZ+fFikYneijLu9Liugz8ZsYz2i8j0hzGv27LWKpEHYRfvj+LyCOSijpcR/2i8RouV+cA=="], + + "@aws-sdk/client-scheduler/@smithy/fetch-http-handler": ["@smithy/fetch-http-handler@5.4.5", "", { "dependencies": { "@smithy/core": "^3.24.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-SK3VMeH0fibgdTg2QeB+O4p7Yy/2E5HBOHJeC58FshkDdeuX8lOgO7PfjYfLyPLP1ch55j91cQqKBzDS0mRjSQ=="], + + "@aws-sdk/client-scheduler/@smithy/node-http-handler": ["@smithy/node-http-handler@4.7.5", "", { "dependencies": { "@smithy/core": "^3.24.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-3dA9TQ+ybRSZ/m0wnbZhiBy4Dezjgq1Ib/ZZrYTpJDBgpoLLU/SDzZc/g0x0MNAdOJe1wPcM+x2PBRmoOur+Sw=="], + "@aws-sdk/client-sfn/@smithy/types": ["@smithy/types@4.14.1", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-59b5HtSVrVR/eYNei3BUj3DCPKD/G7EtDDe7OEJE7i7FtQFugYo6MxbotS8mVJkLNVf8gYaAlEBwwtJ9HzhWSg=="], "@aws-sdk/client-sts/@aws-sdk/core": ["@aws-sdk/core@3.974.15", "", { "dependencies": { "@aws-sdk/types": "^3.973.9", "@aws-sdk/xml-builder": "^3.972.26", "@aws/lambda-invoke-store": "^0.2.2", "@smithy/core": "^3.24.5", "@smithy/signature-v4": "^5.4.5", "@smithy/types": "^4.14.2", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-UpA0rTGW/tHGITcCqHisbuuEPraYg9GG+mWmXjY5+RxZBMLGe6aL9oe0ix50LztwAcPIkGZLH0yWdMIkCM10hw=="], @@ -2337,6 +2352,24 @@ "@aws-sdk/client-s3/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity": ["@aws-sdk/credential-provider-web-identity@3.972.43", "", { "dependencies": { "@aws-sdk/core": "^3.974.13", "@aws-sdk/nested-clients": "^3.997.11", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.3", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-wQtL34lUD/09VXjwAUo2T+I3aEXRDxMB3DKmTJL/Zj0Gi6sLDTrVhae1XVt01yzkquOWajI/sZW72JGDZ1ciTw=="], + "@aws-sdk/client-scheduler/@aws-sdk/core/@aws-sdk/xml-builder": ["@aws-sdk/xml-builder@3.972.26", "", { "dependencies": { "@smithy/types": "^4.14.2", "fast-xml-parser": "5.7.3", "tslib": "^2.6.2" } }, "sha512-cDbrqvDS73whl6YAPSPq0U6whzG6UWI9PuWh0wrUuGoZexhWEqhdunbukV7iBoaWnFV1AODutM5hOD6rtn439g=="], + + "@aws-sdk/client-scheduler/@aws-sdk/core/@smithy/signature-v4": ["@smithy/signature-v4@5.4.5", "", { "dependencies": { "@smithy/core": "^3.24.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-QBJKWGqIknH0dc9LWpfH1mkdokAx6iXYN3UcQ3eY6uIEyScuoQAhfl94ge7ozUy9WgFUdE8xsvwBjaYBbWmPNA=="], + + "@aws-sdk/client-scheduler/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-env": ["@aws-sdk/credential-provider-env@3.972.41", "", { "dependencies": { "@aws-sdk/core": "^3.974.15", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-n1EbJ98yvPWWdHZZv8bRBMqqDQJrtgtxyJ4xLy2Uqrh25BCOZQ7nnS1CsFXvuH8r0b0KVHDZEGEH5FxmEMP8jg=="], + + "@aws-sdk/client-scheduler/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-http": ["@aws-sdk/credential-provider-http@3.972.43", "", { "dependencies": { "@aws-sdk/core": "^3.974.15", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/fetch-http-handler": "^5.4.5", "@smithy/node-http-handler": "^4.7.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-TT76RN1NkI9WoyZqCNxOw6/WBMF7pYOTJcXbMokNFU+euSG40Kaf/t/FhDACVZWP+43wEM6ZynIPIkzS1wR1iA=="], + + "@aws-sdk/client-scheduler/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini": ["@aws-sdk/credential-provider-ini@3.972.46", "", { "dependencies": { "@aws-sdk/core": "^3.974.15", "@aws-sdk/credential-provider-env": "^3.972.41", "@aws-sdk/credential-provider-http": "^3.972.43", "@aws-sdk/credential-provider-login": "^3.972.45", "@aws-sdk/credential-provider-process": "^3.972.41", "@aws-sdk/credential-provider-sso": "^3.972.45", "@aws-sdk/credential-provider-web-identity": "^3.972.45", "@aws-sdk/nested-clients": "^3.997.13", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/credential-provider-imds": "^4.3.6", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-hvcgcwOiS0nb2XFb5Op1Pz/vYaWz5K8kKullziGpdNRuG0NwzRXseuPt2CoBqknHGaSPVesu1aOn2OcctEYdCA=="], + + "@aws-sdk/client-scheduler/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-process": ["@aws-sdk/credential-provider-process@3.972.41", "", { "dependencies": { "@aws-sdk/core": "^3.974.15", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-7I/n1zkysouLOWvkEhjNEP4vMnD2v4kzzr3/3QBdrripEpn7ap1/I5DF3Hou1SUqkKWo1f3oPGMyFAA1FAMvsQ=="], + + "@aws-sdk/client-scheduler/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso": ["@aws-sdk/credential-provider-sso@3.972.45", "", { "dependencies": { "@aws-sdk/core": "^3.974.15", "@aws-sdk/nested-clients": "^3.997.13", "@aws-sdk/token-providers": "3.1056.0", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-oHgbz/eFD8IKiksqDsz9ZMU4A59BpQq4QwJedBnGD80ZqYcHPPHZBwjBnxLVkB7iRVVHWpDclR8yWdD2PkQIUA=="], + + "@aws-sdk/client-scheduler/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity": ["@aws-sdk/credential-provider-web-identity@3.972.45", "", { "dependencies": { "@aws-sdk/core": "^3.974.15", "@aws-sdk/nested-clients": "^3.997.13", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-CDhzKdb2onv5bpnjn/acgdNmJOQthPDLsPizU7rZflsEcgMMp8Mlri+U5hdxf8ldvZJpvM3vLU6D56vfJm5AMQ=="], + + "@aws-sdk/client-scheduler/@aws-sdk/credential-provider-node/@smithy/credential-provider-imds": ["@smithy/credential-provider-imds@4.3.6", "", { "dependencies": { "@smithy/core": "^3.24.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-tHhdiWZfG1ZIh2YcRfPJmY2gHcBmqbAzqm3ER4TIDFYsSEqTD5tICT7cgQ/kI8LRakxp12myOYyK68XPn7MnHw=="], + "@aws-sdk/client-sts/@aws-sdk/core/@aws-sdk/xml-builder": ["@aws-sdk/xml-builder@3.972.26", "", { "dependencies": { "@smithy/types": "^4.14.2", "fast-xml-parser": "5.7.3", "tslib": "^2.6.2" } }, "sha512-cDbrqvDS73whl6YAPSPq0U6whzG6UWI9PuWh0wrUuGoZexhWEqhdunbukV7iBoaWnFV1AODutM5hOD6rtn439g=="], "@aws-sdk/client-sts/@aws-sdk/core/@smithy/signature-v4": ["@smithy/signature-v4@5.4.5", "", { "dependencies": { "@smithy/core": "^3.24.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-QBJKWGqIknH0dc9LWpfH1mkdokAx6iXYN3UcQ3eY6uIEyScuoQAhfl94ge7ozUy9WgFUdE8xsvwBjaYBbWmPNA=="], @@ -2567,6 +2600,16 @@ "@aws-sdk/client-s3/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.11", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.13", "@aws-sdk/signature-v4-multi-region": "^3.996.28", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.3", "@smithy/fetch-http-handler": "^5.4.3", "@smithy/node-http-handler": "^4.7.3", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-nWXXJ1r/r8N2Gw1pWolRgED38/A9A8DHR2ETWIv220zh4PZHcybbR4hUVWWktmNXTRHzDJwRluapHn0rZxuoqA=="], + "@aws-sdk/client-scheduler/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/credential-provider-login": ["@aws-sdk/credential-provider-login@3.972.45", "", { "dependencies": { "@aws-sdk/core": "^3.974.15", "@aws-sdk/nested-clients": "^3.997.13", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-MZQv4SNjByk1iOKmrqmzcUF/uCB05wjvEHyXKxmGQTUANTIVayX6HPUF0bzkWLvtnkH7sAn9kUCfkXbSpj9sDA=="], + + "@aws-sdk/client-scheduler/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.13", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.15", "@aws-sdk/signature-v4-multi-region": "^3.996.30", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/fetch-http-handler": "^5.4.5", "@smithy/node-http-handler": "^4.7.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-2pA6eyb5nSo/ZD2cayhOTEMoGQYgspq0RI05GDLkzQ3ajZ6isS6waV6E92Am/hz4LIlLUTrbwPLurJ/fuiHvkg=="], + + "@aws-sdk/client-scheduler/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.13", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.15", "@aws-sdk/signature-v4-multi-region": "^3.996.30", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/fetch-http-handler": "^5.4.5", "@smithy/node-http-handler": "^4.7.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-2pA6eyb5nSo/ZD2cayhOTEMoGQYgspq0RI05GDLkzQ3ajZ6isS6waV6E92Am/hz4LIlLUTrbwPLurJ/fuiHvkg=="], + + "@aws-sdk/client-scheduler/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1056.0", "", { "dependencies": { "@aws-sdk/core": "^3.974.15", "@aws-sdk/nested-clients": "^3.997.13", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-81duvlltQlsfn5K+o8zILcystBRdbT1G2JJYVCML5NZHBz4CL/zf+sAemCtBh/uh6RQUMyInGeZLQ7/8igZhbA=="], + + "@aws-sdk/client-scheduler/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.13", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.15", "@aws-sdk/signature-v4-multi-region": "^3.996.30", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/fetch-http-handler": "^5.4.5", "@smithy/node-http-handler": "^4.7.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-2pA6eyb5nSo/ZD2cayhOTEMoGQYgspq0RI05GDLkzQ3ajZ6isS6waV6E92Am/hz4LIlLUTrbwPLurJ/fuiHvkg=="], + "@aws-sdk/client-sts/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/credential-provider-login": ["@aws-sdk/credential-provider-login@3.972.45", "", { "dependencies": { "@aws-sdk/core": "^3.974.15", "@aws-sdk/nested-clients": "^3.997.13", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-MZQv4SNjByk1iOKmrqmzcUF/uCB05wjvEHyXKxmGQTUANTIVayX6HPUF0bzkWLvtnkH7sAn9kUCfkXbSpj9sDA=="], "@aws-sdk/client-sts/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.13", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.15", "@aws-sdk/signature-v4-multi-region": "^3.996.30", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/fetch-http-handler": "^5.4.5", "@smithy/node-http-handler": "^4.7.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-2pA6eyb5nSo/ZD2cayhOTEMoGQYgspq0RI05GDLkzQ3ajZ6isS6waV6E92Am/hz4LIlLUTrbwPLurJ/fuiHvkg=="], diff --git a/infrastructure/lib/problem-deploy/disruption-executor-lambda.ts b/infrastructure/lib/problem-deploy/disruption-executor-lambda.ts new file mode 100644 index 000000000..775d01c70 --- /dev/null +++ b/infrastructure/lib/problem-deploy/disruption-executor-lambda.ts @@ -0,0 +1,176 @@ +import * as path from "node:path"; +import { ArnFormat, Duration, Stack } from "aws-cdk-lib"; +import type { ITable } from "aws-cdk-lib/aws-dynamodb"; +import { type IEventBus, Rule } from "aws-cdk-lib/aws-events"; +import { LambdaFunction } from "aws-cdk-lib/aws-events-targets"; +import * as iam from "aws-cdk-lib/aws-iam"; +import { Architecture } from "aws-cdk-lib/aws-lambda"; +import { NodejsFunction } from "aws-cdk-lib/aws-lambda-nodejs"; +import { Construct } from "constructs"; +import { + LAMBDA_NODEJS_BUNDLING_TARGET, + LAMBDA_NODEJS_RUNTIME, + LAMBDA_SOURCE_MAP_ENABLED, +} from "../utils/lambda-runtime.js"; +import { buildExternalIdParameterArnPattern } from "./handlers/shared/external-id-store.js"; + +export interface DisruptionExecutorLambdaProps { + readonly environmentName: string; + /** disruption-fire が `*DisruptionFired` を publish する EventBus。 本 Lambda がその rule の target。 */ + readonly eventBus: IEventBus; + /** team deployment 解決 (GSI1 Query) 用。 */ + readonly deploymentsTable: ITable; + /** EXEC# 冪等行 (conditional Put) 用。 fire の REQUEST#/AUDIT# と同居。 */ + readonly disruptionsTable: ITable; + /** `{ [problemId]: ProblemDisruptionEntry[] }` (action 込)。 build 時 literal 置換で env 4KB を回避。 */ + readonly problemsDisruptions?: Readonly>; +} + +/** + * [ADR-031 / Issue #1419] cross-account disruption executor Lambda (Phase B)。 + * + * EventBridge `tenantcloud.disruptions` source の `*DisruptionFired` を拾い、 該当 team deployment へ + * AssumeRole して実障害を注入し、 ADR-029 INV-2 のため revert を aws-scheduler に予約する。 注入の破壊力は + * **競技者側 CompetitorDeployRole (AdministratorAccess)** に由来し、 本 Lambda 自身の IAM は最小: + * - sts:AssumeRole は `TenkaCloud-*` ロールのみ (= deploy worker / describe-stack と同 scope) + * - ssm:GetParameter + kms:Decrypt は tenant ExternalId の SecureString のみ (describe-stack と同パターン) + * - DDB は deployments の Query (GSI1) + disruptions の PutItem (EXEC# 冪等) のみ + * - scheduler:CreateSchedule + iam:PassRole は revert scheduler role のみ + * SDK の SendCommand / Invoke / UpdateStack 権限は **本 Lambda の role には無い** (= 注入は assumed + * credentials で行う)。 = blast radius を IAM で封じつつ、 破壊操作は competitor の同意済 role に閉じる。 + * + * revert は scheduler が本 Lambda 自身を `mode:"revert"` payload で呼び戻す one-shot (= EXEC# 冪等 name)。 + */ +export class DisruptionExecutorLambda extends Construct { + public readonly fn: NodejsFunction; + public readonly schedulerRole: iam.Role; + + constructor(scope: Construct, id: string, props: DisruptionExecutorLambdaProps) { + super(scope, id); + const stack = Stack.of(this); + + // self-invoke (scheduler → executor) の ARN を循環なしで得るため functionName を固定し、 ARN を構築する。 + const functionName = `${stack.stackName}-disruption-executor`.slice(0, 64); + const executorArn = stack.formatArn({ + service: "lambda", + resource: "function", + resourceName: functionName, + arnFormat: ArnFormat.COLON_RESOURCE_NAME, + }); + + // scheduler が executor を起動するために assume する role (= revert 予約の Target.RoleArn)。 + this.schedulerRole = new iam.Role(this, "RevertSchedulerRole", { + assumedBy: new iam.ServicePrincipal("scheduler.amazonaws.com"), + inlinePolicies: { + InvokeExecutor: new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + actions: ["lambda:InvokeFunction"], + resources: [executorArn], + }), + ], + }), + }, + }); + + this.fn = new NodejsFunction(this, "Function", { + functionName, + runtime: LAMBDA_NODEJS_RUNTIME, + architecture: Architecture.ARM_64, + entry: path.resolve(import.meta.dirname, "handlers/disruption-executor-handler/index.ts"), + handler: "handler", + timeout: Duration.seconds(60), + memorySize: 512, + environment: { + DEPLOYMENTS_TABLE_NAME: props.deploymentsTable.tableName, + DISRUPTIONS_TABLE_NAME: props.disruptionsTable.tableName, + REVERT_SCHEDULER_ROLE_ARN: this.schedulerRole.roleArn, + EXECUTOR_FUNCTION_ARN: executorArn, + NODE_OPTIONS: "--enable-source-maps", + }, + bundling: { + minify: true, + target: LAMBDA_NODEJS_BUNDLING_TARGET, + sourceMap: LAMBDA_SOURCE_MAP_ENABLED, + externalModules: [], + // disruptions catalog (action 込) を build 時 literal 置換 (env 4KB 回避、 fire と同 catalog)。 + define: { + "process.env.BATTLE_PROBLEMS_DISRUPTIONS": JSON.stringify( + JSON.stringify(props.problemsDisruptions ?? {}), + ), + }, + }, + }); + + // --- 最小 IAM --- + const ssmArn = buildExternalIdParameterArnPattern( + stack.region, + stack.account, + props.environmentName, + ); + this.fn.addToRolePolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ["ssm:GetParameter"], + resources: [ssmArn], + }), + ); + this.fn.addToRolePolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ["kms:Decrypt"], + resources: ["*"], + conditions: { StringLike: { "kms:EncryptionContext:PARAMETER_ARN": ssmArn } }, + }), + ); + this.fn.addToRolePolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ["sts:AssumeRole"], + resources: ["arn:aws:iam::*:role/TenkaCloud-*"], + }), + ); + // deployments: team deployment 解決は GSI1 Query のみ。 + this.fn.addToRolePolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ["dynamodb:Query"], + resources: [ + props.deploymentsTable.tableArn, + `${props.deploymentsTable.tableArn}/index/GSI1`, + ], + }), + ); + // disruptions: EXEC# 冪等 claim は conditional PutItem のみ。 + this.fn.addToRolePolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ["dynamodb:PutItem"], + resources: [props.disruptionsTable.tableArn], + }), + ); + // revert 予約 (scheduler) + その実行 role を渡す PassRole。 + this.fn.addToRolePolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ["scheduler:CreateSchedule"], + resources: ["*"], + }), + ); + this.fn.addToRolePolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ["iam:PassRole"], + resources: [this.schedulerRole.roleArn], + conditions: { StringEquals: { "iam:PassedToService": "scheduler.amazonaws.com" } }, + }), + ); + + // `*DisruptionFired` (= disruption-fire の publish) を拾って executor を起動する。 + new Rule(this, "FiredRule", { + eventBus: props.eventBus, + eventPattern: { source: ["tenkacloud.disruptions"] }, + targets: [new LambdaFunction(this.fn)], + }); + } +} diff --git a/infrastructure/lib/problem-deploy/handlers/describe-stack-handler/index.ts b/infrastructure/lib/problem-deploy/handlers/describe-stack-handler/index.ts index 1dd809d7f..848e7891e 100644 --- a/infrastructure/lib/problem-deploy/handlers/describe-stack-handler/index.ts +++ b/infrastructure/lib/problem-deploy/handlers/describe-stack-handler/index.ts @@ -1,7 +1,7 @@ import { CloudFormationClient, DescribeStacksCommand } from "@aws-sdk/client-cloudformation"; -import { GetParameterCommand, SSMClient } from "@aws-sdk/client-ssm"; -import type { Credentials } from "@aws-sdk/client-sts"; -import { AssumeRoleCommand, STSClient } from "@aws-sdk/client-sts"; +import { SSMClient } from "@aws-sdk/client-ssm"; +import { type Credentials, STSClient } from "@aws-sdk/client-sts"; +import { assumeCompetitorRole } from "../shared/assume-competitor-role.js"; import { errorDeployTrace, logDeployTrace } from "../shared/trace-log.js"; export interface DescribeStackStateMachineInput { @@ -32,147 +32,6 @@ function requireString(value: unknown, field: string): string { return value; } -function assertCompleteCredentials(credentials: Credentials | undefined): Credentials { - if (!credentials?.AccessKeyId || !credentials.SecretAccessKey || !credentials.SessionToken) { - throw new Error("AssumeRole returned incomplete credentials"); - } - return credentials; -} - -/** - * Issue #1245 + #856: rotation race の AssumeRole 失敗のうち、 ExternalId mismatch に起因する - * 4xx だけを 1 generation 前で retry する。 Network / Throttling / 5xx 系は retry せず即 fail。 - * - * `verify.ts` 側の `shouldRetryWithPreviousVersion` と同じ error name 集合を共有し、 - * blanket-catch (= 全 error で previous version を試す) のような band-aid を避ける。 - */ -const ASSUME_ROLE_FALLBACK_ERROR_NAMES: ReadonlySet = new Set([ - "AccessDenied", - "AccessDeniedException", - "Forbidden", -]); - -function shouldRetryWithPreviousVersion(err: unknown): boolean { - const name = err instanceof Error ? err.name : ""; - return ASSUME_ROLE_FALLBACK_ERROR_NAMES.has(name); -} - -async function assumeCompetitorRole( - deps: DescribeStackDeps, - params: { - readonly region: string; - readonly jobId: string; - readonly competitorRoleArn?: string; - readonly externalIdParameterName?: string; - }, -): Promise { - const hasRole = - typeof params.competitorRoleArn === "string" && params.competitorRoleArn.length > 0; - const hasExternalId = - typeof params.externalIdParameterName === "string" && params.externalIdParameterName.length > 0; - if (!hasRole && !hasExternalId) return undefined; - if (!hasRole || !hasExternalId) { - throw new Error("competitorRoleArn and externalIdParameterName must be provided together"); - } - // 上の 2 guard で competitorRoleArn / externalIdParameterName が string であることは確定。 - const competitorRoleArn = params.competitorRoleArn as string; - const externalIdParameterName = params.externalIdParameterName as string; - - const externalIdOut = await deps.ssm.send( - new GetParameterCommand({ - Name: externalIdParameterName, - WithDecryption: true, - }), - ); - const externalId = externalIdOut.Parameter?.Value; - if (!externalId) { - throw new Error(`ExternalId not found in SSM SecureString: ${externalIdParameterName}`); - } - - try { - return await assumeRoleWithExternalId(deps, competitorRoleArn, params.jobId, externalId); - } catch (currentErr) { - return await retryWithPreviousExternalId(deps, { - region: params.region, - jobId: params.jobId, - competitorRoleArn, - externalIdParameterName, - currentVersion: Number(externalIdOut.Parameter?.Version ?? 0), - currentErr, - }); - } -} - -/** - * Issue #1245: rotation race の retry path を 1 関数に切り出す。 - * - * 旧 implementation の問題点: - * - 全 error class で blanket fallback (= Throttling / Network 系も前 version で retry していた) - * - 成功時の log が `console.warn` の自由 string であり、 metrics filter が当てづらく silent - * - * 修正後: - * - `shouldRetryWithPreviousVersion` で AccessDenied 系 (= ExternalId mismatch) に絞る - * - 1 generation 前 SSM version が無ければ original error を rethrow (= silent skip しない) - * - 成功時は `errorDeployTrace` で `deploy.describe-stack.assume-role.grace-fallback` を発火し、 - * operator alarm に pick up させる (= grace 多発 = rotation pipeline のバグ可視化) - * - retry でも ExternalId は必ず渡される (= 「ExternalId 無し AssumeRole」は禁止) - */ -async function retryWithPreviousExternalId( - deps: DescribeStackDeps, - args: { - readonly region: string; - readonly jobId: string; - readonly competitorRoleArn: string; - readonly externalIdParameterName: string; - readonly currentVersion: number; - readonly currentErr: unknown; - }, -): Promise { - const { currentErr } = args; - if (!shouldRetryWithPreviousVersion(currentErr)) throw currentErr; - const previousVersion = args.currentVersion - 1; - if (previousVersion <= 0) throw currentErr; - const previousExternalIdOut = await deps.ssm.send( - new GetParameterCommand({ - Name: `${args.externalIdParameterName}:${previousVersion}`, - WithDecryption: true, - }), - ); - const previousExternalId = previousExternalIdOut.Parameter?.Value; - if (!previousExternalId) throw currentErr; - const credentials = await assumeRoleWithExternalId( - deps, - args.competitorRoleArn, - args.jobId, - previousExternalId, - ); - errorDeployTrace("deploy.describe-stack.assume-role.grace-fallback", { - jobId: args.jobId, - correlationId: args.jobId, - region: args.region, - externalIdVersion: previousVersion, - reason: currentErr instanceof Error ? currentErr.name : "Unknown", - }); - return credentials; -} - -async function assumeRoleWithExternalId( - deps: DescribeStackDeps, - roleArn: string, - jobId: string, - externalId: string, -): Promise { - const assumeOut = await deps.sts.send( - new AssumeRoleCommand({ - RoleArn: roleArn, - RoleSessionName: `tenkacloud-describe-stack-${jobId.slice(0, 24)}`, - ExternalId: externalId, - DurationSeconds: 900, - }), - ); - return assertCompleteCredentials(assumeOut.Credentials); -} - /** * Issue (regression 調査): Step Functions 経由で渡される `input.detail.jobId` * が undefined で fail するケースが 1 回観測された (= ジョブ ID は DDB row に @@ -230,6 +89,8 @@ export async function describeStackForDeployment( jobId, competitorRoleArn: detail.competitorRoleArn, externalIdParameterName: detail.externalIdParameterName, + sessionNamePrefix: "tenkacloud-describe-stack-", + graceFallbackTraceEvent: "deploy.describe-stack.assume-role.grace-fallback", }); const cfn = deps.cfnClient({ region, credentials }); const out = await cfn.send(new DescribeStacksCommand({ StackName: stackName })); diff --git a/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/dispatch-command.ts b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/dispatch-command.ts new file mode 100644 index 000000000..9f3e0a3e4 --- /dev/null +++ b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/dispatch-command.ts @@ -0,0 +1,126 @@ +/** + * [ADR-031 / Issue #1419] cross-account disruption executor の **純粋な dispatch core**。 + * + * fired event (= `*DisruptionFired`) が持つ `action` 宣言 + 既に fold 済の `parameters` + + * team の `stackOutputs` を受け取り、 競技者アカウントで実行すべき 1 アクションを **SDK 非依存の + * 正規化記述子** に落とす。 ここでは AWS を一切呼ばない (= AssumeRole / SendCommand 等は handler の + * 責務)。 純関数なので unit test で全分岐を pin できる。 + * + * 設計判断: + * - `targetRef` / `functionRef` は **stackOutputs の key** からのみ解決する (= 任意 resource id を + * 直接実行させない、 ADR-031 の injection 縮小)。 解決できなければ loud に throw (silent fallback 禁止)。 + * - `paramTemplate` の `{{key}}` は fired `parameters` の値でのみ置換する。 値が無い placeholder は + * literal `{{key}}` を競技者アカウントへ送らないよう throw する (= validate-problems の宣言時 allow-list と + * runtime の二重防御)。 + * - revert は注入と同じ kind / target を使い、 `action.revert` の documentName / paramTemplate で + * 上書きする。 `afterSeconds` は scheduling metadata なので記述子には含めない (handler が scheduler に渡す)。 + */ + +import type { + DisruptionAction, + DisruptionActionKind, +} from "../../../utils/discover-problems-catalog.js"; + +/** 競技者アカウントで実行する 1 アクションの正規化記述子 (SDK 非依存)。 */ +export interface DisruptionDispatch { + readonly kind: DisruptionActionKind; + /** 解決済の実行対象 (= ssm: instance ids 文字列 / lambda: function 名 ARN / cfn: stack 名)。 */ + readonly target: string; + /** ssm-run-command の SSM Document 名 (他 kind では undefined)。 */ + readonly documentName?: string; + /** placeholder 置換済の API 引数 (= SSM Parameters / Lambda payload / CFn Parameters の素材)。 */ + readonly params: Record; +} + +const PLACEHOLDER_RE = /\{\{\s*([A-Za-z0-9_]+)\s*\}\}/g; + +function substituteString(value: string, params: Readonly>): string { + return value.replace(PLACEHOLDER_RE, (_match, key: string) => { + if (!(key in params)) { + throw new Error( + `disruption action placeholder {{${key}}} has no value in the fired parameters`, + ); + } + return String(params[key]); + }); +} + +/** paramTemplate を再帰的に walk して string 中の `{{key}}` を fired parameters で置換する。 */ +function substituteValue(value: unknown, params: Readonly>): unknown { + if (typeof value === "string") return substituteString(value, params); + if (Array.isArray(value)) return value.map((v) => substituteValue(v, params)); + if (value && typeof value === "object") { + const out: Record = {}; + for (const [k, v] of Object.entries(value)) out[k] = substituteValue(v, params); + return out; + } + return value; +} + +function substituteTemplate( + template: Readonly> | undefined, + params: Readonly>, +): Record { + if (!template) return {}; + return substituteValue(template, params) as Record; +} + +/** stackOutputs[ref] を解決する。 未宣言 / 空は loud に throw (= 注入対象が無いのに送らない)。 */ +function resolveOutput( + ref: string, + stackOutputs: Readonly>, + label: string, +): string { + const value = stackOutputs[ref]; + if (typeof value !== "string" || value === "") { + throw new Error(`disruption action ${label}="${ref}" not found in team stackOutputs`); + } + return value; +} + +/** + * 注入アクションの記述子を組み立てる。 lambda-invoke は `functionRef ?? targetRef` を、 それ以外は + * `targetRef` を stackOutputs から解決する。 + */ +export function buildDisruptionDispatch( + action: DisruptionAction, + parameters: Readonly>, + stackOutputs: Readonly>, +): DisruptionDispatch { + const targetRef = + action.kind === "lambda-invoke" ? (action.functionRef ?? action.targetRef) : action.targetRef; + const target = resolveOutput( + targetRef, + stackOutputs, + action.kind === "lambda-invoke" ? "functionRef/targetRef" : "targetRef", + ); + return { + kind: action.kind, + target, + ...(action.documentName ? { documentName: action.documentName } : {}), + params: substituteTemplate(action.paramTemplate, parameters), + }; +} + +/** + * 復旧アクションの記述子。 注入と同じ kind / target を使い、 `action.revert` の documentName / + * paramTemplate で上書きする。 `afterSeconds` は記述子に含めない (= handler が scheduler に渡す)。 + */ +export function buildRevertDispatch( + action: DisruptionAction, + parameters: Readonly>, + stackOutputs: Readonly>, +): DisruptionDispatch { + const injected = buildDisruptionDispatch(action, parameters, stackOutputs); + const revert = action.revert; + return { + kind: injected.kind, + target: injected.target, + ...(revert.documentName + ? { documentName: revert.documentName } + : injected.documentName + ? { documentName: injected.documentName } + : {}), + params: substituteTemplate(revert.paramTemplate, parameters), + }; +} diff --git a/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/execute.ts b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/execute.ts new file mode 100644 index 000000000..40dae0f93 --- /dev/null +++ b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/execute.ts @@ -0,0 +1,122 @@ +/** + * [ADR-031 / Issue #1419] cross-account disruption executor の orchestration。 + * + * `*DisruptionFired` event (= disruption-fire が operator account の bus に publish したもの) を 1 件受け、 + * 該当 team の deployment へ実障害を注入し、 ADR-029 INV-2 のため復旧を予約する。 流れ: + * + * 1. catalog で `(problemId, disruptionId)` の `action` を解決。 action 未宣言 = Phase A (監査のみ) で no-op。 + * 2. `EXEC#{requestId}#{teamId}` の conditional claim で per-team 冪等性を取る (= EventBridge at-least-once 対策)。 + * 3. team の deployment row (jobId / region / competitorRoleArn / externalId / stackOutputs) を解決。 + * 未 deploy / 未完了 = 注入対象が無いので no-op (loud にせず skip、 監査は claim 済)。 + * 4. competitor account へ AssumeRole (ExternalId) し、 inject dispatch を送る。 + * 5. revert dispatch を `afterSeconds` 後に予約 (= 必ず復旧する。 永続障害の禁止)。 + * + * I/O 境界 (catalog 解決 / 冪等 claim / deployment 解決 / AssumeRole / 送信 / 予約) はすべて deps として注入し、 + * 本 orchestration は **判断 (順序 / 分岐) だけ** を持つ純粋な関数にする (= describe-stack-handler と同じ DI 方針、 + * unit test で全分岐を mock で pin できる)。 各 dep の具体実装 (SDK mapping / scheduler / DDB query) と + * Lambda + IAM の CDK 配線は、 競技者アカウントへ実 fault を deploy する判断を伴うため別途。 + */ + +import type { + DisruptionAction, + ProblemDisruptionEntry, +} from "../../../utils/discover-problems-catalog.js"; +import { + buildDisruptionDispatch, + buildRevertDispatch, + type DisruptionDispatch, +} from "./dispatch-command.js"; + +/** fired event の Detail (= disruption-fire.publishEntries が JSON.stringify する形)。 */ +export interface DisruptionFiredDetail { + readonly disruptionId: string; + readonly eventId: string; + readonly problemId: string; + readonly tenantId: string; + readonly teamId: string; + readonly parameters: Readonly>; + readonly requestId: string; + readonly firedAt: string; +} + +/** 注入対象 team deployment の解決結果。 */ +export interface DeploymentTarget { + readonly jobId: string; + readonly region: string; + readonly competitorRoleArn: string; + readonly externalIdParameterName: string; + /** CFn Outputs (= deployment row の stackOutputs JSON を parse 済)。 */ + readonly stackOutputs: Readonly>; +} + +export interface ExecutorDeps { + /** problemsDisruptions catalog (= fire と同じ env 由来)。 */ + readonly problemsDisruptions: Readonly>; + /** `EXEC#{requestId}#{teamId}` の conditional claim。 claimed=winner / duplicate=既処理。 */ + readonly claimExecution: (detail: DisruptionFiredDetail) => Promise<"claimed" | "duplicate">; + /** team+problem deployment を解決。 未 deploy / 未完了 / stackOutputs 無しは undefined。 */ + readonly resolveDeployment: ( + detail: DisruptionFiredDetail, + ) => Promise; + /** dispatch を競技者アカウントで実行 (= AssumeRole + SDK send は具体実装側)。 */ + readonly sendDispatch: (dispatch: DisruptionDispatch, target: DeploymentTarget) => Promise; + /** + * revert を afterSeconds 後に予約 (ADR-029 INV-2)。 scheduler 機構は具体実装側。 + * `detail` は冪等な schedule 名 (= EXEC# と対の requestId/teamId) と revert invocation payload の + * 組み立てに必要なため渡す (= 具体実装 scheduleRevert がそれらを使う)。 + */ + readonly scheduleRevert: ( + detail: DisruptionFiredDetail, + dispatch: DisruptionDispatch, + target: DeploymentTarget, + afterSeconds: number, + ) => Promise; +} + +export type DisruptionExecuteOutcome = + | { readonly kind: "ok"; readonly jobId: string } + | { readonly kind: "no_action" } + | { readonly kind: "duplicate" } + | { readonly kind: "unknown_disruption" } + | { readonly kind: "no_deployment" }; + +function resolveAction( + catalog: ExecutorDeps["problemsDisruptions"], + problemId: string, + disruptionId: string, +): DisruptionAction | undefined | "unknown" { + const entries = catalog[problemId]; + if (!entries) return "unknown"; + const declaration = entries.find((d) => d.id === disruptionId); + if (!declaration) return "unknown"; + return declaration.action; +} + +/** + * 1 件の fired disruption を実行する。 副作用は deps 経由のみ。 戻り値で結果を表す + * (= caller の handler が log / metric に使う)。 + */ +export async function executeDisruptionAction( + detail: DisruptionFiredDetail, + deps: ExecutorDeps, +): Promise { + const action = resolveAction(deps.problemsDisruptions, detail.problemId, detail.disruptionId); + if (action === "unknown") return { kind: "unknown_disruption" }; + // action 未宣言 = Phase A 監査のみ。 注入は起こさない (= 後方互換)。 + if (!action) return { kind: "no_action" }; + + // EventBridge at-least-once の再配送を per-team 冪等で弾く。 claim は注入の前に取る。 + if ((await deps.claimExecution(detail)) === "duplicate") return { kind: "duplicate" }; + + const target = await deps.resolveDeployment(detail); + if (!target) return { kind: "no_deployment" }; + + const inject = buildDisruptionDispatch(action, detail.parameters, target.stackOutputs); + await deps.sendDispatch(inject, target); + + // ADR-029 INV-2: 注入したら必ず復旧を予約する (revert は schema 必須)。 + const revert = buildRevertDispatch(action, detail.parameters, target.stackOutputs); + await deps.scheduleRevert(detail, revert, target, action.revert.afterSeconds); + + return { kind: "ok", jobId: target.jobId }; +} diff --git a/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/executor-store.ts b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/executor-store.ts new file mode 100644 index 000000000..ec9274bf8 --- /dev/null +++ b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/executor-store.ts @@ -0,0 +1,105 @@ +/** + * [ADR-031 / Issue #1419] executor の DDB 副作用 dep 実装 (= `executeDisruptionAction` の + * `claimExecution` / `resolveDeployment` の具体実装)。 既存 pattern を踏襲し新 SDK 依存を増やさない: + * - claim: disruption-fire の REQUEST# 冪等 claim と同型 (conditional Put + CCF=duplicate) + * - resolve: leaderboard-score-events の GSI1(TENANT#)+eventId filter query と同型 + * + * AssumeRole / SDK 送信 / scheduler は別 dep (= deploy 判断を伴うため owner)。 ここは DDB のみ。 + */ + +import { ConditionalCheckFailedException } from "@aws-sdk/client-dynamodb"; +import { type DynamoDBDocumentClient, PutCommand, QueryCommand } from "@aws-sdk/lib-dynamodb"; +import type { DeploymentItem } from "../deploy-handler/types.js"; +import { parseStackOutputs } from "../shared/cfn-status.js"; +import type { DeploymentTarget, DisruptionFiredDetail } from "./execute.js"; + +export interface ExecutorResources { + readonly ddb: Pick; + /** Deployments テーブル (= deploy-handler が書く row)。 GSI1 = TENANT#。 */ + readonly deploymentsTableName: string; + /** Disruptions テーブル (= fire の REQUEST#/AUDIT# と同居、 EXEC# 冪等行を置く)。 */ + readonly disruptionsTableName: string; + /** EXEC# 行の TTL 秒数 (省略時 7 日)。 */ + readonly execTtlSeconds?: number; +} + +const DEFAULT_EXEC_TTL_SECONDS = 7 * 24 * 60 * 60; + +/** + * `EXEC#{requestId}#{teamId}` を conditional Put で奪う。 EventBridge at-least-once の再配送に対する + * per-team 冪等性。 ConditionalCheckFailed = 既に処理済 (duplicate)、 それ以外の error は伝播。 + */ +export async function claimExecution( + resources: ExecutorResources, + detail: DisruptionFiredDetail, + nowMs: number, +): Promise<"claimed" | "duplicate"> { + const pk = `EXEC#${detail.requestId}#${detail.teamId}`; + try { + await resources.ddb.send( + new PutCommand({ + TableName: resources.disruptionsTableName, + Item: { + PK: pk, + SK: "METADATA", + disruptionId: detail.disruptionId, + eventId: detail.eventId, + problemId: detail.problemId, + tenantId: detail.tenantId, + teamId: detail.teamId, + requestId: detail.requestId, + firedAt: detail.firedAt, + expiresAt: + Math.floor(nowMs / 1000) + (resources.execTtlSeconds ?? DEFAULT_EXEC_TTL_SECONDS), + }, + ConditionExpression: "attribute_not_exists(PK)", + }), + ); + return "claimed"; + } catch (err) { + if (err instanceof ConditionalCheckFailedException) return "duplicate"; + throw err; + } +} + +/** + * fired `(tenantId, eventId, teamId, problemId)` の **COMPLETE** deployment を GSI1 query で解決し、 + * cross-account 注入に必要な情報が揃った行のみ返す。 未 deploy / 未完了 / cross-account 情報や + * stackOutputs 欠落は undefined (= 注入対象なし、 caller が no-op にする)。 + */ +export async function resolveDeployment( + resources: ExecutorResources, + detail: DisruptionFiredDetail, +): Promise { + const out = await resources.ddb.send( + new QueryCommand({ + TableName: resources.deploymentsTableName, + IndexName: "GSI1", + KeyConditionExpression: "GSI1PK = :pk", + FilterExpression: "eventId = :ev AND teamId = :tid AND problemId = :pid", + ExpressionAttributeValues: { + ":pk": `TENANT#${detail.tenantId}`, + ":ev": detail.eventId, + ":tid": detail.teamId, + ":pid": detail.problemId, + }, + }), + ); + const items = (out.Items ?? []) as Partial[]; + const ready = items.find( + (d) => + d.status === "COMPLETE" && + typeof d.jobId === "string" && + typeof d.region === "string" && + typeof d.competitorRoleArn === "string" && + typeof d.externalIdParameterName === "string", + ); + if (!ready) return undefined; + return { + jobId: ready.jobId as string, + region: ready.region as string, + competitorRoleArn: ready.competitorRoleArn as string, + externalIdParameterName: ready.externalIdParameterName as string, + stackOutputs: parseStackOutputs(ready.stackOutputs), + }; +} diff --git a/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/index.ts b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/index.ts new file mode 100644 index 000000000..37570b7b1 --- /dev/null +++ b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/index.ts @@ -0,0 +1,108 @@ +/** + * [ADR-031 / Issue #1419] cross-account disruption executor Lambda の entry。 + * + * 実 client / AssumeRole を組み立てて `routeDisruptionInvocation` (= 純粋 router) に注入するだけの + * 薄い glue (= describe-stack-handler/index.ts と同じ「testable service + real-deps entry」分離)。 + * 判断ロジック・dispatch mapping・scheduler 呼び出し・DDB アクセスはすべて test 済の module 側にあり、 + * ここは AWS SDK の構築と env 読取に閉じる。 + * + * 2 経路で起動される (route が判別): + * - EventBridge `*DisruptionFired` rule → `{ detail }` envelope (= 注入) + * - aws-scheduler one-shot → `{ mode:"revert", dispatch, target }` (= 復旧) + * revert / inject とも `wiredSendDispatch` が target から都度 AssumeRole する (= 注入時 creds は永続しない)。 + */ + +import { CloudFormationClient } from "@aws-sdk/client-cloudformation"; +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { LambdaClient } from "@aws-sdk/client-lambda"; +import { SchedulerClient } from "@aws-sdk/client-scheduler"; +import { SSMClient } from "@aws-sdk/client-ssm"; +import { STSClient } from "@aws-sdk/client-sts"; +import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb"; +import { parseDisruptionsCatalogEnv } from "../../../utils/discover-problems-catalog.js"; +import { assumeCompetitorRole } from "../shared/assume-competitor-role.js"; +import { logDeployTrace } from "../shared/trace-log.js"; +import type { DeploymentTarget, ExecutorDeps } from "./execute.js"; +import { claimExecution, type ExecutorResources, resolveDeployment } from "./executor-store.js"; +import { type RouteOutcome, routeDisruptionInvocation } from "./route.js"; +import { scheduleRevert } from "./schedule-revert.js"; +import { type DispatchTarget, sendDispatch } from "./send-dispatch.js"; + +const SESSION_NAME_PREFIX = "tc-disruption-"; +const GRACE_FALLBACK_TRACE = "deploy.disruption-executor.assume-role.grace-fallback"; + +const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); +const ssm = new SSMClient({}); +const sts = new STSClient({}); +const scheduler = new SchedulerClient({}); + +const resources: ExecutorResources = { + ddb, + deploymentsTableName: process.env.DEPLOYMENTS_TABLE_NAME ?? "", + disruptionsTableName: process.env.DISRUPTIONS_TABLE_NAME ?? "", +}; + +const problemsDisruptions = parseDisruptionsCatalogEnv(process.env.BATTLE_PROBLEMS_DISRUPTIONS); +const schedulerRoleArn = process.env.REVERT_SCHEDULER_ROLE_ARN ?? ""; +const revertTargetArn = process.env.EXECUTOR_FUNCTION_ARN ?? ""; + +/** target から都度 AssumeRole して competitor account 内で dispatch を送る (= inject / revert 共通)。 */ +async function wiredSendDispatch( + dispatch: Parameters[0], + target: DeploymentTarget, +): Promise { + const credentials = await assumeCompetitorRole( + { ssm, sts }, + { + region: target.region, + jobId: target.jobId, + competitorRoleArn: target.competitorRoleArn, + externalIdParameterName: target.externalIdParameterName, + sessionNamePrefix: SESSION_NAME_PREFIX, + graceFallbackTraceEvent: GRACE_FALLBACK_TRACE, + }, + ); + const dispatchTarget: DispatchTarget = { region: target.region, credentials }; + await sendDispatch(dispatch, dispatchTarget, { + ssmClient: (t) => new SSMClient(sdkClientConfig(t)), + lambdaClient: (t) => new LambdaClient(sdkClientConfig(t)), + cfnClient: (t) => new CloudFormationClient(sdkClientConfig(t)), + }); +} + +/** STS Credentials (PascalCase) を SDK client config の camelCase credentials に写す (describe-stack と同方針)。 */ +function sdkClientConfig(target: DispatchTarget) { + const creds = target.credentials; + return { + region: target.region, + ...(creds + ? { + credentials: { + accessKeyId: creds.AccessKeyId ?? "", + secretAccessKey: creds.SecretAccessKey ?? "", + sessionToken: creds.SessionToken, + }, + } + : {}), + }; +} + +const deps: ExecutorDeps = { + problemsDisruptions, + claimExecution: (detail) => claimExecution(resources, detail, Date.now()), + resolveDeployment: (detail) => resolveDeployment(resources, detail), + sendDispatch: wiredSendDispatch, + scheduleRevert: (detail, dispatch, target, afterSeconds) => + scheduleRevert(dispatch, detail, target, afterSeconds, { + scheduler, + schedulerRoleArn, + revertTargetArn, + }), +}; + +/** Lambda handler。 inject / revert を route が判別し dispatch する。 outcome は observability の trace に出す。 */ +export async function handler(event: unknown): Promise { + const outcome = await routeDisruptionInvocation(event, deps); + logDeployTrace("deploy.disruption-executor.outcome", { kind: outcome.kind }); + return outcome; +} diff --git a/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/route.ts b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/route.ts new file mode 100644 index 000000000..717860b22 --- /dev/null +++ b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/route.ts @@ -0,0 +1,90 @@ +/** + * [ADR-031 / Issue #1419] executor Lambda の invocation router (= handler entry の純粋ロジック)。 + * + * 同じ executor Lambda が 2 経路で起動される: + * 1. EventBridge `*DisruptionFired` rule → `{ ..., detail: DisruptionFiredDetail }` envelope (= 注入) + * 2. aws-scheduler の one-shot → `{ mode: "revert", dispatch, target }` payload (= 復旧、 scheduleRevert が積む) + * + * router は両者を判別し、 注入は `executeDisruptionAction`、 復旧は `sendDispatch` dep をそのまま再利用する + * (= revert の credentials は wired sendDispatch dep が target から都度 AssumeRole する前提なので、 inject と + * 同じ dep で送れる)。 不正 envelope は `invalid_event` (= loud に落とさず handler が log/metric にできる形)。 + * + * 実 client / AssumeRole の組み立ては index.ts (= 別途、 deploy 判断を伴う) が deps として注入する。 + * 本 module は純粋で、 unit test で全分岐を mock で pin できる。 + */ + +import type { DisruptionDispatch } from "./dispatch-command.js"; +import { + type DeploymentTarget, + type DisruptionExecuteOutcome, + type DisruptionFiredDetail, + type ExecutorDeps, + executeDisruptionAction, +} from "./execute.js"; + +interface RevertInvocation { + readonly mode: "revert"; + readonly dispatch: DisruptionDispatch; + readonly target: DeploymentTarget; +} + +export type RouteOutcome = + | DisruptionExecuteOutcome + | { readonly kind: "reverted" } + | { readonly kind: "invalid_event" }; + +function asNonEmptyString(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** EventBridge envelope の `detail` を DisruptionFiredDetail へ narrow。 必須 string 欠落は undefined。 */ +export function parseDisruptionFiredDetail(event: unknown): DisruptionFiredDetail | undefined { + if (!isObject(event) || !isObject(event.detail)) return undefined; + const d = event.detail; + const disruptionId = asNonEmptyString(d.disruptionId); + const eventId = asNonEmptyString(d.eventId); + const problemId = asNonEmptyString(d.problemId); + const tenantId = asNonEmptyString(d.tenantId); + const teamId = asNonEmptyString(d.teamId); + const requestId = asNonEmptyString(d.requestId); + const firedAt = asNonEmptyString(d.firedAt); + if (!disruptionId || !eventId || !problemId || !tenantId || !teamId || !requestId || !firedAt) { + return undefined; + } + return { + disruptionId, + eventId, + problemId, + tenantId, + teamId, + requestId, + firedAt, + parameters: isObject(d.parameters) ? d.parameters : {}, + }; +} + +function isRevertInvocation(event: unknown): event is RevertInvocation { + return ( + isObject(event) && event.mode === "revert" && isObject(event.dispatch) && isObject(event.target) + ); +} + +/** executor Lambda の 1 invocation を inject / revert に振り分けて実行する。 */ +export async function routeDisruptionInvocation( + event: unknown, + deps: ExecutorDeps, +): Promise { + if (isRevertInvocation(event)) { + // revert: scheduleRevert が積んだ構築済 dispatch を、 inject と同じ sendDispatch dep で送る + // (= dep が target から都度 AssumeRole する。 注入時 creds は永続しない)。 + await deps.sendDispatch(event.dispatch, event.target); + return { kind: "reverted" }; + } + const detail = parseDisruptionFiredDetail(event); + if (!detail) return { kind: "invalid_event" }; + return executeDisruptionAction(detail, deps); +} diff --git a/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/schedule-revert.ts b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/schedule-revert.ts new file mode 100644 index 000000000..3435cabc6 --- /dev/null +++ b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/schedule-revert.ts @@ -0,0 +1,72 @@ +/** + * [ADR-031 / ADR-029 INV-2 / Issue #1419] executor の `scheduleRevert` dep 具体実装。 + * + * 注入と同時に「afterSeconds 後に 1 度だけ復旧する」 one-shot schedule を aws-scheduler に登録する + * (= ADR-031 で選定した機構)。 schedule は executor Lambda 自身を `mode:"revert"` payload で呼び戻す: + * revert 時刻には注入時の credentials は失効しているため、 payload は DeploymentTarget (= roleArn / + * externalIdParameterName / region / stackOutputs) と **構築済の revert dispatch** を運び、 handler は + * 再 AssumeRole して送るだけにする (= re-lookup 不要、 注入時の決定を凍結)。 + * + * - name は `EXEC#` と対の冪等キー (= 同 requestId/teamId の重複 fire で同名 → CreateSchedule が衝突を弾く)。 + * - FlexibleTimeWindow=OFF / ActionAfterCompletion=DELETE (= 発火後に自動削除、 schedule が溜まらない)。 + * - schedule が target を起動するための RoleArn は construct が作り env 経由で注入 (= deps.schedulerRoleArn)。 + * + * SDK error は握り潰さず伝播 (= 復旧予約に失敗したら loud にする。 INV-2 を黙って破らない)。 + */ + +import { + ActionAfterCompletion, + CreateScheduleCommand, + FlexibleTimeWindowMode, + type SchedulerClient, + ScheduleState, +} from "@aws-sdk/client-scheduler"; +import type { DisruptionDispatch } from "./dispatch-command.js"; +import type { DeploymentTarget, DisruptionFiredDetail } from "./execute.js"; + +export interface ScheduleRevertDeps { + readonly scheduler: Pick; + /** schedule が target を起動するために assume する role の ARN (= construct が作成、 env 注入)。 */ + readonly schedulerRoleArn: string; + /** revert を実行する target (= executor Lambda 自身) の ARN。 */ + readonly revertTargetArn: string; +} + +const MAX_SCHEDULE_NAME = 64; + +/** `EXEC#{requestId}#{teamId}` と対の冪等な schedule 名。 aws-scheduler の name 制約に sanitize。 */ +export function revertScheduleName(detail: DisruptionFiredDetail): string { + return `tc-revert-${detail.requestId}-${detail.teamId}` + .replace(/[^0-9A-Za-z\-_.]/g, "-") + .slice(0, MAX_SCHEDULE_NAME); +} + +/** firedAt + afterSeconds の UTC 時刻を aws-scheduler の `at(...)` 式 (秒精度) に。 */ +export function revertAtExpression(firedAtIso: string, afterSeconds: number): string { + const at = new Date(new Date(firedAtIso).getTime() + afterSeconds * 1000); + return `at(${at.toISOString().slice(0, 19)})`; +} + +export async function scheduleRevert( + revert: DisruptionDispatch, + detail: DisruptionFiredDetail, + target: DeploymentTarget, + afterSeconds: number, + deps: ScheduleRevertDeps, +): Promise { + await deps.scheduler.send( + new CreateScheduleCommand({ + Name: revertScheduleName(detail), + ScheduleExpression: revertAtExpression(detail.firedAt, afterSeconds), + ScheduleExpressionTimezone: "UTC", + FlexibleTimeWindow: { Mode: FlexibleTimeWindowMode.OFF }, + State: ScheduleState.ENABLED, + ActionAfterCompletion: ActionAfterCompletion.DELETE, + Target: { + Arn: deps.revertTargetArn, + RoleArn: deps.schedulerRoleArn, + Input: JSON.stringify({ mode: "revert", detail, dispatch: revert, target }), + }, + }), + ); +} diff --git a/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/send-dispatch.ts b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/send-dispatch.ts new file mode 100644 index 000000000..8180e894f --- /dev/null +++ b/infrastructure/lib/problem-deploy/handlers/disruption-executor-handler/send-dispatch.ts @@ -0,0 +1,107 @@ +/** + * [ADR-031 / Issue #1419] executor の `sendDispatch` dep 具体実装 = 正規化記述子 (DisruptionDispatch) を + * 競技者アカウントの実 SDK command に map して送る。 client は describe-stack-handler と同じ factory dep + * (`{ region, credentials }` を受けて assumed-credential client を返す) で注入し、 unit test では mock する。 + * + * 新 SDK 依存は無し (ssm / lambda / cloudformation client は infra に既存)。 SDK の振る舞い選択: + * - ssm-run-command: target = stackOutputs の instance ids (comma 区切り) を SendCommand の InstanceIds に。 + * params は SSM Parameters (= Record) へ coerce。 + * - lambda-invoke: InvocationType="Event" の **非同期** 投入 (= fault 注入の完了は待たない、 executor を塞がない)。 + * - cfn-stack-update: UsePreviousTemplate=true + params を CFn Parameters に。 既存 stack を parameter 上書き + * だけで degrade させる前提。 IAM capability は competitor stack が named role を含みうるため明示。 + */ + +import { + Capability, + type CloudFormationClient, + UpdateStackCommand, +} from "@aws-sdk/client-cloudformation"; +import { InvokeCommand, type LambdaClient } from "@aws-sdk/client-lambda"; +import { SendCommandCommand, type SSMClient } from "@aws-sdk/client-ssm"; +import type { Credentials } from "@aws-sdk/client-sts"; +import type { DisruptionDispatch } from "./dispatch-command.js"; + +export interface DispatchTarget { + readonly region: string; + readonly credentials?: Credentials; +} + +export interface SendDispatchDeps { + readonly ssmClient: (target: DispatchTarget) => Pick; + readonly lambdaClient: (target: DispatchTarget) => Pick; + readonly cfnClient: (target: DispatchTarget) => Pick; +} + +const DEFAULT_SSM_DOCUMENT = "AWS-RunShellScript"; +const CFN_UPDATE_CAPABILITIES: Capability[] = [ + Capability.CAPABILITY_IAM, + Capability.CAPABILITY_NAMED_IAM, + Capability.CAPABILITY_AUTO_EXPAND, +]; + +/** dispatch.params の各値を SSM Parameters の Record へ coerce。 */ +function toSsmParameters(params: Readonly>): Record { + const out: Record = {}; + for (const [key, value] of Object.entries(params)) { + out[key] = Array.isArray(value) ? value.map((v) => String(v)) : [String(value)]; + } + return out; +} + +/** dispatch.params を CFn UpdateStack の Parameters ({ParameterKey, ParameterValue}[]) へ。 */ +function toCfnParameters( + params: Readonly>, +): { ParameterKey: string; ParameterValue: string }[] { + return Object.entries(params).map(([key, value]) => ({ + ParameterKey: key, + ParameterValue: String(value), + })); +} + +/** comma 区切りの instance ids 文字列を trim + 空要素除去で配列化。 */ +function toInstanceIds(target: string): string[] { + return target + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0); +} + +/** + * 1 つの dispatch 記述子を competitor account で実行する。 AssumeRole 済の credentials は target に乗る + * 前提 (= 解決は caller=handler 側)。 SDK error は握り潰さず伝播 (= 注入失敗を loud にする)。 + */ +export async function sendDispatch( + dispatch: DisruptionDispatch, + target: DispatchTarget, + deps: SendDispatchDeps, +): Promise { + if (dispatch.kind === "ssm-run-command") { + await deps.ssmClient(target).send( + new SendCommandCommand({ + DocumentName: dispatch.documentName ?? DEFAULT_SSM_DOCUMENT, + InstanceIds: toInstanceIds(dispatch.target), + Parameters: toSsmParameters(dispatch.params), + }), + ); + return; + } + if (dispatch.kind === "lambda-invoke") { + await deps.lambdaClient(target).send( + new InvokeCommand({ + FunctionName: dispatch.target, + InvocationType: "Event", + Payload: new TextEncoder().encode(JSON.stringify(dispatch.params)), + }), + ); + return; + } + // cfn-stack-update + await deps.cfnClient(target).send( + new UpdateStackCommand({ + StackName: dispatch.target, + UsePreviousTemplate: true, + Parameters: toCfnParameters(dispatch.params), + Capabilities: CFN_UPDATE_CAPABILITIES, + }), + ); +} diff --git a/infrastructure/lib/problem-deploy/handlers/shared/assume-competitor-role.ts b/infrastructure/lib/problem-deploy/handlers/shared/assume-competitor-role.ts new file mode 100644 index 000000000..49fe1fb74 --- /dev/null +++ b/infrastructure/lib/problem-deploy/handlers/shared/assume-competitor-role.ts @@ -0,0 +1,169 @@ +/** + * Cross-account AssumeRole into the competitor's `CompetitorDeployRole`, shared across handlers. + * + * 以前は describe-stack-handler が自前で持ち、 verify.ts / participant SSO も類似ロジックを別個に + * 抱えていた (= rotation-race retry / ExternalId mismatch fallback の error name 集合がコメントで + * 「同じものを共有」 と書かれつつ実体は重複していた)。 ここに 1 本化して **単一の監査点** にする + * (= 資格情報経路の重複は security リスク。 #856 / #1245 で hardening した retry を 1 箇所に集約)。 + * + * caller 固有の cosmetic だけを param 化する (= 振る舞いは不変): + * - `sessionNamePrefix`: RoleSessionName の prefix (例: describe-stack / disruption-executor) + * - `graceFallbackTraceEvent`: grace-fallback 成功時に発火する trace event 名 (operator alarm が key にする) + * + * Issue #1245 + #856 の方針はそのまま: + * - ExternalId は SSM SecureString から都度 decrypt (= コードに埋め込まない) + * - AssumeRole 失敗のうち AccessDenied 系 (= ExternalId mismatch) だけ 1 generation 前で 1 度 retry + * - Network / Throttling / 5xx は retry せず即 rethrow (= blanket fallback band-aid を避ける) + * - grace-fallback 成功は errorDeployTrace で発火し operator alarm に拾わせる + * - retry でも ExternalId は必ず渡す (= 「ExternalId 無し AssumeRole」 は禁止) + */ + +import { GetParameterCommand, type SSMClient } from "@aws-sdk/client-ssm"; +import { AssumeRoleCommand, type Credentials, type STSClient } from "@aws-sdk/client-sts"; +import { errorDeployTrace } from "./trace-log.js"; + +export interface AssumeCompetitorRoleDeps { + readonly ssm: Pick; + readonly sts: Pick; +} + +export interface AssumeCompetitorRoleParams { + readonly region: string; + readonly jobId: string; + readonly competitorRoleArn?: string; + readonly externalIdParameterName?: string; + /** RoleSessionName prefix (caller 識別)。 例: "tenkacloud-describe-stack-" / "tc-disruption-"。 */ + readonly sessionNamePrefix: string; + /** grace-fallback 成功時の trace event 名 (= operator alarm の key)。 */ + readonly graceFallbackTraceEvent: string; +} + +const ASSUME_ROLE_FALLBACK_ERROR_NAMES: ReadonlySet = new Set([ + "AccessDenied", + "AccessDeniedException", + "Forbidden", +]); + +/** AccessDenied 系 (= ExternalId mismatch) のみ 1 generation 前での retry 対象。 */ +export function shouldRetryWithPreviousExternalIdVersion(err: unknown): boolean { + const name = err instanceof Error ? err.name : ""; + return ASSUME_ROLE_FALLBACK_ERROR_NAMES.has(name); +} + +function assertCompleteCredentials(credentials: Credentials | undefined): Credentials { + if (!credentials?.AccessKeyId || !credentials.SecretAccessKey || !credentials.SessionToken) { + throw new Error("AssumeRole returned incomplete credentials"); + } + return credentials; +} + +async function assumeRoleWithExternalId( + deps: AssumeCompetitorRoleDeps, + args: { + readonly roleArn: string; + readonly jobId: string; + readonly externalId: string; + readonly sessionNamePrefix: string; + }, +): Promise { + const assumeOut = await deps.sts.send( + new AssumeRoleCommand({ + RoleArn: args.roleArn, + RoleSessionName: `${args.sessionNamePrefix}${args.jobId.slice(0, 24)}`, + ExternalId: args.externalId, + DurationSeconds: 900, + }), + ); + return assertCompleteCredentials(assumeOut.Credentials); +} + +async function retryWithPreviousExternalId( + deps: AssumeCompetitorRoleDeps, + args: { + readonly region: string; + readonly jobId: string; + readonly competitorRoleArn: string; + readonly externalIdParameterName: string; + readonly currentVersion: number; + readonly currentErr: unknown; + readonly sessionNamePrefix: string; + readonly graceFallbackTraceEvent: string; + }, +): Promise { + const { currentErr } = args; + if (!shouldRetryWithPreviousExternalIdVersion(currentErr)) throw currentErr; + const previousVersion = args.currentVersion - 1; + if (previousVersion <= 0) throw currentErr; + const previousExternalIdOut = await deps.ssm.send( + new GetParameterCommand({ + Name: `${args.externalIdParameterName}:${previousVersion}`, + WithDecryption: true, + }), + ); + const previousExternalId = previousExternalIdOut.Parameter?.Value; + if (!previousExternalId) throw currentErr; + const credentials = await assumeRoleWithExternalId(deps, { + roleArn: args.competitorRoleArn, + jobId: args.jobId, + externalId: previousExternalId, + sessionNamePrefix: args.sessionNamePrefix, + }); + errorDeployTrace(args.graceFallbackTraceEvent, { + jobId: args.jobId, + correlationId: args.jobId, + region: args.region, + externalIdVersion: previousVersion, + reason: currentErr instanceof Error ? currentErr.name : "Unknown", + }); + return credentials; +} + +/** + * competitor account の `CompetitorDeployRole` を ExternalId 付きで AssumeRole する。 + * competitorRoleArn / externalIdParameterName の双方が無ければ undefined (= same-account 経路)、 + * 片方だけは config error。 ExternalId mismatch (rotation race) は 1 generation 前で 1 度 retry。 + */ +export async function assumeCompetitorRole( + deps: AssumeCompetitorRoleDeps, + params: AssumeCompetitorRoleParams, +): Promise { + const hasRole = + typeof params.competitorRoleArn === "string" && params.competitorRoleArn.length > 0; + const hasExternalId = + typeof params.externalIdParameterName === "string" && params.externalIdParameterName.length > 0; + if (!hasRole && !hasExternalId) return undefined; + if (!hasRole || !hasExternalId) { + throw new Error("competitorRoleArn and externalIdParameterName must be provided together"); + } + // 上の 2 guard で competitorRoleArn / externalIdParameterName が string であることは確定。 + const competitorRoleArn = params.competitorRoleArn as string; + const externalIdParameterName = params.externalIdParameterName as string; + + const externalIdOut = await deps.ssm.send( + new GetParameterCommand({ Name: externalIdParameterName, WithDecryption: true }), + ); + const externalId = externalIdOut.Parameter?.Value; + if (!externalId) { + throw new Error(`ExternalId not found in SSM SecureString: ${externalIdParameterName}`); + } + + try { + return await assumeRoleWithExternalId(deps, { + roleArn: competitorRoleArn, + jobId: params.jobId, + externalId, + sessionNamePrefix: params.sessionNamePrefix, + }); + } catch (currentErr) { + return await retryWithPreviousExternalId(deps, { + region: params.region, + jobId: params.jobId, + competitorRoleArn, + externalIdParameterName, + currentVersion: Number(externalIdOut.Parameter?.Version ?? 0), + currentErr, + sessionNamePrefix: params.sessionNamePrefix, + graceFallbackTraceEvent: params.graceFallbackTraceEvent, + }); + } +} diff --git a/infrastructure/lib/problem-deploy/problem-deploy-backend-stack.ts b/infrastructure/lib/problem-deploy/problem-deploy-backend-stack.ts index dbf14d482..4808eec34 100644 --- a/infrastructure/lib/problem-deploy/problem-deploy-backend-stack.ts +++ b/infrastructure/lib/problem-deploy/problem-deploy-backend-stack.ts @@ -22,6 +22,7 @@ import { } from "./deploy-event-rule.js"; import { DeploymentsTable } from "./deployments-table.js"; import { DescribeStackLambda } from "./describe-stack-lambda.js"; +import { DisruptionExecutorLambda } from "./disruption-executor-lambda.js"; import { DisruptionsTable } from "./disruptions-table.js"; import { EventApiLambda } from "./event-api-lambda.js"; import { EventsTable } from "./events-table.js"; @@ -349,6 +350,17 @@ export class ProblemDeployBackendStack extends cdk.Stack { }); this.eventApiLambda = eventApi.fn; + // [ADR-031 / Issue #1419] Disruption Phase B: operator fire が publish した `*DisruptionFired` を + // 拾い、 team deployment へ AssumeRole して実障害を注入し、 revert を予約する cross-account executor。 + // action 未宣言の disruption は no-op (= Phase A 監査のみ、 後方互換)。 + new DisruptionExecutorLambda(this, "DisruptionExecutor", { + environmentName: props.environmentName, + eventBus, + deploymentsTable: deployments.table, + disruptionsTable: disruptions.table, + problemsDisruptions: (props.problemsDisruptions ?? {}) as Readonly>, + }); + // Issue #459 / ADR-002 Phase 2.1: Competitor Accounts CRUD + STS verify Lambda。 // 独立 Lambda にする理由: SSM SecureString R/W + STS AssumeRole の IAM scope を最小化するため。 const competitorAccountsApi = new CompetitorAccountsApiLambda(this, "CompetitorAccountsApi", { diff --git a/infrastructure/package.json b/infrastructure/package.json index 40026ead0..77875031c 100644 --- a/infrastructure/package.json +++ b/infrastructure/package.json @@ -50,6 +50,7 @@ "@aws-sdk/client-cloudwatch-logs": "^3.1053.0", "@aws-sdk/client-codebuild": "^3.1048.0", "@aws-sdk/client-codepipeline": "^3.1048.0", + "@aws-sdk/client-scheduler": "^3.1048.0", "@aws-sdk/client-sfn": "^3.1048.0", "@cdklabs/sbt-aws": "0.3.9", "@tenkacloud/coordination-plugin-sdk": "workspace:*", diff --git a/infrastructure/test/problem-deploy-backend-stack-events.test.ts b/infrastructure/test/problem-deploy-backend-stack-events.test.ts index 2071d2ac9..22bbea237 100644 --- a/infrastructure/test/problem-deploy-backend-stack-events.test.ts +++ b/infrastructure/test/problem-deploy-backend-stack-events.test.ts @@ -5,14 +5,15 @@ import { synthDefault } from "./problem-deploy-backend-stack.test-helpers"; describe("ProblemDeployBackendStack (MVP-1) — EventBridge Rules", () => { const tpl = synthDefault(); - it("should have 7 EventBridge Rules (Create / Delete / BulkCreate / GenericScoring / ExternalIdAudit schedule / SystemAuditWriter (Issue #1034) / CodeBuildFailure (Issue #1029))", () => { + it("should have 8 EventBridge Rules (Create / Delete / BulkCreate / GenericScoring / ExternalIdAudit schedule / SystemAuditWriter (Issue #1034) / CodeBuildFailure (Issue #1029) / DisruptionExecutor (ADR-031 #1419))", () => { // 旧 2 (Create / Delete state-machine event rules) // + BulkCreate (Issue #910 Phase 2.C: BulkDeployCreateRequested → Distributed Map) // + GenericScoring schedule rate(1 minute) (= ADR-012 Phase 3.B、 旧 HealthCheck 後継) // + ExternalIdAudit schedule rate(1 day) (= Phase 3.2 / Issue #603 で追加) // = 5。GenericScoring は scoring 問題が無い tenant でも reconcile 用に常時 instantiate される。 // 旧 5 + Issue #1034 SystemAuditWriter (SBT bus) + Issue #1029 CodeBuildFailure (default bus) - tpl.resourceCountIs("AWS::Events::Rule", 7); + // + ADR-031 #1419 DisruptionExecutor (tenkacloud.disruptions → cross-account fault executor) + tpl.resourceCountIs("AWS::Events::Rule", 8); tpl.hasResourceProperties( "AWS::Events::Rule", Match.objectLike({ diff --git a/infrastructure/test/problem-deploy/assume-competitor-role.test.ts b/infrastructure/test/problem-deploy/assume-competitor-role.test.ts new file mode 100644 index 000000000..2586f9d9c --- /dev/null +++ b/infrastructure/test/problem-deploy/assume-competitor-role.test.ts @@ -0,0 +1,133 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const errorDeployTrace = vi.fn(); +vi.mock("../../lib/problem-deploy/handlers/shared/trace-log", () => ({ + errorDeployTrace: (...args: unknown[]) => errorDeployTrace(...args), + logDeployTrace: vi.fn(), +})); + +import { + type AssumeCompetitorRoleDeps, + assumeCompetitorRole, + shouldRetryWithPreviousExternalIdVersion, +} from "../../lib/problem-deploy/handlers/shared/assume-competitor-role"; + +/** + * 共有 assumeCompetitorRole の挙動 pin。 describe-stack-handler の既存 test が回帰網だが、 ここでは + * 共有契約 + parameterize した cosmetic (sessionNamePrefix / graceFallbackTraceEvent) を直接確認する。 + */ + +const CREDS = { AccessKeyId: "AK", SecretAccessKey: "SK", SessionToken: "ST" }; + +function makeDeps( + ssmSend: ReturnType, + stsSend: ReturnType, +): AssumeCompetitorRoleDeps { + return { + ssm: { send: ssmSend } as unknown as AssumeCompetitorRoleDeps["ssm"], + sts: { send: stsSend } as unknown as AssumeCompetitorRoleDeps["sts"], + }; +} + +const baseParams = { + region: "ap-northeast-1", + jobId: "job-1234567890123456789012345678", + competitorRoleArn: "arn:aws:iam::111122223333:role/TenkaCloud-CompetitorDeploy-Role", + externalIdParameterName: "/tenkacloud/tenant-1/external-id", + sessionNamePrefix: "tc-disruption-", + graceFallbackTraceEvent: "deploy.disruption-executor.assume-role.grace-fallback", +}; + +describe("assumeCompetitorRole (shared)", () => { + beforeEach(() => vi.clearAllMocks()); + + it("should return undefined when neither competitorRoleArn nor externalIdParameterName is given", async () => { + const deps = makeDeps(vi.fn(), vi.fn()); + expect( + await assumeCompetitorRole(deps, { + ...baseParams, + competitorRoleArn: undefined, + externalIdParameterName: undefined, + }), + ).toBeUndefined(); + }); + + it("should throw when only one side of the cross-account metadata is present", async () => { + const deps = makeDeps(vi.fn(), vi.fn()); + await expect( + assumeCompetitorRole(deps, { ...baseParams, externalIdParameterName: undefined }), + ).rejects.toThrow("must be provided together"); + }); + + it("should throw when the SSM ExternalId parameter has no value", async () => { + const deps = makeDeps(vi.fn().mockResolvedValue({ Parameter: {} }), vi.fn()); + await expect(assumeCompetitorRole(deps, baseParams)).rejects.toThrow( + "ExternalId not found in SSM SecureString", + ); + }); + + it("should AssumeRole with the caller's session-name prefix (truncated jobId) and return the creds", async () => { + const ssm = vi.fn().mockResolvedValue({ Parameter: { Value: "ext-id", Version: 3 } }); + const sts = vi.fn().mockResolvedValue({ Credentials: CREDS }); + expect(await assumeCompetitorRole(makeDeps(ssm, sts), baseParams)).toEqual(CREDS); + const input = sts.mock.calls[0][0].input; + expect(input.RoleSessionName).toBe(`tc-disruption-${baseParams.jobId.slice(0, 24)}`); + expect(input.ExternalId).toBe("ext-id"); + expect(input.DurationSeconds).toBe(900); + }); + + it("should throw when AssumeRole returns incomplete credentials", async () => { + const ssm = vi.fn().mockResolvedValue({ Parameter: { Value: "ext-id", Version: 1 } }); + const sts = vi.fn().mockResolvedValue({ Credentials: { AccessKeyId: "AK" } }); + await expect(assumeCompetitorRole(makeDeps(ssm, sts), baseParams)).rejects.toThrow( + "incomplete credentials", + ); + }); + + it("should grace-fallback to the previous ExternalId version on AccessDenied and fire the caller's trace event", async () => { + const ssm = vi + .fn() + .mockResolvedValueOnce({ Parameter: { Value: "new-id", Version: 4 } }) + .mockResolvedValueOnce({ Parameter: { Value: "old-id", Version: 3 } }); + const denied = Object.assign(new Error("denied"), { name: "AccessDenied" }); + const sts = vi.fn().mockRejectedValueOnce(denied).mockResolvedValueOnce({ Credentials: CREDS }); + expect(await assumeCompetitorRole(makeDeps(ssm, sts), baseParams)).toEqual(CREDS); + expect(ssm.mock.calls[1][0].input.Name).toBe(`${baseParams.externalIdParameterName}:3`); + expect(errorDeployTrace).toHaveBeenCalledWith( + "deploy.disruption-executor.assume-role.grace-fallback", + expect.objectContaining({ externalIdVersion: 3, reason: "AccessDenied" }), + ); + }); + + it("should rethrow immediately on a non-AccessDenied error (no blanket band-aid)", async () => { + const ssm = vi.fn().mockResolvedValue({ Parameter: { Value: "id", Version: 2 } }); + const sts = vi + .fn() + .mockRejectedValue(Object.assign(new Error("slow"), { name: "ThrottlingException" })); + await expect(assumeCompetitorRole(makeDeps(ssm, sts), baseParams)).rejects.toThrow("slow"); + expect(errorDeployTrace).not.toHaveBeenCalled(); + }); + + it("should rethrow the original error when there is no previous version to fall back to", async () => { + const ssm = vi.fn().mockResolvedValue({ Parameter: { Value: "id", Version: 1 } }); + const denied = Object.assign(new Error("denied"), { name: "AccessDenied" }); + const sts = vi.fn().mockRejectedValue(denied); + await expect(assumeCompetitorRole(makeDeps(ssm, sts), baseParams)).rejects.toThrow("denied"); + }); +}); + +describe("shouldRetryWithPreviousExternalIdVersion", () => { + it("should retry only on the AccessDenied family", () => { + for (const name of ["AccessDenied", "AccessDeniedException", "Forbidden"]) { + expect(shouldRetryWithPreviousExternalIdVersion(Object.assign(new Error(), { name }))).toBe( + true, + ); + } + expect( + shouldRetryWithPreviousExternalIdVersion( + Object.assign(new Error(), { name: "ThrottlingException" }), + ), + ).toBe(false); + expect(shouldRetryWithPreviousExternalIdVersion("not-an-error")).toBe(false); + }); +}); diff --git a/infrastructure/test/problem-deploy/disruption-dispatch-command.test.ts b/infrastructure/test/problem-deploy/disruption-dispatch-command.test.ts new file mode 100644 index 000000000..fe6088d01 --- /dev/null +++ b/infrastructure/test/problem-deploy/disruption-dispatch-command.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, it } from "vitest"; +import { + buildDisruptionDispatch, + buildRevertDispatch, +} from "../../lib/problem-deploy/handlers/disruption-executor-handler/dispatch-command"; +import type { DisruptionAction } from "../../lib/utils/discover-problems-catalog"; + +/** + * [ADR-031 / #1419] executor の純粋 dispatch core を pin する。 + * - targetRef / functionRef を stackOutputs から解決 (未解決は throw) + * - paramTemplate の {{key}} を fired parameters で置換 (値無しは throw) + * - revert は同 kind/target + revert.paramTemplate/documentName で上書き + */ + +const ssmAction: DisruptionAction = { + kind: "ssm-run-command", + targetRef: "WorkerInstanceIds", + documentName: "AWS-RunShellScript", + paramTemplate: { commands: ["tc qdisc add dev {{device}} root netem delay {{delayMs}}ms"] }, + revert: { + afterSeconds: 600, + documentName: "AWS-RunShellScript", + paramTemplate: { commands: ["tc qdisc del dev {{device}} root"] }, + }, +}; + +const stackOutputs = { WorkerInstanceIds: "i-aaa,i-bbb", FaultFn: "tc-fault-fn" }; +const parameters = { device: "eth0", delayMs: 200 }; + +describe("buildDisruptionDispatch (ADR-031 #1419)", () => { + it("should resolve targetRef from stackOutputs and substitute placeholders from fired parameters", () => { + expect(buildDisruptionDispatch(ssmAction, parameters, stackOutputs)).toEqual({ + kind: "ssm-run-command", + target: "i-aaa,i-bbb", + documentName: "AWS-RunShellScript", + params: { commands: ["tc qdisc add dev eth0 root netem delay 200ms"] }, + }); + }); + + it("should resolve a lambda-invoke functionRef (preferred over targetRef) from stackOutputs", () => { + const action: DisruptionAction = { + kind: "lambda-invoke", + targetRef: "WorkerInstanceIds", + functionRef: "FaultFn", + paramTemplate: { mode: "fail", device: "{{device}}" }, + revert: { afterSeconds: 30 }, + }; + expect(buildDisruptionDispatch(action, parameters, stackOutputs)).toEqual({ + kind: "lambda-invoke", + target: "tc-fault-fn", + params: { mode: "fail", device: "eth0" }, + }); + }); + + it("should fall back to targetRef for lambda-invoke when functionRef is absent", () => { + const action: DisruptionAction = { + kind: "lambda-invoke", + targetRef: "FaultFn", + revert: { afterSeconds: 30 }, + }; + expect(buildDisruptionDispatch(action, parameters, stackOutputs).target).toBe("tc-fault-fn"); + }); + + it("should return empty params when no paramTemplate is declared", () => { + const action: DisruptionAction = { + kind: "cfn-stack-update", + targetRef: "WorkerInstanceIds", + revert: { afterSeconds: 30 }, + }; + const dispatch = buildDisruptionDispatch(action, parameters, stackOutputs); + expect(dispatch.params).toEqual({}); + expect(dispatch.documentName).toBeUndefined(); + }); + + it("should throw when targetRef cannot be resolved from stackOutputs", () => { + expect(() => buildDisruptionDispatch(ssmAction, parameters, { Other: "x" })).toThrow( + /targetRef="WorkerInstanceIds" not found/, + ); + }); + + it("should throw when a placeholder has no value in the fired parameters", () => { + expect(() => buildDisruptionDispatch(ssmAction, { device: "eth0" }, stackOutputs)).toThrow( + /\{\{delayMs\}\} has no value/, + ); + }); + + it("should substitute placeholders nested inside arrays and objects", () => { + const action: DisruptionAction = { + kind: "ssm-run-command", + targetRef: "WorkerInstanceIds", + paramTemplate: { nested: { list: ["{{device}}", { deep: "delay-{{delayMs}}" }] } }, + revert: { afterSeconds: 1 }, + }; + expect(buildDisruptionDispatch(action, parameters, stackOutputs).params).toEqual({ + nested: { list: ["eth0", { deep: "delay-200" }] }, + }); + }); + + it("should pass non-string leaves (number / boolean / null) through unchanged", () => { + const action: DisruptionAction = { + kind: "lambda-invoke", + targetRef: "FaultFn", + paramTemplate: { count: 3, enabled: true, note: null, label: "{{device}}" }, + revert: { afterSeconds: 1 }, + }; + expect(buildDisruptionDispatch(action, parameters, stackOutputs).params).toEqual({ + count: 3, + enabled: true, + note: null, + label: "eth0", + }); + }); +}); + +describe("buildRevertDispatch (ADR-031 #1419, ADR-029 INV-2)", () => { + it("should reuse the inject target/kind and apply the revert paramTemplate + documentName", () => { + expect(buildRevertDispatch(ssmAction, parameters, stackOutputs)).toEqual({ + kind: "ssm-run-command", + target: "i-aaa,i-bbb", + documentName: "AWS-RunShellScript", + params: { commands: ["tc qdisc del dev eth0 root"] }, + }); + }); + + it("should inherit the inject documentName when the revert omits it, and yield empty params with no revert template", () => { + const action: DisruptionAction = { + kind: "ssm-run-command", + targetRef: "WorkerInstanceIds", + documentName: "AWS-RunShellScript", + revert: { afterSeconds: 60 }, + }; + expect(buildRevertDispatch(action, parameters, stackOutputs)).toEqual({ + kind: "ssm-run-command", + target: "i-aaa,i-bbb", + documentName: "AWS-RunShellScript", + params: {}, + }); + }); + + it("should yield no documentName when neither inject nor revert declares one", () => { + const action: DisruptionAction = { + kind: "lambda-invoke", + targetRef: "FaultFn", + revert: { afterSeconds: 60, paramTemplate: { mode: "recover" } }, + }; + const revert = buildRevertDispatch(action, parameters, stackOutputs); + expect(revert.documentName).toBeUndefined(); + expect(revert.params).toEqual({ mode: "recover" }); + }); +}); diff --git a/infrastructure/test/problem-deploy/disruption-execute.test.ts b/infrastructure/test/problem-deploy/disruption-execute.test.ts new file mode 100644 index 000000000..217d5ccde --- /dev/null +++ b/infrastructure/test/problem-deploy/disruption-execute.test.ts @@ -0,0 +1,135 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + type DeploymentTarget, + type DisruptionFiredDetail, + type ExecutorDeps, + executeDisruptionAction, +} from "../../lib/problem-deploy/handlers/disruption-executor-handler/execute"; +import type { ProblemDisruptionEntry } from "../../lib/utils/discover-problems-catalog"; + +/** + * [ADR-031 / #1419] executor orchestration を pin する。 副作用は deps 経由のみなので、 + * 各 dep を mock して分岐 (no_action / duplicate / no_deployment / ok + 順序) を観察する。 + */ + +const withAction: ProblemDisruptionEntry = { + id: "ec2-latency-injection", + name: "latency", + eventDetailType: "DegradedDisruptionFired", + parameters: { delayMs: 200, device: "eth0" }, + action: { + kind: "ssm-run-command", + targetRef: "WorkerInstanceIds", + documentName: "AWS-RunShellScript", + paramTemplate: { commands: ["tc qdisc add dev {{device}} root netem delay {{delayMs}}ms"] }, + revert: { + afterSeconds: 600, + paramTemplate: { commands: ["tc qdisc del dev {{device}} root"] }, + }, + }, +}; + +const detail: DisruptionFiredDetail = { + disruptionId: "ec2-latency-injection", + eventId: "evt-1", + problemId: "microservice-migration-battle", + tenantId: "tenant-1", + teamId: "team-1", + parameters: { delayMs: 200, device: "eth0" }, + requestId: "req-1", + firedAt: "2026-06-02T00:00:00.000Z", +}; + +const target: DeploymentTarget = { + jobId: "job-1", + region: "ap-northeast-1", + competitorRoleArn: "arn:aws:iam::111122223333:role/TenkaCloud-CompetitorDeploy-Role", + externalIdParameterName: "/tenkacloud/tenant-1/external-id", + stackOutputs: { WorkerInstanceIds: "i-aaa,i-bbb" }, +}; + +function makeDeps(over: Partial = {}): ExecutorDeps { + return { + problemsDisruptions: { "microservice-migration-battle": [withAction] }, + claimExecution: vi.fn().mockResolvedValue("claimed"), + resolveDeployment: vi.fn().mockResolvedValue(target), + sendDispatch: vi.fn().mockResolvedValue(undefined), + scheduleRevert: vi.fn().mockResolvedValue(undefined), + ...over, + }; +} + +describe("executeDisruptionAction (ADR-031 #1419)", () => { + beforeEach(() => vi.clearAllMocks()); + + it("should inject the built dispatch then schedule the revert on the happy path", async () => { + const deps = makeDeps(); + const outcome = await executeDisruptionAction(detail, deps); + expect(outcome).toEqual({ kind: "ok", jobId: "job-1" }); + + expect(deps.sendDispatch).toHaveBeenCalledTimes(1); + const [injected, sentTarget] = (deps.sendDispatch as ReturnType).mock.calls[0]; + expect(injected).toEqual({ + kind: "ssm-run-command", + target: "i-aaa,i-bbb", + documentName: "AWS-RunShellScript", + params: { commands: ["tc qdisc add dev eth0 root netem delay 200ms"] }, + }); + expect(sentTarget).toBe(target); + + expect(deps.scheduleRevert).toHaveBeenCalledTimes(1); + const [detailArg, revert, , afterSeconds] = (deps.scheduleRevert as ReturnType) + .mock.calls[0]; + expect(detailArg).toBe(detail); // revert payload / idempotent schedule name 用に detail を渡す + expect(revert).toEqual({ + kind: "ssm-run-command", + target: "i-aaa,i-bbb", + documentName: "AWS-RunShellScript", + params: { commands: ["tc qdisc del dev eth0 root"] }, + }); + expect(afterSeconds).toBe(600); + }); + + it("should be a no-op (no_action) when the disruption declares no action (Phase A)", async () => { + const noAction: ProblemDisruptionEntry = { ...withAction, action: undefined }; + const deps = makeDeps({ + problemsDisruptions: { "microservice-migration-battle": [noAction] }, + }); + expect(await executeDisruptionAction(detail, deps)).toEqual({ kind: "no_action" }); + expect(deps.claimExecution).not.toHaveBeenCalled(); + expect(deps.sendDispatch).not.toHaveBeenCalled(); + }); + + it("should return unknown_disruption when the problem / disruption is not in the catalog", async () => { + expect(await executeDisruptionAction(detail, makeDeps({ problemsDisruptions: {} }))).toEqual({ + kind: "unknown_disruption", + }); + const otherId = makeDeps({ + problemsDisruptions: { "microservice-migration-battle": [{ ...withAction, id: "other" }] }, + }); + expect(await executeDisruptionAction(detail, otherId)).toEqual({ kind: "unknown_disruption" }); + }); + + it("should stop at the idempotency claim when the execution is a duplicate", async () => { + const deps = makeDeps({ claimExecution: vi.fn().mockResolvedValue("duplicate") }); + expect(await executeDisruptionAction(detail, deps)).toEqual({ kind: "duplicate" }); + expect(deps.resolveDeployment).not.toHaveBeenCalled(); + expect(deps.sendDispatch).not.toHaveBeenCalled(); + expect(deps.scheduleRevert).not.toHaveBeenCalled(); + }); + + it("should be a no-op (no_deployment) when the team has no resolvable deployment", async () => { + const deps = makeDeps({ resolveDeployment: vi.fn().mockResolvedValue(undefined) }); + expect(await executeDisruptionAction(detail, deps)).toEqual({ kind: "no_deployment" }); + expect(deps.sendDispatch).not.toHaveBeenCalled(); + expect(deps.scheduleRevert).not.toHaveBeenCalled(); + }); + + it("should not schedule the revert when the inject send fails (error propagates)", async () => { + const deps = makeDeps({ + sendDispatch: vi.fn().mockRejectedValue(new Error("SendCommand denied")), + }); + await expect(executeDisruptionAction(detail, deps)).rejects.toThrow("SendCommand denied"); + expect(deps.scheduleRevert).not.toHaveBeenCalled(); + }); +}); diff --git a/infrastructure/test/problem-deploy/disruption-executor-lambda.test.ts b/infrastructure/test/problem-deploy/disruption-executor-lambda.test.ts new file mode 100644 index 000000000..c1bfdc1b0 --- /dev/null +++ b/infrastructure/test/problem-deploy/disruption-executor-lambda.test.ts @@ -0,0 +1,136 @@ +import { App, Stack } from "aws-cdk-lib"; +import { Match, Template } from "aws-cdk-lib/assertions"; +import { AttributeType, Table } from "aws-cdk-lib/aws-dynamodb"; +import { EventBus } from "aws-cdk-lib/aws-events"; +import { describe, expect, it } from "vitest"; +import { DisruptionExecutorLambda } from "../../lib/problem-deploy/disruption-executor-lambda"; + +/** + * [ADR-031 / #1419] cross-account disruption executor の CDK 境界を pin する。 核心は + * 「自前 role は最小 (sts:AssumeRole は TenkaCloud-* のみ、 SendCommand/Invoke/UpdateStack は **持たない**)」 + * = 破壊力は assumed の CompetitorDeployRole に閉じ、 executor 自身の blast radius は IAM で封じる。 + */ + +const SYNTH_TIMEOUT_MS = 120_000; + +function synth(): Template { + const app = new App(); + const stack = new Stack(app, "TestStack"); + const deployments = new Table(stack, "Deployments", { + partitionKey: { name: "PK", type: AttributeType.STRING }, + }); + const disruptions = new Table(stack, "Disruptions", { + partitionKey: { name: "PK", type: AttributeType.STRING }, + sortKey: { name: "SK", type: AttributeType.STRING }, + }); + new DisruptionExecutorLambda(stack, "Executor", { + environmentName: "development", + eventBus: new EventBus(stack, "Bus"), + deploymentsTable: deployments, + disruptionsTable: disruptions, + problemsDisruptions: { "microservice-migration-battle": [{ id: "x" }] }, + }); + return Template.fromStack(stack); +} + +/** 全 IAM Role の inline policy (= 権限境界。 trust policy は除外) の action を集める。 */ +function inlineActions(tpl: Template): string[] { + return Object.values(tpl.findResources("AWS::IAM::Policy")).flatMap((p) => + ( + (p as { Properties?: { PolicyDocument?: { Statement?: unknown[] } } }).Properties + ?.PolicyDocument?.Statement ?? [] + ).flatMap((s) => { + const a = (s as { Action?: string | string[] }).Action; + return Array.isArray(a) ? a : typeof a === "string" ? [a] : []; + }), + ); +} + +describe("DisruptionExecutorLambda (ADR-031 #1419)", () => { + it( + "should provision a Node.js / arm64 executor Lambda with the wiring env", + () => { + const tpl = synth(); + tpl.hasResourceProperties( + "AWS::Lambda::Function", + Match.objectLike({ + Runtime: "nodejs22.x", + Architectures: ["arm64"], + Environment: Match.objectLike({ + Variables: Match.objectLike({ + DEPLOYMENTS_TABLE_NAME: Match.anyValue(), + DISRUPTIONS_TABLE_NAME: Match.anyValue(), + REVERT_SCHEDULER_ROLE_ARN: Match.anyValue(), + EXECUTOR_FUNCTION_ARN: Match.anyValue(), + }), + }), + }), + ); + }, + SYNTH_TIMEOUT_MS, + ); + + it( + "should scope sts:AssumeRole to TenkaCloud-* roles (not a wildcard)", + () => { + const tpl = synth(); + tpl.hasResourceProperties( + "AWS::IAM::Policy", + Match.objectLike({ + PolicyDocument: Match.objectLike({ + Statement: Match.arrayWith([ + Match.objectLike({ + Action: "sts:AssumeRole", + Resource: "arn:aws:iam::*:role/TenkaCloud-*", + }), + ]), + }), + }), + ); + }, + SYNTH_TIMEOUT_MS, + ); + + it( + "should NOT grant the executor's own role the destructive cross-account actions (they ride the assumed role)", + () => { + const actions = inlineActions(synth()); + // 注入/復旧の破壊操作は assumed CompetitorDeployRole 経由。 executor 自身の role には付けない。 + expect(actions).not.toContain("ssm:SendCommand"); + expect(actions).not.toContain("lambda:InvokeFunction"); // 自前 role には無い (= scheduler role 側のみ) + expect(actions).not.toContain("cloudformation:UpdateStack"); + // 最小権限は揃っている。 + expect(actions).toContain("dynamodb:Query"); + expect(actions).toContain("dynamodb:PutItem"); + expect(actions).toContain("scheduler:CreateSchedule"); + expect(actions).toContain("iam:PassRole"); + expect(actions).toContain("ssm:GetParameter"); + }, + SYNTH_TIMEOUT_MS, + ); + + it( + "should route tenkacloud.disruptions events to the executor and provision a scheduler-assumable revert role", + () => { + const tpl = synth(); + tpl.hasResourceProperties( + "AWS::Events::Rule", + Match.objectLike({ EventPattern: { source: ["tenkacloud.disruptions"] } }), + ); + // revert scheduler role: scheduler.amazonaws.com が assume + lambda:InvokeFunction を持つ。 + tpl.hasResourceProperties( + "AWS::IAM::Role", + Match.objectLike({ + AssumeRolePolicyDocument: Match.objectLike({ + Statement: Match.arrayWith([ + Match.objectLike({ + Principal: { Service: "scheduler.amazonaws.com" }, + }), + ]), + }), + }), + ); + }, + SYNTH_TIMEOUT_MS, + ); +}); diff --git a/infrastructure/test/problem-deploy/disruption-executor-store.test.ts b/infrastructure/test/problem-deploy/disruption-executor-store.test.ts new file mode 100644 index 000000000..def6f6074 --- /dev/null +++ b/infrastructure/test/problem-deploy/disruption-executor-store.test.ts @@ -0,0 +1,125 @@ +import { ConditionalCheckFailedException } from "@aws-sdk/client-dynamodb"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { DisruptionFiredDetail } from "../../lib/problem-deploy/handlers/disruption-executor-handler/execute"; +import { + claimExecution, + type ExecutorResources, + resolveDeployment, +} from "../../lib/problem-deploy/handlers/disruption-executor-handler/executor-store"; + +/** + * [ADR-031 / #1419] executor の DDB dep 実装。 claimExecution (冪等 Put) と resolveDeployment + * (GSI1 query + COMPLETE filter + stackOutputs parse) を mocked ddb で pin する。 + */ + +const detail: DisruptionFiredDetail = { + disruptionId: "ec2-latency-injection", + eventId: "evt-1", + problemId: "microservice-migration-battle", + tenantId: "tenant-1", + teamId: "team-1", + parameters: { delayMs: 200 }, + requestId: "req-1", + firedAt: "2026-06-02T00:00:00.000Z", +}; + +function makeResources(send: ReturnType): ExecutorResources { + return { + ddb: { send } as unknown as ExecutorResources["ddb"], + deploymentsTableName: "Deployments", + disruptionsTableName: "Disruptions", + }; +} + +describe("claimExecution (ADR-031 #1419)", () => { + beforeEach(() => vi.clearAllMocks()); + + it("should Put the EXEC# row with attribute_not_exists and return claimed", async () => { + const send = vi.fn().mockResolvedValue({}); + expect(await claimExecution(makeResources(send), detail, 1_000_000)).toBe("claimed"); + const put = send.mock.calls[0][0]; + expect(put.input.TableName).toBe("Disruptions"); + expect(put.input.Item.PK).toBe("EXEC#req-1#team-1"); + expect(put.input.ConditionExpression).toBe("attribute_not_exists(PK)"); + expect(put.input.Item.expiresAt).toBe(Math.floor(1_000_000 / 1000) + 7 * 24 * 60 * 60); + }); + + it("should return duplicate on ConditionalCheckFailed", async () => { + const send = vi + .fn() + .mockRejectedValue(new ConditionalCheckFailedException({ message: "exists", $metadata: {} })); + expect(await claimExecution(makeResources(send), detail, 0)).toBe("duplicate"); + }); + + it("should propagate non-conditional errors", async () => { + const send = vi.fn().mockRejectedValue(new Error("throttled")); + await expect(claimExecution(makeResources(send), detail, 0)).rejects.toThrow("throttled"); + }); + + it("should honor a custom execTtlSeconds", async () => { + const send = vi.fn().mockResolvedValue({}); + const resources = { ...makeResources(send), execTtlSeconds: 60 }; + await claimExecution(resources, detail, 5000); + expect(send.mock.calls[0][0].input.Item.expiresAt).toBe(Math.floor(5000 / 1000) + 60); + }); +}); + +describe("resolveDeployment (ADR-031 #1419)", () => { + beforeEach(() => vi.clearAllMocks()); + + const completeRow = { + status: "COMPLETE", + jobId: "job-1", + region: "ap-northeast-1", + competitorRoleArn: "arn:aws:iam::111122223333:role/TenkaCloud-CompetitorDeploy-Role", + externalIdParameterName: "/tenkacloud/tenant-1/external-id", + teamId: "team-1", + problemId: "microservice-migration-battle", + eventId: "evt-1", + stackOutputs: JSON.stringify({ WorkerInstanceIds: "i-aaa,i-bbb" }), + }; + + it("should query GSI1 by tenant + filter event/team/problem and return the parsed target", async () => { + const send = vi.fn().mockResolvedValue({ Items: [completeRow] }); + const target = await resolveDeployment(makeResources(send), detail); + expect(target).toEqual({ + jobId: "job-1", + region: "ap-northeast-1", + competitorRoleArn: "arn:aws:iam::111122223333:role/TenkaCloud-CompetitorDeploy-Role", + externalIdParameterName: "/tenkacloud/tenant-1/external-id", + stackOutputs: { WorkerInstanceIds: "i-aaa,i-bbb" }, + }); + const query = send.mock.calls[0][0]; + expect(query.input.IndexName).toBe("GSI1"); + expect(query.input.ExpressionAttributeValues).toMatchObject({ + ":pk": "TENANT#tenant-1", + ":ev": "evt-1", + ":tid": "team-1", + ":pid": "microservice-migration-battle", + }); + }); + + it("should return undefined when no row is COMPLETE", async () => { + const send = vi.fn().mockResolvedValue({ Items: [{ ...completeRow, status: "IN_PROGRESS" }] }); + expect(await resolveDeployment(makeResources(send), detail)).toBeUndefined(); + }); + + it("should skip a COMPLETE row missing cross-account fields", async () => { + const send = vi + .fn() + .mockResolvedValue({ Items: [{ ...completeRow, competitorRoleArn: undefined }] }); + expect(await resolveDeployment(makeResources(send), detail)).toBeUndefined(); + }); + + it("should default stackOutputs to {} when the row has no stackOutputs", async () => { + const send = vi + .fn() + .mockResolvedValue({ Items: [{ ...completeRow, stackOutputs: undefined }] }); + expect((await resolveDeployment(makeResources(send), detail))?.stackOutputs).toEqual({}); + }); + + it("should return undefined on an empty result set", async () => { + const send = vi.fn().mockResolvedValue({}); + expect(await resolveDeployment(makeResources(send), detail)).toBeUndefined(); + }); +}); diff --git a/infrastructure/test/problem-deploy/disruption-route.test.ts b/infrastructure/test/problem-deploy/disruption-route.test.ts new file mode 100644 index 000000000..89957df59 --- /dev/null +++ b/infrastructure/test/problem-deploy/disruption-route.test.ts @@ -0,0 +1,126 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { ExecutorDeps } from "../../lib/problem-deploy/handlers/disruption-executor-handler/execute"; +import { + parseDisruptionFiredDetail, + routeDisruptionInvocation, +} from "../../lib/problem-deploy/handlers/disruption-executor-handler/route"; +import type { ProblemDisruptionEntry } from "../../lib/utils/discover-problems-catalog"; + +/** + * [ADR-031 / #1419] executor router: EventBridge inject envelope と scheduler revert payload を + * 判別し、 それぞれ executeDisruptionAction / sendDispatch dep に振り分けることを pin する。 + */ + +const action = { + kind: "ssm-run-command" as const, + targetRef: "WorkerInstanceIds", + paramTemplate: { commands: ["delay {{delayMs}}"] }, + revert: { afterSeconds: 600 }, +}; +const withAction: ProblemDisruptionEntry = { + id: "ec2-latency-injection", + name: "latency", + eventDetailType: "DegradedDisruptionFired", + parameters: { delayMs: 200 }, + action, +}; + +const firedDetail = { + disruptionId: "ec2-latency-injection", + eventId: "evt-1", + problemId: "microservice-migration-battle", + tenantId: "tenant-1", + teamId: "team-1", + parameters: { delayMs: 200 }, + requestId: "req-1", + firedAt: "2026-06-02T00:00:00.000Z", +}; + +const deploymentTarget = { + jobId: "job-1", + region: "ap-northeast-1", + competitorRoleArn: "arn:aws:iam::111122223333:role/TenkaCloud-CompetitorDeploy-Role", + externalIdParameterName: "/tenkacloud/tenant-1/external-id", + stackOutputs: { WorkerInstanceIds: "i-aaa" }, +}; + +function makeDeps(over: Partial = {}): ExecutorDeps { + return { + problemsDisruptions: { "microservice-migration-battle": [withAction] }, + claimExecution: vi.fn().mockResolvedValue("claimed"), + resolveDeployment: vi.fn().mockResolvedValue(deploymentTarget), + sendDispatch: vi.fn().mockResolvedValue(undefined), + scheduleRevert: vi.fn().mockResolvedValue(undefined), + ...over, + }; +} + +describe("parseDisruptionFiredDetail", () => { + it("should narrow a valid EventBridge envelope detail", () => { + expect(parseDisruptionFiredDetail({ "detail-type": "X", detail: firedDetail })).toEqual( + firedDetail, + ); + }); + + it("should default parameters to {} when absent / non-object", () => { + const { parameters, ...rest } = firedDetail; + expect(parseDisruptionFiredDetail({ detail: rest })?.parameters).toEqual({}); + expect( + parseDisruptionFiredDetail({ detail: { ...rest, parameters: ["x"] } })?.parameters, + ).toEqual({}); + }); + + it("should return undefined for a missing detail or a missing required field", () => { + expect(parseDisruptionFiredDetail(undefined)).toBeUndefined(); + expect(parseDisruptionFiredDetail({})).toBeUndefined(); + expect(parseDisruptionFiredDetail({ detail: { ...firedDetail, teamId: "" } })).toBeUndefined(); + }); +}); + +describe("routeDisruptionInvocation (ADR-031 #1419)", () => { + beforeEach(() => vi.clearAllMocks()); + + it("should route an EventBridge fired envelope to executeDisruptionAction (inject)", async () => { + const deps = makeDeps(); + const outcome = await routeDisruptionInvocation( + { "detail-type": "DegradedDisruptionFired", detail: firedDetail }, + deps, + ); + expect(outcome).toEqual({ kind: "ok", jobId: "job-1" }); + expect(deps.sendDispatch).toHaveBeenCalledTimes(1); // the inject send + expect(deps.scheduleRevert).toHaveBeenCalledTimes(1); + }); + + it("should route a scheduler revert payload straight to sendDispatch (no re-execution)", async () => { + const deps = makeDeps(); + const revertDispatch = { + kind: "ssm-run-command" as const, + target: "i-aaa", + params: { commands: ["undo"] }, + }; + const outcome = await routeDisruptionInvocation( + { mode: "revert", dispatch: revertDispatch, target: deploymentTarget }, + deps, + ); + expect(outcome).toEqual({ kind: "reverted" }); + expect(deps.sendDispatch).toHaveBeenCalledWith(revertDispatch, deploymentTarget); + expect(deps.claimExecution).not.toHaveBeenCalled(); + expect(deps.scheduleRevert).not.toHaveBeenCalled(); + }); + + it("should return invalid_event for a malformed envelope (neither revert nor a valid detail)", async () => { + const deps = makeDeps(); + expect(await routeDisruptionInvocation({ detail: { teamId: "team-1" } }, deps)).toEqual({ + kind: "invalid_event", + }); + expect(deps.sendDispatch).not.toHaveBeenCalled(); + }); + + it("should treat a revert payload missing dispatch/target as a (failing) inject parse", async () => { + const deps = makeDeps(); + expect(await routeDisruptionInvocation({ mode: "revert" }, deps)).toEqual({ + kind: "invalid_event", + }); + expect(deps.sendDispatch).not.toHaveBeenCalled(); + }); +}); diff --git a/infrastructure/test/problem-deploy/disruption-schedule-revert.test.ts b/infrastructure/test/problem-deploy/disruption-schedule-revert.test.ts new file mode 100644 index 000000000..4664e3a3e --- /dev/null +++ b/infrastructure/test/problem-deploy/disruption-schedule-revert.test.ts @@ -0,0 +1,103 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { DisruptionDispatch } from "../../lib/problem-deploy/handlers/disruption-executor-handler/dispatch-command"; +import type { + DeploymentTarget, + DisruptionFiredDetail, +} from "../../lib/problem-deploy/handlers/disruption-executor-handler/execute"; +import { + revertAtExpression, + revertScheduleName, + type ScheduleRevertDeps, + scheduleRevert, +} from "../../lib/problem-deploy/handlers/disruption-executor-handler/schedule-revert"; + +/** + * [ADR-031 / ADR-029 INV-2 / #1419] scheduleRevert: one-shot aws-scheduler 登録を mocked client で pin。 + * 冪等 name / at(...) 式 / DELETE-after-completion / revert payload を観察する。 + */ + +const detail: DisruptionFiredDetail = { + disruptionId: "ec2-latency-injection", + eventId: "evt-1", + problemId: "microservice-migration-battle", + tenantId: "tenant-1", + teamId: "team-1", + parameters: { delayMs: 200 }, + requestId: "req-1", + firedAt: "2026-06-02T00:00:00.000Z", +}; + +const target: DeploymentTarget = { + jobId: "job-1", + region: "ap-northeast-1", + competitorRoleArn: "arn:aws:iam::111122223333:role/TenkaCloud-CompetitorDeploy-Role", + externalIdParameterName: "/tenkacloud/tenant-1/external-id", + stackOutputs: { WorkerInstanceIds: "i-aaa" }, +}; + +const revert: DisruptionDispatch = { + kind: "ssm-run-command", + target: "i-aaa", + documentName: "AWS-RunShellScript", + params: { commands: ["tc qdisc del dev eth0 root"] }, +}; + +function makeDeps(send = vi.fn().mockResolvedValue({})): { + deps: ScheduleRevertDeps; + send: typeof send; +} { + return { + deps: { + scheduler: { send } as unknown as ScheduleRevertDeps["scheduler"], + schedulerRoleArn: "arn:aws:iam::444455556666:role/tc-disruption-scheduler", + revertTargetArn: "arn:aws:lambda:ap-northeast-1:444455556666:function:tc-disruption-executor", + }, + send, + }; +} + +describe("revertScheduleName", () => { + it("should be the idempotent EXEC# twin sanitized to the scheduler name charset / length", () => { + expect(revertScheduleName(detail)).toBe("tc-revert-req-1-team-1"); + const dirty = { ...detail, requestId: "req/1 weird", teamId: "team#1" }; + expect(revertScheduleName(dirty)).toBe("tc-revert-req-1-weird-team-1"); + const long = { ...detail, requestId: "r".repeat(100) }; + expect(revertScheduleName(long).length).toBe(64); + }); +}); + +describe("revertAtExpression", () => { + it("should produce a UTC at(...) expression at firedAt + afterSeconds (second precision)", () => { + expect(revertAtExpression("2026-06-02T00:00:00.000Z", 600)).toBe("at(2026-06-02T00:10:00)"); + }); +}); + +describe("scheduleRevert (ADR-031 #1419)", () => { + beforeEach(() => vi.clearAllMocks()); + + it("should create a one-shot DELETE-after schedule invoking the target with the revert payload", async () => { + const { deps, send } = makeDeps(); + await scheduleRevert(revert, detail, target, 600, deps); + const input = send.mock.calls[0][0].input; + expect(input.Name).toBe("tc-revert-req-1-team-1"); + expect(input.ScheduleExpression).toBe("at(2026-06-02T00:10:00)"); + expect(input.FlexibleTimeWindow).toEqual({ Mode: "OFF" }); + expect(input.ActionAfterCompletion).toBe("DELETE"); + expect(input.State).toBe("ENABLED"); + expect(input.Target.Arn).toBe(deps.revertTargetArn); + expect(input.Target.RoleArn).toBe(deps.schedulerRoleArn); + expect(JSON.parse(input.Target.Input)).toEqual({ + mode: "revert", + detail, + dispatch: revert, + target, + }); + }); + + it("should propagate scheduler errors (revert scheduling failure is loud — INV-2)", async () => { + const { deps } = makeDeps(vi.fn().mockRejectedValue(new Error("ConflictException"))); + await expect(scheduleRevert(revert, detail, target, 600, deps)).rejects.toThrow( + "ConflictException", + ); + }); +}); diff --git a/infrastructure/test/problem-deploy/disruption-send-dispatch.test.ts b/infrastructure/test/problem-deploy/disruption-send-dispatch.test.ts new file mode 100644 index 000000000..2b3cefdfe --- /dev/null +++ b/infrastructure/test/problem-deploy/disruption-send-dispatch.test.ts @@ -0,0 +1,102 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { DisruptionDispatch } from "../../lib/problem-deploy/handlers/disruption-executor-handler/dispatch-command"; +import { + type DispatchTarget, + type SendDispatchDeps, + sendDispatch, +} from "../../lib/problem-deploy/handlers/disruption-executor-handler/send-dispatch"; + +/** + * [ADR-031 / #1419] sendDispatch: DisruptionDispatch → 実 SDK command の mapping を mocked client で pin。 + * client factory は assumed-credential 付きで呼ばれること + 各 kind の command input を観察する。 + */ + +const target: DispatchTarget = { + region: "ap-northeast-1", + credentials: { AccessKeyId: "AK", SecretAccessKey: "SK", SessionToken: "ST" }, +}; + +function makeDeps(): { + deps: SendDispatchDeps; + ssm: ReturnType; + lambda: ReturnType; + cfn: ReturnType; + factories: Record>; +} { + const ssm = vi.fn().mockResolvedValue({}); + const lambda = vi.fn().mockResolvedValue({}); + const cfn = vi.fn().mockResolvedValue({}); + const ssmFactory = vi.fn().mockReturnValue({ send: ssm }); + const lambdaFactory = vi.fn().mockReturnValue({ send: lambda }); + const cfnFactory = vi.fn().mockReturnValue({ send: cfn }); + return { + deps: { ssmClient: ssmFactory, lambdaClient: lambdaFactory, cfnClient: cfnFactory }, + ssm, + lambda, + cfn, + factories: { ssmFactory, lambdaFactory, cfnFactory }, + }; +} + +describe("sendDispatch (ADR-031 #1419)", () => { + beforeEach(() => vi.clearAllMocks()); + + it("should send SSM SendCommand with split InstanceIds and coerced string[] Parameters", async () => { + const { deps, ssm, factories } = makeDeps(); + const dispatch: DisruptionDispatch = { + kind: "ssm-run-command", + target: "i-aaa, i-bbb ,", + documentName: "AWS-RunShellScript", + params: { commands: ["echo hi"], timeout: 30 }, + }; + await sendDispatch(dispatch, target, deps); + expect(factories.ssmFactory).toHaveBeenCalledWith(target); + const input = ssm.mock.calls[0][0].input; + expect(input.DocumentName).toBe("AWS-RunShellScript"); + expect(input.InstanceIds).toEqual(["i-aaa", "i-bbb"]); + expect(input.Parameters).toEqual({ commands: ["echo hi"], timeout: ["30"] }); + }); + + it("should default the SSM document name when none is declared", async () => { + const { deps, ssm } = makeDeps(); + await sendDispatch({ kind: "ssm-run-command", target: "i-x", params: {} }, target, deps); + expect(ssm.mock.calls[0][0].input.DocumentName).toBe("AWS-RunShellScript"); + }); + + it("should invoke Lambda asynchronously (Event) with the params as JSON payload", async () => { + const { deps, lambda, factories } = makeDeps(); + await sendDispatch( + { kind: "lambda-invoke", target: "fault-fn", params: { mode: "fail" } }, + target, + deps, + ); + expect(factories.lambdaFactory).toHaveBeenCalledWith(target); + const input = lambda.mock.calls[0][0].input; + expect(input.FunctionName).toBe("fault-fn"); + expect(input.InvocationType).toBe("Event"); + expect(new TextDecoder().decode(input.Payload)).toBe(JSON.stringify({ mode: "fail" })); + }); + + it("should update the CFn stack with UsePreviousTemplate + mapped Parameters + IAM capabilities", async () => { + const { deps, cfn, factories } = makeDeps(); + await sendDispatch( + { kind: "cfn-stack-update", target: "team-stack", params: { DesiredCount: 0 } }, + target, + deps, + ); + expect(factories.cfnFactory).toHaveBeenCalledWith(target); + const input = cfn.mock.calls[0][0].input; + expect(input.StackName).toBe("team-stack"); + expect(input.UsePreviousTemplate).toBe(true); + expect(input.Parameters).toEqual([{ ParameterKey: "DesiredCount", ParameterValue: "0" }]); + expect(input.Capabilities).toContain("CAPABILITY_NAMED_IAM"); + }); + + it("should propagate SDK errors (fault injection failure is loud)", async () => { + const { deps, ssm } = makeDeps(); + ssm.mockRejectedValueOnce(new Error("AccessDenied")); + await expect( + sendDispatch({ kind: "ssm-run-command", target: "i-x", params: {} }, target, deps), + ).rejects.toThrow("AccessDenied"); + }); +});