diff --git a/.buildkite/pipelines/evals/llm_evals.yml b/.buildkite/pipelines/evals/llm_evals.yml index c9f8ae42623c6..7daea3e879062 100644 --- a/.buildkite/pipelines/evals/llm_evals.yml +++ b/.buildkite/pipelines/evals/llm_evals.yml @@ -242,6 +242,29 @@ steps: - exit_status: '-1' limit: 3 + - label: 'Evals: PCI Compliance' + key: kbn-evals-weekly-pci-compliance + command: bash .buildkite/scripts/steps/evals/run_suite.sh + env: + KBN_EVALS: '1' + FTR_EIS_CCM: '1' + EVAL_SUITE_ID: 'pci-compliance' + EVAL_FANOUT: '1' + EVAL_INCLUDE_EIS_MODELS: '1' + EVAL_MODEL_GROUPS: *weekly_eis_core_models + EVAL_SERVER_CONFIG_SET: 'evals_pci_compliance' + timeout_in_minutes: 60 + agents: + image: family/kibana-ubuntu-2404 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + preemptible: true + retry: + automatic: + - exit_status: '-1' + limit: 3 + - label: 'Evals: Entity Analytics' key: kbn-evals-weekly-entity-analytics command: bash .buildkite/scripts/steps/evals/run_suite.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ca8e9fc97e726..13cfdb7147a37 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1284,6 +1284,7 @@ x-pack/solutions/security/packages/kbn-cloud-security-posture/public @elastic/co x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery @elastic/security-generative-ai x-pack/solutions/security/packages/kbn-evals-suite-endpoint @elastic/security-defend-workflows x-pack/solutions/security/packages/kbn-evals-suite-entity-analytics @elastic/security-entity-analytics +x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance @elastic/security-defend-workflows x-pack/solutions/security/packages/kbn-evals-suite-security-ai-rules @elastic/security-detection-engine x-pack/solutions/security/packages/kbn-scout-security @elastic/appex-qa @elastic/security-engineering-productivity x-pack/solutions/security/packages/kbn-securitysolution-autocomplete @elastic/security-detection-engine diff --git a/package.json b/package.json index 68c49ef8fb5a8..80db7d716a5e1 100644 --- a/package.json +++ b/package.json @@ -1710,6 +1710,7 @@ "@kbn/evals-suite-llm-tasks": "link:x-pack/platform/packages/shared/ai-infra/kbn-evals-suite-llm-tasks", "@kbn/evals-suite-obs-ai-assistant": "link:x-pack/solutions/observability/packages/kbn-evals-suite-obs-ai-assistant", "@kbn/evals-suite-observability-ai": "link:x-pack/solutions/observability/packages/kbn-evals-suite-observability-ai", + "@kbn/evals-suite-pci-compliance": "link:x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance", "@kbn/evals-suite-security-ai-rules": "link:x-pack/solutions/security/packages/kbn-evals-suite-security-ai-rules", "@kbn/evals-suite-significant-events": "link:x-pack/platform/packages/shared/kbn-evals-suite-significant-events", "@kbn/evals-suite-streams": "link:x-pack/platform/packages/shared/kbn-evals-suite-streams", diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_pci_compliance/stateful/classic.stateful.config.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_pci_compliance/stateful/classic.stateful.config.ts new file mode 100644 index 0000000000000..ee8f01a05a1bb --- /dev/null +++ b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_pci_compliance/stateful/classic.stateful.config.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { ScoutServerConfig } from '../../../../../types'; +import { servers as evalsTracingConfig } from '../../evals_tracing/stateful/classic.stateful.config'; + +/** + * Custom Scout stateful server configuration for PCI DSS v4.0.1 compliance evals. + * Enables the Agent Builder experimental features UI setting and the + * pciComplianceAgentBuilder experimental flag in Security Solution. + * + * Usage: + * node scripts/scout start-server --arch stateful --domain classic --serverConfigSet evals_pci_compliance + */ +export const servers: ScoutServerConfig = { + ...evalsTracingConfig, + kbnTestServer: { + ...evalsTracingConfig.kbnTestServer, + serverArgs: [ + ...evalsTracingConfig.kbnTestServer.serverArgs, + '--uiSettings.overrides.agentBuilder:experimentalFeatures=true', + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'pciComplianceAgentBuilder', + ])}`, + ], + }, +}; diff --git a/tsconfig.base.json b/tsconfig.base.json index 56678132d4844..e5641d0e8fdee 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1172,6 +1172,8 @@ "@kbn/evals-suite-obs-ai-assistant/*": ["x-pack/solutions/observability/packages/kbn-evals-suite-obs-ai-assistant/*"], "@kbn/evals-suite-observability-ai": ["x-pack/solutions/observability/packages/kbn-evals-suite-observability-ai"], "@kbn/evals-suite-observability-ai/*": ["x-pack/solutions/observability/packages/kbn-evals-suite-observability-ai/*"], + "@kbn/evals-suite-pci-compliance": ["x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance"], + "@kbn/evals-suite-pci-compliance/*": ["x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/*"], "@kbn/evals-suite-security-ai-rules": ["x-pack/solutions/security/packages/kbn-evals-suite-security-ai-rules"], "@kbn/evals-suite-security-ai-rules/*": ["x-pack/solutions/security/packages/kbn-evals-suite-security-ai-rules/*"], "@kbn/evals-suite-significant-events": ["x-pack/platform/packages/shared/kbn-evals-suite-significant-events"], diff --git a/x-pack/platform/packages/shared/agent-builder/agent-builder-server/allow_lists.ts b/x-pack/platform/packages/shared/agent-builder/agent-builder-server/allow_lists.ts index 72ca1528bee10..5af68015fd91a 100644 --- a/x-pack/platform/packages/shared/agent-builder/agent-builder-server/allow_lists.ts +++ b/x-pack/platform/packages/shared/agent-builder/agent-builder-server/allow_lists.ts @@ -45,8 +45,7 @@ export const AGENT_BUILDER_BUILTIN_TOOLS = [ `${internalNamespaces.security}.get_entity`, `${internalNamespaces.security}.search_entities`, `${internalNamespaces.security}.pci_scope_discovery`, - `${internalNamespaces.security}.pci_compliance_check`, - `${internalNamespaces.security}.pci_compliance_report`, + `${internalNamespaces.security}.pci_compliance`, `${internalNamespaces.security}.pci_field_mapper`, // Streams diff --git a/x-pack/platform/packages/shared/kbn-evals/evals.suites.json b/x-pack/platform/packages/shared/kbn-evals/evals.suites.json index e155227d43d20..6776c2a7b24f4 100644 --- a/x-pack/platform/packages/shared/kbn-evals/evals.suites.json +++ b/x-pack/platform/packages/shared/kbn-evals/evals.suites.json @@ -169,6 +169,15 @@ "tags": ["platform", "workflows"], "ciLabels": ["evals:workflows"], "serverConfigSet": "evals_workflows" + }, + { + "id": "pci-compliance", + "name": "PCI DSS v4.0.1 Compliance", + "slackChannel": "#security-defend-workflows-tests", + "configPath": "x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/playwright.config.ts", + "tags": ["security", "pci-compliance"], + "ciLabels": ["evals:pci-compliance"], + "serverConfigSet": "evals_pci_compliance" } ] } diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/README.md b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/README.md new file mode 100644 index 0000000000000..15a333354efe1 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/README.md @@ -0,0 +1,101 @@ +# @kbn/evals-suite-pci-compliance + +End-to-end evaluation suite for the **PCI DSS v4.0.1 compliance** Agent Builder +skill. It exercises the consolidated `pci_compliance` tool along with +`pci_scope_discovery` and `pci_field_mapper` against a small, deterministic +dataset and asserts on scoring, evidence, scope claims, and the mandatory QSA +disclaimer. + +The suite is modeled on `@kbn/evals-suite-endpoint` so traces, spans, and +evaluator fields are directly comparable across security eval suites. + +## Prerequisites + +- The feature flag `pciComplianceAgentBuilder` must be enabled on the Kibana + test server. This is handled automatically when the suite runs through the + `evals_pci_compliance` Scout `serverConfigSet` + (`src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_pci_compliance`). +- An AI connector must be available (see the `@kbn/evals` docs for the + standard connector setup). +- The Agent Builder experimental features UI setting is also enabled by that + config set. + +## Running + +From the Kibana repo root: + +```sh +# Start the Kibana + ES test server with the PCI compliance config set +node scripts/scout start-server \ + --arch stateful \ + --domain classic \ + --serverConfigSet evals_pci_compliance + +# In another terminal, run the suite +node scripts/evals start --suite pci-compliance +``` + +All evaluation specs live under [`evals/pci_compliance`](./evals/pci_compliance). + +### Seeding a dev cluster manually + +To import the eval data into a remote dev cluster (e.g. Elastic Cloud): + +```sh +# Requires x-pack/.env with: Elasticsearch=, username=, password= +./scripts/seed_dev_cluster.sh # seed data +./scripts/seed_dev_cluster.sh --cleanup # delete data streams +``` + +## Scenarios + +| Spec | Skill / Tool | What it asserts | +| --------------------------------------- | ----------------------------------- | -------------------------------------------------------------------------------------------------- | +| `full compliance report` | `pci_compliance` (`mode: "report"`) | Full scorecard across all 12 requirements with correct RED/AMBER/GREEN status. | +| `requirement 8.3.4 — brute force` | `pci_compliance` (`mode: "check"`) | Detects 7 failed logins for jdoe (exceeds threshold of 6), RED status. | +| `requirement 4.1 — weak TLS` | `pci_compliance` (`mode: "check"`) | Flags TLS 1.0, TLS 1.1, and plain HTTP as violations. | +| `requirement 2.2.4 — default accounts` | `pci_compliance` (`mode: "check"`) | Flags admin and root successful logins as default-account violations. | +| `scope discovery` | `pci_scope_discovery` | Identifies 4 ECS indices and classifies them (identity, network, endpoint). | +| `field mapping for custom data` | `pci_field_mapper` | Suggests correct ECS targets for non-ECS fields (username → user.name, etc.). | +| `scoped check (auth-only)` | `pci_compliance` | Auth requirements produce real findings; network/vuln/malware requirements are NOT_ASSESSABLE. | +| `requirement 9 — no matching data` | `pci_compliance` (`mode: "check"`) | Returns AMBER/NOT_ASSESSABLE when no physical access events exist. | + +## Data generators + +Deterministic seed data lives in +[`src/data_generators/pci_data.ts`](./src/data_generators/pci_data.ts). It +provisions five data streams: + +| Index | Contents | Doc count | +| ------------------------- | ---------------------------------------------------- | --------- | +| `logs-pci-auth-eval` | ECS auth events: 7 failed logins (jdoe), admin/root successes, IAM events | 13 | +| `logs-pci-network-eval` | TLS 1.3/1.2 (good), TLS 1.0/1.1 (weak), plain HTTP | 6 | +| `logs-pci-vuln-eval` | Critical/high CVEs, IDS alerts (exploit, port scan) | 4 | +| `logs-pci-endpoint-eval` | Malware detection, suspicious process execution | 2 | +| `logs-pci-custom-eval` | Non-ECS legacy fields for field-mapper tests | 4 | + +The generators expose `seedPciEvalData()` and `cleanupPciEvalData()` so each +spec owns its lifecycle without leaking indices between runs. + +## Evaluator + +The suite uses a PCI-specific criteria evaluator +(`src/evaluate_dataset.ts#createPciCriteriaEvaluator`) that pins a baseline +(`BASELINE_PCI_CRITERIA`) asserting: + +- The DSS version (`4.0.1`) is referenced. +- The response declines to act as QSA attestation (non-attestation disclaimer). +- A structured `scopeClaim` payload is emitted alongside any finding. + +Scenario-specific criteria layer on top of the baseline. + +## Why a dedicated suite + +- **Determinism**: PCI findings depend heavily on the data shape. Seeding a + small known-good dataset is far more reliable than reusing generic logs. +- **Scope-claim parity**: Every PCI tool response ships a scope claim with + DSS version, indices, time range, evaluated requirements, checked fields, + and a disclaimer. The suite asserts on this for every scenario. +- **Feature flag isolation**: The `pciComplianceAgentBuilder` flag is + off-by-default in Kibana; the `evals_pci_compliance` config set isolates + the suite from the rest of the eval runners. diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/evals/pci_compliance/pci_compliance.spec.ts b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/evals/pci_compliance/pci_compliance.spec.ts new file mode 100644 index 0000000000000..e81d010143ff4 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/evals/pci_compliance/pci_compliance.spec.ts @@ -0,0 +1,280 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { tags } from '@kbn/scout'; +import { evaluate } from '../../src/evaluate'; +import { + PCI_INDICES, + cleanupPciEvalData, + seedPciEvalData, +} from '../../src/data_generators/pci_data'; + +const ALL_ECS_INDICES = `${PCI_INDICES.auth},${PCI_INDICES.network},${PCI_INDICES.vuln},${PCI_INDICES.endpoint}`; + +evaluate.describe('PCI DSS v4.0.1 Compliance', { tag: tags.stateful.classic }, () => { + evaluate.beforeAll(async ({ internalEsClient, chatClient, log }) => { + await seedPciEvalData({ esClient: internalEsClient, log }); + + try { + await chatClient.converse({ message: 'hello' }); + } catch (e) { + log.warning(`Warmup failed: ${e}`); + } + }); + + evaluate.afterAll(async ({ internalEsClient, log }) => { + await cleanupPciEvalData({ esClient: internalEsClient, log }); + }); + + // --------------------------------------------------------------------------- + // Scenario 1: Full Compliance Report + // --------------------------------------------------------------------------- + evaluate('full compliance report across all requirements', async ({ evaluateDataset }) => { + await evaluateDataset({ + dataset: { + name: 'pci-compliance: full report', + description: + 'Validates the agent produces a full PCI DSS v4.0.1 posture report covering all 12 ' + + 'requirements. Expected flow: scope_discovery → compliance_check (all) → compliance_report. ' + + 'Req 4.1 and 8.3.4 should be RED, req 2.2.4 RED (default accounts), ' + + 'non-assessable requirements (3, 9, 12) should be AMBER/NOT_ASSESSABLE.', + examples: [ + { + input: { + question: `Run a full PCI DSS compliance report using indices ${ALL_ECS_INDICES} for the last hour.`, + }, + output: { + criteria: [ + 'Called the pci_compliance tool in report mode (not just a single check).', + 'Produced a scorecard covering requirements 1–12 (by id or by name).', + 'Assigned RED or violation status to requirement 8 (or 8.3.4) due to the brute-force data for user "jdoe".', + 'Assigned RED or violation status to requirement 4 (or 4.1) due to weak TLS 1.0, TLS 1.1, and plain HTTP traffic.', + 'Flagged requirement 2.2.4 as RED or violation due to "admin" and "root" successful logins.', + 'Marked requirements with no matching data (e.g. 3 — stored data, 9 — physical access, 12 — policies) as AMBER, NOT_ASSESSABLE, or similar non-GREEN status.', + 'Reported an overall confidence rollup (HIGH / MEDIUM / LOW / NOT_ASSESSABLE) rather than a single score alone.', + ], + }, + }, + ], + }, + }); + }); + + // --------------------------------------------------------------------------- + // Scenario 2: Single Requirement — Brute Force (Req 8.3.4) + // --------------------------------------------------------------------------- + evaluate('requirement 8.3.4 — brute-force detection', async ({ evaluateDataset }) => { + await evaluateDataset({ + dataset: { + name: 'pci-compliance: requirement 8.3.4 brute force', + description: + 'Validates the agent detects repeated failed logins exceeding the threshold of 10 ' + + 'for PCI DSS requirement 8.3.4. Expected: RED status, HIGH confidence. ' + + 'Evidence: jdoe with 12 failed login attempts from 192.168.1.100.', + examples: [ + { + input: { + question: `Check PCI DSS requirement 8.3.4 against ${PCI_INDICES.auth} for the last hour and surface any violations with evidence.`, + }, + output: { + criteria: [ + 'Called the pci_compliance tool in check mode for requirement 8.3.4 (or requirement 8).', + `Passed the index pattern ${PCI_INDICES.auth} (or an equivalent) to the tool.`, + 'Surfaced the repeated failed logins for user "jdoe" as a RED / violation finding.', + 'The evidence shows at least 12 (or more than 10) failed authentication attempts for user "jdoe".', + 'The source IP 192.168.1.100 is mentioned in the brute-force evidence.', + ], + }, + }, + ], + }, + }); + }); + + // --------------------------------------------------------------------------- + // Scenario 3: Single Requirement — Weak TLS (Req 4.1) + // --------------------------------------------------------------------------- + evaluate('requirement 4.1 — weak TLS detection', async ({ evaluateDataset }) => { + await evaluateDataset({ + dataset: { + name: 'pci-compliance: requirement 4.1 weak TLS', + description: + 'Validates the agent surfaces legacy TLS/SSL and unencrypted HTTP connections as ' + + 'PCI DSS requirement 4.1 violations. Expected: RED. ' + + 'Violations: 203.0.113.51 (TLS 1.0), 203.0.113.52 (TLS 1.1), 198.51.100.10 (HTTP/no TLS).', + examples: [ + { + input: { + question: `Check PCI requirement 4.1 against ${PCI_INDICES.network} for the last hour. Are there any weak TLS or unencrypted connections?`, + }, + output: { + criteria: [ + 'Called the pci_compliance tool in check mode for requirement 4.1 (or requirement 4).', + 'Identified TLS 1.0 connections (destination 203.0.113.51) as a violation.', + 'Identified TLS 1.1 connections (destination 203.0.113.52) as a violation.', + 'Identified plain HTTP traffic (destination 198.51.100.10, no TLS) as a violation.', + 'Did not report the strong TLS 1.2 and TLS 1.3 traffic as violations.', + ], + }, + }, + ], + }, + }); + }); + + // --------------------------------------------------------------------------- + // Scenario 4: Single Requirement — Default Accounts (Req 2.2.4) + // --------------------------------------------------------------------------- + evaluate('requirement 2.2.4 — default account detection', async ({ evaluateDataset }) => { + await evaluateDataset({ + dataset: { + name: 'pci-compliance: requirement 2.2.4 default accounts', + description: + 'Validates the agent detects successful logins from default/well-known accounts ' + + '(admin, root) as a PCI DSS requirement 2.2.4 violation. Expected: RED.', + examples: [ + { + input: { + question: `Check PCI DSS requirement 2.2.4 against ${PCI_INDICES.auth} for the last hour.`, + }, + output: { + criteria: [ + 'Called the pci_compliance tool in check mode for requirement 2.2.4 (or requirement 2).', + 'Identified successful authentication events for "admin" as a violation — default accounts should not be in active use.', + 'Identified successful authentication events for "root" as a violation — default accounts should not be in active use.', + ], + }, + }, + ], + }, + }); + }); + + // --------------------------------------------------------------------------- + // Scenario 5: Scope Discovery + // --------------------------------------------------------------------------- + evaluate('scope discovery across PCI-relevant data', async ({ evaluateDataset }) => { + await evaluateDataset({ + dataset: { + name: 'pci-compliance: scope discovery', + description: + 'Validates the agent inventories which indices contain PCI-relevant telemetry before ' + + 'running checks. Expected: 4 ECS indices discovered with correct categories — ' + + 'auth → identity, network → network, vuln → endpoint, endpoint → endpoint.', + examples: [ + { + input: { + question: `What PCI-relevant data do I have in ${PCI_INDICES.auth},${PCI_INDICES.network},${PCI_INDICES.vuln},${PCI_INDICES.endpoint}?`, + }, + output: { + criteria: [ + 'Called pci_scope_discovery (rather than running compliance checks directly).', + `Reported ${PCI_INDICES.auth} as PCI-relevant, classified under "identity" or auth category.`, + `Reported ${PCI_INDICES.network} as PCI-relevant, classified under "network" category.`, + `Reported ${PCI_INDICES.vuln} as PCI-relevant. The tool classified it under one or more of: "vulnerability", "endpoint", "identity", "network" (the exact category names from pci_scope_discovery).`, + `Reported ${PCI_INDICES.endpoint} as PCI-relevant, classified under "endpoint" or malware category.`, + ], + }, + }, + ], + }, + }); + }); + + // --------------------------------------------------------------------------- + // Scenario 6: Field Mapper — Non-ECS Data + // --------------------------------------------------------------------------- + evaluate('field mapping for non-ECS custom data', async ({ evaluateDataset }) => { + await evaluateDataset({ + dataset: { + name: 'pci-compliance: field mapping', + description: + 'Validates the agent maps non-ECS fields in the custom legacy index to their ECS ' + + 'equivalents. Expected mappings: username → user.name, src_ip → source.ip, ' + + 'auth_result → event.outcome, operation → event.action, hostname → host.name, ' + + 'cve → vulnerability.id, severity → vulnerability.severity, program → process.name.', + examples: [ + { + input: { + question: `Map the fields in ${PCI_INDICES.custom} to ECS for PCI compliance queries.`, + }, + output: { + criteria: [ + 'Called the pci_field_mapper tool against the supplied custom index.', + 'Suggested mapping "username" → "user.name".', + 'Suggested mapping "src_ip" → "source.ip".', + 'Suggested mapping "hostname" → "host.name".', + 'Suggested mapping "cve" → "vulnerability.id".', + 'Suggested mapping "severity" to the ECS field "vulnerability.severity".', + 'All suggested mappings target valid ECS field names (e.g., user.name, source.ip, host.name, vulnerability.id, vulnerability.severity, event.outcome, event.action, process.name).', + ], + }, + }, + ], + }, + }); + }); + + // --------------------------------------------------------------------------- + // Scenario 7: Scoped to Specific Index (auth-only) + // --------------------------------------------------------------------------- + evaluate('scoped check using only the auth index', async ({ evaluateDataset }) => { + await evaluateDataset({ + dataset: { + name: 'pci-compliance: scoped to auth index', + description: + 'Validates that when scoped to only the auth index, auth-related requirements ' + + '(8, 8.3.4, 8.3.9, 2.2.4) produce real results while network/vuln/malware ' + + 'requirements are AMBER or NOT_ASSESSABLE.', + examples: [ + { + input: { + question: `Check PCI compliance using only ${PCI_INDICES.auth} for the last hour.`, + }, + output: { + criteria: [ + `Scoped the check to ${PCI_INDICES.auth} (not logs-* or a broader pattern).`, + 'Produced real findings for authentication-related requirements (requirement 8 and/or 2.2.4).', + 'Marked network-related requirements (e.g. 1, 4) as AMBER, NOT_ASSESSABLE, or similar since network data is not in the auth index.', + 'Marked vulnerability/malware requirements (e.g. 5, 6, 11) as AMBER, NOT_ASSESSABLE, or similar since that data is not in the auth index.', + ], + }, + }, + ], + }, + }); + }); + + // --------------------------------------------------------------------------- + // Scenario 8: No Matching Data (Req 9 — Physical Access) + // --------------------------------------------------------------------------- + evaluate('requirement 9 — no matching physical access data', async ({ evaluateDataset }) => { + await evaluateDataset({ + dataset: { + name: 'pci-compliance: no matching data', + description: + 'Validates the agent handles missing data gracefully. Requirement 9 (physical access) ' + + 'has no matching events in any test index. Expected: AMBER or NOT_ASSESSABLE.', + examples: [ + { + input: { + question: `Check PCI requirement 9 against ${ALL_ECS_INDICES} for the last hour.`, + }, + output: { + criteria: [ + 'Called the pci_compliance tool in check mode for requirement 9.', + 'Returned AMBER, NOT_ASSESSABLE, or an equivalent non-GREEN / non-RED status.', + 'Explained that no physical access or badge events were found in the evaluated indices.', + 'Did not fabricate violations or evidence — the finding reflects the actual absence of data.', + ], + }, + }, + ], + }, + }); + }); +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/jest.config.js b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/jest.config.js new file mode 100644 index 0000000000000..dc4e366be7989 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../../../../..', + roots: ['/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance'], +}; diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/kibana.jsonc b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/kibana.jsonc new file mode 100644 index 0000000000000..6706f5fc4fbee --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/kibana.jsonc @@ -0,0 +1,7 @@ +{ + "type": "functional-tests", + "id": "@kbn/evals-suite-pci-compliance", + "owner": "@elastic/security-defend-workflows", + "group": "security", + "visibility": "private" +} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/moon.yml b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/moon.yml new file mode 100644 index 0000000000000..d96a6d1a0108d --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/moon.yml @@ -0,0 +1,40 @@ +# This file is generated by the @kbn/moon package. Any manual edits will be erased! +# To extend this, write your extensions/overrides to 'moon.extend.yml' +# then regenerate this file with: 'node scripts/regenerate_moon_projects.js --update --filter @kbn/evals-suite-pci-compliance' + +$schema: https://moonrepo.dev/schemas/project.json +id: '@kbn/evals-suite-pci-compliance' +layer: unknown +owners: + defaultOwner: '@elastic/security-defend-workflows' +toolchains: + default: node +language: typescript +project: + title: '@kbn/evals-suite-pci-compliance' + description: Moon project for @kbn/evals-suite-pci-compliance + channel: '' + owner: '@elastic/security-defend-workflows' + sourceRoot: x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance +dependsOn: + - '@kbn/evals' + - '@kbn/scout' + - '@kbn/agent-builder-common' + - '@kbn/tooling-log' + - '@kbn/core' + - '@kbn/test' + - '@kbn/security-solution-plugin' +tags: + - functional-tests + - package + - prod + - group-security + - private + - jest-unit-tests +fileGroups: + src: + - '**/*.ts' + - '!target/**/*' + jest-config: + - jest.config.js +tasks: {} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/package.json b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/package.json new file mode 100644 index 0000000000000..1ece28160988e --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/evals-suite-pci-compliance", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/playwright.config.ts b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/playwright.config.ts new file mode 100644 index 0000000000000..137c2134aff28 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/playwright.config.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createPlaywrightEvalsConfig } from '@kbn/evals'; + +export default createPlaywrightEvalsConfig({ + testDir: `${__dirname}/evals`, + timeout: 30 * 60_000, +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/scripts/seed_dev_cluster.sh b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/scripts/seed_dev_cluster.sh new file mode 100755 index 0000000000000..1a5d22c919f6a --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/scripts/seed_dev_cluster.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# +# Seed (or cleanup) PCI compliance eval data on a dev Elasticsearch cluster. +# +# Usage: +# ./scripts/seed_dev_cluster.sh # seed data +# ./scripts/seed_dev_cluster.sh --cleanup # delete indices +# +# Reads credentials from x-pack/.env (Elasticsearch URL + basic auth). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ENV_FILE="${SCRIPT_DIR}/../../../../../.env" + +if [[ ! -f "$ENV_FILE" ]]; then + echo "ERROR: .env file not found at $ENV_FILE" + exit 1 +fi + +# shellcheck source=/dev/null +source "$ENV_FILE" + +ES_URL="${Elasticsearch:?Elasticsearch URL not set in .env}" +ES_USER="${username:-elastic}" +ES_PASS="${password:?password not set in .env}" +AUTH="${ES_USER}:${ES_PASS}" + +INDICES=( + "logs-pci-auth-eval" + "logs-pci-network-eval" + "logs-pci-vuln-eval" + "logs-pci-endpoint-eval" + "logs-pci-custom-eval" +) + +minute_ago() { + local offset_min=$1 + if date --version >/dev/null 2>&1; then + date -u -d "-${offset_min} minutes" '+%Y-%m-%dT%H:%M:%SZ' + else + date -u -v-"${offset_min}"M '+%Y-%m-%dT%H:%M:%SZ' + fi +} + +cleanup() { + echo "Deleting PCI eval data streams..." + for idx in "${INDICES[@]}"; do + local code + code=$(curl -s -o /dev/null -w "%{http_code}" -u "$AUTH" -X DELETE "${ES_URL}/_data_stream/${idx}") + if [[ "$code" == "404" ]]; then + code=$(curl -s -o /dev/null -w "%{http_code}" -u "$AUTH" -X DELETE "${ES_URL}/${idx}?ignore_unavailable=true") + fi + echo " $idx → HTTP $code" + done + echo "Done." +} + +seed() { + echo "Seeding PCI compliance eval data..." + echo " ES: $ES_URL" + echo " User: $ES_USER" + echo "" + + # --- Auth index --- + local auth_body="" + for i in $(seq 0 11); do + local ts + ts=$(minute_ago $((60 - i))) + auth_body+='{"create":{"_index":"logs-pci-auth-eval"}} +{"@timestamp":"'"$ts"'","event":{"category":"authentication","outcome":"failure","action":"user_login"},"user":{"name":"jdoe"},"source":{"ip":"192.168.1.100"}} +' + done + + for pair in "50:admin:10.0.0.5" "49:root:10.0.0.6" "48:alice:10.0.0.7" "47:bob:10.0.0.8"; do + IFS=: read -r offset user ip <<< "$pair" + local ts + ts=$(minute_ago "$offset") + auth_body+='{"create":{"_index":"logs-pci-auth-eval"}} +{"@timestamp":"'"$ts"'","event":{"category":"authentication","outcome":"success","action":"user_login"},"user":{"name":"'"$user"'"},"source":{"ip":"'"$ip"'"}} +' + done + + local ts + ts=$(minute_ago 46) + auth_body+='{"create":{"_index":"logs-pci-auth-eval"}} +{"@timestamp":"'"$ts"'","event":{"category":"iam","action":"password_change"},"user":{"name":"alice"},"source":{"ip":"10.0.0.7"}} +' + ts=$(minute_ago 45) + auth_body+='{"create":{"_index":"logs-pci-auth-eval"}} +{"@timestamp":"'"$ts"'","event":{"category":"iam","action":"mfa_enroll"},"user":{"name":"bob"},"source":{"ip":"10.0.0.8"}} +' + + echo " Indexing auth events..." + curl -s -u "$AUTH" -X POST "${ES_URL}/_bulk?refresh=true" \ + -H 'Content-Type: application/x-ndjson' \ + --data-binary "$auth_body" | jq -r '" errors: \(.errors), items: \(.items | length)"' + + # --- Network index --- + local net_body="" + local net_data=( + "40:10.0.0.1:203.0.113.50:1.3:https" + "39:10.0.0.2:203.0.113.51:1.0:https" + "38:10.0.0.3:203.0.113.52:1.1:https" + "37:10.0.0.4:198.51.100.10::http" + "36:10.0.0.5:203.0.113.53:1.2:https" + "35:10.0.0.6:203.0.113.54:1.3:https" + ) + for entry in "${net_data[@]}"; do + IFS=: read -r offset src dst tls_ver proto <<< "$entry" + ts=$(minute_ago "$offset") + if [[ -n "$tls_ver" ]]; then + net_body+='{"create":{"_index":"logs-pci-network-eval"}} +{"@timestamp":"'"$ts"'","event":{"category":"network"},"source":{"ip":"'"$src"'"},"destination":{"ip":"'"$dst"'"},"tls":{"version":"'"$tls_ver"'"},"network":{"protocol":"'"$proto"'"}} +' + else + net_body+='{"create":{"_index":"logs-pci-network-eval"}} +{"@timestamp":"'"$ts"'","event":{"category":"network"},"source":{"ip":"'"$src"'"},"destination":{"ip":"'"$dst"'"},"network":{"protocol":"'"$proto"'"}} +' + fi + done + + echo " Indexing network events..." + curl -s -u "$AUTH" -X POST "${ES_URL}/_bulk?refresh=true" \ + -H 'Content-Type: application/x-ndjson' \ + --data-binary "$net_body" | jq -r '" errors: \(.errors), items: \(.items | length)"' + + # --- Vuln index --- + local vuln_body="" + ts=$(minute_ago 30) + vuln_body+='{"create":{"_index":"logs-pci-vuln-eval"}} +{"@timestamp":"'"$ts"'","event":{"category":"vulnerability"},"vulnerability":{"id":"CVE-2024-5678","severity":"critical"},"host":{"name":"db-server-01"}} +' + ts=$(minute_ago 29) + vuln_body+='{"create":{"_index":"logs-pci-vuln-eval"}} +{"@timestamp":"'"$ts"'","event":{"category":"vulnerability"},"vulnerability":{"id":"CVE-2024-9999","severity":"high"},"host":{"name":"web-server-02"}} +' + ts=$(minute_ago 28) + vuln_body+='{"create":{"_index":"logs-pci-vuln-eval"}} +{"@timestamp":"'"$ts"'","event":{"category":"intrusion_detection","kind":"alert","action":"exploit_attempt"},"host":{"name":"web-server-02"}} +' + ts=$(minute_ago 27) + vuln_body+='{"create":{"_index":"logs-pci-vuln-eval"}} +{"@timestamp":"'"$ts"'","event":{"category":"intrusion_detection","kind":"alert","action":"port_scan"},"host":{"name":"web-server-03"}} +' + + echo " Indexing vulnerability/IDS events..." + curl -s -u "$AUTH" -X POST "${ES_URL}/_bulk?refresh=true" \ + -H 'Content-Type: application/x-ndjson' \ + --data-binary "$vuln_body" | jq -r '" errors: \(.errors), items: \(.items | length)"' + + # --- Endpoint index --- + local ep_body="" + ts=$(minute_ago 20) + ep_body+='{"create":{"_index":"logs-pci-endpoint-eval"}} +{"@timestamp":"'"$ts"'","event":{"category":"malware","module":"endpoint","action":"malware_detected"},"host":{"name":"workstation-01"},"process":{"name":"suspicious.exe"}} +' + ts=$(minute_ago 19) + ep_body+='{"create":{"_index":"logs-pci-endpoint-eval"}} +{"@timestamp":"'"$ts"'","event":{"category":"process","module":"endpoint","action":"process_started"},"host":{"name":"workstation-02"},"process":{"name":"powershell.exe"}} +' + + echo " Indexing endpoint events..." + curl -s -u "$AUTH" -X POST "${ES_URL}/_bulk?refresh=true" \ + -H 'Content-Type: application/x-ndjson' \ + --data-binary "$ep_body" | jq -r '" errors: \(.errors), items: \(.items | length)"' + + # --- Custom legacy index --- + local custom_body="" + ts=$(minute_ago 25) + custom_body+='{"create":{"_index":"logs-pci-custom-eval"}} +{"@timestamp":"'"$ts"'","username":"jsmith","src_ip":"172.16.0.1","auth_result":"pass","operation":"login","hostname":"app-01"} +' + ts=$(minute_ago 24) + custom_body+='{"create":{"_index":"logs-pci-custom-eval"}} +{"@timestamp":"'"$ts"'","username":"jdoe","src_ip":"172.16.0.2","auth_result":"fail","operation":"login","hostname":"app-01"} +' + ts=$(minute_ago 23) + custom_body+='{"create":{"_index":"logs-pci-custom-eval"}} +{"@timestamp":"'"$ts"'","hostname":"web-server-01","severity":"high","cve":"CVE-2024-1234","program":"nginx"} +' + ts=$(minute_ago 22) + custom_body+='{"create":{"_index":"logs-pci-custom-eval"}} +{"@timestamp":"'"$ts"'","username":"admin","src_ip":"172.16.0.3","auth_result":"pass","operation":"sudo","hostname":"db-01"} +' + + echo " Indexing custom legacy events..." + curl -s -u "$AUTH" -X POST "${ES_URL}/_bulk?refresh=true" \ + -H 'Content-Type: application/x-ndjson' \ + --data-binary "$custom_body" | jq -r '" errors: \(.errors), items: \(.items | length)"' + + echo "" + echo "Done! Seeded all 5 PCI eval indices." + echo "" + echo "Indices:" + for idx in "${INDICES[@]}"; do + local count + count=$(curl -s -u "$AUTH" "${ES_URL}/${idx}/_count" | jq -r '.count // "N/A"') + echo " $idx → $count docs" + done +} + +case "${1:-}" in + --cleanup|-c) + cleanup + ;; + *) + seed + ;; +esac diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/src/chat_client.ts b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/src/chat_client.ts new file mode 100644 index 0000000000000..17391aaaeeb4f --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/src/chat_client.ts @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ToolingLog } from '@kbn/tooling-log'; +import type { HttpHandler } from '@kbn/core/public'; +import { agentBuilderDefaultAgentId } from '@kbn/agent-builder-common'; +import pRetry from 'p-retry'; + +const RETRIES = 2; +const MIN_TIMEOUT = 2000; + +export interface ConverseParams { + message: string; + conversationId?: string; +} + +export interface ConverseResponse { + conversationId?: string; + messages: Array<{ message: string }>; + steps?: unknown[]; + errors: unknown[]; + traceId?: string; +} + +/** + * Thin wrapper around the Agent Builder `converse` endpoint. Mirrors the Endpoint suite + * (see @kbn/evals-suite-endpoint) so PCI evals produce comparable traces and metrics. + */ +export class PciEvalChatClient { + constructor( + private readonly fetch: HttpHandler, + private readonly log: ToolingLog, + private readonly connectorId: string + ) {} + + async converse({ message, conversationId }: ConverseParams): Promise { + // Read per-call so env-var overrides (e.g. AGENT_BUILDER_AGENT_ID) are honoured between tests. + const agentId = process.env.AGENT_BUILDER_AGENT_ID ?? agentBuilderDefaultAgentId; + + this.log.info('Calling converse'); + + const callConverseApi = async (): Promise => { + const response = await this.fetch('/api/agent_builder/converse', { + method: 'POST', + version: '2023-10-31', + body: JSON.stringify({ + agent_id: agentId, + connector_id: this.connectorId, + conversation_id: conversationId, + input: message, + }), + }); + + const chatResponse = response as { + conversation_id: string; + trace_id?: string; + steps: unknown[]; + response: { message: string }; + }; + + const { + conversation_id: conversationIdFromResponse, + response: latestResponse, + steps, + trace_id: traceId, + } = chatResponse; + + return { + conversationId: conversationIdFromResponse, + messages: [{ message }, latestResponse], + steps, + errors: [], + traceId, + }; + }; + + try { + return await pRetry(callConverseApi, { + retries: RETRIES, + minTimeout: MIN_TIMEOUT, + onFailedAttempt: (error) => { + const isLastAttempt = error.retriesLeft === 0; + if (isLastAttempt) { + this.log.error( + new Error(`Failed to call converse API after ${error.attemptNumber} attempts`, { + cause: error, + }) + ); + } else { + this.log.warning( + new Error(`Converse API call failed on attempt ${error.attemptNumber}; retrying...`, { + cause: error, + }) + ); + } + }, + }); + } catch (error) { + this.log.error('Error occurred while calling converse API'); + return { + conversationId, + steps: [], + messages: [ + { message }, + { + message: + 'This question could not be answered as an internal error occurred. Please try again.', + }, + ], + errors: [ + { + error: { + message: error instanceof Error ? error.message : 'Unknown error', + stack: error instanceof Error ? error.stack : undefined, + }, + type: 'error', + }, + ], + }; + } + } +} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/src/data_generators/pci_data.ts b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/src/data_generators/pci_data.ts new file mode 100644 index 0000000000000..10be924cf30b0 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/src/data_generators/pci_data.ts @@ -0,0 +1,318 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; +import type { ToolingLog } from '@kbn/tooling-log'; + +/** + * Deterministic seed data for PCI compliance evals. + * + * Index names are **randomized per test run** (e.g. `logs-a7f3b2-auth`) so the + * PCI skill cannot rely on predictable names and must use ECS field caps / scope + * discovery to identify the right data. This ensures the evals validate real + * skill behaviour rather than name-pattern matching. + * + * Five indices covering the core PCI DSS v4.0.1 requirement families: + * + * - auth — Authentication / IAM events (Req 2, 8): + * 12 failed logins for "jdoe" (brute-force trigger, exceeds PCI 8.3.4 + * threshold of 10), successful logins for "admin" and "root" + * (default-account violation), plus password_change and mfa_enroll. + * + * - network — Network + TLS metadata (Req 1, 4): + * TLS 1.3, 1.2 (good), TLS 1.0 + 1.1 (weak), plain HTTP (no TLS). + * + * - vuln — Vulnerability + IDS events (Req 6, 11): + * Critical/high CVEs and intrusion detection alerts. + * + * - endpoint — Endpoint / malware events (Req 5): + * Malware detection and suspicious process execution. + * + * - custom — Non-ECS legacy fields for field-mapper tests: + * Flat field names (username, src_ip, hostname, cve, severity, program). + * + * Timestamps use `Date.now()` offsets so data always falls within lookback windows. + * Kept intentionally small (tens of docs each) so evals are fast and reproducible. + */ + +const RANDOM_PREFIX = Math.random().toString(36).substring(2, 8); + +export const PCI_INDICES = { + auth: `logs-${RANDOM_PREFIX}-auth`, + network: `logs-${RANDOM_PREFIX}-network`, + vuln: `logs-${RANDOM_PREFIX}-vuln`, + endpoint: `logs-${RANDOM_PREFIX}-endpoint`, + custom: `logs-${RANDOM_PREFIX}-custom`, +} as const; + +const MINUTE = 60_000; +const recentTimestamp = (offsetMinutes: number) => + new Date(Date.now() - offsetMinutes * MINUTE).toISOString(); + +type Doc = Record; + +async function bulkIndex(esClient: Client, index: string, docs: Doc[]): Promise { + if (docs.length === 0) return; + const body = docs.flatMap((doc) => [{ create: { _index: index } }, doc]); + const response = await esClient.bulk({ refresh: true, operations: body }); + if (response.errors) { + const firstError = response.items.find((item) => { + const op = Object.values(item)[0]; + return op && 'error' in op && op.error; + }); + throw new Error(`Bulk indexing into ${index} failed: ${JSON.stringify(firstError, null, 2)}`); + } +} + +function buildAuthDocs(): Doc[] { + const docs: Doc[] = []; + + // 12 failed logins for "jdoe" from the same IP — exceeds PCI 8.3.4 lockout threshold of 10 + for (let i = 0; i < 12; i++) { + docs.push({ + '@timestamp': recentTimestamp(60 - i), + event: { category: 'authentication', outcome: 'failure', action: 'user_login' }, + user: { name: 'jdoe' }, + source: { ip: '192.168.1.100' }, + }); + } + + // Successful logins — "admin" and "root" trigger default-account violation (Req 2.2.4) + docs.push( + { + '@timestamp': recentTimestamp(50), + event: { category: 'authentication', outcome: 'success', action: 'user_login' }, + user: { name: 'admin' }, + source: { ip: '10.0.0.5' }, + }, + { + '@timestamp': recentTimestamp(49), + event: { category: 'authentication', outcome: 'success', action: 'user_login' }, + user: { name: 'root' }, + source: { ip: '10.0.0.6' }, + }, + { + '@timestamp': recentTimestamp(48), + event: { category: 'authentication', outcome: 'success', action: 'user_login' }, + user: { name: 'alice' }, + source: { ip: '10.0.0.7' }, + }, + { + '@timestamp': recentTimestamp(47), + event: { category: 'authentication', outcome: 'success', action: 'user_login' }, + user: { name: 'bob' }, + source: { ip: '10.0.0.8' }, + } + ); + + // IAM events — password change and MFA enroll (Req 8.3.6, 8.3.9) + docs.push( + { + '@timestamp': recentTimestamp(46), + event: { category: 'iam', action: 'password_change' }, + user: { name: 'alice' }, + source: { ip: '10.0.0.7' }, + }, + { + '@timestamp': recentTimestamp(45), + event: { category: 'iam', action: 'mfa_enroll' }, + user: { name: 'bob' }, + source: { ip: '10.0.0.8' }, + } + ); + + return docs; +} + +function buildNetworkDocs(): Doc[] { + return [ + { + '@timestamp': recentTimestamp(40), + event: { category: 'network' }, + source: { ip: '10.0.0.1' }, + destination: { ip: '203.0.113.50' }, + tls: { version: '1.3' }, + network: { protocol: 'https' }, + }, + // Weak: TLS 1.0 + { + '@timestamp': recentTimestamp(39), + event: { category: 'network' }, + source: { ip: '10.0.0.2' }, + destination: { ip: '203.0.113.51' }, + tls: { version: '1.0' }, + network: { protocol: 'https' }, + }, + // Weak: TLS 1.1 + { + '@timestamp': recentTimestamp(38), + event: { category: 'network' }, + source: { ip: '10.0.0.3' }, + destination: { ip: '203.0.113.52' }, + tls: { version: '1.1' }, + network: { protocol: 'https' }, + }, + // Weak: plain HTTP — no TLS at all + { + '@timestamp': recentTimestamp(37), + event: { category: 'network' }, + source: { ip: '10.0.0.4' }, + destination: { ip: '198.51.100.10' }, + network: { protocol: 'http' }, + }, + { + '@timestamp': recentTimestamp(36), + event: { category: 'network' }, + source: { ip: '10.0.0.5' }, + destination: { ip: '203.0.113.53' }, + tls: { version: '1.2' }, + network: { protocol: 'https' }, + }, + { + '@timestamp': recentTimestamp(35), + event: { category: 'network' }, + source: { ip: '10.0.0.6' }, + destination: { ip: '203.0.113.54' }, + tls: { version: '1.3' }, + network: { protocol: 'https' }, + }, + ]; +} + +function buildVulnDocs(): Doc[] { + return [ + { + '@timestamp': recentTimestamp(30), + event: { category: 'vulnerability' }, + vulnerability: { id: 'CVE-2024-5678', severity: 'critical' }, + host: { name: 'db-server-01' }, + }, + { + '@timestamp': recentTimestamp(29), + event: { category: 'vulnerability' }, + vulnerability: { id: 'CVE-2024-9999', severity: 'high' }, + host: { name: 'web-server-02' }, + }, + { + '@timestamp': recentTimestamp(28), + event: { category: 'intrusion_detection', kind: 'alert', action: 'exploit_attempt' }, + host: { name: 'web-server-02' }, + }, + { + '@timestamp': recentTimestamp(27), + event: { category: 'intrusion_detection', kind: 'alert', action: 'port_scan' }, + host: { name: 'web-server-03' }, + }, + ]; +} + +function buildEndpointDocs(): Doc[] { + return [ + { + '@timestamp': recentTimestamp(20), + event: { category: 'malware', module: 'endpoint', action: 'malware_detected' }, + host: { name: 'workstation-01' }, + process: { name: 'suspicious.exe' }, + }, + { + '@timestamp': recentTimestamp(19), + event: { category: 'process', module: 'endpoint', action: 'process_started' }, + host: { name: 'workstation-02' }, + process: { name: 'powershell.exe' }, + }, + ]; +} + +function buildCustomLegacyDocs(): Doc[] { + return [ + { + '@timestamp': recentTimestamp(25), + username: 'jsmith', + src_ip: '172.16.0.1', + auth_result: 'pass', + operation: 'login', + hostname: 'app-01', + }, + { + '@timestamp': recentTimestamp(24), + username: 'jdoe', + src_ip: '172.16.0.2', + auth_result: 'fail', + operation: 'login', + hostname: 'app-01', + }, + { + '@timestamp': recentTimestamp(23), + hostname: 'web-server-01', + severity: 'high', + cve: 'CVE-2024-1234', + program: 'nginx', + }, + { + '@timestamp': recentTimestamp(22), + username: 'admin', + src_ip: '172.16.0.3', + auth_result: 'pass', + operation: 'sudo', + hostname: 'db-01', + }, + ]; +} + +export async function seedPciEvalData({ + esClient, + log, +}: { + esClient: Client; + log: ToolingLog; +}): Promise { + log.info('Seeding PCI compliance eval data'); + + const authDocs = buildAuthDocs(); + const networkDocs = buildNetworkDocs(); + const vulnDocs = buildVulnDocs(); + const endpointDocs = buildEndpointDocs(); + const customDocs = buildCustomLegacyDocs(); + + await bulkIndex(esClient, PCI_INDICES.auth, authDocs); + await bulkIndex(esClient, PCI_INDICES.network, networkDocs); + await bulkIndex(esClient, PCI_INDICES.vuln, vulnDocs); + await bulkIndex(esClient, PCI_INDICES.endpoint, endpointDocs); + await bulkIndex(esClient, PCI_INDICES.custom, customDocs); + + log.info( + `Seeded ${authDocs.length} auth, ${networkDocs.length} network, ` + + `${vulnDocs.length} vuln, ${endpointDocs.length} endpoint, ${customDocs.length} custom docs` + ); +} + +export async function cleanupPciEvalData({ + esClient, + log, +}: { + esClient: Client; + log: ToolingLog; +}): Promise { + log.info('Cleaning up PCI compliance eval data'); + const indices = Object.values(PCI_INDICES); + + for (const index of indices) { + try { + await esClient.indices.deleteDataStream({ name: index }); + } catch { + try { + await esClient.indices.delete({ index, ignore_unavailable: true }); + } catch (error) { + log.warning( + `Failed to delete PCI eval index ${index}: ${ + error instanceof Error ? error.message : error + }` + ); + } + } + } +} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/src/evaluate.ts b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/src/evaluate.ts new file mode 100644 index 0000000000000..4773df6986900 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/src/evaluate.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; +import { evaluate as base } from '@kbn/evals'; +import { createEsClientForTesting, systemIndicesSuperuser } from '@kbn/test'; +import { PciEvalChatClient } from './chat_client'; +import type { EvaluatePciDataset } from './evaluate_dataset'; +import { createEvaluatePciDataset } from './evaluate_dataset'; + +export const evaluate = base.extend< + {}, + { + chatClient: PciEvalChatClient; + evaluateDataset: EvaluatePciDataset; + internalEsClient: Client; + } +>({ + chatClient: [ + async ({ fetch, log, connector }, use) => { + const chatClient = new PciEvalChatClient(fetch, log, connector.id); + await use(chatClient); + }, + { scope: 'worker' }, + ], + evaluateDataset: [ + ({ chatClient, evaluators, executorClient, traceEsClient, log }, use) => { + use( + createEvaluatePciDataset({ + chatClient, + evaluators, + executorClient, + traceEsClient, + log, + }) + ); + }, + { scope: 'worker' }, + ], + internalEsClient: [ + async ({ config }, use) => { + const { username, password } = systemIndicesSuperuser; + const esUrl = new URL(config.hosts.elasticsearch); + esUrl.username = username; + esUrl.password = password; + const client = createEsClientForTesting({ + esUrl: esUrl.toString(), + isCloud: config.isCloud, + }); + const alive = await client.ping().catch(() => false); + if (!alive) { + throw new Error( + `internalEsClient: unable to reach Elasticsearch as "${username}". ` + + 'Ensure the system_indices_superuser user exists (created during test cluster startup).' + ); + } + await use(client); + await client.close(); + }, + { scope: 'worker' }, + ], +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/src/evaluate_dataset.ts b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/src/evaluate_dataset.ts new file mode 100644 index 0000000000000..d1703c9abde00 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/src/evaluate_dataset.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client as EsClient } from '@elastic/elasticsearch'; +import type { ToolingLog } from '@kbn/tooling-log'; +import type { + DefaultEvaluators, + EvaluationDataset, + Evaluator, + EvalsExecutorClient, + Example, +} from '@kbn/evals'; +import { createSkillInvocationEvaluator } from '@kbn/evals'; +import type { PciEvalChatClient } from './chat_client'; + +export interface PciDatasetExample extends Example { + input: { + question: string; + }; + output: { + criteria: string[]; + }; +} + +export type EvaluatePciDataset = (options: { + dataset: { + name: string; + description: string; + examples: PciDatasetExample[]; + }; +}) => Promise; + +/** + * Criteria baked into every PCI example. The PCI skill guarantees: + * - PCI DSS v4.0.1 is cited (or `4.0.1`) in the answer. + * - The QSA disclaimer is surfaced so the answer never reads as a compliance + * determination. + * - A `scopeClaim` describing the indices + time range actually evaluated is referenced, + * either verbatim or by paraphrase. + * + * Keeping these as shared criteria (vs. copy-pasting into every spec) prevents drift when + * the contract evolves. + */ +export const BASELINE_PCI_CRITERIA = [ + 'The response explicitly references PCI DSS v4.0.1 (or the string "4.0.1").', + 'The response includes a QSA / Qualified Security Assessor disclaimer — the agent does not claim the finding is a formal compliance determination.', + 'The response references a scope (evaluated indices and the time range) the tool actually used, rather than a vague "I checked everything".', +]; + +export function createPciCriteriaEvaluator({ + evaluators, + extraCriteria = [], +}: { + evaluators: DefaultEvaluators; + extraCriteria?: string[]; +}): Evaluator { + return { + name: 'PCI Criteria', + kind: 'LLM' as const, + evaluate: async ({ expected, ...rest }) => { + const exampleCriteria: string[] = (expected as PciDatasetExample['output'])?.criteria ?? []; + const allCriteria = [...BASELINE_PCI_CRITERIA, ...extraCriteria, ...exampleCriteria]; + return evaluators.criteria(allCriteria).evaluate({ expected, ...rest }); + }, + }; +} + +export function createEvaluatePciDataset({ + evaluators, + executorClient, + chatClient, + traceEsClient, + log, +}: { + evaluators: DefaultEvaluators; + executorClient: EvalsExecutorClient; + chatClient: PciEvalChatClient; + traceEsClient: EsClient; + log: ToolingLog; +}): EvaluatePciDataset { + return async function evaluatePciDataset({ + dataset: { name, description, examples }, + }: { + dataset: { + name: string; + description: string; + examples: PciDatasetExample[]; + }; + }) { + const dataset = { + name, + description, + examples, + } satisfies EvaluationDataset; + + await executorClient.runExperiment( + { + dataset, + task: async ({ input }) => { + const response = await chatClient.converse({ message: input.question }); + + return { + messages: response.messages, + steps: response.steps, + errors: response.errors, + traceId: response.traceId, + }; + }, + }, + [ + createPciCriteriaEvaluator({ evaluators }), + createSkillInvocationEvaluator({ + traceEsClient, + log, + skillName: 'pci-compliance', + }), + ] + ); + }; +} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/tsconfig.json b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/tsconfig.json new file mode 100644 index 0000000000000..928aba06e01f4 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "@kbn/tsconfig-base/tsconfig.json", + "compilerOptions": { + "outDir": "target/types", + "types": ["jest", "node"] + }, + "include": ["**/*.ts"], + "exclude": ["target/**/*"], + "kbn_references": [ + "@kbn/evals", + "@kbn/scout", + "@kbn/agent-builder-common", + "@kbn/tooling-log", + "@kbn/core", + "@kbn/test", + ] +} diff --git a/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts b/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts index 98b6801aa9578..851de4555f925 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts @@ -237,6 +237,12 @@ export const allowedExperimentalValues = Object.freeze({ */ automaticTroubleshootingSkill: false, + /** + * Enables the PCI DSS v4.0.1 Compliance Agent Builder skill and its backing tools. + * Gates skill + tool registration so the feature can ship dark and be enabled per environment. + */ + pciComplianceAgentBuilder: true, + /** * Enables the new flyout using the EUI flyout system */ diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/pci_compliance/pci_compliance_skill.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/pci_compliance/pci_compliance_skill.test.ts index 4e6892d3505f7..e84f5e5ac158d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/pci_compliance/pci_compliance_skill.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/pci_compliance/pci_compliance_skill.test.ts @@ -6,12 +6,16 @@ */ import { platformCoreTools } from '@kbn/agent-builder-common'; -import { pciComplianceSkill } from './pci_compliance_skill'; -import { PCI_COMPLIANCE_CHECK_TOOL_ID } from '../../tools/pci_compliance_check_tool'; -import { PCI_COMPLIANCE_REPORT_TOOL_ID } from '../../tools/pci_compliance_report_tool'; +import { pciComplianceSkill, PCI_COMPLIANCE_SKILL_TOOL_IDS } from './pci_compliance_skill'; +import { PCI_COMPLIANCE_TOOL_ID } from '../../tools/pci_compliance_tool'; import { PCI_SCOPE_DISCOVERY_TOOL_ID } from '../../tools/pci_scope_discovery_tool'; import { PCI_FIELD_MAPPER_TOOL_ID } from '../../tools/pci_field_mapper_tool'; +/** + * Skill-level contract tests. The Agent Builder tool-selection guideline caps skills at + * roughly 5 registry tool references because tool-selection accuracy degrades past that. + * We assert the cap here rather than relying on a soft guideline. + */ describe('pciComplianceSkill', () => { it('has the correct skill id', () => { expect(pciComplianceSkill.id).toBe('pci-compliance'); @@ -21,62 +25,58 @@ describe('pciComplianceSkill', () => { expect(pciComplianceSkill.basePath).toBe('skills/security/compliance'); }); - it('has a non-empty description', () => { + it('has a non-empty description referencing PCI DSS v4.0.1', () => { expect(pciComplianceSkill.description.length).toBeGreaterThan(0); expect(pciComplianceSkill.description).toContain('PCI DSS v4.0.1'); }); - it('has non-empty content with instructions', () => { - expect(pciComplianceSkill.content.length).toBeGreaterThan(0); - }); - - it('references PCI DSS v4.0.1 in content', () => { + it('references PCI DSS v4.0.1 and key v4.0.1 clarifications in content', () => { expect(pciComplianceSkill.content).toContain('v4.0.1'); + expect(pciComplianceSkill.content).toContain('critical-severity only'); + expect(pciComplianceSkill.content).toContain('ALL CDE access'); + expect(pciComplianceSkill.content).toContain('FIDO2'); }); - it('includes compliance workflow guidance', () => { - expect(pciComplianceSkill.content).toContain('Compliance Assessment Workflow'); - }); - - it('includes confidence interpretation guidance', () => { + it('documents confidence interpretation', () => { expect(pciComplianceSkill.content).toContain('GREEN + HIGH confidence'); expect(pciComplianceSkill.content).toContain('RED + HIGH confidence'); expect(pciComplianceSkill.content).toContain('NOT_ASSESSABLE'); }); - it('documents v4.0.1 clarifications', () => { - expect(pciComplianceSkill.content).toContain('critical-severity only'); - expect(pciComplianceSkill.content).toContain('ALL CDE access'); - expect(pciComplianceSkill.content).toContain('FIDO2'); - }); - - it('includes deduplication guidance', () => { + it('includes deduplication guidance and the consolidated tool workflow', () => { expect(pciComplianceSkill.content).toContain('Deduplication'); + expect(pciComplianceSkill.content).toContain(PCI_COMPLIANCE_TOOL_ID); + expect(pciComplianceSkill.content).toContain(PCI_SCOPE_DISCOVERY_TOOL_ID); + expect(pciComplianceSkill.content).toContain(PCI_FIELD_MAPPER_TOOL_ID); }); - it('includes non-ECS data workflow', () => { - expect(pciComplianceSkill.content).toContain(PCI_FIELD_MAPPER_TOOL_ID); + it('documents the scopeClaim provenance record', () => { + expect(pciComplianceSkill.content).toContain('scopeClaim'); }); describe('getRegistryTools', () => { - it('returns all PCI tool IDs', () => { - const toolIds = pciComplianceSkill.getRegistryTools!(); + const toolIds = pciComplianceSkill.getRegistryTools!() as string[]; + + it('exposes the consolidated PCI tool set plus ES|QL generators', () => { + expect(toolIds).toEqual(expect.arrayContaining([...PCI_COMPLIANCE_SKILL_TOOL_IDS])); expect(toolIds).toContain(PCI_SCOPE_DISCOVERY_TOOL_ID); - expect(toolIds).toContain(PCI_COMPLIANCE_CHECK_TOOL_ID); - expect(toolIds).toContain(PCI_COMPLIANCE_REPORT_TOOL_ID); + expect(toolIds).toContain(PCI_COMPLIANCE_TOOL_ID); expect(toolIds).toContain(PCI_FIELD_MAPPER_TOOL_ID); - }); - - it('returns platform core tools for data exploration', () => { - const toolIds = pciComplianceSkill.getRegistryTools!(); expect(toolIds).toContain(platformCoreTools.generateEsql); expect(toolIds).toContain(platformCoreTools.executeEsql); - expect(toolIds).toContain(platformCoreTools.listIndices); }); - it('stays within the 25 registry tool limit', () => { - const toolIds = pciComplianceSkill.getRegistryTools!(); - expect((toolIds as string[]).length).toBeLessThanOrEqual(25); + it('does not advertise the deprecated split check/report tool IDs', () => { + expect(toolIds).not.toContain('security.pci_compliance_check'); + expect(toolIds).not.toContain('security.pci_compliance_report'); + }); + + it('stays within the 5 registry tool tool-selection cap', () => { + expect(toolIds.length).toBeLessThanOrEqual(5); + }); + + it('has no duplicate entries', () => { + expect(new Set(toolIds).size).toBe(toolIds.length); }); }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/pci_compliance/pci_compliance_skill.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/pci_compliance/pci_compliance_skill.ts index 2f0e74ad6b1cf..9cd29cf28daec 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/pci_compliance/pci_compliance_skill.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/pci_compliance/pci_compliance_skill.ts @@ -8,34 +8,32 @@ import { platformCoreTools } from '@kbn/agent-builder-common'; import { defineSkillType } from '@kbn/agent-builder-server/skills/type_definition'; import { - PCI_COMPLIANCE_CHECK_TOOL_ID, - PCI_COMPLIANCE_REPORT_TOOL_ID, + PCI_COMPLIANCE_TOOL_ID, PCI_FIELD_MAPPER_TOOL_ID, PCI_SCOPE_DISCOVERY_TOOL_ID, - SECURITY_ALERTS_TOOL_ID, - SECURITY_ENTITY_RISK_SCORE_TOOL_ID, } from '../../tools'; -const PCI_TOOL_IDS = [ +/** + * Registry-scoped tool IDs advertised by this skill. + * + * Keep this list small. The Agent Builder guideline caps a skill at ~5 registry tool references + * because each additional tool materially degrades the LLM's tool-selection accuracy. The + * consolidated `pci_compliance` tool (with `mode: check | report`) replaces what was previously + * two separate tools, which keeps us within the cap. + */ +export const PCI_COMPLIANCE_SKILL_TOOL_IDS = [ PCI_SCOPE_DISCOVERY_TOOL_ID, - PCI_COMPLIANCE_CHECK_TOOL_ID, - PCI_COMPLIANCE_REPORT_TOOL_ID, + PCI_COMPLIANCE_TOOL_ID, PCI_FIELD_MAPPER_TOOL_ID, - SECURITY_ALERTS_TOOL_ID, - SECURITY_ENTITY_RISK_SCORE_TOOL_ID, - platformCoreTools.search, - platformCoreTools.listIndices, - platformCoreTools.getIndexMapping, - platformCoreTools.getDocumentById, - platformCoreTools.cases, - platformCoreTools.productDocumentation, platformCoreTools.generateEsql, platformCoreTools.executeEsql, -]; +] as const; + +export const PCI_COMPLIANCE_SKILL_ID = 'pci-compliance'; export const pciComplianceSkill = defineSkillType({ - id: 'pci-compliance', - name: 'pci-compliance', + id: PCI_COMPLIANCE_SKILL_ID, + name: PCI_COMPLIANCE_SKILL_ID, basePath: 'skills/security/compliance', description: 'PCI DSS v4.0.1 compliance assessments with violation detection, confidence scoring, ' + @@ -54,22 +52,21 @@ Use this skill when: Do **not** use this skill when: - The user is asking about general security threats unrelated to PCI compliance - The user needs threat hunting or attack investigation (use security alerts tools instead) +- The user is asking about SOC 2, HIPAA, GDPR, or other non-PCI compliance frameworks ## Available Tools -- **${PCI_SCOPE_DISCOVERY_TOOL_ID}**: Discover PCI-relevant data coverage across indices -- **${PCI_COMPLIANCE_CHECK_TOOL_ID}**: Run compliance checks with violation detection and confidence scoring -- **${PCI_COMPLIANCE_REPORT_TOOL_ID}**: Generate compliance reports with visual scorecards -- **${PCI_FIELD_MAPPER_TOOL_ID}**: Map non-ECS fields to ECS equivalents for custom data sources -- **${platformCoreTools.generateEsql}**: Generate ES|QL queries for adapted compliance checks -- **${platformCoreTools.executeEsql}**: Execute ES|QL queries and return tabular results +- **${PCI_SCOPE_DISCOVERY_TOOL_ID}**: Discover PCI-relevant indices and classify them by scope area (network, identity, endpoint, cloud, application). Uses a single batched field-capabilities call. +- **${PCI_COMPLIANCE_TOOL_ID}**: Unified PCI DSS evaluation. Pass \`mode: "check"\` to run violation/coverage/preflight per requirement and return findings with evidence, or \`mode: "report"\` to produce a scorecard roll-up across requirements. +- **${PCI_FIELD_MAPPER_TOOL_ID}**: Inspect non-ECS fields and suggest ECS mappings when scope discovery reports low ECS coverage. +- **${platformCoreTools.generateEsql}**: Generate ES|QL queries for adapted compliance checks when mapped fields differ from ECS. +- **${platformCoreTools.executeEsql}**: Execute ES|QL queries against discovered data. ## Compliance Assessment Workflow -1. **Discover available data** — call ${PCI_SCOPE_DISCOVERY_TOOL_ID} to identify indices and data coverage. -2. **Run compliance checks** — call ${PCI_COMPLIANCE_CHECK_TOOL_ID} for individual or full requirement assessments. -3. **Generate reports** — call ${PCI_COMPLIANCE_REPORT_TOOL_ID} for structured compliance reports with visual scorecards. -4. **Handle non-ECS data** — if scope discovery reports low ECS coverage, call ${PCI_FIELD_MAPPER_TOOL_ID} to discover field mappings, then use ${platformCoreTools.generateEsql} with those mappings. +1. **Discover available data** — call ${PCI_SCOPE_DISCOVERY_TOOL_ID} to identify indices and data coverage. Inspect \`scopeClaim\` in the response to verify which indices were evaluated. +2. **Run checks or reports** — call ${PCI_COMPLIANCE_TOOL_ID}. Choose \`mode: "check"\` when the user wants per-requirement findings with evidence, or \`mode: "report"\` when they want a posture snapshot or executive summary. +3. **Handle non-ECS data** — if scope discovery reports low ECS coverage, call ${PCI_FIELD_MAPPER_TOOL_ID} to discover field mappings, then use ${platformCoreTools.generateEsql} with those mappings. ## Interpreting Results @@ -79,17 +76,19 @@ Do **not** use this skill when: - **AMBER** = partial data or no matching events; widen time range or check index patterns - **NOT_ASSESSABLE** = required fields missing from indices; data source may need onboarding +Every response includes a \`scopeClaim\` object that cites the DSS version, indices, time range, requirement IDs actually evaluated, and the fields that were probed. Surface this to the user verbatim when producing audit-facing output — it is the provenance record for the assessment. + ## Deduplication -If violation counts seem inflated or the user mentions re-indexing or data migration, recommend specifying exact index patterns via the indices parameter to avoid double-counting from overlapping patterns. +If violation counts seem inflated or the user mentions re-indexing or data migration, recommend specifying exact index patterns via the \`indices\` parameter to avoid double-counting from overlapping patterns. ## Timeframes -Each check has a recommended lookback period (e.g. 7 days for brute-force, 365 days for stale accounts). User-provided timeRange overrides defaults. +Each check has a recommended lookback period (e.g. 7 days for brute-force, 365 days for stale accounts). User-provided \`timeRange\` overrides defaults. Time range values are bound as ES|QL parameters, not string-interpolated, so user-supplied timestamps cannot alter the query structure. ## Requirements Outside SIEM Scope -Requirements 3 (stored data), 9 (physical access), and 12 (policies) are primarily process-based. Report available telemetry but always note that manual process verification is required for full compliance. +Requirements 3 (stored data), 9 (physical access), and 12 (policies) are primarily process-based. Report available telemetry but always note that manual process verification by a Qualified Security Assessor (QSA) is required for full compliance. ## PCI DSS Version @@ -98,5 +97,5 @@ All checks reference PCI DSS v4.0.1 (published June 2024). v4.0 was retired Dece - Req 8.4.2: MFA required for ALL CDE access, not just administrative - Phishing-resistant auth (FIDO2/WebAuthn) can substitute for traditional MFA for non-admin CDE access `, - getRegistryTools: () => PCI_TOOL_IDS, + getRegistryTools: () => [...PCI_COMPLIANCE_SKILL_TOOL_IDS], }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/register_skills.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/register_skills.ts index 8d80101821034..133cdcf350d47 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/register_skills.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/register_skills.ts @@ -60,5 +60,8 @@ export const registerSkills = async ({ await agentBuilder.skills.register(threatHuntingSkill); await agentBuilder.skills.register(alertAnalysisSkill); - agentBuilder.skills.register(pciComplianceSkill); + + if (experimentalFeatures.pciComplianceAgentBuilder) { + agentBuilder.skills.register(pciComplianceSkill); + } }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/index.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/index.ts index 2ae658c9a172c..58296844657a5 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/index.ts @@ -24,9 +24,5 @@ export { SECURITY_CREATE_DETECTION_RULE_TOOL_ID, } from './create_detection_rule_tool'; export { pciScopeDiscoveryTool, PCI_SCOPE_DISCOVERY_TOOL_ID } from './pci_scope_discovery_tool'; -export { pciComplianceCheckTool, PCI_COMPLIANCE_CHECK_TOOL_ID } from './pci_compliance_check_tool'; -export { - pciComplianceReportTool, - PCI_COMPLIANCE_REPORT_TOOL_ID, -} from './pci_compliance_report_tool'; +export { pciComplianceTool, PCI_COMPLIANCE_TOOL_ID } from './pci_compliance_tool'; export { pciFieldMapperTool, PCI_FIELD_MAPPER_TOOL_ID } from './pci_field_mapper_tool'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_check_tool.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_check_tool.test.ts deleted file mode 100644 index c4399dc29e52c..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_check_tool.test.ts +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ToolResultType } from '@kbn/agent-builder-common'; -import { executeEsql } from '@kbn/agent-builder-genai-utils'; -import type { ToolHandlerStandardReturn } from '@kbn/agent-builder-server/tools'; -import { createToolHandlerContext, createToolTestMocks } from '../__mocks__/test_helpers'; -import { pciComplianceCheckTool, PCI_COMPLIANCE_CHECK_TOOL_ID } from './pci_compliance_check_tool'; - -jest.mock('@kbn/agent-builder-genai-utils', () => ({ - executeEsql: jest.fn(), -})); - -describe('pciComplianceCheckTool', () => { - const { mockCore, mockLogger, mockEsClient, mockRequest } = createToolTestMocks(); - const tool = pciComplianceCheckTool(mockCore, mockLogger); - - beforeEach(() => { - jest.clearAllMocks(); - mockEsClient.asCurrentUser.fieldCaps.mockResolvedValue({ - fields: { - 'event.category': { keyword: { type: 'keyword' } }, - 'event.outcome': { keyword: { type: 'keyword' } }, - 'user.name': { keyword: { type: 'keyword' } }, - 'source.ip': { ip: { type: 'ip' } }, - '@timestamp': { date: { type: 'date' } }, - }, - } as never); - }); - - describe('schema', () => { - it('accepts valid requirement request', () => { - const result = tool.schema.safeParse({ requirement: '8' }); - expect(result.success).toBe(true); - }); - - it('accepts sub-requirement request', () => { - const result = tool.schema.safeParse({ requirement: '8.3.4' }); - expect(result.success).toBe(true); - }); - - it('rejects missing requirement', () => { - const result = tool.schema.safeParse({}); - expect(result.success).toBe(false); - }); - }); - - describe('properties', () => { - it('returns correct id', () => { - expect(tool.id).toBe(PCI_COMPLIANCE_CHECK_TOOL_ID); - }); - - it('references PCI DSS v4.0.1 in description', () => { - expect(tool.description).toContain('v4.0.1'); - }); - }); - - describe('handler', () => { - it('returns error for invalid requirement', async () => { - const result = (await tool.handler( - { requirement: '99', includeEvidence: false }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; - - expect(result.results[0].type).toBe(ToolResultType.error); - }); - - it('returns RED when violation query finds violations (8.3.4 brute force)', async () => { - (executeEsql as jest.Mock).mockResolvedValue({ - columns: [ - { name: 'user.name', type: 'keyword' }, - { name: 'source.ip', type: 'ip' }, - { name: 'failed_attempts', type: 'long' }, - ], - values: [ - ['admin', '10.0.0.1', 15], - ['root', '192.168.1.5', 12], - ], - }); - - const result = (await tool.handler( - { requirement: '8.3.4', includeEvidence: true }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; - - const payload = result.results.find((r) => r.type === ToolResultType.other)?.data as { - overallStatus: string; - overallConfidence: string; - requirementResults: Array<{ status: string; confidence: string }>; - }; - expect(payload.overallStatus).toBe('RED'); - expect(payload.requirementResults[0].status).toBe('RED'); - expect(payload.requirementResults[0].confidence).toBe('HIGH'); - }); - - it('returns GREEN with HIGH confidence when violation query returns 0 rows and coverage exists', async () => { - (executeEsql as jest.Mock) - .mockResolvedValueOnce({ columns: [], values: [] }) - .mockResolvedValueOnce({ - columns: [{ name: 'matching_events', type: 'long' }], - values: [[500]], - }); - - const result = (await tool.handler( - { requirement: '8.3.4', includeEvidence: false }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; - - const payload = result.results.find((r) => r.type === ToolResultType.other)?.data as { - overallStatus: string; - requirementResults: Array<{ status: string; confidence: string }>; - }; - expect(payload.overallStatus).toBe('GREEN'); - expect(payload.requirementResults[0].status).toBe('GREEN'); - expect(payload.requirementResults[0].confidence).toBe('HIGH'); - }); - - it('returns NOT_ASSESSABLE when fields are missing from index', async () => { - mockEsClient.asCurrentUser.fieldCaps.mockResolvedValue({ fields: {} } as never); - - (executeEsql as jest.Mock) - .mockResolvedValueOnce({ columns: [], values: [] }) - .mockResolvedValueOnce({ - columns: [{ name: 'matching_events', type: 'long' }], - values: [[0]], - }); - - const result = (await tool.handler( - { requirement: '8.3.4', includeEvidence: false }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; - - const payload = result.results.find((r) => r.type === ToolResultType.other)?.data as { - requirementResults: Array<{ status: string; confidence: string }>; - }; - expect(payload.requirementResults[0].status).toBe('NOT_ASSESSABLE'); - expect(payload.requirementResults[0].confidence).toBe('NOT_ASSESSABLE'); - }); - - it('returns AMBER when coverage query finds no data but fields exist', async () => { - (executeEsql as jest.Mock) - .mockResolvedValueOnce({ columns: [], values: [] }) - .mockResolvedValueOnce({ - columns: [{ name: 'matching_events', type: 'long' }], - values: [[0]], - }); - - const result = (await tool.handler( - { requirement: '8.3.4', includeEvidence: false }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; - - const payload = result.results.find((r) => r.type === ToolResultType.other)?.data as { - requirementResults: Array<{ status: string }>; - }; - expect(payload.requirementResults[0].status).toBe('AMBER'); - }); - - it('includes visual esqlResults when RED violations are found', async () => { - (executeEsql as jest.Mock).mockResolvedValue({ - columns: [ - { name: 'user.name', type: 'keyword' }, - { name: 'failed_attempts', type: 'long' }, - ], - values: [ - ['admin', 15], - ['root', 12], - ], - }); - - const result = (await tool.handler( - { requirement: '8.3.4', includeEvidence: true }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; - - const esqlResult = result.results.find((r) => r.type === ToolResultType.esqlResults); - expect(esqlResult).toBeDefined(); - expect(esqlResult?.tool_result_id).toBeDefined(); - }); - - it('evaluates all requirements with concurrency when "all" is specified', async () => { - (executeEsql as jest.Mock).mockResolvedValue({ - columns: [{ name: 'matching_events', type: 'long' }], - values: [[5]], - }); - - const result = (await tool.handler( - { requirement: 'all', includeEvidence: false }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; - - expect(executeEsql).toHaveBeenCalled(); - const payload = result.results.find((r) => r.type === ToolResultType.other)?.data as { - requirementResults: unknown[]; - }; - expect(payload.requirementResults.length).toBeGreaterThan(12); - }); - - it('expands top-level requirement to include sub-requirements', async () => { - (executeEsql as jest.Mock).mockResolvedValue({ - columns: [{ name: 'matching_events', type: 'long' }], - values: [[10]], - }); - - const result = (await tool.handler( - { requirement: '8', includeEvidence: false }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; - - const payload = result.results.find((r) => r.type === ToolResultType.other)?.data as { - requirementResults: Array<{ requirement: string }>; - }; - const reqIds = payload.requirementResults.map((r) => r.requirement); - expect(reqIds).toContain('8'); - expect(reqIds).toContain('8.3.4'); - expect(reqIds).toContain('8.4.2'); - }); - - it('uses per-check timeframe when no user timeRange is provided', async () => { - (executeEsql as jest.Mock).mockResolvedValue({ - columns: [], - values: [], - }); - - mockEsClient.asCurrentUser.fieldCaps.mockResolvedValue({ - fields: { - 'event.category': { keyword: { type: 'keyword' } }, - 'event.outcome': { keyword: { type: 'keyword' } }, - 'user.name': { keyword: { type: 'keyword' } }, - '@timestamp': { date: { type: 'date' } }, - }, - } as never); - - await tool.handler( - { requirement: '8.2.4', includeEvidence: false }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - ); - - const callArgs = (executeEsql as jest.Mock).mock.calls[0][0].query as string; - const today = new Date(); - const yearAgo = new Date(today.getTime() - 365 * 24 * 60 * 60 * 1000); - const yearStr = yearAgo.toISOString().substring(0, 4); - expect(callArgs).toContain(yearStr); - }); - }); -}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_check_tool.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_check_tool.ts deleted file mode 100644 index fdaa66e9cf290..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_check_tool.ts +++ /dev/null @@ -1,463 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { z } from '@kbn/zod'; -import { ToolType, ToolResultType } from '@kbn/agent-builder-common'; -import type { BuiltinToolDefinition } from '@kbn/agent-builder-server'; -import { getToolResultId } from '@kbn/agent-builder-server/tools'; -import type { ElasticsearchClient } from '@kbn/core/server'; -import { executeEsql } from '@kbn/agent-builder-genai-utils'; -import type { Logger } from '@kbn/logging'; -import type { SecuritySolutionPluginCoreSetupDependencies } from '../../plugin_contract'; -import { getAgentBuilderResourceAvailability } from '../utils/get_agent_builder_resource_availability'; -import { securityTool } from './constants'; -import { - type ComplianceStatus, - type ComplianceConfidence, - type PciRequirementDefinition, - getIndexPattern, - getTimeRangeForCheck, - normalizeRequirementId, - resolveRequirementIds, - PCI_REQUIREMENTS, -} from './pci_compliance_requirements'; - -const pciComplianceCheckSchema = z.object({ - requirement: z - .string() - .describe( - 'PCI DSS requirement identifier. Use "all" for a full assessment, major requirements like "8", or sub-requirements like "8.3.4".' - ), - timeRange: z - .object({ - from: z.string(), - to: z.string(), - }) - .optional() - .describe( - 'Optional ISO time range. If omitted, each check uses its recommended lookback period (e.g. 7 days for brute-force, 365 days for stale accounts).' - ), - indices: z - .array(z.string().min(1)) - .optional() - .describe( - 'Specify exact index patterns to avoid duplicate counts from overlapping patterns. ' + - 'Critical during re-indexing to prevent false results. Example: use "logs-okta.system-default" ' + - 'instead of "logs-*" when multiple indices contain the same events.' - ), - includeEvidence: z - .boolean() - .optional() - .default(true) - .describe('Include tabular ES|QL evidence in findings. Defaults to true.'), -}); - -export const PCI_COMPLIANCE_CHECK_TOOL_ID = securityTool('pci_compliance_check'); - -interface RequirementCheckResult { - requirement: string; - name: string; - pciReference: string; - status: ComplianceStatus; - confidence: ComplianceConfidence; - summary: string; - caveats: string[]; - findings: Array<{ - check: string; - status: ComplianceStatus; - detail: string; - evidence?: { - query: string; - columns: Array<{ name: string; type: string }>; - values: unknown[][]; - }; - }>; - recommendations: string[]; - dataGaps: string[]; -} - -const CONCURRENCY_LIMIT = 4; - -async function runWithConcurrency( - tasks: Array<() => Promise>, - limit: number -): Promise { - const results: T[] = new Array(tasks.length); - const executing: Set> = new Set(); - - for (let i = 0; i < tasks.length; i++) { - const index = i; - const p = tasks[index]().then((result) => { - results[index] = result; - executing.delete(p); - }); - executing.add(p); - - if (executing.size >= limit) { - await Promise.race(executing); - } - } - await Promise.all(executing); - return results; -} - -async function runPreflight( - definition: PciRequirementDefinition, - indexPattern: string, - esClient: ElasticsearchClient -): Promise<{ - confidence: ComplianceConfidence; - missingFields: string[]; - fieldFillRate?: number; -}> { - try { - const fieldCaps = await esClient.fieldCaps({ - index: indexPattern, - fields: definition.requiredFields, - ignore_unavailable: true, - allow_no_indices: true, - }); - - const existingFields = Object.keys(fieldCaps.fields ?? {}); - const missing = definition.requiredFields.filter( - (f) => f !== '@timestamp' && !existingFields.includes(f) - ); - - const requiredWithoutTimestamp = definition.requiredFields.filter((f) => f !== '@timestamp'); - if (requiredWithoutTimestamp.length > 0 && missing.length === requiredWithoutTimestamp.length) { - return { confidence: 'NOT_ASSESSABLE', missingFields: missing }; - } - if (missing.length > 0) { - return { confidence: 'MEDIUM', missingFields: missing }; - } - return { confidence: 'HIGH', missingFields: [] }; - } catch { - return { confidence: 'LOW', missingFields: [] }; - } -} - -async function evaluateRequirement({ - requirementId, - indexPattern, - from, - to, - includeEvidence, - esClient, -}: { - requirementId: string; - indexPattern: string; - from: string; - to: string; - includeEvidence: boolean; - esClient: ElasticsearchClient; -}): Promise { - const definition = PCI_REQUIREMENTS[requirementId]; - const caveats: string[] = []; - let status: ComplianceStatus; - let confidence: ComplianceConfidence; - const findings: RequirementCheckResult['findings'] = []; - - // Layer 1: Run violation query if available - if (definition.buildViolationEsql) { - const violationEsql = definition.buildViolationEsql(indexPattern, from, to); - try { - const violationResult = await executeEsql({ query: violationEsql, esClient }); - const rowCount = violationResult.values?.length ?? 0; - - if (definition.verdict === 'rows_mean_violation' && rowCount > 0) { - status = 'RED'; - confidence = 'HIGH'; - findings.push({ - check: `${definition.id} violation detection`, - status: 'RED', - detail: `Found ${rowCount} violation(s) for ${definition.name}.`, - ...(includeEvidence - ? { - evidence: { - query: violationEsql, - columns: violationResult.columns, - values: violationResult.values.slice(0, 50), - }, - } - : {}), - }); - - return buildResult(definition, status, confidence, findings, caveats); - } - - if (definition.verdict === 'rows_mean_evidence' && rowCount > 0) { - status = 'GREEN'; - confidence = 'HIGH'; - findings.push({ - check: `${definition.id} evidence detection`, - status: 'GREEN', - detail: `Found ${rowCount} evidence record(s) for ${definition.name}.`, - ...(includeEvidence - ? { - evidence: { - query: violationEsql, - columns: violationResult.columns, - values: violationResult.values.slice(0, 50), - }, - } - : {}), - }); - - return buildResult(definition, status, confidence, findings, caveats); - } - } catch (error) { - caveats.push( - `Violation query failed: ${error instanceof Error ? error.message : 'Unknown error'}` - ); - } - } - - // Layer 2: Run coverage query - const coverageEsql = definition.buildCoverageEsql(indexPattern, from, to); - try { - const coverageResult = await executeEsql({ query: coverageEsql, esClient }); - const count = toNumber(coverageResult.values?.[0]?.[0]); - - if (count > 0) { - if (definition.verdict === 'rows_mean_violation') { - status = 'GREEN'; - confidence = 'HIGH'; - findings.push({ - check: `${definition.id} coverage check`, - status: 'GREEN', - detail: `${count} related events found with no violations detected for ${definition.name}.`, - ...(includeEvidence - ? { evidence: { query: coverageEsql, columns: coverageResult.columns, values: coverageResult.values.slice(0, 10) } } - : {}), - }); - } else { - status = 'GREEN'; - confidence = 'MEDIUM'; - caveats.push('Coverage data found but no specific violation check was run.'); - findings.push({ - check: `${definition.id} telemetry coverage`, - status: 'GREEN', - detail: `${count} matching events found for ${definition.name}.`, - }); - } - return buildResult(definition, status, confidence, findings, caveats); - } - } catch (error) { - caveats.push( - `Coverage query failed: ${error instanceof Error ? error.message : 'Unknown error'}` - ); - } - - // Layer 3: Preflight — check if fields even exist - const preflight = await runPreflight(definition, indexPattern, esClient); - if (preflight.confidence === 'NOT_ASSESSABLE') { - status = 'NOT_ASSESSABLE'; - confidence = 'NOT_ASSESSABLE'; - findings.push({ - check: `${definition.id} data availability`, - status: 'NOT_ASSESSABLE', - detail: `Required fields are missing from the index: ${preflight.missingFields.join(', ')}. Cannot assess this requirement.`, - }); - } else { - status = 'AMBER'; - confidence = preflight.confidence; - const detail = - preflight.missingFields.length > 0 - ? `Fields exist but no matching events found. Missing fields: ${preflight.missingFields.join(', ')}.` - : 'Fields exist in the index but no matching events found in the selected time range.'; - findings.push({ - check: `${definition.id} data availability`, - status: 'AMBER', - detail, - }); - caveats.push('No matching data in time range. Consider widening the time window or checking index patterns.'); - } - - return buildResult(definition, status, confidence, findings, caveats); -} - -function buildResult( - definition: PciRequirementDefinition, - status: ComplianceStatus, - confidence: ComplianceConfidence, - findings: RequirementCheckResult['findings'], - caveats: string[] -): RequirementCheckResult { - const statusLabel = - status === 'GREEN' - ? 'compliant' - : status === 'RED' - ? 'non-compliant' - : status === 'AMBER' - ? 'partially assessable' - : 'not assessable'; - - return { - requirement: definition.id, - name: definition.name, - pciReference: definition.pciReference, - status, - confidence, - summary: `Requirement ${definition.id} is ${statusLabel} (confidence: ${confidence}).`, - caveats, - findings, - recommendations: definition.recommendations, - dataGaps: status === 'GREEN' ? [] : definition.requiredFields, - }; -} - -const toNumber = (value: unknown): number => { - if (typeof value === 'number') { - return value; - } - if (typeof value === 'string') { - const parsed = Number(value); - return Number.isFinite(parsed) ? parsed : 0; - } - return 0; -}; - -export const pciComplianceCheckTool = ( - core: SecuritySolutionPluginCoreSetupDependencies, - logger: Logger -): BuiltinToolDefinition => { - return { - id: PCI_COMPLIANCE_CHECK_TOOL_ID, - type: ToolType.builtin, - description: - 'Run PCI DSS v4.0.1 compliance checks with violation detection, data quality preflight, ' + - 'and confidence scoring. Supports individual requirements (e.g. "8.3.4"), top-level categories (e.g. "8"), or full assessments ("all").', - schema: pciComplianceCheckSchema, - availability: { - cacheMode: 'space', - handler: async ({ request }) => { - return getAgentBuilderResourceAvailability({ core, request, logger }); - }, - }, - handler: async ({ requirement, timeRange, indices, includeEvidence = true }, { esClient }) => { - const normalized = normalizeRequirementId(requirement); - if (!normalized) { - return { - results: [ - { - type: ToolResultType.error, - data: { - message: `Unsupported PCI requirement "${requirement}". Use "all", "1".."12", or sub-requirements like "8.3.4".`, - }, - }, - ], - }; - } - - const indexPattern = getIndexPattern(indices); - const requirementIds = resolveRequirementIds( - normalized === 'all' ? undefined : [normalized] - ); - - const tasks = requirementIds.map((reqId) => async () => { - const { from, to } = getTimeRangeForCheck(reqId, timeRange); - return evaluateRequirement({ - requirementId: reqId, - indexPattern, - from, - to, - includeEvidence, - esClient: esClient.asCurrentUser, - }); - }); - - const requirementResults = await runWithConcurrency(tasks, CONCURRENCY_LIMIT); - - const statusCounts = requirementResults.reduce( - (acc, result) => { - acc[result.status] = (acc[result.status] ?? 0) + 1; - return acc; - }, - {} as Record - ); - - const overallStatus: ComplianceStatus = - (statusCounts.RED ?? 0) > 0 - ? 'RED' - : (statusCounts.AMBER ?? 0) > 0 || (statusCounts.NOT_ASSESSABLE ?? 0) > 0 - ? 'AMBER' - : 'GREEN'; - - const confidenceCounts = requirementResults.reduce( - (acc, result) => { - acc[result.confidence] = (acc[result.confidence] ?? 0) + 1; - return acc; - }, - {} as Record - ); - const overallConfidence: ComplianceConfidence = - (confidenceCounts.NOT_ASSESSABLE ?? 0) > requirementResults.length / 2 - ? 'NOT_ASSESSABLE' - : (confidenceCounts.LOW ?? 0) + (confidenceCounts.NOT_ASSESSABLE ?? 0) > - requirementResults.length / 2 - ? 'LOW' - : (confidenceCounts.HIGH ?? 0) >= requirementResults.length / 2 - ? 'HIGH' - : 'MEDIUM'; - - const results: Array<{ - type: ToolResultType; - data: Record; - tool_result_id?: string; - }> = []; - - const redFindings = requirementResults.filter((r) => r.status === 'RED'); - if (redFindings.length > 0) { - const evidenceRows = redFindings.flatMap((r) => - r.findings - .filter((f) => f.evidence && f.evidence.values.length > 0) - .flatMap((f) => f.evidence!.values.map((row) => [r.requirement, r.name, ...row])) - ); - - if (evidenceRows.length > 0) { - const firstEvidence = redFindings[0]?.findings.find((f) => f.evidence); - const evidenceColumns = [ - { name: 'requirement', type: 'keyword' }, - { name: 'check', type: 'keyword' }, - ...(firstEvidence?.evidence?.columns ?? []), - ]; - - results.push({ - type: ToolResultType.query, - data: { esql: 'PCI DSS v4.0.1 Compliance Violations' }, - }); - results.push({ - tool_result_id: getToolResultId(), - type: ToolResultType.esqlResults, - data: { - query: 'PCI DSS v4.0.1 Compliance Check - Violations Found', - columns: evidenceColumns, - values: evidenceRows.slice(0, 100), - }, - }); - } - } - - results.push({ - type: ToolResultType.other, - data: { - request: { - requirement, - indices: indices ?? [...indexPattern.split(',')], - }, - overallStatus, - overallConfidence, - statusCounts, - requirementResults, - }, - }); - - return { results }; - }, - tags: ['security', 'compliance', 'pci', 'audit'], - }; -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_evaluator.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_evaluator.test.ts new file mode 100644 index 0000000000000..af6fb1e6bcb26 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_evaluator.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; + +jest.mock('@kbn/agent-builder-genai-utils', () => ({ + executeEsql: jest.fn(), +})); + +import { executeEsql } from '@kbn/agent-builder-genai-utils'; +import { evaluateRequirement, runWithConcurrency } from './pci_compliance_evaluator'; + +const mockExecuteEsql = executeEsql as jest.MockedFunction; + +const createEsClient = (overrides: Partial = {}): ElasticsearchClient => + ({ + fieldCaps: jest.fn().mockResolvedValue({ fields: {} }), + ...overrides, + } as unknown as ElasticsearchClient); + +describe('evaluateRequirement — ES|QL parameter binding', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('binds the user time range via ?_tstart / ?_tend without interpolating it into the query', async () => { + mockExecuteEsql.mockResolvedValue({ + columns: [{ name: 'matching_events', type: 'long' }], + values: [[5]], + } as never); + + await evaluateRequirement({ + requirementId: '1', + indexPattern: 'logs-*', + from: '2024-01-01T00:00:00Z', + to: '2024-01-08T00:00:00Z', + includeEvidence: false, + esClient: createEsClient(), + }); + + expect(mockExecuteEsql).toHaveBeenCalled(); + const call = mockExecuteEsql.mock.calls[0][0]; + expect(call.query).toContain('?_tstart'); + expect(call.query).toContain('?_tend'); + expect(call.query).not.toContain('2024-01-01T00:00:00Z'); + expect(call.params).toEqual([ + { _tstart: '2024-01-01T00:00:00Z' }, + { _tend: '2024-01-08T00:00:00Z' }, + ]); + }); + + it('returns NOT_ASSESSABLE when coverage is empty and every required field is missing', async () => { + mockExecuteEsql.mockResolvedValue({ + columns: [{ name: 'matching_events', type: 'long' }], + values: [[0]], + } as never); + + const esClient = createEsClient({ + // No fields exist -> preflight reports every non-@timestamp required field as missing. + fieldCaps: jest.fn().mockResolvedValue({ fields: {} }), + } as unknown as Partial); + + const result = await evaluateRequirement({ + requirementId: '1', + indexPattern: 'logs-*', + from: '2024-01-01T00:00:00Z', + to: '2024-01-08T00:00:00Z', + includeEvidence: false, + esClient, + }); + + expect(result.status).toBe('NOT_ASSESSABLE'); + expect(result.confidence).toBe('NOT_ASSESSABLE'); + }); + + it('returns GREEN when the coverage query finds matching events for a rows_mean_evidence requirement', async () => { + mockExecuteEsql.mockResolvedValue({ + columns: [{ name: 'matching_events', type: 'long' }], + values: [[42]], + } as never); + + const result = await evaluateRequirement({ + requirementId: '1', + indexPattern: 'logs-*', + from: '2024-01-01T00:00:00Z', + to: '2024-01-08T00:00:00Z', + includeEvidence: false, + esClient: createEsClient(), + }); + + expect(result.status).toBe('GREEN'); + expect(result.evidenceCount).toBe(42); + }); + + it('omits evidence payloads when includeEvidence is false', async () => { + mockExecuteEsql.mockResolvedValue({ + columns: [{ name: 'matching_events', type: 'long' }], + values: [[3]], + } as never); + + const result = await evaluateRequirement({ + requirementId: '1', + indexPattern: 'logs-*', + from: '2024-01-01T00:00:00Z', + to: '2024-01-08T00:00:00Z', + includeEvidence: false, + esClient: createEsClient(), + }); + + for (const finding of result.findings) { + expect(finding.evidence).toBeUndefined(); + } + }); +}); + +describe('runWithConcurrency', () => { + it('preserves input order regardless of completion order', async () => { + const order = [30, 10, 20, 5]; + const tasks = order.map((delay, idx) => async () => { + await new Promise((r) => setTimeout(r, delay)); + return idx; + }); + + const result = await runWithConcurrency(tasks, 2); + expect(result).toEqual([0, 1, 2, 3]); + }); + + it('caps in-flight tasks at the configured limit', async () => { + let inFlight = 0; + let peak = 0; + const tasks = new Array(8).fill(null).map(() => async () => { + inFlight++; + peak = Math.max(peak, inFlight); + await new Promise((r) => setTimeout(r, 5)); + inFlight--; + return true; + }); + + await runWithConcurrency(tasks, 3); + expect(peak).toBeLessThanOrEqual(3); + }); + + it('rejects limit <= 0', async () => { + await expect(runWithConcurrency([async () => 1], 0)).rejects.toThrow('limit must be > 0'); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_evaluator.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_evaluator.ts new file mode 100644 index 0000000000000..853cda41e5924 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_evaluator.ts @@ -0,0 +1,337 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; +import { executeEsql } from '@kbn/agent-builder-genai-utils'; +import { + type ComplianceConfidence, + type ComplianceStatus, + type PciRequirementDefinition, + buildPciTimeRangeParams, + PCI_REQUIREMENTS, +} from './pci_compliance_requirements'; + +export interface EvaluateRequirementArgs { + requirementId: string; + indexPattern: string; + from: string; + to: string; + includeEvidence: boolean; + esClient: ElasticsearchClient; +} + +export interface RequirementFinding { + check: string; + status: ComplianceStatus; + detail: string; + evidence?: { + query: string; + columns: Array<{ name: string; type: string }>; + values: unknown[][]; + }; +} + +export interface EvaluatedRequirement { + requirement: string; + name: string; + pciReference: string; + status: ComplianceStatus; + confidence: ComplianceConfidence; + summary: string; + caveats: string[]; + findings: RequirementFinding[]; + recommendations: string[]; + dataGaps: string[]; + evidenceCount: number; + /** Numeric 0-100 score factoring confidence into status. */ + score: number; +} + +const toNumber = (value: unknown): number => { + if (typeof value === 'number') return value; + if (typeof value === 'string') { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : 0; + } + return 0; +}; + +async function runPreflight( + definition: PciRequirementDefinition, + indexPattern: string, + esClient: ElasticsearchClient +): Promise<{ confidence: ComplianceConfidence; missingFields: string[] }> { + try { + const fieldCaps = await esClient.fieldCaps({ + index: indexPattern, + fields: definition.requiredFields, + ignore_unavailable: true, + allow_no_indices: true, + }); + + const existingFields = Object.keys(fieldCaps.fields ?? {}); + const missing = definition.requiredFields.filter( + (f) => f !== '@timestamp' && !existingFields.includes(f) + ); + + const requiredWithoutTimestamp = definition.requiredFields.filter((f) => f !== '@timestamp'); + if (requiredWithoutTimestamp.length > 0 && missing.length === requiredWithoutTimestamp.length) { + return { confidence: 'NOT_ASSESSABLE', missingFields: missing }; + } + if (missing.length > 0) { + return { confidence: 'MEDIUM', missingFields: missing }; + } + return { confidence: 'HIGH', missingFields: [] }; + } catch { + return { confidence: 'LOW', missingFields: [] }; + } +} + +function statusToScore(status: ComplianceStatus, confidence: ComplianceConfidence): number { + const baseScore = status === 'GREEN' ? 100 : status === 'AMBER' ? 50 : status === 'RED' ? 0 : 25; + const confidenceWeight = + confidence === 'HIGH' ? 1.0 : confidence === 'MEDIUM' ? 0.8 : confidence === 'LOW' ? 0.5 : 0.3; + return Math.round(baseScore * confidenceWeight); +} + +function buildResult( + definition: PciRequirementDefinition, + status: ComplianceStatus, + confidence: ComplianceConfidence, + findings: RequirementFinding[], + caveats: string[], + evidenceCount: number +): EvaluatedRequirement { + const statusLabel = + status === 'GREEN' + ? 'compliant' + : status === 'RED' + ? 'non-compliant' + : status === 'AMBER' + ? 'partially assessable' + : 'not assessable'; + + return { + requirement: definition.id, + name: definition.name, + pciReference: definition.pciReference, + status, + confidence, + summary: `Requirement ${definition.id} is ${statusLabel} (confidence: ${confidence}).`, + caveats, + findings, + recommendations: definition.recommendations, + dataGaps: status === 'GREEN' ? [] : definition.requiredFields, + evidenceCount, + score: statusToScore(status, confidence), + }; +} + +/** + * Evaluate a single PCI DSS requirement against an index pattern + time range. + * + * Implementation notes: + * - ES|QL queries reference `?_tstart` / `?_tend` named parameters. Time-range values are + * bound via the ES|QL params array (never interpolated into the query string), which is + * the security boundary against injection in user-supplied time ranges. + * - The index pattern is interpolated into `FROM` because ES|QL cannot parameterise the + * target index. The caller MUST have validated the pattern against + * `pciIndexPatternSchema` before this function is reached. + * - Layering is intentional: (1) violation query, (2) coverage query, (3) field preflight. + * Each layer tightens confidence and narrows the RED / AMBER / GREEN verdict. + */ +export async function evaluateRequirement({ + requirementId, + indexPattern, + from, + to, + includeEvidence, + esClient, +}: EvaluateRequirementArgs): Promise { + const definition = PCI_REQUIREMENTS[requirementId]; + const params = buildPciTimeRangeParams({ from, to }); + const caveats: string[] = []; + const findings: RequirementFinding[] = []; + let status: ComplianceStatus = 'AMBER'; + let confidence: ComplianceConfidence = 'LOW'; + let evidenceCount = 0; + + // Layer 1 — violation query (if the requirement defines one) + if (definition.buildViolationEsql) { + const violationEsql = definition.buildViolationEsql(indexPattern); + try { + const violationResult = await executeEsql({ query: violationEsql, params, esClient }); + const rowCount = violationResult.values?.length ?? 0; + + if (definition.verdict === 'rows_mean_violation' && rowCount > 0) { + status = 'RED'; + confidence = 'HIGH'; + evidenceCount = rowCount; + findings.push({ + check: `${definition.id} violation detection`, + status: 'RED', + detail: `Found ${rowCount} violation(s) for ${definition.name}.`, + ...(includeEvidence + ? { + evidence: { + query: violationEsql, + columns: violationResult.columns, + values: violationResult.values.slice(0, 50), + }, + } + : {}), + }); + return buildResult(definition, status, confidence, findings, caveats, evidenceCount); + } + + if (definition.verdict === 'rows_mean_evidence' && rowCount > 0) { + status = 'GREEN'; + confidence = 'HIGH'; + evidenceCount = rowCount; + findings.push({ + check: `${definition.id} evidence detection`, + status: 'GREEN', + detail: `Found ${rowCount} evidence record(s) for ${definition.name}.`, + ...(includeEvidence + ? { + evidence: { + query: violationEsql, + columns: violationResult.columns, + values: violationResult.values.slice(0, 50), + }, + } + : {}), + }); + return buildResult(definition, status, confidence, findings, caveats, evidenceCount); + } + } catch (error) { + caveats.push( + `Violation query failed: ${error instanceof Error ? error.message : 'Unknown error'}` + ); + } + } + + // Layer 2 — coverage query + const coverageEsql = definition.buildCoverageEsql(indexPattern); + try { + const coverageResult = await executeEsql({ query: coverageEsql, params, esClient }); + const count = toNumber(coverageResult.values?.[0]?.[0]); + + if (count > 0) { + if (definition.verdict === 'rows_mean_violation') { + status = 'GREEN'; + confidence = 'HIGH'; + evidenceCount = count; + findings.push({ + check: `${definition.id} coverage check`, + status: 'GREEN', + detail: `${count} related events found with no violations detected for ${definition.name}.`, + ...(includeEvidence + ? { + evidence: { + query: coverageEsql, + columns: coverageResult.columns, + values: coverageResult.values.slice(0, 10), + }, + } + : {}), + }); + } else { + status = 'GREEN'; + confidence = definition.buildViolationEsql ? 'HIGH' : 'MEDIUM'; + evidenceCount = count; + caveats.push('Coverage data found but no specific violation check was run.'); + findings.push({ + check: `${definition.id} telemetry coverage`, + status: 'GREEN', + detail: `${count} matching events found for ${definition.name}.`, + }); + } + return buildResult(definition, status, confidence, findings, caveats, evidenceCount); + } + } catch (error) { + caveats.push( + `Coverage query failed: ${error instanceof Error ? error.message : 'Unknown error'}` + ); + } + + // Layer 3 — preflight against mappings + const preflight = await runPreflight(definition, indexPattern, esClient); + if (preflight.confidence === 'NOT_ASSESSABLE') { + status = 'NOT_ASSESSABLE'; + confidence = 'NOT_ASSESSABLE'; + findings.push({ + check: `${definition.id} data availability`, + status: 'NOT_ASSESSABLE', + detail: `Required fields are missing from the index: ${preflight.missingFields.join( + ', ' + )}. Cannot assess this requirement.`, + }); + } else { + status = 'AMBER'; + confidence = preflight.confidence; + const detail = + preflight.missingFields.length > 0 + ? `Fields exist but no matching events found. Missing fields: ${preflight.missingFields.join( + ', ' + )}.` + : 'Fields exist in the index but no matching events found in the selected time range.'; + findings.push({ + check: `${definition.id} data availability`, + status: 'AMBER', + detail, + }); + caveats.push( + 'No matching data in time range. Consider widening the time window or checking index patterns.' + ); + } + + return buildResult(definition, status, confidence, findings, caveats, evidenceCount); +} + +/** + * Default per-call concurrency for requirement evaluation. + * + * Each requirement issues at most 3 sequential ES|QL / field-caps round-trips (violation, + * coverage, preflight). Running them 4-wide keeps a full PCI posture report (~20 + * requirements) under ~5× the latency of a single requirement while staying well below + * Elasticsearch's default `indices.query.bool.max_clause_count` and the ES|QL task-queue + * thresholds observed in `@kbn/evals-suite-pci-compliance` runs. Raise cautiously if the + * target cluster has dedicated search capacity. + */ +export const PCI_REQUIREMENT_CONCURRENCY = 4; + +/** + * Concurrency-limited task runner that preserves input order in the result array. + * + * Used by the consolidated PCI tool so both check-mode and report-mode outputs are stable + * regardless of which requirement finishes first. + */ +export async function runWithConcurrency( + tasks: Array<() => Promise>, + limit: number +): Promise { + if (limit <= 0) { + throw new Error('runWithConcurrency: limit must be > 0'); + } + const results: T[] = new Array(tasks.length); + const executing = new Set>(); + + for (let i = 0; i < tasks.length; i++) { + const index = i; + const p = tasks[index]().then((result) => { + results[index] = result; + executing.delete(p); + }); + executing.add(p); + if (executing.size >= limit) { + await Promise.race(executing); + } + } + await Promise.all(executing); + return results; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_report_tool.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_report_tool.test.ts deleted file mode 100644 index 310145002422b..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_report_tool.test.ts +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ToolResultType } from '@kbn/agent-builder-common'; -import { executeEsql } from '@kbn/agent-builder-genai-utils'; -import type { ToolHandlerStandardReturn } from '@kbn/agent-builder-server/tools'; -import { createToolHandlerContext, createToolTestMocks } from '../__mocks__/test_helpers'; -import { - pciComplianceReportTool, - PCI_COMPLIANCE_REPORT_TOOL_ID, -} from './pci_compliance_report_tool'; - -jest.mock('@kbn/agent-builder-genai-utils', () => ({ - executeEsql: jest.fn(), -})); - -describe('pciComplianceReportTool', () => { - const { mockCore, mockLogger, mockEsClient, mockRequest } = createToolTestMocks(); - const tool = pciComplianceReportTool(mockCore, mockLogger); - - beforeEach(() => { - jest.clearAllMocks(); - }); - - describe('schema', () => { - it('accepts valid report request', () => { - const result = tool.schema.safeParse({ format: 'executive' }); - expect(result.success).toBe(true); - }); - - it('accepts empty request for full report', () => { - const result = tool.schema.safeParse({}); - expect(result.success).toBe(true); - }); - }); - - describe('properties', () => { - it('returns correct id', () => { - expect(tool.id).toBe(PCI_COMPLIANCE_REPORT_TOOL_ID); - }); - - it('references PCI DSS v4.0.1 in description', () => { - expect(tool.description).toContain('v4.0.1'); - }); - }); - - describe('handler', () => { - it('returns error for invalid requirement', async () => { - const result = (await tool.handler( - { requirements: ['99'], format: 'summary', includeRecommendations: true }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; - - expect(result.results[0].type).toBe(ToolResultType.error); - }); - - it('generates executive report with correct shape', async () => { - (executeEsql as jest.Mock).mockResolvedValue({ - columns: [{ name: 'matching_events', type: 'long' }], - values: [[100]], - }); - - const result = (await tool.handler( - { requirements: ['1'], format: 'executive', includeRecommendations: false }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; - - const payload = result.results.find((r) => r.type === ToolResultType.other)?.data as { - format: string; - overallScore: number; - overallStatus: string; - overallConfidence: string; - requirements: Array<{ id: string; status: string; confidence: string }>; - }; - - expect(payload.format).toBe('executive'); - expect(payload.overallScore).toBeGreaterThan(0); - expect(payload.overallConfidence).toBeDefined(); - }); - - it('includes visual scorecard as esqlResults', async () => { - (executeEsql as jest.Mock).mockResolvedValue({ - columns: [{ name: 'matching_events', type: 'long' }], - values: [[50]], - }); - - const result = (await tool.handler( - { requirements: ['1'], format: 'summary', includeRecommendations: true }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; - - const esqlResult = result.results.find((r) => r.type === ToolResultType.esqlResults); - expect(esqlResult).toBeDefined(); - expect(esqlResult?.tool_result_id).toBeDefined(); - - const data = esqlResult?.data as { - columns: Array<{ name: string }>; - values: unknown[][]; - }; - expect(data.columns.map((c) => c.name)).toEqual( - expect.arrayContaining(['Requirement', 'Status', 'Confidence', 'Score']) - ); - }); - - it('degrades score when no evidence is found', async () => { - (executeEsql as jest.Mock).mockResolvedValue({ - columns: [{ name: 'matching_events', type: 'long' }], - values: [[0]], - }); - - const result = (await tool.handler( - { format: 'summary', includeRecommendations: false }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; - - const payload = result.results.find((r) => r.type === ToolResultType.other)?.data as { - overallScore: number; - overallStatus: string; - }; - expect(payload.overallScore).toBeLessThan(85); - }); - - it('includes confidence in report rows', async () => { - (executeEsql as jest.Mock).mockResolvedValue({ - columns: [{ name: 'matching_events', type: 'long' }], - values: [[100]], - }); - - const result = (await tool.handler( - { requirements: ['8'], format: 'detailed', includeRecommendations: true }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; - - const payload = result.results.find((r) => r.type === ToolResultType.other)?.data as { - requirements: Array<{ confidence: string; pciReference: string }>; - }; - for (const req of payload.requirements) { - expect(req.confidence).toBeDefined(); - expect(req.pciReference).toContain('v4.0.1'); - } - }); - - it('runs checks in parallel with concurrency control', async () => { - let maxConcurrent = 0; - let currentConcurrent = 0; - - (executeEsql as jest.Mock).mockImplementation(async () => { - currentConcurrent++; - maxConcurrent = Math.max(maxConcurrent, currentConcurrent); - await new Promise((resolve) => setTimeout(resolve, 10)); - currentConcurrent--; - return { - columns: [{ name: 'matching_events', type: 'long' }], - values: [[5]], - }; - }); - - await tool.handler( - { format: 'summary', includeRecommendations: false }, - createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - ); - - expect(maxConcurrent).toBeLessThanOrEqual(4); - expect(maxConcurrent).toBeGreaterThan(1); - }); - }); -}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_report_tool.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_report_tool.ts deleted file mode 100644 index a903e1d6d85c1..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_report_tool.ts +++ /dev/null @@ -1,388 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { z } from '@kbn/zod'; -import { ToolType, ToolResultType } from '@kbn/agent-builder-common'; -import type { BuiltinToolDefinition } from '@kbn/agent-builder-server'; -import { getToolResultId } from '@kbn/agent-builder-server/tools'; -import type { ElasticsearchClient } from '@kbn/core/server'; -import { executeEsql } from '@kbn/agent-builder-genai-utils'; -import type { Logger } from '@kbn/logging'; -import type { SecuritySolutionPluginCoreSetupDependencies } from '../../plugin_contract'; -import { getAgentBuilderResourceAvailability } from '../utils/get_agent_builder_resource_availability'; -import { securityTool } from './constants'; -import { - type ComplianceStatus, - type ComplianceConfidence, - type PciRequirementDefinition, - getIndexPattern, - getTimeRangeForCheck, - normalizeRequirementId, - PCI_REQUIREMENTS, - resolveRequirementIds, -} from './pci_compliance_requirements'; - -const pciComplianceReportSchema = z.object({ - requirements: z - .array(z.string()) - .optional() - .describe('Optional list of requirements to include. Defaults to all.'), - format: z - .enum(['summary', 'detailed', 'executive']) - .optional() - .default('summary') - .describe('Report level: summary, detailed, or executive.'), - timeRange: z - .object({ - from: z.string(), - to: z.string(), - }) - .optional() - .describe( - 'Optional ISO time range. If omitted, each check uses its recommended lookback period.' - ), - indices: z - .array(z.string().min(1)) - .optional() - .describe( - 'Specify exact index patterns to avoid duplicate counts. Use specific patterns during re-indexing.' - ), - includeRecommendations: z - .boolean() - .optional() - .default(true) - .describe('Include recommendation output in report sections.'), -}); - -export const PCI_COMPLIANCE_REPORT_TOOL_ID = securityTool('pci_compliance_report'); - -const CONCURRENCY_LIMIT = 4; - -async function runWithConcurrency( - tasks: Array<() => Promise>, - limit: number -): Promise { - const results: T[] = []; - const executing: Set> = new Set(); - - for (const task of tasks) { - const p = task().then((result) => { - results.push(result); - executing.delete(p); - }); - executing.add(p); - - if (executing.size >= limit) { - await Promise.race(executing); - } - } - await Promise.all(executing); - return results; -} - -interface RequirementReportRow { - id: string; - name: string; - pciReference: string; - status: ComplianceStatus; - confidence: ComplianceConfidence; - score: number; - evidenceCount: number; - topFindings: string[]; - recommendations: string[]; -} - -const toNumber = (value: unknown): number => { - if (typeof value === 'number') return value; - if (typeof value === 'string') { - const parsed = Number(value); - return Number.isFinite(parsed) ? parsed : 0; - } - return 0; -}; - -async function evaluateRequirementForReport({ - definition, - indexPattern, - from, - to, - esClient, -}: { - definition: PciRequirementDefinition; - indexPattern: string; - from: string; - to: string; - esClient: ElasticsearchClient; -}): Promise { - let status: ComplianceStatus = 'AMBER'; - let confidence: ComplianceConfidence = 'LOW'; - let evidenceCount = 0; - const topFindings: string[] = []; - - // Run violation query if available - if (definition.buildViolationEsql) { - try { - const violationResult = await executeEsql({ - query: definition.buildViolationEsql(indexPattern, from, to), - esClient, - }); - const rowCount = violationResult.values?.length ?? 0; - - if (definition.verdict === 'rows_mean_violation' && rowCount > 0) { - status = 'RED'; - confidence = 'HIGH'; - evidenceCount = rowCount; - topFindings.push(`${rowCount} violation(s) detected for ${definition.name}.`); - return buildRow(definition, status, confidence, evidenceCount, topFindings); - } - - if (definition.verdict === 'rows_mean_evidence' && rowCount > 0) { - status = 'GREEN'; - confidence = 'HIGH'; - evidenceCount = rowCount; - topFindings.push(`${rowCount} evidence record(s) found for ${definition.name}.`); - return buildRow(definition, status, confidence, evidenceCount, topFindings); - } - } catch (error) { - topFindings.push( - `Violation query failed: ${error instanceof Error ? error.message : 'Unknown error'}` - ); - } - } - - // Run coverage query - try { - const coverageResult = await executeEsql({ - query: definition.buildCoverageEsql(indexPattern, from, to), - esClient, - }); - const count = toNumber(coverageResult.values?.[0]?.[0]); - - if (count > 0) { - status = 'GREEN'; - confidence = definition.buildViolationEsql ? 'HIGH' : 'MEDIUM'; - evidenceCount = count; - topFindings.push(`${count} matching events found for ${definition.name}.`); - } else { - topFindings.push(`No telemetry evidence found for ${definition.name}.`); - } - } catch (error) { - status = 'AMBER'; - confidence = 'NOT_ASSESSABLE'; - topFindings.push( - `Query failed: ${error instanceof Error ? error.message : 'Unknown error'}` - ); - } - - return buildRow(definition, status, confidence, evidenceCount, topFindings); -} - -function buildRow( - definition: PciRequirementDefinition, - status: ComplianceStatus, - confidence: ComplianceConfidence, - evidenceCount: number, - topFindings: string[] -): RequirementReportRow { - return { - id: definition.id, - name: definition.name, - pciReference: definition.pciReference, - status, - confidence, - score: statusToScore(status, confidence), - evidenceCount, - topFindings, - recommendations: definition.recommendations, - }; -} - -function statusToScore(status: ComplianceStatus, confidence: ComplianceConfidence): number { - const baseScore = - status === 'GREEN' ? 100 : status === 'AMBER' ? 50 : status === 'RED' ? 0 : 25; - - const confidenceWeight = - confidence === 'HIGH' - ? 1.0 - : confidence === 'MEDIUM' - ? 0.8 - : confidence === 'LOW' - ? 0.5 - : 0.3; - - return Math.round(baseScore * confidenceWeight); -} - -const calculateOverallScore = (rows: RequirementReportRow[]): number => { - if (rows.length === 0) return 0; - const total = rows.reduce((sum, row) => sum + row.score, 0); - return Math.round(total / rows.length); -}; - -const scoreToStatus = (score: number): ComplianceStatus => { - if (score >= 85) return 'GREEN'; - if (score >= 60) return 'AMBER'; - return 'RED'; -}; - -export const pciComplianceReportTool = ( - core: SecuritySolutionPluginCoreSetupDependencies, - logger: Logger -): BuiltinToolDefinition => { - return { - id: PCI_COMPLIANCE_REPORT_TOOL_ID, - type: ToolType.builtin, - description: - 'Generate PCI DSS v4.0.1 compliance reports with Red/Amber/Green status, confidence scoring, ' + - 'and visual scorecards across all requirements or a selected subset.', - schema: pciComplianceReportSchema, - availability: { - cacheMode: 'space', - handler: async ({ request }) => { - return getAgentBuilderResourceAvailability({ core, request, logger }); - }, - }, - handler: async ( - { requirements, format = 'summary', timeRange, indices, includeRecommendations = true }, - { esClient } - ) => { - if (requirements?.length) { - for (const raw of requirements) { - if (!normalizeRequirementId(raw)) { - return { - results: [ - { - type: ToolResultType.error, - data: { message: 'One or more PCI requirement identifiers are invalid.' }, - }, - ], - }; - } - } - } - - const requested = resolveRequirementIds(requirements); - const normalized = requested.map((reqId) => normalizeRequirementId(reqId)); - - if (normalized.some((reqId) => !reqId)) { - return { - results: [ - { - type: ToolResultType.error, - data: { message: 'One or more PCI requirement identifiers are invalid.' }, - }, - ], - }; - } - - const requirementIds = [...new Set(normalized.map((reqId) => reqId as string))]; - const indexPattern = getIndexPattern(indices); - - const tasks = requirementIds.map((reqId) => async () => { - const definition = PCI_REQUIREMENTS[reqId]; - const { from, to } = getTimeRangeForCheck(reqId, timeRange); - return evaluateRequirementForReport({ - definition, - indexPattern, - from, - to, - esClient: esClient.asCurrentUser, - }); - }); - - const rows = await runWithConcurrency(tasks, CONCURRENCY_LIMIT); - - const overallScore = calculateOverallScore(rows); - const overallStatus = scoreToStatus(overallScore); - const redCount = rows.filter((r) => r.status === 'RED').length; - const amberCount = rows.filter((r) => r.status === 'AMBER').length; - const greenCount = rows.filter((r) => r.status === 'GREEN').length; - const notAssessableCount = rows.filter((r) => r.confidence === 'NOT_ASSESSABLE').length; - - const highConfCount = rows.filter((r) => r.confidence === 'HIGH').length; - const overallConfidence: ComplianceConfidence = - highConfCount >= rows.length / 2 ? 'HIGH' : 'MEDIUM'; - - const requirementRows = rows.map((row) => ({ - ...row, - recommendations: includeRecommendations ? row.recommendations : [], - })); - - const results: Array<{ - type: ToolResultType; - data: Record; - tool_result_id?: string; - }> = []; - - // Visual scorecard table - const scorecardColumns = [ - { name: 'Requirement', type: 'keyword' }, - { name: 'Check', type: 'keyword' }, - { name: 'Status', type: 'keyword' }, - { name: 'Confidence', type: 'keyword' }, - { name: 'Score', type: 'long' }, - { name: 'Findings', type: 'long' }, - ]; - const scorecardValues = rows.map((r) => [ - r.id, - r.name, - r.status, - r.confidence, - r.score, - r.evidenceCount, - ]); - - results.push({ - type: ToolResultType.query, - data: { esql: 'PCI DSS v4.0.1 Compliance Report' }, - }); - results.push({ - tool_result_id: getToolResultId(), - type: ToolResultType.esqlResults, - data: { - query: 'PCI DSS v4.0.1 Compliance Scorecard', - columns: scorecardColumns, - values: scorecardValues, - }, - }); - - results.push({ - type: ToolResultType.other, - data: { - format, - generatedAt: new Date().toISOString(), - overallScore, - overallStatus, - overallConfidence, - summary: `PCI DSS v4.0.1 posture is ${overallStatus} with score ${overallScore}/100. Requirements: ${greenCount} GREEN, ${amberCount} AMBER, ${redCount} RED, ${notAssessableCount} NOT ASSESSABLE.`, - requirements: - format === 'executive' - ? requirementRows.map(({ id, name, status, confidence, score, evidenceCount }) => ({ - id, - name, - status, - confidence, - score, - evidenceCount, - })) - : requirementRows, - dataCoverage: { - indexPattern, - totalRequirements: requirementRows.length, - greenCount, - amberCount, - redCount, - notAssessableCount, - }, - }, - }); - - return { results }; - }, - tags: ['security', 'compliance', 'pci', 'reporting'], - }; -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_requirements.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_requirements.test.ts new file mode 100644 index 0000000000000..6d6ff5ef644a9 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_requirements.test.ts @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + DEFAULT_PCI_INDEX_PATTERNS, + PCI_REQUIREMENTS, + buildPciTimeRangeParams, + getIndexList, + getIndexPattern, + getTimeRangeForCheck, + normalizeRequirementId, + resolveRequirementIds, +} from './pci_compliance_requirements'; + +describe('PCI compliance requirement builders', () => { + const indexPattern = 'logs-*'; + + /** + * Requirements that intentionally scan the full index rather than the caller-supplied + * time window. These queries must still be injection-safe (no interpolated time values), + * but they do not — and should not — reference ?_tstart / ?_tend. + */ + const FULL_INDEX_REQUIREMENT_IDS = new Set(['10.5']); + + describe('ES|QL query safety', () => { + it('uses ?_tstart / ?_tend placeholders in every time-scoped coverage query', () => { + for (const [id, def] of Object.entries(PCI_REQUIREMENTS)) { + const esql = def.buildCoverageEsql(indexPattern); + + if (FULL_INDEX_REQUIREMENT_IDS.has(id)) { + // Full-index scans have no user-supplied time value to interpolate, but they + // must still not leak any time literal or unbound `${...}` template marker. + expect(esql).not.toMatch(/\?_tstart|\?_tend/); + } else { + expect(esql).toContain('?_tstart'); + expect(esql).toContain('?_tend'); + } + + // No time string interpolated into the query. + expect(esql).not.toMatch(/\d{4}-\d{2}-\d{2}T/); + // No unbound ${...} interpolation markers remaining. + expect(esql).not.toMatch(/\$\{[^}]+\}/); + expect(esql).toContain(`FROM ${indexPattern}`); + // Sanity: the id is referenced in the requirement def. + expect(def.id).toBe(id); + } + }); + + it('uses ?_tstart / ?_tend placeholders in every violation query', () => { + for (const def of Object.values(PCI_REQUIREMENTS)) { + const builder = def.buildViolationEsql; + if (builder) { + const esql = builder(indexPattern); + expect(esql).toContain('?_tstart'); + expect(esql).toContain('?_tend'); + expect(esql).not.toMatch(/\d{4}-\d{2}-\d{2}T/); + expect(esql).not.toMatch(/\$\{[^}]+\}/); + expect(esql).toContain(`FROM ${indexPattern}`); + } + } + }); + + it('forwards the pre-validated index pattern verbatim into FROM', () => { + const customPattern = 'cluster-a:logs-pci-*'; + for (const def of Object.values(PCI_REQUIREMENTS)) { + expect(def.buildCoverageEsql(customPattern)).toContain(`FROM ${customPattern}`); + } + }); + }); +}); + +describe('buildPciTimeRangeParams', () => { + it('emits a positional params array with _tstart then _tend', () => { + const params = buildPciTimeRangeParams({ + from: '2024-01-01T00:00:00Z', + to: '2024-01-08T00:00:00Z', + }); + expect(params).toEqual([ + { _tstart: '2024-01-01T00:00:00Z' }, + { _tend: '2024-01-08T00:00:00Z' }, + ]); + }); +}); + +describe('getTimeRangeForCheck', () => { + it('returns the explicit user time range when provided', () => { + const explicit = { + from: '2024-01-01T00:00:00Z', + to: '2024-01-08T00:00:00Z', + }; + expect(getTimeRangeForCheck('8', explicit)).toEqual(explicit); + }); + + it('falls back to the per-requirement default lookback window', () => { + const range = getTimeRangeForCheck('8'); + const diffDays = (Date.parse(range.to) - Date.parse(range.from)) / (24 * 60 * 60 * 1000); + const expectedDays = PCI_REQUIREMENTS['8'].defaultLookbackDays; + expect(Math.round(diffDays)).toBe(expectedDays); + }); +}); + +describe('normalizeRequirementId', () => { + it('accepts "all"', () => { + expect(normalizeRequirementId('all')).toBe('all'); + }); + + it('accepts a top-level id', () => { + expect(normalizeRequirementId('8')).toBe('8'); + }); + + it('normalises a sub-requirement to its top-level id when no exact definition exists', () => { + const result = normalizeRequirementId('1.2.3'); + expect(result === '1' || result === '1.2.3').toBe(true); + if (result !== null) { + expect(PCI_REQUIREMENTS[result] ?? PCI_REQUIREMENTS[result.split('.')[0]]).toBeDefined(); + } + }); + + it('returns null for unknown ids', () => { + expect(normalizeRequirementId('99')).toBeNull(); + expect(normalizeRequirementId('drop')).toBeNull(); + }); +}); + +describe('resolveRequirementIds', () => { + it('returns every requirement when the list is empty or contains "all"', () => { + const allKeys = Object.keys(PCI_REQUIREMENTS); + expect(resolveRequirementIds()).toEqual(allKeys); + expect(resolveRequirementIds([])).toEqual(allKeys); + expect(resolveRequirementIds(['all'])).toEqual(allKeys); + }); + + it('expands a top-level id to include its sub-requirements', () => { + const resolved = resolveRequirementIds(['8']); + expect(resolved).toContain('8'); + // Every returned id must live in PCI_REQUIREMENTS. + for (const id of resolved) { + expect(PCI_REQUIREMENTS[id]).toBeDefined(); + } + }); + + it('silently drops unknown ids', () => { + expect(resolveRequirementIds(['99', 'drop'])).toEqual([]); + }); +}); + +describe('getIndexPattern / getIndexList', () => { + it('defaults to DEFAULT_PCI_INDEX_PATTERNS when no indices are provided', () => { + expect(getIndexList()).toEqual([...DEFAULT_PCI_INDEX_PATTERNS]); + expect(getIndexPattern()).toBe([...DEFAULT_PCI_INDEX_PATTERNS].join(',')); + }); + + it('honours explicit indices', () => { + const indices = ['logs-custom-a*', 'logs-custom-b*']; + expect(getIndexList(indices)).toEqual(indices); + expect(getIndexPattern(indices)).toBe(indices.join(',')); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_requirements.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_requirements.ts index 22b59eb3eb4a0..97a03c820bbba 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_requirements.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_requirements.ts @@ -11,6 +11,17 @@ export type ComplianceConfidence = 'HIGH' | 'MEDIUM' | 'LOW' | 'NOT_ASSESSABLE'; export type VerdictType = 'rows_mean_violation' | 'rows_mean_evidence'; +/** + * PCI compliance ES|QL queries are built as static templates that reference two named + * parameters, `?_tstart` and `?_tend`. The time-range values are **never** interpolated + * into the query string — they are bound by the Elasticsearch ES|QL engine via the + * `params` array on the request. This is the equivalent of SQL prepared statements and is + * the security boundary against ES|QL injection attempts in user-supplied time ranges. + * + * The index pattern in the `FROM` clause cannot be parameterised by ES|QL today, so the + * caller must ensure the value has been validated against {@link pciIndexPatternSchema} + * in `./pci_compliance_schemas.ts` before being passed to these builders. + */ export interface PciRequirementDefinition { id: string; name: string; @@ -20,21 +31,21 @@ export interface PciRequirementDefinition { verdict: VerdictType; defaultLookbackDays: number; recommendations: string[]; - buildViolationEsql?: (indexPattern: string, from: string, to: string) => string; - buildCoverageEsql: (indexPattern: string, from: string, to: string) => string; + buildViolationEsql?: (indexPattern: string) => string; + buildCoverageEsql: (indexPattern: string) => string; } export const DEFAULT_PCI_INDEX_PATTERNS = ['logs-*', 'metrics-*', 'endgame-*'] as const; -const escapeEsqlString = (s: string): string => s.replace(/"/g, '\\"'); +/** + * Shared WHERE fragment that constrains every PCI compliance query to the caller's time + * range. The `?_tstart` / `?_tend` placeholders are bound as ES|QL parameters — see + * {@link buildPciTimeRangeParams}. + */ +const TIME_WINDOW = '@timestamp >= ?_tstart AND @timestamp <= ?_tend'; -const coverageQuery = ( - indexPattern: string, - from: string, - to: string, - whereClause: string -): string => - `FROM ${indexPattern} | WHERE @timestamp >= "${escapeEsqlString(from)}" AND @timestamp <= "${escapeEsqlString(to)}" AND ${whereClause} | STATS matching_events = COUNT(*) | LIMIT 1`; +const coverageQuery = (indexPattern: string, whereClause: string): string => + `FROM ${indexPattern} | WHERE ${TIME_WINDOW} AND ${whereClause} | STATS matching_events = COUNT(*) | LIMIT 1`; export const PCI_REQUIREMENTS: Record = { // ── Top-level coverage checks (requirements 1–12) ────────────────────── @@ -52,8 +63,7 @@ export const PCI_REQUIREMENTS: Record = { 'Ensure network segmentation and firewall policy updates are continuously logged.', 'Review denied network traffic trends and tune policies for high-risk paths.', ], - buildCoverageEsql: (i, f, t) => - coverageQuery(i, f, t, 'event.category == "network"'), + buildCoverageEsql: (i) => coverageQuery(i, 'event.category == "network"'), }, '2': { id: '2', @@ -67,8 +77,8 @@ export const PCI_REQUIREMENTS: Record = { 'Centralize secure baseline changes and monitor drift across critical systems.', 'Track hardening exceptions with expiration and compensating controls.', ], - buildCoverageEsql: (i, f, t) => - coverageQuery(i, f, t, 'event.category == "configuration" OR event.action LIKE "*config*"'), + buildCoverageEsql: (i) => + coverageQuery(i, 'event.category == "configuration" OR event.action LIKE "*config*"'), }, '3': { id: '3', @@ -83,8 +93,8 @@ export const PCI_REQUIREMENTS: Record = { 'Monitor sensitive data access and verify retention/deletion controls are audited.', 'Supplement with manual evidence: data-flow diagrams, encryption key inventories, and PAN discovery scans.', ], - buildCoverageEsql: (i, f, t) => - coverageQuery(i, f, t, 'event.category == "database" OR event.action LIKE "*data*access*"'), + buildCoverageEsql: (i) => + coverageQuery(i, 'event.category == "database" OR event.action LIKE "*data*access*"'), }, '4': { id: '4', @@ -98,8 +108,8 @@ export const PCI_REQUIREMENTS: Record = { 'Ensure TLS metadata is ingested and monitor weak/legacy protocol usage.', 'Alert on unencrypted transport observed in cardholder-data paths.', ], - buildCoverageEsql: (i, f, t) => - coverageQuery(i, f, t, 'tls.version IS NOT NULL OR network.protocol IS NOT NULL'), + buildCoverageEsql: (i) => + coverageQuery(i, 'tls.version IS NOT NULL OR network.protocol IS NOT NULL'), }, '5': { id: '5', @@ -113,8 +123,8 @@ export const PCI_REQUIREMENTS: Record = { 'Confirm endpoint telemetry for malware prevention and detection is complete.', 'Investigate hosts repeatedly reporting malware indicators.', ], - buildCoverageEsql: (i, f, t) => - coverageQuery(i, f, t, 'event.category == "malware" OR event.module == "endpoint"'), + buildCoverageEsql: (i) => + coverageQuery(i, 'event.category == "malware" OR event.module == "endpoint"'), }, '6': { id: '6', @@ -128,8 +138,8 @@ export const PCI_REQUIREMENTS: Record = { 'Track remediation SLAs for critical vulnerabilities and overdue patches.', 'Correlate vulnerability findings with internet-facing assets.', ], - buildCoverageEsql: (i, f, t) => - coverageQuery(i, f, t, 'vulnerability.id IS NOT NULL OR event.action LIKE "*patch*"'), + buildCoverageEsql: (i) => + coverageQuery(i, 'vulnerability.id IS NOT NULL OR event.action LIKE "*patch*"'), }, '7': { id: '7', @@ -143,11 +153,9 @@ export const PCI_REQUIREMENTS: Record = { 'Review privilege grants and access exceptions for least-privilege alignment.', 'Monitor unusual privileged access across critical systems.', ], - buildCoverageEsql: (i, f, t) => + buildCoverageEsql: (i) => coverageQuery( i, - f, - t, 'event.category == "iam" OR event.action LIKE "*role*" OR event.action LIKE "*privilege*"' ), }, @@ -163,13 +171,8 @@ export const PCI_REQUIREMENTS: Record = { 'Ensure MFA-related events are ingested for interactive user authentication.', 'Investigate concentrated failed authentication activity by user or source.', ], - buildCoverageEsql: (i, f, t) => - coverageQuery( - i, - f, - t, - 'event.category == "authentication" OR event.action LIKE "*login*"' - ), + buildCoverageEsql: (i) => + coverageQuery(i, 'event.category == "authentication" OR event.action LIKE "*login*"'), }, '9': { id: '9', @@ -184,13 +187,8 @@ export const PCI_REQUIREMENTS: Record = { 'Integrate badge/physical access systems where available for end-to-end traceability.', 'Document manual evidence collection for physical controls not represented in logs.', ], - buildCoverageEsql: (i, f, t) => - coverageQuery( - i, - f, - t, - 'event.category == "physical_access" OR event.action LIKE "*badge*"' - ), + buildCoverageEsql: (i) => + coverageQuery(i, 'event.category == "physical_access" OR event.action LIKE "*badge*"'), }, '10': { id: '10', @@ -204,8 +202,7 @@ export const PCI_REQUIREMENTS: Record = { 'Validate audit logging across critical systems and identity providers.', 'Monitor ingestion gaps and logging outages as priority control failures.', ], - buildCoverageEsql: (i, f, t) => - coverageQuery(i, f, t, 'event.category IS NOT NULL'), + buildCoverageEsql: (i) => coverageQuery(i, 'event.category IS NOT NULL'), }, '11': { id: '11', @@ -219,13 +216,8 @@ export const PCI_REQUIREMENTS: Record = { 'Track recurring security test cadence and unresolved high-risk findings.', 'Correlate intrusion alerts with control test outcomes for validation.', ], - buildCoverageEsql: (i, f, t) => - coverageQuery( - i, - f, - t, - 'event.category == "intrusion_detection" OR vulnerability.id IS NOT NULL' - ), + buildCoverageEsql: (i) => + coverageQuery(i, 'event.category == "intrusion_detection" OR vulnerability.id IS NOT NULL'), }, '12': { id: '12', @@ -240,8 +232,8 @@ export const PCI_REQUIREMENTS: Record = { 'Maintain periodic policy review records and map owners to each PCI control area.', 'Supplement telemetry-based checks with documented procedural evidence for audits.', ], - buildCoverageEsql: (i, f, t) => - coverageQuery(i, f, t, 'event.action LIKE "*policy*" OR event.category == "configuration"'), + buildCoverageEsql: (i) => + coverageQuery(i, 'event.action LIKE "*policy*" OR event.category == "configuration"'), }, // ── Sub-requirement violation checks ─────────────────────────────────── @@ -259,15 +251,13 @@ export const PCI_REQUIREMENTS: Record = { 'Ensure all NSC changes are correlated with approved change tickets.', 'Flag changes made outside of approved change windows.', ], - buildCoverageEsql: (i, f, t) => + buildCoverageEsql: (i) => coverageQuery( i, - f, - t, 'event.category == "configuration" AND (event.action LIKE "*security_group*" OR event.action LIKE "*firewall*" OR event.action LIKE "*network_acl*" OR event.action LIKE "*rule*")' ), - buildViolationEsql: (i, f, t) => - `FROM ${i} | WHERE @timestamp >= "${f}" AND @timestamp <= "${t}" AND event.category == "configuration" AND (event.action LIKE "*security_group*" OR event.action LIKE "*firewall*" OR event.action LIKE "*network_acl*") | STATS change_count = COUNT(*) BY event.action, user.name | SORT change_count DESC | LIMIT 50`, + buildViolationEsql: (i) => + `FROM ${i} | WHERE ${TIME_WINDOW} AND event.category == "configuration" AND (event.action LIKE "*security_group*" OR event.action LIKE "*firewall*" OR event.action LIKE "*network_acl*") | STATS change_count = COUNT(*) BY event.action, user.name | SORT change_count DESC | LIMIT 50`, }, '4.2.1': { @@ -283,10 +273,13 @@ export const PCI_REQUIREMENTS: Record = { 'Disable TLS 1.0 and 1.1 on all systems processing cardholder data.', 'Upgrade to TLS 1.2 or 1.3 and restrict cipher suites to strong algorithms.', ], - buildViolationEsql: (i, f, t) => - `FROM ${i} | WHERE @timestamp >= "${f}" AND @timestamp <= "${t}" AND tls.version IS NOT NULL AND tls.version IN ("1.0", "1.1", "SSLv3", "SSLv2") | STATS connection_count = COUNT(*) BY destination.ip, tls.version | SORT connection_count DESC | LIMIT 50`, - buildCoverageEsql: (i, f, t) => - coverageQuery(i, f, t, 'tls.version IS NOT NULL'), + buildViolationEsql: (i) => + `FROM ${i} | WHERE ${TIME_WINDOW} AND (` + + `(tls.version IS NOT NULL AND tls.version IN ("1.0", "1.1", "SSLv3", "SSLv2")) OR ` + + `(network.protocol == "http" AND tls.version IS NULL)` + + `) | STATS connection_count = COUNT(*) BY destination.ip, tls.version | SORT connection_count DESC | LIMIT 50`, + buildCoverageEsql: (i) => + coverageQuery(i, 'tls.version IS NOT NULL OR network.protocol IS NOT NULL'), }, '5.2.1': { @@ -302,11 +295,9 @@ export const PCI_REQUIREMENTS: Record = { 'Ensure all in-scope endpoints report anti-malware telemetry.', 'Investigate hosts missing malware prevention event coverage.', ], - buildCoverageEsql: (i, f, t) => + buildCoverageEsql: (i) => coverageQuery( i, - f, - t, 'event.category == "malware" OR event.module == "endpoint" OR event.action LIKE "*malware*" OR event.action LIKE "*virus*"' ), }, @@ -324,10 +315,9 @@ export const PCI_REQUIREMENTS: Record = { 'Prioritize remediation of critical-severity vulnerabilities within 30 days.', 'Establish compensating controls for vulnerabilities that cannot be patched within the SLA.', ], - buildViolationEsql: (i, f, t) => - `FROM ${i} | WHERE @timestamp >= "${f}" AND @timestamp <= "${t}" AND vulnerability.id IS NOT NULL AND vulnerability.severity == "critical" | STATS vuln_count = COUNT(*) BY vulnerability.id, host.name | SORT vuln_count DESC | LIMIT 50`, - buildCoverageEsql: (i, f, t) => - coverageQuery(i, f, t, 'vulnerability.id IS NOT NULL'), + buildViolationEsql: (i) => + `FROM ${i} | WHERE ${TIME_WINDOW} AND vulnerability.id IS NOT NULL AND vulnerability.severity == "critical" | STATS vuln_count = COUNT(*) BY vulnerability.id, host.name | SORT vuln_count DESC | LIMIT 50`, + buildCoverageEsql: (i) => coverageQuery(i, 'vulnerability.id IS NOT NULL'), }, '7.2.2': { @@ -343,15 +333,13 @@ export const PCI_REQUIREMENTS: Record = { 'Review privilege grants and ensure they align with least-privilege principles.', 'Alert on role assignments to highly privileged groups outside of change windows.', ], - buildCoverageEsql: (i, f, t) => + buildCoverageEsql: (i) => coverageQuery( i, - f, - t, 'event.category == "iam" AND (event.action LIKE "*role*" OR event.action LIKE "*group*" OR event.action LIKE "*privilege*" OR event.action LIKE "*permission*")' ), - buildViolationEsql: (i, f, t) => - `FROM ${i} | WHERE @timestamp >= "${f}" AND @timestamp <= "${t}" AND event.category == "iam" AND (event.action LIKE "*role*assign*" OR event.action LIKE "*group*add*" OR event.action LIKE "*privilege*grant*") | STATS change_count = COUNT(*) BY user.name, event.action | SORT change_count DESC | LIMIT 50`, + buildViolationEsql: (i) => + `FROM ${i} | WHERE ${TIME_WINDOW} AND event.category == "iam" AND (event.action LIKE "*role*assign*" OR event.action LIKE "*group*add*" OR event.action LIKE "*privilege*grant*") | STATS change_count = COUNT(*) BY user.name, event.action | SORT change_count DESC | LIMIT 50`, }, '8.2.4': { @@ -367,15 +355,10 @@ export const PCI_REQUIREMENTS: Record = { 'Disable or remove accounts with no successful authentication in 90+ days.', 'Implement automated account lifecycle management with periodic reviews.', ], - buildViolationEsql: (i, f, t) => - `FROM ${i} | WHERE @timestamp >= "${f}" AND @timestamp <= "${t}" AND event.category == "authentication" AND event.outcome == "success" | STATS last_login = MAX(@timestamp) BY user.name | EVAL days_inactive = DATE_DIFF("day", last_login, NOW()) | WHERE days_inactive > 90 | SORT days_inactive DESC | LIMIT 50`, - buildCoverageEsql: (i, f, t) => - coverageQuery( - i, - f, - t, - 'event.category == "authentication" AND event.outcome == "success"' - ), + buildViolationEsql: (i) => + `FROM ${i} | WHERE ${TIME_WINDOW} AND event.category == "authentication" AND event.outcome == "success" | STATS last_login = MAX(@timestamp) BY user.name | EVAL days_inactive = DATE_DIFF("day", last_login, NOW()) | WHERE days_inactive > 90 | SORT days_inactive DESC | LIMIT 50`, + buildCoverageEsql: (i) => + coverageQuery(i, 'event.category == "authentication" AND event.outcome == "success"'), }, '8.3.4': { @@ -391,15 +374,10 @@ export const PCI_REQUIREMENTS: Record = { 'Configure account lockout after no more than 10 invalid login attempts per PCI DSS v4.0.1.', 'Ensure lockout duration is at least 30 minutes or requires administrator unlock with identity verification.', ], - buildViolationEsql: (i, f, t) => - `FROM ${i} | WHERE @timestamp >= "${f}" AND @timestamp <= "${t}" AND event.category == "authentication" AND event.outcome == "failure" | STATS failed_attempts = COUNT(*) BY user.name, source.ip | WHERE failed_attempts > 10 | SORT failed_attempts DESC | LIMIT 50`, - buildCoverageEsql: (i, f, t) => - coverageQuery( - i, - f, - t, - 'event.category == "authentication" AND event.outcome == "failure"' - ), + buildViolationEsql: (i) => + `FROM ${i} | WHERE ${TIME_WINDOW} AND event.category == "authentication" AND event.outcome == "failure" | STATS failed_attempts = COUNT(*) BY user.name, source.ip | WHERE failed_attempts > 10 | SORT failed_attempts DESC | LIMIT 50`, + buildCoverageEsql: (i) => + coverageQuery(i, 'event.category == "authentication" AND event.outcome == "failure"'), }, '8.4.2': { @@ -416,11 +394,9 @@ export const PCI_REQUIREMENTS: Record = { 'Consider phishing-resistant factors (FIDO2/WebAuthn) which satisfy MFA requirements per v4.0.1.', 'Ensure MFA-related events (challenge, verify, enroll) are ingested into SIEM.', ], - buildCoverageEsql: (i, f, t) => + buildCoverageEsql: (i) => coverageQuery( i, - f, - t, 'event.category == "authentication" AND (event.action LIKE "*mfa*" OR event.action LIKE "*multi_factor*" OR event.action LIKE "*2fa*" OR event.action LIKE "*totp*" OR event.action LIKE "*fido*" OR event.action LIKE "*webauthn*" OR event.action LIKE "*verify*factor*")' ), }, @@ -438,10 +414,9 @@ export const PCI_REQUIREMENTS: Record = { 'Investigate any audit log stop, pause, or deletion events immediately.', 'Implement write-once log storage to prevent tampering.', ], - buildViolationEsql: (i, f, t) => - `FROM ${i} | WHERE @timestamp >= "${f}" AND @timestamp <= "${t}" AND (event.action LIKE "*audit*stop*" OR event.action LIKE "*audit*delete*" OR event.action LIKE "*audit*pause*" OR event.action LIKE "*log*clear*" OR event.action LIKE "*log*delete*" OR event.action LIKE "*trail*stop*") | STATS event_count = COUNT(*) BY event.action, host.name, user.name | SORT event_count DESC | LIMIT 50`, - buildCoverageEsql: (i, f, t) => - coverageQuery(i, f, t, 'event.category IS NOT NULL'), + buildViolationEsql: (i) => + `FROM ${i} | WHERE ${TIME_WINDOW} AND (event.action LIKE "*audit*stop*" OR event.action LIKE "*audit*delete*" OR event.action LIKE "*audit*pause*" OR event.action LIKE "*log*clear*" OR event.action LIKE "*log*delete*" OR event.action LIKE "*trail*stop*") | STATS event_count = COUNT(*) BY event.action, host.name, user.name | SORT event_count DESC | LIMIT 50`, + buildCoverageEsql: (i) => coverageQuery(i, 'event.category IS NOT NULL'), }, '10.2.2': { @@ -457,11 +432,9 @@ export const PCI_REQUIREMENTS: Record = { 'Ensure all administrative actions including config changes, user management, and system modifications are logged.', 'Correlate admin actions with change management records.', ], - buildCoverageEsql: (i, f, t) => + buildCoverageEsql: (i) => coverageQuery( i, - f, - t, 'event.category == "configuration" OR event.category == "iam" OR event.action LIKE "*admin*" OR event.action LIKE "*sudo*" OR event.action LIKE "*root*"' ), }, @@ -479,8 +452,14 @@ export const PCI_REQUIREMENTS: Record = { 'Ensure log retention policies maintain at least 12 months of audit logs.', 'Verify that the most recent 3 months of logs are immediately available for analysis.', ], - buildCoverageEsql: (i, _f, _t) => - `FROM ${i} | STATS oldest_log = MIN(@timestamp), newest_log = MAX(@timestamp), total_events = COUNT(*) | EVAL retention_days = DATE_DIFF("day", oldest_log, newest_log) | LIMIT 1`, + // Retention intentionally spans the full index rather than the caller-supplied window, + // so this query has no WHERE clause on @timestamp — there is no user-supplied time + // value to bind. `total_events` is projected first so the evaluator's generic + // `values[0][0]` count-based scoring path treats "any events exist" as evidence; + // `oldest_log`, `newest_log`, and `retention_days` remain available as context for + // reviewers inspecting raw evidence. + buildCoverageEsql: (i) => + `FROM ${i} | STATS total_events = COUNT(*), oldest_log = MIN(@timestamp), newest_log = MAX(@timestamp) | EVAL retention_days = DATE_DIFF("day", oldest_log, newest_log)`, }, '11.5': { @@ -496,10 +475,9 @@ export const PCI_REQUIREMENTS: Record = { 'Investigate and resolve active IDS/IPS alerts promptly.', 'Tune detection rules to reduce false positives while maintaining coverage.', ], - buildViolationEsql: (i, f, t) => - `FROM ${i} | WHERE @timestamp >= "${f}" AND @timestamp <= "${t}" AND event.category == "intrusion_detection" AND event.kind == "alert" | STATS alert_count = COUNT(*) BY host.name, event.action | SORT alert_count DESC | LIMIT 50`, - buildCoverageEsql: (i, f, t) => - coverageQuery(i, f, t, 'event.category == "intrusion_detection"'), + buildViolationEsql: (i) => + `FROM ${i} | WHERE ${TIME_WINDOW} AND event.category == "intrusion_detection" AND event.kind == "alert" | STATS alert_count = COUNT(*) BY host.name, event.action | SORT alert_count DESC | LIMIT 50`, + buildCoverageEsql: (i) => coverageQuery(i, 'event.category == "intrusion_detection"'), }, '2.2.4': { @@ -515,15 +493,10 @@ export const PCI_REQUIREMENTS: Record = { 'Remove or disable all default and vendor-supplied accounts before deploying systems.', 'If a default account cannot be removed, change the password and restrict access.', ], - buildViolationEsql: (i, f, t) => - `FROM ${i} | WHERE @timestamp >= "${f}" AND @timestamp <= "${t}" AND event.category == "authentication" AND event.outcome == "success" AND (user.name == "admin" OR user.name == "administrator" OR user.name == "root" OR user.name == "guest" OR user.name == "default" OR user.name == "test" OR user.name == "sa" OR user.name == "postgres" OR user.name == "oracle") | STATS login_count = COUNT(*) BY user.name, source.ip | SORT login_count DESC | LIMIT 50`, - buildCoverageEsql: (i, f, t) => - coverageQuery( - i, - f, - t, - 'event.category == "authentication" AND event.outcome == "success"' - ), + buildViolationEsql: (i) => + `FROM ${i} | WHERE ${TIME_WINDOW} AND event.category == "authentication" AND event.outcome == "success" AND (user.name == "admin" OR user.name == "administrator" OR user.name == "root" OR user.name == "guest" OR user.name == "default" OR user.name == "test" OR user.name == "sa" OR user.name == "postgres" OR user.name == "oracle") | STATS login_count = COUNT(*) BY user.name, source.ip | SORT login_count DESC | LIMIT 50`, + buildCoverageEsql: (i) => + coverageQuery(i, 'event.category == "authentication" AND event.outcome == "success"'), }, '8.3.6': { @@ -540,11 +513,9 @@ export const PCI_REQUIREMENTS: Record = { 'If systems cannot support 12 characters, document compensating controls and enforce 8-character minimum.', 'Ensure password policy change events are ingested into SIEM for auditability.', ], - buildCoverageEsql: (i, f, t) => + buildCoverageEsql: (i) => coverageQuery( i, - f, - t, 'event.category == "iam" AND (event.action LIKE "*password*policy*" OR event.action LIKE "*password*change*" OR event.action LIKE "*password*reset*" OR event.action LIKE "*credential*")' ), }, @@ -563,11 +534,9 @@ export const PCI_REQUIREMENTS: Record = { 'PCI DSS v4.0.1 eliminated the password-only option: MFA is the preferred path.', 'Ensure password change and MFA enrollment events are ingested for audit evidence.', ], - buildCoverageEsql: (i, f, t) => + buildCoverageEsql: (i) => coverageQuery( i, - f, - t, 'event.category == "iam" AND (event.action LIKE "*password*change*" OR event.action LIKE "*password*reset*" OR event.action LIKE "*mfa*enroll*" OR event.action LIKE "*mfa*register*" OR event.action LIKE "*2fa*" OR event.action LIKE "*totp*")' ), }, @@ -578,23 +547,17 @@ export const PCI_REQUIREMENTS: Record = { description: 'Verify that audit log entries contain required detail: user ID, event type, date/time, success or failure indication, origination, and identity or name of affected data/resource. Check field fill rates for completeness.', pciReference: 'PCI DSS v4.0.1 Section 10.3', - requiredFields: [ - 'user.name', - 'event.category', - 'event.action', - 'event.outcome', - '@timestamp', - ], + requiredFields: ['user.name', 'event.category', 'event.action', 'event.outcome', '@timestamp'], verdict: 'rows_mean_evidence', defaultLookbackDays: 7, recommendations: [ 'Ensure all audit log entries include user ID, event type, timestamp, outcome, and source.', 'Investigate data sources with low field fill rates for required audit trail fields.', ], - buildViolationEsql: (i, f, t) => - `FROM ${i} | WHERE @timestamp >= "${f}" AND @timestamp <= "${t}" | STATS total = COUNT(*), has_user = COUNT(user.name), has_action = COUNT(event.action), has_outcome = COUNT(event.outcome) | EVAL user_pct = ROUND((has_user * 100.0) / total), action_pct = ROUND((has_action * 100.0) / total), outcome_pct = ROUND((has_outcome * 100.0) / total) | LIMIT 1`, - buildCoverageEsql: (i, f, t) => - coverageQuery(i, f, t, 'event.category IS NOT NULL AND user.name IS NOT NULL'), + buildViolationEsql: (i) => + `FROM ${i} | WHERE ${TIME_WINDOW} | STATS total = COUNT(*), has_user = COUNT(user.name), has_action = COUNT(event.action), has_outcome = COUNT(event.outcome) | EVAL user_pct = ROUND((has_user * 100.0) / total), action_pct = ROUND((has_action * 100.0) / total), outcome_pct = ROUND((has_outcome * 100.0) / total) | LIMIT 1`, + buildCoverageEsql: (i) => + coverageQuery(i, 'event.category IS NOT NULL AND user.name IS NOT NULL'), }, '11.6': { @@ -611,18 +574,29 @@ export const PCI_REQUIREMENTS: Record = { 'Deploy change-detection mechanisms that alert on unauthorized script or header modifications.', 'This requirement became mandatory March 31, 2025 per PCI DSS v4.0.1.', ], - buildViolationEsql: (i, f, t) => - `FROM ${i} | WHERE @timestamp >= "${f}" AND @timestamp <= "${t}" AND (event.action LIKE "*tamper*" OR event.action LIKE "*integrity*violation*" OR event.action LIKE "*csp*violation*" OR event.action LIKE "*script*inject*" OR event.action LIKE "*page*change*" OR event.action LIKE "*skimmer*") | STATS alert_count = COUNT(*) BY url.domain, event.action | SORT alert_count DESC | LIMIT 50`, - buildCoverageEsql: (i, f, t) => + buildViolationEsql: (i) => + `FROM ${i} | WHERE ${TIME_WINDOW} AND (event.action LIKE "*tamper*" OR event.action LIKE "*integrity*violation*" OR event.action LIKE "*csp*violation*" OR event.action LIKE "*script*inject*" OR event.action LIKE "*page*change*" OR event.action LIKE "*skimmer*") | STATS alert_count = COUNT(*) BY url.domain, event.action | SORT alert_count DESC | LIMIT 50`, + buildCoverageEsql: (i) => coverageQuery( i, - f, - t, 'event.action LIKE "*csp*" OR event.action LIKE "*integrity*" OR event.action LIKE "*tamper*" OR event.action LIKE "*payment*page*"' ), }, }; +/** + * Build the ES|QL `params` array for the shared `?_tstart` / `?_tend` placeholders. Pass + * the result straight to {@link executeEsql} so time-range values flow through + * parameter binding instead of string interpolation. + */ +export const buildPciTimeRangeParams = ({ + from, + to, +}: { + from: string; + to: string; +}): Array> => [{ _tstart: from }, { _tend: to }]; + export const getTimeRangeForCheck = ( checkId: string, userTimeRange?: { from: string; to: string } @@ -677,3 +651,6 @@ export const getIndexPattern = (indices?: string[]): string => { const selected = indices && indices.length > 0 ? indices : [...DEFAULT_PCI_INDEX_PATTERNS]; return selected.join(','); }; + +export const getIndexList = (indices?: string[]): string[] => + indices && indices.length > 0 ? [...indices] : [...DEFAULT_PCI_INDEX_PATTERNS]; diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_schemas.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_schemas.test.ts new file mode 100644 index 0000000000000..da6fde5d20ac1 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_schemas.test.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + PCI_AUTOMATED_ASSESSMENT_DISCLAIMER, + PCI_DSS_VERSION, + buildScopeClaim, + pciIndexPatternSchema, + pciRequirementIdSchema, + pciTimeRangeSchema, +} from './pci_compliance_schemas'; + +describe('pciIndexPatternSchema', () => { + it.each(['logs-*', 'metrics-*', 'cluster-a:logs-pci-*', 'logs-custom.myapp-2024.01'])( + 'accepts %j', + (value) => { + expect(pciIndexPatternSchema.safeParse(value).success).toBe(true); + } + ); + + it.each([ + '', + ' logs-*', + 'logs-*"; DROP INDEX', + 'logs-*\n| FROM malicious', + 'logs-*\t| LIMIT 1', + 'logs-* | LIMIT 1', + '../escape', + 'logs-*#frag', + 'bad\u0000index', + 'logs-*,metrics-*', + // 256 chars -> exceeds max + 'a'.repeat(256), + ])('rejects %j', (value) => { + expect(pciIndexPatternSchema.safeParse(value).success).toBe(false); + }); +}); + +describe('pciTimeRangeSchema', () => { + it('accepts a valid ISO-8601 range with offset', () => { + expect( + pciTimeRangeSchema.safeParse({ + from: '2024-01-01T00:00:00Z', + to: '2024-02-01T00:00:00+02:00', + }).success + ).toBe(true); + }); + + it('accepts from === to', () => { + expect( + pciTimeRangeSchema.safeParse({ + from: '2024-01-01T00:00:00Z', + to: '2024-01-01T00:00:00Z', + }).success + ).toBe(true); + }); + + it('rejects a non-ISO string', () => { + expect(pciTimeRangeSchema.safeParse({ from: 'yesterday', to: 'now' }).success).toBe(false); + }); + + it('rejects when `from` is after `to`', () => { + expect( + pciTimeRangeSchema.safeParse({ + from: '2024-02-01T00:00:00Z', + to: '2024-01-01T00:00:00Z', + }).success + ).toBe(false); + }); +}); + +describe('pciRequirementIdSchema', () => { + it('accepts "all"', () => { + expect(pciRequirementIdSchema.safeParse('all').success).toBe(true); + }); + + it('accepts a top-level requirement id', () => { + expect(pciRequirementIdSchema.safeParse('8').success).toBe(true); + }); + + it('rejects made-up ids', () => { + expect(pciRequirementIdSchema.safeParse('99').success).toBe(false); + expect(pciRequirementIdSchema.safeParse('drop').success).toBe(false); + }); +}); + +describe('buildScopeClaim', () => { + it('pins the DSS version and attaches the QSA disclaimer', () => { + const claim = buildScopeClaim({ + indices: ['logs-*'], + from: '2024-01-01T00:00:00Z', + to: '2024-01-08T00:00:00Z', + requirementsEvaluated: ['8', '8'], + requiredFieldsChecked: ['user.name', 'event.outcome', 'user.name'], + }); + expect(claim.pciDssVersion).toBe(PCI_DSS_VERSION); + expect(claim.pciDssVersion).toBe('4.0.1'); + expect(claim.disclaimer).toBe(PCI_AUTOMATED_ASSESSMENT_DISCLAIMER); + }); + + it('deduplicates and sorts requirementsEvaluated + requiredFieldsChecked', () => { + const claim = buildScopeClaim({ + indices: ['logs-*'], + from: '2024-01-01T00:00:00Z', + to: '2024-01-02T00:00:00Z', + requirementsEvaluated: ['10', '8', '8', '1'], + requiredFieldsChecked: ['user.name', 'event.outcome', 'user.name', '@timestamp'], + }); + expect(claim.requirementsEvaluated).toEqual(['1', '10', '8']); + expect(claim.requiredFieldsChecked).toEqual(['@timestamp', 'event.outcome', 'user.name']); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_schemas.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_schemas.ts new file mode 100644 index 0000000000000..769c58972bdb0 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_schemas.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { PCI_REQUIREMENTS } from './pci_compliance_requirements'; + +/** + * Allowed index-pattern shape. Elasticsearch index names disallow the characters + * \ / ? " < > | , #, and we additionally disallow whitespace and any control character + * to prevent attempts to inject a second FROM clause. Wildcards (*) are allowed because + * data streams such as `logs-*` rely on them. Cross-cluster patterns using `:` are allowed. + * + * Because the FROM clause of an ES|QL query cannot be parameterised, this regex is the + * security boundary for any value that ultimately flows into `FROM ` — treat it + * with the same care as a SQL prepared-statement table whitelist. + */ +const INDEX_PATTERN_REGEX = /^[a-z0-9][a-z0-9._+\-*:]{0,254}$/i; + +export const pciIndexPatternSchema = z + .string() + .min(1) + .max(255) + .regex( + INDEX_PATTERN_REGEX, + 'Index pattern may only contain letters, digits, and the characters . _ + - * : and must start with a letter or digit.' + ); + +/** + * Strict ISO-8601 time-range validation. ES|QL ingests these via parameter binding + * (?_tstart / ?_tend), but tightening here still gives the LLM clear, early feedback + * when it fabricates a time range. + */ +export const pciTimeRangeSchema = z + .object({ + from: z.string().datetime({ offset: true }), + to: z.string().datetime({ offset: true }), + }) + .refine((value) => new Date(value.from) <= new Date(value.to), { + message: '`from` must be earlier than or equal to `to`.', + }); + +/** + * Closed union of PCI DSS requirement identifiers — derived from {@link PCI_REQUIREMENTS} + * so we cannot drift from the implementation. `"all"` selects every requirement. + */ +const requirementLiterals = [ + 'all', + ...(Object.keys(PCI_REQUIREMENTS) as Array), +] as const; + +export const pciRequirementIdSchema = z.enum( + requirementLiterals as unknown as readonly [string, ...string[]] +); + +export type PciRequirementIdInput = z.infer; + +/** + * Reusable scope-claim shape attached to every PCI tool response. Agents must cite this + * back in natural-language answers so downstream users can verify what was actually + * evaluated and where the QSA still needs to act. + */ +export const PCI_DSS_VERSION = '4.0.1' as const; + +export const PCI_AUTOMATED_ASSESSMENT_DISCLAIMER = + 'Automated PCI DSS telemetry assessment only. Formal compliance determination requires a ' + + 'Qualified Security Assessor (QSA) and process-level verification. Use these findings as ' + + 'input to, not a replacement for, a scoped PCI DSS audit.'; + +export interface PciScopeClaim { + pciDssVersion: typeof PCI_DSS_VERSION; + indices: string[]; + timeRange: { from: string; to: string }; + requirementsEvaluated: string[]; + requiredFieldsChecked: string[]; + disclaimer: typeof PCI_AUTOMATED_ASSESSMENT_DISCLAIMER; +} + +export interface BuildScopeClaimArgs { + indices: string[]; + from: string; + to: string; + requirementsEvaluated: string[]; + requiredFieldsChecked: string[]; +} + +export const buildScopeClaim = ({ + indices, + from, + to, + requirementsEvaluated, + requiredFieldsChecked, +}: BuildScopeClaimArgs): PciScopeClaim => ({ + pciDssVersion: PCI_DSS_VERSION, + indices, + timeRange: { from, to }, + requirementsEvaluated: [...new Set(requirementsEvaluated)].sort(), + requiredFieldsChecked: [...new Set(requiredFieldsChecked)].sort(), + disclaimer: PCI_AUTOMATED_ASSESSMENT_DISCLAIMER, +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_tool.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_tool.test.ts new file mode 100644 index 0000000000000..7c757f12de84e --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_tool.test.ts @@ -0,0 +1,220 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +jest.mock('@kbn/agent-builder-genai-utils', () => ({ + executeEsql: jest.fn(), +})); + +import { ToolResultType } from '@kbn/agent-builder-common'; +import type { ToolHandlerStandardReturn } from '@kbn/agent-builder-server/tools'; +import { executeEsql } from '@kbn/agent-builder-genai-utils'; +import { createToolHandlerContext, createToolTestMocks } from '../__mocks__/test_helpers'; +import { PCI_COMPLIANCE_TOOL_ID, pciComplianceTool } from './pci_compliance_tool'; + +const mockExecuteEsql = executeEsql as jest.MockedFunction; + +describe('pciComplianceTool (consolidated)', () => { + const { mockCore, mockLogger, mockEsClient, mockRequest } = createToolTestMocks(); + const tool = pciComplianceTool(mockCore, mockLogger); + + beforeEach(() => { + jest.clearAllMocks(); + // Default: coverage returns 1 matching event so we exit layer 2 cleanly. + mockExecuteEsql.mockResolvedValue({ + columns: [{ name: 'matching_events', type: 'long' }], + values: [[1]], + } as never); + mockEsClient.asCurrentUser.fieldCaps.mockResolvedValue({ + fields: { '@timestamp': { date: { type: 'date' } } }, + } as never); + }); + + describe('properties', () => { + it('returns the consolidated tool id', () => { + expect(tool.id).toBe(PCI_COMPLIANCE_TOOL_ID); + }); + }); + + describe('schema', () => { + it('rejects missing mode', () => { + expect(tool.schema.safeParse({}).success).toBe(false); + }); + + it('rejects unknown mode', () => { + expect(tool.schema.safeParse({ mode: 'invalid' }).success).toBe(false); + }); + + it('rejects indices that look like injection attempts', () => { + const result = tool.schema.safeParse({ + mode: 'check', + indices: ['logs-*"; DROP'], + }); + expect(result.success).toBe(false); + }); + + it('rejects an inverted timeRange', () => { + const result = tool.schema.safeParse({ + mode: 'check', + timeRange: { from: '2024-03-01T00:00:00Z', to: '2024-01-01T00:00:00Z' }, + }); + expect(result.success).toBe(false); + }); + + it('rejects unknown requirement ids', () => { + const result = tool.schema.safeParse({ + mode: 'check', + requirements: ['99'], + }); + expect(result.success).toBe(false); + }); + + it('accepts a well-formed check request', () => { + const result = tool.schema.safeParse({ + mode: 'check', + requirements: ['8'], + indices: ['logs-*'], + timeRange: { from: '2024-01-01T00:00:00Z', to: '2024-01-08T00:00:00Z' }, + }); + expect(result.success).toBe(true); + }); + + it('accepts a well-formed report request with `all`', () => { + const result = tool.schema.safeParse({ + mode: 'report', + requirements: ['all'], + }); + expect(result.success).toBe(true); + }); + }); + + describe('ES|QL injection protection', () => { + it('binds ?_tstart / ?_tend from timeRange without interpolating', async () => { + await tool.handler( + { + mode: 'check', + requirements: ['1'], + indices: ['logs-*'], + timeRange: { from: '2024-01-01T00:00:00Z', to: '2024-01-08T00:00:00Z' }, + includeEvidence: false, + format: 'summary', + includeRecommendations: true, + }, + createToolHandlerContext(mockRequest, mockEsClient, mockLogger) + ); + + expect(mockExecuteEsql).toHaveBeenCalled(); + const call = mockExecuteEsql.mock.calls[0][0]; + expect(call.query).toContain('?_tstart'); + expect(call.query).toContain('?_tend'); + expect(call.query).not.toContain('2024-01-01T00:00:00Z'); + expect(call.params).toEqual([ + { _tstart: '2024-01-01T00:00:00Z' }, + { _tend: '2024-01-08T00:00:00Z' }, + ]); + }); + + it('uses the validated index pattern verbatim in FROM', async () => { + await tool.handler( + { + mode: 'check', + requirements: ['1'], + indices: ['cluster-a:logs-pci-*'], + timeRange: { from: '2024-01-01T00:00:00Z', to: '2024-01-08T00:00:00Z' }, + includeEvidence: false, + format: 'summary', + includeRecommendations: true, + }, + createToolHandlerContext(mockRequest, mockEsClient, mockLogger) + ); + + const call = mockExecuteEsql.mock.calls[0][0]; + expect(call.query).toContain('FROM cluster-a:logs-pci-*'); + }); + }); + + describe('scope claim', () => { + it('attaches a scopeClaim describing what was evaluated in check mode', async () => { + const result = (await tool.handler( + { + mode: 'check', + requirements: ['1'], + indices: ['logs-custom-a*'], + timeRange: { from: '2024-01-01T00:00:00Z', to: '2024-01-08T00:00:00Z' }, + includeEvidence: false, + format: 'summary', + includeRecommendations: true, + }, + createToolHandlerContext(mockRequest, mockEsClient, mockLogger) + )) as ToolHandlerStandardReturn; + + const summary = result.results.find((r) => r.type === ToolResultType.other); + const data = summary?.data as { + scopeClaim: { + pciDssVersion: string; + indices: string[]; + requirementsEvaluated: string[]; + timeRange: { from: string; to: string }; + disclaimer: string; + }; + }; + expect(data.scopeClaim.pciDssVersion).toBe('4.0.1'); + expect(data.scopeClaim.indices).toEqual(['logs-custom-a*']); + expect(data.scopeClaim.requirementsEvaluated).toContain('1'); + expect(data.scopeClaim.timeRange).toEqual({ + from: '2024-01-01T00:00:00Z', + to: '2024-01-08T00:00:00Z', + }); + expect(data.scopeClaim.disclaimer).toContain('Qualified Security Assessor'); + }); + + it('attaches a scopeClaim in report mode too', async () => { + const result = (await tool.handler( + { + mode: 'report', + requirements: ['1'], + indices: ['logs-custom-a*'], + timeRange: { from: '2024-01-01T00:00:00Z', to: '2024-01-08T00:00:00Z' }, + format: 'summary', + includeRecommendations: true, + includeEvidence: false, + }, + createToolHandlerContext(mockRequest, mockEsClient, mockLogger) + )) as ToolHandlerStandardReturn; + + const summary = result.results.find((r) => r.type === ToolResultType.other); + const data = summary?.data as { + mode: string; + scopeClaim: { pciDssVersion: string; disclaimer: string }; + }; + expect(data.mode).toBe('report'); + expect(data.scopeClaim.pciDssVersion).toBe('4.0.1'); + expect(data.scopeClaim.disclaimer).toContain('Qualified Security Assessor'); + }); + }); + + describe('response shape', () => { + it('returns an ESQL scorecard followed by a summary in report mode', async () => { + const result = (await tool.handler( + { + mode: 'report', + requirements: ['1'], + indices: ['logs-*'], + timeRange: { from: '2024-01-01T00:00:00Z', to: '2024-01-08T00:00:00Z' }, + format: 'summary', + includeRecommendations: true, + includeEvidence: false, + }, + createToolHandlerContext(mockRequest, mockEsClient, mockLogger) + )) as ToolHandlerStandardReturn; + + const types = result.results.map((r) => r.type); + expect(types).toContain(ToolResultType.query); + expect(types).toContain(ToolResultType.esqlResults); + expect(types).toContain(ToolResultType.other); + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_tool.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_tool.ts new file mode 100644 index 0000000000000..1f430aacf2965 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_compliance_tool.ts @@ -0,0 +1,450 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { ToolType, ToolResultType } from '@kbn/agent-builder-common'; +import type { BuiltinToolDefinition } from '@kbn/agent-builder-server'; +import { getToolResultId } from '@kbn/agent-builder-server/tools'; +import type { Logger } from '@kbn/logging'; +import type { SecuritySolutionPluginCoreSetupDependencies } from '../../plugin_contract'; +import { getAgentBuilderResourceAvailability } from '../utils/get_agent_builder_resource_availability'; +import { securityTool } from './constants'; +import { + type ComplianceStatus, + type ComplianceConfidence, + getIndexList, + getIndexPattern, + getTimeRangeForCheck, + normalizeRequirementId, + resolveRequirementIds, + PCI_REQUIREMENTS, +} from './pci_compliance_requirements'; +import { + pciIndexPatternSchema, + pciRequirementIdSchema, + pciTimeRangeSchema, + buildScopeClaim, +} from './pci_compliance_schemas'; +import { + type EvaluatedRequirement, + evaluateRequirement, + runWithConcurrency, + PCI_REQUIREMENT_CONCURRENCY, +} from './pci_compliance_evaluator'; + +const REPORT_FORMATS = ['summary', 'detailed', 'executive'] as const; + +const pciComplianceSchema = z + .object({ + mode: z + .enum(['check', 'report']) + .describe( + '`check` runs violation + coverage + preflight for one or more requirements and returns findings with evidence. ' + + '`report` produces a roll-up scorecard (RED/AMBER/GREEN + confidence + score) across the requested set. ' + + 'Use `check` when you need actionable findings; use `report` when you need a posture snapshot.' + ), + requirements: z + .array(pciRequirementIdSchema) + .min(1) + .optional() + .describe( + 'Requirement identifiers to evaluate. Accepts "all", top-level ("1".."12"), or sub-requirements ("8.3.4"). ' + + 'Defaults to ["all"].' + ), + timeRange: pciTimeRangeSchema + .optional() + .describe( + 'Optional ISO-8601 time range (`from` must be <= `to`). If omitted, each requirement ' + + 'uses its recommended lookback window (e.g. 7 days for brute-force, 365 days for stale accounts).' + ), + indices: z + .array(pciIndexPatternSchema) + .min(1) + .optional() + .describe( + 'Index patterns to query. Specify exact patterns to avoid overlap/double-counting during ' + + 're-indexing. Defaults to logs-*, metrics-*, endgame-*.' + ), + includeEvidence: z + .boolean() + .optional() + .default(true) + .describe('[check mode] Include tabular ES|QL evidence in findings. Ignored in report mode.'), + format: z + .enum(REPORT_FORMATS) + .optional() + .default('summary') + .describe('[report mode] Report depth: summary, detailed, or executive.'), + includeRecommendations: z + .boolean() + .optional() + .default(true) + .describe('[report mode] Include recommendation text on each requirement row.'), + }) + .describe( + 'Run PCI DSS v4.0.1 compliance checks (`mode: check`) or generate a compliance report (`mode: report`). ' + + 'One tool replaces the prior pci_compliance_check / pci_compliance_report pair.' + ); + +export const PCI_COMPLIANCE_TOOL_ID = securityTool('pci_compliance'); + +const scoreToStatus = (score: number): ComplianceStatus => { + if (score >= 85) return 'GREEN'; + if (score >= 60) return 'AMBER'; + return 'RED'; +}; + +const rollupConfidence = (rows: EvaluatedRequirement[]): ComplianceConfidence => { + if (rows.length === 0) return 'NOT_ASSESSABLE'; + const counts = rows.reduce((acc, r) => { + acc[r.confidence] = (acc[r.confidence] ?? 0) + 1; + return acc; + }, {} as Record); + if ((counts.NOT_ASSESSABLE ?? 0) > rows.length / 2) return 'NOT_ASSESSABLE'; + if ((counts.LOW ?? 0) + (counts.NOT_ASSESSABLE ?? 0) > rows.length / 2) return 'LOW'; + if ((counts.HIGH ?? 0) >= rows.length / 2) return 'HIGH'; + return 'MEDIUM'; +}; + +const rollupOverallStatus = (rows: EvaluatedRequirement[]): ComplianceStatus => { + const statusCounts = rows.reduce((acc, r) => { + acc[r.status] = (acc[r.status] ?? 0) + 1; + return acc; + }, {} as Record); + if ((statusCounts.RED ?? 0) > 0) return 'RED'; + if ((statusCounts.AMBER ?? 0) > 0 || (statusCounts.NOT_ASSESSABLE ?? 0) > 0) return 'AMBER'; + return 'GREEN'; +}; + +/** + * Consolidated PCI DSS v4.0.1 compliance tool. + * + * Security properties: + * 1. `timeRange.from` / `timeRange.to` are validated as ISO-8601 with a `from <= to` refinement + * and bound into ES|QL via `?_tstart` / `?_tend` named parameters — never interpolated. + * 2. Index patterns are validated against a strict regex before being placed into the ES|QL + * `FROM` clause (which cannot be parameterised today). + * 3. `requirements` entries are validated against the enum of known requirement IDs plus + * the sentinel `"all"`. Unknown IDs are rejected at the schema boundary. + * + * Every response includes a `scopeClaim` object citing the DSS version, the indices and + * time range that were actually evaluated, the requirement IDs that were run, the fields + * that were checked, and a QSA disclaimer so downstream consumers can cite provenance. + */ +export const pciComplianceTool = ( + core: SecuritySolutionPluginCoreSetupDependencies, + logger: Logger +): BuiltinToolDefinition => { + return { + id: PCI_COMPLIANCE_TOOL_ID, + type: ToolType.builtin, + description: + 'Run PCI DSS v4.0.1 compliance checks or reports. Mode "check" returns per-requirement ' + + 'findings with ES|QL evidence. Mode "report" returns a visual scorecard with confidence-weighted ' + + 'scores across the requested requirements. All responses include a scopeClaim describing which ' + + 'indices, time range, and requirement IDs were actually evaluated.', + schema: pciComplianceSchema, + availability: { + cacheMode: 'space', + handler: async ({ request }) => { + return getAgentBuilderResourceAvailability({ core, request, logger }); + }, + }, + handler: async ( + { + mode, + requirements, + timeRange, + indices, + includeEvidence = true, + format = 'summary', + includeRecommendations = true, + }, + { esClient } + ) => { + const requestedRaw = requirements && requirements.length > 0 ? requirements : ['all']; + + const normalizedRaw = requestedRaw.map((req) => normalizeRequirementId(req)); + if (normalizedRaw.some((id) => id === null)) { + const invalid = requestedRaw.filter((_, i) => normalizedRaw[i] === null); + return { + results: [ + { + type: ToolResultType.error, + data: { + message: `Unsupported PCI requirement(s): ${invalid.join( + ', ' + )}. Use "all", top-level ("1".."12"), or sub-requirements like "8.3.4".`, + }, + }, + ], + }; + } + + const requestedIds = normalizedRaw.filter((id): id is string => id !== null); + const wantAll = requestedIds.includes('all'); + const requirementIds = resolveRequirementIds( + wantAll ? undefined : Array.from(new Set(requestedIds)) + ); + + if (requirementIds.length === 0) { + return { + results: [ + { + type: ToolResultType.error, + data: { message: 'No PCI DSS requirements resolved for evaluation.' }, + }, + ], + }; + } + + const indexList = getIndexList(indices); + const indexPattern = getIndexPattern(indices); + + const tasks = requirementIds.map((reqId) => async () => { + const { from, to } = getTimeRangeForCheck(reqId, timeRange); + return evaluateRequirement({ + requirementId: reqId, + indexPattern, + from, + to, + includeEvidence: mode === 'check' ? includeEvidence : false, + esClient: esClient.asCurrentUser, + }); + }); + + const rows = await runWithConcurrency(tasks, PCI_REQUIREMENT_CONCURRENCY); + + const requiredFieldsChecked = Array.from( + new Set(requirementIds.flatMap((id) => PCI_REQUIREMENTS[id]?.requiredFields ?? [])) + ); + + const resolvedTimeRange = + timeRange ?? + (() => { + const ranges = requirementIds.map((id) => getTimeRangeForCheck(id)); + const from = ranges.reduce( + (earliest, r) => (r.from < earliest ? r.from : earliest), + ranges[0].from + ); + const to = ranges.reduce((latest, r) => (r.to > latest ? r.to : latest), ranges[0].to); + return { from, to }; + })(); + + const scopeClaim = buildScopeClaim({ + indices: indexList, + from: resolvedTimeRange.from, + to: resolvedTimeRange.to, + requirementsEvaluated: requirementIds, + requiredFieldsChecked, + }); + + if (mode === 'check') { + return buildCheckResponse({ + rows, + scopeClaim, + indexPattern, + indexList, + requirements: requestedRaw, + }); + } + + return buildReportResponse({ + rows, + scopeClaim, + indexPattern, + format, + includeRecommendations, + }); + }, + tags: ['security', 'compliance', 'pci', 'audit'], + }; +}; + +function buildCheckResponse({ + rows, + scopeClaim, + indexPattern, + indexList, + requirements, +}: { + rows: EvaluatedRequirement[]; + scopeClaim: ReturnType; + indexPattern: string; + indexList: string[]; + requirements: string[]; +}) { + const statusCounts = rows.reduce((acc, r) => { + acc[r.status] = (acc[r.status] ?? 0) + 1; + return acc; + }, {} as Record); + + const overallStatus = rollupOverallStatus(rows); + const overallConfidence = rollupConfidence(rows); + + const results: Array<{ + type: ToolResultType; + data: Record; + tool_result_id?: string; + }> = []; + + const redFindings = rows.filter((r) => r.status === 'RED'); + if (redFindings.length > 0) { + const evidenceRows = redFindings.flatMap((r) => + r.findings + .filter((f) => f.evidence && f.evidence.values.length > 0) + .flatMap((f) => f.evidence?.values.map((row) => [r.requirement, r.name, ...row]) ?? []) + ); + if (evidenceRows.length > 0) { + const firstEvidence = redFindings + .flatMap((r) => r.findings) + .find((f) => f.evidence && f.evidence.values.length > 0); + const evidenceColumns = [ + { name: 'requirement', type: 'keyword' }, + { name: 'check', type: 'keyword' }, + ...(firstEvidence?.evidence?.columns ?? []), + ]; + + results.push({ + type: ToolResultType.query, + data: { esql: 'PCI DSS v4.0.1 Compliance Violations' }, + }); + results.push({ + tool_result_id: getToolResultId(), + type: ToolResultType.esqlResults, + data: { + query: 'PCI DSS v4.0.1 Compliance Check - Violations Found', + columns: evidenceColumns, + values: evidenceRows.slice(0, 100), + }, + }); + } + } + + results.push({ + type: ToolResultType.other, + data: { + mode: 'check', + request: { requirements, indices: indexList, indexPattern }, + overallStatus, + overallConfidence, + statusCounts, + requirementResults: rows, + scopeClaim, + }, + }); + + return { results }; +} + +function buildReportResponse({ + rows, + scopeClaim, + indexPattern, + format, + includeRecommendations, +}: { + rows: EvaluatedRequirement[]; + scopeClaim: ReturnType; + indexPattern: string; + format: (typeof REPORT_FORMATS)[number]; + includeRecommendations: boolean; +}) { + const overallScore = + rows.length === 0 ? 0 : Math.round(rows.reduce((sum, r) => sum + r.score, 0) / rows.length); + const overallStatus = scoreToStatus(overallScore); + const overallConfidence = rollupConfidence(rows); + + const greenCount = rows.filter((r) => r.status === 'GREEN').length; + const amberCount = rows.filter((r) => r.status === 'AMBER').length; + const redCount = rows.filter((r) => r.status === 'RED').length; + const notAssessableCount = rows.filter((r) => r.status === 'NOT_ASSESSABLE').length; + + const scorecardColumns = [ + { name: 'Requirement', type: 'keyword' }, + { name: 'Check', type: 'keyword' }, + { name: 'Status', type: 'keyword' }, + { name: 'Confidence', type: 'keyword' }, + { name: 'Score', type: 'long' }, + { name: 'Findings', type: 'long' }, + ]; + const scorecardValues = rows.map((r) => [ + r.requirement, + r.name, + r.status, + r.confidence, + r.score, + r.evidenceCount, + ]); + + const results: Array<{ + type: ToolResultType; + data: Record; + tool_result_id?: string; + }> = [ + { + type: ToolResultType.query, + data: { esql: 'PCI DSS v4.0.1 Compliance Report' }, + }, + { + tool_result_id: getToolResultId(), + type: ToolResultType.esqlResults, + data: { + query: 'PCI DSS v4.0.1 Compliance Scorecard', + columns: scorecardColumns, + values: scorecardValues, + }, + }, + ]; + + const requirementRows = rows.map((row) => ({ + id: row.requirement, + name: row.name, + pciReference: row.pciReference, + status: row.status, + confidence: row.confidence, + score: row.score, + evidenceCount: row.evidenceCount, + topFindings: row.findings.map((f) => f.detail), + recommendations: includeRecommendations ? row.recommendations : [], + })); + + results.push({ + type: ToolResultType.other, + data: { + mode: 'report', + format, + generatedAt: new Date().toISOString(), + overallScore, + overallStatus, + overallConfidence, + summary: `PCI DSS v4.0.1 posture is ${overallStatus} with score ${overallScore}/100. Requirements: ${greenCount} GREEN, ${amberCount} AMBER, ${redCount} RED, ${notAssessableCount} NOT ASSESSABLE.`, + requirements: + format === 'executive' + ? requirementRows.map(({ id, name, status, confidence, score, evidenceCount }) => ({ + id, + name, + status, + confidence, + score, + evidenceCount, + })) + : requirementRows, + dataCoverage: { + indexPattern, + totalRequirements: requirementRows.length, + greenCount, + amberCount, + redCount, + notAssessableCount, + }, + scopeClaim, + }, + }); + + return { results }; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_field_mapper_tool.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_field_mapper_tool.test.ts index 174bdc085e98d..c177cc77de29b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_field_mapper_tool.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_field_mapper_tool.test.ts @@ -19,19 +19,45 @@ describe('pciFieldMapperTool', () => { }); describe('schema', () => { - it('accepts valid index pattern', () => { + it('accepts a valid index pattern', () => { const result = tool.schema.safeParse({ indexPattern: 'logs-custom-myapp*' }); expect(result.success).toBe(true); }); - it('rejects missing index pattern', () => { + it('rejects a missing index pattern', () => { const result = tool.schema.safeParse({}); expect(result.success).toBe(false); }); + + it.each([ + 'logs-*"; DROP INDEX', + 'logs-*\n| FROM malicious', + '../../etc/passwd', + '\u0000logs-*', + ])('rejects malicious index pattern %j', (indexPattern) => { + const result = tool.schema.safeParse({ indexPattern }); + expect(result.success).toBe(false); + }); + + it('accepts an explicit ISO time range', () => { + const result = tool.schema.safeParse({ + indexPattern: 'logs-*', + timeRange: { from: '2024-01-01T00:00:00Z', to: '2024-01-08T00:00:00Z' }, + }); + expect(result.success).toBe(true); + }); + + it('rejects an inverted time range', () => { + const result = tool.schema.safeParse({ + indexPattern: 'logs-*', + timeRange: { from: '2024-02-01T00:00:00Z', to: '2024-01-01T00:00:00Z' }, + }); + expect(result.success).toBe(false); + }); }); describe('properties', () => { - it('returns correct id', () => { + it('returns the expected tool id', () => { expect(tool.id).toBe(PCI_FIELD_MAPPER_TOOL_ID); }); }); @@ -50,9 +76,7 @@ describe('pciFieldMapperTool', () => { mockEsClient.asCurrentUser.search.mockResolvedValue({ hits: { - hits: [ - { _source: { username: 'admin', src_ip: '10.0.0.1' } }, - ], + hits: [{ _source: { username: 'admin', src_ip: '10.0.0.1' } }], }, } as never); @@ -67,14 +91,9 @@ describe('pciFieldMapperTool', () => { suggestedEcsField: string; confidence: number; }>; - ecsCoveragePercent: number; }; - expect(payload.suggestedMappings.length).toBeGreaterThan(0); - - const userNameMapping = payload.suggestedMappings.find( - (m) => m.sourceField === 'username' - ); + const userNameMapping = payload.suggestedMappings.find((m) => m.sourceField === 'username'); expect(userNameMapping?.suggestedEcsField).toBe('user.name'); expect(userNameMapping?.confidence).toBeGreaterThanOrEqual(0.5); }); @@ -90,9 +109,7 @@ describe('pciFieldMapperTool', () => { }, } as never); - mockEsClient.asCurrentUser.search.mockResolvedValue({ - hits: { hits: [] }, - } as never); + mockEsClient.asCurrentUser.search.mockResolvedValue({ hits: { hits: [] } } as never); const result = (await tool.handler( { indexPattern: 'logs-payments*' }, @@ -112,67 +129,67 @@ describe('pciFieldMapperTool', () => { expect(sensitiveFields).toHaveLength(0); }); - it('returns error when fieldCaps fails', async () => { - mockEsClient.asCurrentUser.fieldCaps.mockRejectedValue(new Error('index_not_found')); + it('constrains the sample search by the provided time range', async () => { + mockEsClient.asCurrentUser.fieldCaps.mockResolvedValue({ + fields: { username: { keyword: { type: 'keyword' } } }, + } as never); + mockEsClient.asCurrentUser.search.mockResolvedValue({ hits: { hits: [] } } as never); - const result = (await tool.handler( - { indexPattern: 'nonexistent-*' }, + await tool.handler( + { + indexPattern: 'logs-custom*', + timeRange: { from: '2024-01-01T00:00:00Z', to: '2024-01-08T00:00:00Z' }, + }, createToolHandlerContext(mockRequest, mockEsClient, mockLogger) - )) as ToolHandlerStandardReturn; + ); - expect(result.results[0].type).toBe(ToolResultType.error); + const searchCall = (mockEsClient.asCurrentUser.search as unknown as jest.Mock).mock + .calls[0][0]; + expect(searchCall.query).toEqual({ + range: { + '@timestamp': { + gte: '2024-01-01T00:00:00Z', + lte: '2024-01-08T00:00:00Z', + }, + }, + }); }); - it('calculates ECS coverage correctly', async () => { + it('returns a scopeClaim with the PCI DSS version + disclaimer', async () => { mockEsClient.asCurrentUser.fieldCaps.mockResolvedValue({ - fields: { - 'user.name': { keyword: { type: 'keyword' } }, - 'event.action': { keyword: { type: 'keyword' } }, - 'event.category': { keyword: { type: 'keyword' } }, - '@timestamp': { date: { type: 'date' } }, - }, - } as never); - - mockEsClient.asCurrentUser.search.mockResolvedValue({ - hits: { hits: [] }, + fields: { username: { keyword: { type: 'keyword' } } }, } as never); + mockEsClient.asCurrentUser.search.mockResolvedValue({ hits: { hits: [] } } as never); const result = (await tool.handler( - { indexPattern: 'logs-*' }, + { indexPattern: 'logs-custom*' }, createToolHandlerContext(mockRequest, mockEsClient, mockLogger) )) as ToolHandlerStandardReturn; - const payload = result.results[0].data as { ecsCoveragePercent: number }; - expect(payload.ecsCoveragePercent).toBeGreaterThan(0); - expect(payload.ecsCoveragePercent).toBeLessThanOrEqual(100); - }); + const payload = result.results[0].data as { + scopeClaim: { + pciDssVersion: string; + indices: string[]; + disclaimer: string; + requiredFieldsChecked: string[]; + }; + }; - it('avoids false positive matches for similar field names', async () => { - mockEsClient.asCurrentUser.fieldCaps.mockResolvedValue({ - fields: { - user_agent: { keyword: { type: 'keyword' } }, - user_type: { keyword: { type: 'keyword' } }, - '@timestamp': { date: { type: 'date' } }, - }, - } as never); + expect(payload.scopeClaim.pciDssVersion).toBe('4.0.1'); + expect(payload.scopeClaim.indices).toEqual(['logs-custom*']); + expect(payload.scopeClaim.disclaimer).toContain('Qualified Security Assessor'); + expect(payload.scopeClaim.requiredFieldsChecked.length).toBeGreaterThan(0); + }); - mockEsClient.asCurrentUser.search.mockResolvedValue({ - hits: { hits: [] }, - } as never); + it('returns an error when fieldCaps fails', async () => { + mockEsClient.asCurrentUser.fieldCaps.mockRejectedValue(new Error('index_not_found')); const result = (await tool.handler( - { indexPattern: 'logs-*' }, + { indexPattern: 'nonexistent-*' }, createToolHandlerContext(mockRequest, mockEsClient, mockLogger) )) as ToolHandlerStandardReturn; - const payload = result.results[0].data as { - suggestedMappings: Array<{ sourceField: string; suggestedEcsField: string }>; - }; - - const userNameMappings = payload.suggestedMappings.filter( - (m) => m.suggestedEcsField === 'user.name' - ); - expect(userNameMappings).toHaveLength(0); + expect(result.results[0].type).toBe(ToolResultType.error); }); }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_field_mapper_tool.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_field_mapper_tool.ts index 7b60494886139..b2a21476d49e6 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_field_mapper_tool.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_field_mapper_tool.ts @@ -12,17 +12,31 @@ import type { Logger } from '@kbn/logging'; import type { SecuritySolutionPluginCoreSetupDependencies } from '../../plugin_contract'; import { getAgentBuilderResourceAvailability } from '../utils/get_agent_builder_resource_availability'; import { securityTool } from './constants'; +import { + pciIndexPatternSchema, + pciTimeRangeSchema, + buildScopeClaim, +} from './pci_compliance_schemas'; + +const DEFAULT_SAMPLE_LOOKBACK_DAYS = 7; +const SAMPLE_HIT_COUNT = 3; +const SAMPLE_SOURCE_FIELD_LIMIT = 20; const pciFieldMapperSchema = z.object({ - indexPattern: z - .string() - .describe('Index pattern to inspect for field mapping (e.g. "logs-custom-myapp*").'), + indexPattern: pciIndexPatternSchema.describe( + 'Index pattern to inspect for field mapping (e.g. "logs-custom-myapp*").' + ), targetFields: z - .array(z.string()) + .array(z.string().min(1).max(256)) .min(1) + .max(50) + .optional() + .describe('Optional list of ECS fields to map to. Defaults to common PCI-relevant ECS fields.'), + timeRange: pciTimeRangeSchema .optional() .describe( - 'Optional list of ECS fields to map to. Defaults to common PCI-relevant ECS fields.' + 'Optional ISO-8601 time range for the sample hit lookup. Defaults to the last 7 days. ' + + 'Keeping a narrow window avoids scanning frozen/cold data when looking for representative rows.' ), }); @@ -57,8 +71,25 @@ const DEFAULT_ECS_TARGETS = [ ]; const FIELD_MAPPING_HINTS: Record = { - 'user.name': ['username', 'user_name', 'login', 'account', 'principal', 'actor', 'userid', 'user_id'], - 'source.ip': ['src_ip', 'src_addr', 'source_ip', 'client_ip', 'remote_addr', 'remote_ip', 'origin_ip'], + 'user.name': [ + 'username', + 'user_name', + 'login', + 'account', + 'principal', + 'actor', + 'userid', + 'user_id', + ], + 'source.ip': [ + 'src_ip', + 'src_addr', + 'source_ip', + 'client_ip', + 'remote_addr', + 'remote_ip', + 'origin_ip', + ], 'destination.ip': ['dst_ip', 'dst_addr', 'dest_ip', 'server_ip', 'target_ip'], 'event.outcome': ['outcome', 'result', 'status', 'success', 'auth_result', 'login_result'], 'event.action': ['action', 'event_type', 'operation', 'activity', 'method', 'api_call'], @@ -108,6 +139,12 @@ function matchFieldToEcs( return null; } +const defaultTimeRange = (): { from: string; to: string } => { + const to = new Date(); + const from = new Date(to.getTime() - DEFAULT_SAMPLE_LOOKBACK_DAYS * 24 * 60 * 60 * 1000); + return { from: from.toISOString(), to: to.toISOString() }; +}; + export const pciFieldMapperTool = ( core: SecuritySolutionPluginCoreSetupDependencies, logger: Logger @@ -117,7 +154,8 @@ export const pciFieldMapperTool = ( type: ToolType.builtin, description: 'Inspect non-ECS index fields and suggest mappings to ECS fields for PCI compliance queries. ' + - 'Use this when the scope discovery tool reports low ECS coverage for an index.', + 'Use this when the scope discovery tool reports low ECS coverage for an index. Bounded by a ' + + 'short time window to avoid scanning cold/frozen data when sampling rows.', schema: pciFieldMapperSchema, availability: { cacheMode: 'space', @@ -125,8 +163,9 @@ export const pciFieldMapperTool = ( return getAgentBuilderResourceAvailability({ core, request, logger }); }, }, - handler: async ({ indexPattern, targetFields }, { esClient }) => { + handler: async ({ indexPattern, targetFields, timeRange }, { esClient }) => { const ecsTargets = targetFields ?? DEFAULT_ECS_TARGETS; + const resolvedRange = timeRange ?? defaultTimeRange(); let allFields: string[]; try { @@ -142,7 +181,9 @@ export const pciFieldMapperTool = ( results: [ { type: ToolResultType.error, - data: { message: `Unable to inspect fields for index pattern "${indexPattern}".` }, + data: { + message: `Unable to inspect fields for index pattern "${indexPattern}".`, + }, }, ], }; @@ -163,43 +204,60 @@ export const pciFieldMapperTool = ( }> = []; for (const field of nonEcsFields) { - if (isSensitiveField(field)) continue; - - for (const ecsTarget of ecsMissing) { - const match = matchFieldToEcs(field, ecsTarget); - if (match && match.score >= 0.5) { - mappings.push({ - sourceField: field, - suggestedEcsField: ecsTarget, - confidence: match.score, - reason: match.reason, - }); + if (!isSensitiveField(field)) { + for (const ecsTarget of ecsMissing) { + const match = matchFieldToEcs(field, ecsTarget); + if (match && match.score >= 0.5) { + mappings.push({ + sourceField: field, + suggestedEcsField: ecsTarget, + confidence: match.score, + reason: match.reason, + }); + } } } } mappings.sort((a, b) => b.confidence - a.confidence); + // Best-effort sample constrained to the provided time window so we don't scan cold/frozen data. let sampleFields: string[] = []; try { const sampleResponse = await esClient.asCurrentUser.search({ index: indexPattern, - size: 3, + size: SAMPLE_HIT_COUNT, _source_includes: nonEcsFields .filter((f) => !isSensitiveField(f)) - .slice(0, 20), + .slice(0, SAMPLE_SOURCE_FIELD_LIMIT), + query: { + range: { + '@timestamp': { + gte: resolvedRange.from, + lte: resolvedRange.to, + }, + }, + }, + ignore_unavailable: true, + allow_no_indices: true, }); if (sampleResponse.hits?.hits?.length) { sampleFields = [ - ...new Set( - sampleResponse.hits.hits.flatMap((hit) => Object.keys(hit._source ?? {})) - ), + ...new Set(sampleResponse.hits.hits.flatMap((hit) => Object.keys(hit._source ?? {}))), ]; } } catch { - // Sample retrieval is best-effort + // Sample retrieval is best-effort (e.g. indices without @timestamp will error here). } + const scopeClaim = buildScopeClaim({ + indices: [indexPattern], + from: resolvedRange.from, + to: resolvedRange.to, + requirementsEvaluated: [], + requiredFieldsChecked: ecsTargets, + }); + return { results: [ { @@ -209,9 +267,7 @@ export const pciFieldMapperTool = ( totalFields: allFields.length, ecsFieldsPresent, ecsMissing, - ecsCoveragePercent: Math.round( - (ecsFieldsPresent.length / ecsTargets.length) * 100 - ), + ecsCoveragePercent: Math.round((ecsFieldsPresent.length / ecsTargets.length) * 100), suggestedMappings: mappings.slice(0, 20), sampleFieldNames: sampleFields.slice(0, 30), guidance: @@ -219,6 +275,7 @@ export const pciFieldMapperTool = ( ? 'Use the generateEsql tool to create adapted queries using the suggested field mappings above. ' + 'For example, if "username" maps to "user.name", use RENAME or reference the source field directly.' : 'No automatic mappings found. Inspect the sample field names and create manual field mappings.', + scopeClaim, }, }, ], diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_scope_discovery_tool.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_scope_discovery_tool.test.ts index c2999902ce0bf..aee1747e3af4d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_scope_discovery_tool.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_scope_discovery_tool.test.ts @@ -19,12 +19,12 @@ describe('pciScopeDiscoveryTool', () => { }); describe('schema', () => { - it('accepts default input', () => { + it('accepts the default input', () => { const result = tool.schema.safeParse({}); expect(result.success).toBe(true); }); - it('accepts custom indices', () => { + it('accepts valid custom indices', () => { const result = tool.schema.safeParse({ scopeType: 'network', customIndices: ['custom-firewall-*'], @@ -32,62 +32,76 @@ describe('pciScopeDiscoveryTool', () => { expect(result.success).toBe(true); }); - it('rejects invalid scope type', () => { - const result = tool.schema.safeParse({ - scopeType: 'invalid', - }); + it('rejects an invalid scope type', () => { + const result = tool.schema.safeParse({ scopeType: 'invalid' }); expect(result.success).toBe(false); }); + + it.each(['custom-index"; DROP', 'bad\u0000index', 'line\nbreak', '../escape'])( + 'rejects malicious custom index %j', + (bad) => { + const result = tool.schema.safeParse({ customIndices: [bad] }); + expect(result.success).toBe(false); + } + ); }); describe('properties', () => { - it('returns correct id', () => { + it('returns the expected tool id', () => { expect(tool.id).toBe(PCI_SCOPE_DISCOVERY_TOOL_ID); }); }); describe('handler', () => { - it('classifies indices by field and name hints', async () => { + it('uses a single batched fieldCaps call across the discovered indices', async () => { (mockEsClient.asCurrentUser.cat.indices as unknown as jest.Mock).mockResolvedValue([ - { index: 'packetbeat-network-*' }, - { index: 'custom-auth-*' }, + { index: 'packetbeat-network-1' }, + { index: 'auth-logs-1' }, ]); - (mockEsClient.asCurrentUser.fieldCaps as unknown as jest.Mock) - .mockResolvedValueOnce({ - fields: { - 'event.category': {}, - 'source.ip': {}, - 'destination.ip': {}, + (mockEsClient.asCurrentUser.fieldCaps as unknown as jest.Mock).mockResolvedValue({ + fields: { + 'source.ip': { ip: { type: 'ip', indices: ['packetbeat-network-1'] } }, + 'destination.ip': { ip: { type: 'ip', indices: ['packetbeat-network-1'] } }, + 'user.name': { keyword: { type: 'keyword', indices: ['auth-logs-1'] } }, + 'event.outcome': { + keyword: { type: 'keyword', indices: ['auth-logs-1'] }, }, - }) - .mockResolvedValueOnce({ - fields: { - 'event.category': {}, - 'user.name': {}, - 'event.outcome': {}, - }, - }); + 'event.category': { keyword: { type: 'keyword' } }, + }, + }); const result = (await tool.handler( { scopeType: 'all' }, createToolHandlerContext(mockRequest, mockEsClient, mockLogger) )) as ToolHandlerStandardReturn; + expect(mockEsClient.asCurrentUser.fieldCaps).toHaveBeenCalledTimes(1); + const call = (mockEsClient.asCurrentUser.fieldCaps as unknown as jest.Mock).mock.calls[0][0]; + expect(call.index).toEqual(['packetbeat-network-1', 'auth-logs-1']); + expect(result.results).toHaveLength(1); expect(result.results[0].type).toBe(ToolResultType.other); - expect((result.results[0].data as { matchedIndices: number }).matchedIndices).toBe(2); + const data = result.results[0].data as { + matchedIndices: number; + discovered: Array<{ index: string; categories: string[] }>; + }; + expect(data.matchedIndices).toBe(2); }); - it('filters by requested scope type', async () => { + it('filters results by requested scope type', async () => { (mockEsClient.asCurrentUser.cat.indices as unknown as jest.Mock).mockResolvedValue([ - { index: 'packetbeat-network-*' }, - { index: 'custom-auth-*' }, + { index: 'packetbeat-network-1' }, + { index: 'auth-logs-1' }, ]); - (mockEsClient.asCurrentUser.fieldCaps as unknown as jest.Mock) - .mockResolvedValueOnce({ fields: { 'source.ip': {}, 'destination.ip': {} } }) - .mockResolvedValueOnce({ fields: { 'user.name': {}, 'event.outcome': {} } }); + (mockEsClient.asCurrentUser.fieldCaps as unknown as jest.Mock).mockResolvedValue({ + fields: { + 'source.ip': { ip: { type: 'ip', indices: ['packetbeat-network-1'] } }, + 'destination.ip': { ip: { type: 'ip', indices: ['packetbeat-network-1'] } }, + 'user.name': { keyword: { type: 'keyword', indices: ['auth-logs-1'] } }, + }, + }); const result = (await tool.handler( { scopeType: 'network' }, @@ -99,7 +113,79 @@ describe('pciScopeDiscoveryTool', () => { discovered: Array<{ index: string }>; }; expect(data.matchedIndices).toBe(1); - expect(data.discovered[0].index).toContain('packetbeat-network'); + expect(data.discovered[0].index).toBe('packetbeat-network-1'); + }); + + it('resolves custom wildcard patterns to concrete indices instead of storing the pattern', async () => { + (mockEsClient.asCurrentUser.cat.indices as unknown as jest.Mock).mockResolvedValue([ + { index: 'custom-firewall-2024' }, + { index: 'custom-firewall-2025' }, + { index: 'unrelated-index' }, + ]); + + (mockEsClient.asCurrentUser.fieldCaps as unknown as jest.Mock).mockResolvedValue({ + fields: { + 'source.ip': { + ip: { type: 'ip', indices: ['custom-firewall-2024', 'custom-firewall-2025'] }, + }, + 'destination.ip': { + ip: { type: 'ip', indices: ['custom-firewall-2024', 'custom-firewall-2025'] }, + }, + }, + }); + + const result = (await tool.handler( + { scopeType: 'all', customIndices: ['custom-firewall-*'] }, + createToolHandlerContext(mockRequest, mockEsClient, mockLogger) + )) as ToolHandlerStandardReturn; + + const call = (mockEsClient.asCurrentUser.fieldCaps as unknown as jest.Mock).mock.calls[0][0]; + // The pattern should NOT appear as a literal index in the fieldCaps call + expect(call.index).not.toContain('custom-firewall-*'); + expect(call.index).toContain('custom-firewall-2024'); + expect(call.index).toContain('custom-firewall-2025'); + + const data = result.results[0].data as { + discovered: Array<{ index: string; ecsCoveragePercent: number }>; + }; + const firewallIndices = data.discovered.filter((d) => d.index.startsWith('custom-firewall-')); + // Both concrete indices should be discovered with non-zero coverage + expect(firewallIndices).toHaveLength(2); + for (const idx of firewallIndices) { + expect(idx.ecsCoveragePercent).toBeGreaterThan(0); + } + }); + + it('attaches a scopeClaim with the PCI DSS version + disclaimer', async () => { + (mockEsClient.asCurrentUser.cat.indices as unknown as jest.Mock).mockResolvedValue([ + { index: 'packetbeat-network-1' }, + ]); + + (mockEsClient.asCurrentUser.fieldCaps as unknown as jest.Mock).mockResolvedValue({ + fields: { + 'source.ip': { ip: { type: 'ip' } }, + 'destination.ip': { ip: { type: 'ip' } }, + }, + }); + + const result = (await tool.handler( + { scopeType: 'all' }, + createToolHandlerContext(mockRequest, mockEsClient, mockLogger) + )) as ToolHandlerStandardReturn; + + const data = result.results[0].data as { + scopeClaim: { + pciDssVersion: string; + disclaimer: string; + requiredFieldsChecked: string[]; + }; + }; + + expect(data.scopeClaim.pciDssVersion).toBe('4.0.1'); + expect(data.scopeClaim.disclaimer).toContain('Qualified Security Assessor'); + expect(data.scopeClaim.requiredFieldsChecked).toEqual( + expect.arrayContaining(['source.ip', 'destination.ip']) + ); }); }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_scope_discovery_tool.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_scope_discovery_tool.ts index df306404fe35c..8913b0c179291 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_scope_discovery_tool.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/pci_scope_discovery_tool.ts @@ -13,8 +13,17 @@ import type { Logger } from '@kbn/logging'; import type { SecuritySolutionPluginCoreSetupDependencies } from '../../plugin_contract'; import { getAgentBuilderResourceAvailability } from '../utils/get_agent_builder_resource_availability'; import { securityTool } from './constants'; +import { pciIndexPatternSchema, buildScopeClaim } from './pci_compliance_schemas'; -const pciScopeType = z.enum(['all', 'network', 'identity', 'endpoint', 'cloud', 'application']); +const pciScopeType = z.enum([ + 'all', + 'network', + 'identity', + 'endpoint', + 'cloud', + 'application', + 'vulnerability', +]); const pciScopeDiscoverySchema = z.object({ scopeType: pciScopeType @@ -24,10 +33,12 @@ const pciScopeDiscoverySchema = z.object({ 'Scope focus area for discovery: all, network, identity, endpoint, cloud, or application' ), customIndices: z - .array(z.string().min(1)) + .array(pciIndexPatternSchema) + .min(1) + .max(50) .optional() .describe( - 'Optional custom index patterns to include for environments with non-native ingestion' + 'Optional custom index patterns to include for environments with non-native ingestion.' ), }); @@ -66,48 +77,88 @@ const SCOPE_RULES: Record< fieldHints: ['event.category', 'url.domain', 'http.request.method', 'service.name'], nameHints: ['app', 'web', 'nginx', 'apache'], }, + vulnerability: { + fieldHints: ['vulnerability.id', 'vulnerability.severity', 'event.kind'], + nameHints: ['vuln', 'vulnerability', 'cve', 'ids', 'intrusion'], + }, }; -const detectCategories = (index: string, fields: string[]): ScopeCategory[] => { - const lowerIndex = index.toLowerCase(); - const categoryMatches = (Object.keys(SCOPE_RULES) as Array>).filter( - (category) => { - const { fieldHints, nameHints } = SCOPE_RULES[category]; - const hasFieldMatch = fieldHints.some((field) => fields.includes(field)); - const hasNameMatch = nameHints.some((hint) => lowerIndex.includes(hint)); - return hasFieldMatch || hasNameMatch; - } - ); - return categoryMatches; -}; - -const calculateCoverage = (fields: string[]): number => { - const ecsHints = new Set( +const ALL_FIELD_HINTS = Array.from( + new Set( (Object.keys(SCOPE_RULES) as Array>).flatMap( (category) => SCOPE_RULES[category].fieldHints ) - ); + ) +); - if (ecsHints.size === 0) { - return 0; - } +const MAX_INDICES_INSPECTED = 200; - const present = [...ecsHints].filter((field) => fields.includes(field)).length; - return Math.round((present / ecsHints.size) * 100); +const detectCategories = (index: string, fields: Set): ScopeCategory[] => { + const lowerIndex = index.toLowerCase(); + return (Object.keys(SCOPE_RULES) as Array>).filter((category) => { + const { fieldHints, nameHints } = SCOPE_RULES[category]; + const hasFieldMatch = fieldHints.some((field) => fields.has(field)); + const hasNameMatch = nameHints.some((hint) => lowerIndex.includes(hint)); + return hasFieldMatch || hasNameMatch; + }); +}; + +const calculateCoverage = (fields: Set): number => { + if (ALL_FIELD_HINTS.length === 0) return 0; + const present = ALL_FIELD_HINTS.filter((field) => fields.has(field)).length; + return Math.round((present / ALL_FIELD_HINTS.length) * 100); }; -const getFieldList = async (index: string, esClient: ElasticsearchClient): Promise => { +/** + * Build a per-index map of available fields from a single batched `fieldCaps` call. + * + * The prior implementation fired one `fieldCaps` request per discovered index which became + * O(thousands) of sequential RTTs on large clusters. By issuing a single call across the + * consolidated index set and keying on `field.indices` (populated when a field exists in + * only a subset of the requested indices) plus a fallback of "present everywhere", we + * reduce this to a single round-trip. + */ +const fetchFieldsByIndex = async ( + indices: string[], + esClient: ElasticsearchClient +): Promise>> => { + const byIndex = new Map>(); + for (const idx of indices) byIndex.set(idx, new Set()); + + if (indices.length === 0) return byIndex; + try { const response = await esClient.fieldCaps({ - index, + index: indices, fields: ['*'], + include_unmapped: false, ignore_unavailable: true, allow_no_indices: true, }); - return Object.keys(response.fields ?? {}); + + const fields = response.fields ?? {}; + for (const [fieldName, fieldTypes] of Object.entries(fields)) { + const typeEntries = Object.values(fieldTypes ?? {}); + // If any type entry omits `indices`, the field exists across every requested index. + const presentEverywhere = typeEntries.some((entry) => !entry?.indices); + if (presentEverywhere) { + for (const set of byIndex.values()) set.add(fieldName); + } else { + for (const entry of typeEntries) { + const entryIndices = entry?.indices ?? []; + const arr = Array.isArray(entryIndices) ? entryIndices : [entryIndices]; + for (const idx of arr) { + const set = byIndex.get(idx); + if (set) set.add(fieldName); + } + } + } + } } catch { - return []; + // Fall through with empty maps; callers will simply treat indices as having no known fields. } + + return byIndex; }; export const pciScopeDiscoveryTool = ( @@ -118,7 +169,9 @@ export const pciScopeDiscoveryTool = ( id: PCI_SCOPE_DISCOVERY_TOOL_ID, type: ToolType.builtin, description: - 'Discover PCI-relevant data coverage across indices, including custom-ingested data, and classify by scope area.', + 'Discover PCI-relevant data coverage across indices, including custom-ingested data, and ' + + 'classify by scope area. Uses a single batched fieldCaps call across up to ' + + `${MAX_INDICES_INSPECTED} indices rather than per-index round-trips.`, schema: pciScopeDiscoverySchema, availability: { cacheMode: 'space', @@ -133,14 +186,38 @@ export const pciScopeDiscoveryTool = ( expand_wildcards: 'all', })) as Array<{ index: string }>; - const indexSet = new Set(indicesResponse.map(({ index }) => index).filter(Boolean)); + const indexSet = new Set(); + for (const { index } of indicesResponse) { + if (index) indexSet.add(index); + } for (const customIndex of customIndices ?? []) { - indexSet.add(customIndex); + if (customIndex.includes('*') || customIndex.includes('?')) { + // Patterns are resolved via the fieldCaps call, but fieldCaps returns + // concrete index names — not the pattern itself. To avoid a key mismatch + // in the byIndex map, resolve the pattern against the concrete index list + // from cat.indices (which already contains every matching index). + const pattern = new RegExp( + `^${customIndex + .replace(/[.+^${}()|[\]\\]/g, '\\$&') + .replace(/\*/g, '.*') + .replace(/\?/g, '.')}$` + ); + for (const existing of indexSet) { + if (pattern.test(existing)) indexSet.add(existing); + } + } else { + indexSet.add(customIndex); + } } + const indices = Array.from(indexSet).slice(0, MAX_INDICES_INSPECTED); + const truncated = indexSet.size > MAX_INDICES_INSPECTED; + + const fieldsByIndex = await fetchFieldsByIndex(indices, esClient.asCurrentUser); + const discovered: DiscoveredIndex[] = []; - for (const index of indexSet) { - const fields = await getFieldList(index, esClient.asCurrentUser); + for (const index of indices) { + const fields = fieldsByIndex.get(index) ?? new Set(); const categories = detectCategories(index, fields); const shouldInclude = categories.length > 0 && (scopeType === 'all' || categories.includes(scopeType)); @@ -149,20 +226,30 @@ export const pciScopeDiscoveryTool = ( index, categories, ecsCoveragePercent: calculateCoverage(fields), - availableFields: fields.slice(0, 50), + availableFields: Array.from(fields).slice(0, 50), }); } } + const scopeClaim = buildScopeClaim({ + indices: discovered.map((d) => d.index), + from: new Date(0).toISOString(), + to: new Date().toISOString(), + requirementsEvaluated: [], + requiredFieldsChecked: ALL_FIELD_HINTS, + }); + return { results: [ { type: ToolResultType.other, data: { scopeType, - totalIndicesInspected: indexSet.size, + totalIndicesInspected: indices.length, + indicesTruncated: truncated, matchedIndices: discovered.length, discovered, + scopeClaim, }, }, ], diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/register_tools.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/register_tools.ts index e7898cebf8231..fa544c964e2d7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/register_tools.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/register_tools.ts @@ -13,14 +13,16 @@ import { attackDiscoverySearchTool } from './attack_discovery_search_tool'; import { entityRiskScoreTool, getEntityTool, searchEntitiesTool } from './entity_analytics'; import { alertsTool } from './alerts_tool'; import { createDetectionRuleTool } from './create_detection_rule_tool'; -import { pciComplianceCheckTool } from './pci_compliance_check_tool'; -import { pciComplianceReportTool } from './pci_compliance_report_tool'; +import { pciComplianceTool } from './pci_compliance_tool'; import { pciScopeDiscoveryTool } from './pci_scope_discovery_tool'; import { pciFieldMapperTool } from './pci_field_mapper_tool'; import type { SecuritySolutionPluginCoreSetupDependencies } from '../../plugin_contract'; /** - * Registers all security agent builder tools with the agentBuilder plugin + * Registers all security agent builder tools with the agentBuilder plugin. + * + * PCI compliance tools are gated behind `experimentalFeatures.pciComplianceAgentBuilder` so + * the feature can ship dark and be enabled per environment. */ export const registerTools = async ( agentBuilder: AgentBuilderPluginSetup, @@ -35,8 +37,10 @@ export const registerTools = async ( agentBuilder.tools.register(alertsTool(core, logger)); agentBuilder.tools.register(getEntityTool(core, logger, experimentalFeatures)); agentBuilder.tools.register(searchEntitiesTool(core, logger, experimentalFeatures)); - agentBuilder.tools.register(pciScopeDiscoveryTool(core, logger)); - agentBuilder.tools.register(pciComplianceCheckTool(core, logger)); - agentBuilder.tools.register(pciComplianceReportTool(core, logger)); - agentBuilder.tools.register(pciFieldMapperTool(core, logger)); + + if (experimentalFeatures.pciComplianceAgentBuilder) { + agentBuilder.tools.register(pciScopeDiscoveryTool(core, logger)); + agentBuilder.tools.register(pciComplianceTool(core, logger)); + agentBuilder.tools.register(pciFieldMapperTool(core, logger)); + } }; diff --git a/yarn.lock b/yarn.lock index 11ab57e2fb359..3520b0260a704 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6883,6 +6883,10 @@ version "0.0.0" uid "" +"@kbn/evals-suite-pci-compliance@link:x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance": + version "0.0.0" + uid "" + "@kbn/evals-suite-security-ai-rules@link:x-pack/solutions/security/packages/kbn-evals-suite-security-ai-rules": version "0.0.0" uid ""