feat(evals): autonomous skill discovery and generation system

docker-compose.aesop-spike.yml

@@ -0,0 +1,109 @@
+version: '3.8'
+
+services:
+  # ═══════════════════════════════════════════════════════════════
+  # Elasticsearch - Primary data store
+  # ═══════════════════════════════════════════════════════════════
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.4.0-SNAPSHOT
+    container_name: aesop-elasticsearch
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=true
+      - ELASTIC_PASSWORD=changeme
+      - xpack.license.self_generated.type=trial
+      - xpack.ml.enabled=true
+      - cluster.name=aesop-cluster
+    ports:
+      - "9200:9200"
+    volumes:
+      - es-data:/usr/share/elasticsearch/data
+    networks:
+      - aesop-network
+    healthcheck:
+      test: ["CMD-SHELL", "curl -u elastic:changeme -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\\|yellow\"'"]
+      interval: 10s
+      timeout: 5s
+      retries: 30
+
+  # ═══════════════════════════════════════════════════════════════
+  # EDOT Collector - O11y trace collection
+  # ═══════════════════════════════════════════════════════════════
+  edot-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    container_name: aesop-edot-collector
+    command: ["--config=/etc/otel/config.yaml"]
+    ports:
+      - "4318:4318"  # OTLP HTTP receiver
+      - "4317:4317"  # OTLP gRPC receiver
+    volumes:
+      - ./docker/edot-config.yaml:/etc/otel/config.yaml:ro
+    networks:
+      - aesop-network
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+
+  # ═══════════════════════════════════════════════════════════════
+  # Kibana - UI and API server
+  # ═══════════════════════════════════════════════════════════════
+  kibana:
+    image: docker.elastic.co/kibana/kibana:9.4.0-SNAPSHOT
+    container_name: aesop-kibana
+    environment:
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+      - ELASTICSEARCH_USERNAME=elastic
+      - ELASTICSEARCH_PASSWORD=changeme
+      - xpack.evals.enabled=true
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://edot-collector:4318
+      - OTEL_TRACES_EXPORTER=otlp
+      - OTEL_METRICS_EXPORTER=none
+      - OTEL_LOGS_EXPORTER=none
+    ports:
+      - "5601:5601"
+    networks:
+      - aesop-network
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+      edot-collector:
+        condition: service_started
+    healthcheck:
+      test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status | grep -q '\"level\":\"available\"'"]
+      interval: 15s
+      timeout: 10s
+      retries: 30
+
+  # ═══════════════════════════════════════════════════════════════
+  # Data Generator - Loads demo data for hypothesis testing
+  # ═══════════════════════════════════════════════════════════════
+  data-generator:
+    image: node:18-alpine
+    container_name: aesop-data-generator
+    working_dir: /app
+    volumes:
+      - ./x-pack/solutions/security/plugins/security_solution/scripts/aesop_demo:/app
+      - ./scripts:/scripts
+    command: sh -c "
+      echo 'Waiting for Elasticsearch...' &&
+      sleep 30 &&
+      echo 'Generating demo data for hypothesis validation...' &&
+      node data_generator.ts --mode hypothesis-validation
+    "
+    networks:
+      - aesop-network
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+    environment:
+      - ES_URL=http://elasticsearch:9200
+      - ES_USERNAME=elastic
+      - ES_PASSWORD=changeme
+
+networks:
+  aesop-network:
+    driver: bridge
+
+volumes:
+  es-data:
+    driver: local

docker/edot-config.yaml

@@ -0,0 +1,51 @@
+# EDOT Collector Configuration for AESOP Spike
+# Collects OTEL traces from Kibana Agent Builder agents and exports to Elasticsearch
+
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: 0.0.0.0:4318
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+processors:
+  batch:
+    timeout: 10s
+    send_batch_size: 1024
+
+  # Add AESOP-specific attributes
+  attributes:
+    actions:
+      - key: deployment.environment
+        value: aesop-spike-local
+        action: insert
+
+exporters:
+  elasticsearch:
+    endpoints: ["http://elasticsearch:9200"]
+    auth:
+      authenticator: basicauth
+    traces_index: traces-apm.otel-default
+    mapping:
+      mode: ecs
+
+  # Debug exporter for troubleshooting
+  debug:
+    verbosity: detailed
+    sampling_initial: 5
+    sampling_thereafter: 200
+
+extensions:
+  basicauth:
+    client_auth:
+      username: elastic
+      password: changeme
+
+service:
+  extensions: [basicauth]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch, attributes]
+      exporters: [elasticsearch, debug]

x-pack/platform/packages/shared/kbn-llm-batch-processing/__tests__/orchestrator.test.ts

@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { batchProcess } from '../src/orchestrator';
+
+describe('batchProcess', () => {
+  it('should process batches concurrently and merge results', async () => {
+    const items = Array.from({ length: 100 }, (_, i) => i);
+    const processFn = jest.fn(async (batch: number[]) => batch.reduce((a, b) => a + b, 0));
+    const mergeFn = jest.fn(async ([a, b]: [number, number]) => a + b);
+
+    const result = await batchProcess({
+      input: items,
+      splitStrategy: 'item-based',
+      maxItemsPerBatch: 25,
+      processFn,
+      mergeFn,
+      maxConcurrentBatches: 3,
+    });
+
+    // 100 items / 25 per batch = 4 batches
+    expect(processFn).toHaveBeenCalledTimes(4);
+
+    // Hierarchical merge: 4 outputs -> 2 -> 1 = 3 merge calls
+    expect(mergeFn).toHaveBeenCalledTimes(3);
+
+    // Sum of 0..99 = 4950
+    expect(result.output).toBe(4950);
+    expect(result.stats.batches).toBe(4);
+    expect(result.stats.mergeRounds).toBe(2);
+  });
+
+  it('should respect maxConcurrentBatches', async () => {
+    const items = Array.from({ length: 10 }, (_, i) => i);
+    let concurrent = 0;
+    let maxConcurrent = 0;
+
+    const processFn = async (batch: number[]) => {
+      concurrent++;
+      maxConcurrent = Math.max(maxConcurrent, concurrent);
+      await new Promise((resolve) => setTimeout(resolve, 10));
+      concurrent--;
+      return batch.reduce((a, b) => a + b, 0);
+    };
+
+    await batchProcess({
+      input: items,
+      splitStrategy: 'item-based',
+      maxItemsPerBatch: 1,
+      processFn,
+      mergeFn: async ([a, b]) => a + b,
+      maxConcurrentBatches: 2,
+    });
+
+    expect(maxConcurrent).toBeLessThanOrEqual(2);
+  });
+
+  it('should call onProgress callback', async () => {
+    const items = [1, 2, 3, 4];
+    const progressUpdates: Array<{ completed: number; total: number }> = [];
+
+    await batchProcess({
+      input: items,
+      splitStrategy: 'item-based',
+      maxItemsPerBatch: 1,
+      processFn: async (batch) => batch[0],
+      mergeFn: async ([a, b]) => a + b,
+      onProgress: (completed, total) => {
+        progressUpdates.push({ completed, total });
+      },
+    });
+
+    expect(progressUpdates).toEqual([
+      { completed: 1, total: 4 },
+      { completed: 2, total: 4 },
+      { completed: 3, total: 4 },
+      { completed: 4, total: 4 },
+    ]);
+  });
+
+  it('should throw on invalid split strategy', async () => {
+    await expect(
+      batchProcess({
+        input: [1, 2, 3],
+        splitStrategy: 'invalid' as any,
+        processFn: async (batch) => batch[0],
+        mergeFn: async ([a, b]) => a + b,
+      })
+    ).rejects.toThrow('Unknown split strategy');
+  });
+});

x-pack/platform/packages/shared/kbn-llm-batch-processing/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../../../../..',
+  roots: ['<rootDir>/x-pack/platform/packages/shared/kbn-llm-batch-processing'],
+};

x-pack/platform/packages/shared/kbn-llm-batch-processing/kibana.jsonc

@@ -0,0 +1,7 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/llm-batch-processing",
+  "owner": "@elastic/security-generative-ai",
+  "group": "platform",
+  "visibility": "shared"
+}

x-pack/platform/packages/shared/kbn-llm-batch-processing/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/llm-batch-processing",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/platform/packages/shared/kbn-llm-batch-processing/src/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// Main entry point
+export { batchProcess } from './orchestrator';
+
+// Low-level utilities
+export { tokenBasedSplit, itemBasedSplit } from './split';
+export { hierarchicalMerge } from './merge';
+
+// Types
+export type { BatchConfig, BatchResult, BatchStats, SplitStrategy, MergeStrategy } from './types';

x-pack/platform/packages/shared/kbn-llm-batch-processing/src/merge.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * Merge batch outputs hierarchically (tournament-style pairwise merge)
+ *
+ * @param outputs - Array of batch outputs to merge
+ * @param mergeFn - Function to merge two outputs
+ * @returns Final merged output
+ *
+ * @example
+ * ```typescript
+ * const batches = ['batch1', 'batch2', 'batch3', 'batch4'];
+ * const result = await hierarchicalMerge(batches, async ([a, b]) => {
+ *   return await llm.summarize([a, b]);
+ * });
+ * ```
+ */
+export async function hierarchicalMerge<T>(
+  outputs: T[],
+  mergeFn: (pair: [T, T]) => Promise<T>
+): Promise<T> {
+  if (outputs.length === 0) {
+    throw new Error('Cannot merge empty array');
+  }
+
+  if (outputs.length === 1) {
+    return outputs[0];
+  }
+
+  let current = outputs;
+
+  while (current.length > 1) {
+    const nextRound: T[] = [];
+
+    for (let i = 0; i < current.length; i += 2) {
+      if (i + 1 < current.length) {
+        // Merge pair
+        const merged = await mergeFn([current[i], current[i + 1]]);
+        nextRound.push(merged);
+      } else {
+        // Odd one out, pass through to next round
+        nextRound.push(current[i]);
+      }
+    }
+
+    current = nextRound;
+  }
+
+  return current[0];
+}