[AI Infra] Update scripts that generates ES|QL documentation for AI Assistants

x-pack/platform/plugins/shared/inference/moon.yml

@@ -44,6 +44,7 @@ dependsOn:
   - '@kbn/core-ui-settings-common'
   - '@kbn/setup-node-env'
   - '@kbn/es-errors'
+  - '@kbn/dev-utils'
 tags:
   - plugin
   - prod

x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/README.md

@@ -5,18 +5,21 @@ The generated documentation is validated and will emit warnings when invalid que
 
 ## Requirements
 
-- checked out `built-docs` repo in the same folder as the `kibana` repository
 - a running Kibana instance
-- an installed Generative AI connector
 
 ### Run script to generate ES|QL docs and verify syntax
 
+To deterministically get the ES|QL docs from the Elastic's documentation markdown files, without modification from LLMs, you can run:
 ```
 node x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/index.js
 ```
 
-The script will also generate a report of syntax errors found during the generation process, located at
-`x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/__tmp__/syntax-errors.json`. This file will not be checked into git.
+To connect to a connector/LLM to read the built docs and then enrich the extracted docs, you must first have an installed Generative AI connector. Then, pass in the connectorId. Enrichment involves explaining in natural language what the ES|QL examples are doing.
+
+```
+node x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/index.js --connectorId example-connector-id
+```
+
 
 ### Checking syntax errors for generated files
 

x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/enrich_documentation.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ScriptInferenceClient } from '../util/kibana_client';
+import { enrichDocumentationPrompt } from './prompts';
+import { bindOutput } from './utils/output_executor';
+
+/**
+ * Enriches documentation by adding natural language descriptions for each ES|QL query example.
+ * Uses the connectorId from the inferenceClient to connect and enrich the extracted content.
+ *
+ * @param content - The markdown content to enrich
+ * @param inferenceClient - The inference client with connectorId and output API
+ * @returns The enriched content with natural language descriptions for ES|QL queries
+ */
+export async function enrichDocumentation({
+  content,
+  inferenceClient,
+}: {
+  content: string;
+  inferenceClient: ScriptInferenceClient;
+}): Promise<string> {
+  const callOutput = bindOutput({
+    connectorId: inferenceClient.getConnectorId(),
+    output: inferenceClient.output,
+  });
+
+  const enrichedContent = await callOutput(
+    enrichDocumentationPrompt({
+      content,
+    })
+  );
+
+  return enrichedContent;
+}

x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/generate_doc.ts

@@ -33,11 +33,21 @@ export const generateDoc = async ({
 }) => {
   const filesToWrite: FileToWrite[] = [];
 
-  const limiter = pLimit(10);
+  // Reduce concurrency to avoid hitting rate limits (429 errors)
+  // Lower concurrency = fewer simultaneous requests = less chance of rate limits
+  const limiter = pLimit(3);
 
+  // Configure retry logic to handle 429 (Too Many Requests) errors
+  // 429 errors are retryable and will be handled with exponential backoff
   const callOutput = bindOutput({
     connectorId: inferenceClient.getConnectorId(),
     output: inferenceClient.output,
+    maxRetries: 5, // Retry up to 5 times for rate limit errors
+    retryConfiguration: {
+      retryOn: 'auto', // Will retry 429 errors (not in STATUS_NO_RETRY list)
+      initialDelay: 2000, // Start with 2 second delay
+      backoffMultiplier: 2, // Double the delay on each retry
+    },
   });
 
   const documentation = documentationForFunctionRewrite(extraction);