DOC-1455 End to end config property automation

.github/workflows/test.yaml

@@ -222,19 +222,6 @@ jobs:
       - name: Run all tests
         run: npm test
 
-      - name: Generate coverage report
-        if: matrix.node-version == '18'
-        run: npm test -- --coverage --coverageReporters=lcov
-
-      - name: Upload coverage to Codecov
-        if: matrix.node-version == '18'
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          file: ./coverage/lcov.info
-          flags: unittests
-          name: codecov-umbrella
-
   # Summary job that all other jobs depend on
   test-summary:
     if: always()

.gitignore

@@ -1,4 +1,3 @@
-/node_modules/*
 /wip/*
 /docs/*
 .vscode
@@ -10,4 +9,5 @@ gen/
 tree-sitter/
 test/
 modules/
-cloud-controlplane/
+cloud-controlplane/
+node_modules

__tests__/docs-data/examples/admin-example.adoc

@@ -0,0 +1,15 @@
+.Example Configuration for Admin API
+This example shows how to configure the admin API endpoint with custom addressing.
+
+[,yaml]
+----
+redpanda:
+  admin:
+    - address: "0.0.0.0"
+      port: 9644
+    - address: "127.0.0.1"
+      port: 9645
+----
+
+You can specify multiple admin endpoints to provide redundancy and load balancing.
+The admin API is used for cluster management operations.

__tests__/docs-data/property-overrides.json

@@ -0,0 +1,53 @@
+{
+  "properties": {
+    "admin": {
+      "description": "Network addresses for Admin API servers with version info.",
+      "version": "v23.1.0",
+      "example_file": "examples/admin-example.adoc"
+    },
+    "admin_api_tls": {
+      "description": "TLS configuration for the Admin API endpoints.",
+      "version": "v23.2.0",
+      "example_yaml": {
+        "title": "Example TLS Configuration",
+        "description": "This example shows how to configure TLS for the Admin API with client authentication.",
+        "config": {
+          "redpanda": {
+            "admin_api_tls": [
+              {
+                "name": "internal-admin",
+                "enabled": true,
+                "cert_file": "/etc/redpanda/certs/admin.crt",
+                "key_file": "/etc/redpanda/certs/admin.key",
+                "truststore_file": "/etc/redpanda/certs/ca.crt",
+                "require_client_auth": true
+              }
+            ]
+          }
+        }
+      }
+    },
+    "abort_index_segment_size": {
+      "description": "Segment size for transaction abort index. Controls how large each abort index segment can grow.",
+      "example": [
+        ".Example: Setting abort index segment size",
+        "[,yaml]",
+        "----",
+        "redpanda:",
+        "  abort_index_segment_size: 134217728  # 128MB",
+        "----",
+        "",
+        "This setting controls the maximum size of abort index segments.",
+        "Smaller segments may improve memory usage but increase overhead."
+      ]
+    },
+    "append_chunk_size": {
+      "description": "Size of data chunks for append operations.",
+      "version": "v24.1.0",
+      "example": "67108864"
+    },
+    "cloud_storage_access_key": {
+      "description": "Access key for cloud storage authentication. Used to authenticate with S3-compatible object storage services."
+    }
+  }
+}

__tests__/tools/property-docs-overrides.test.js

@@ -0,0 +1,87 @@
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const repoRoot = path.resolve(__dirname, '..', '..');
+const overridesFile = path.join(repoRoot, '__tests__', 'docs-data', 'property-overrides.json');
+
+describe('property-docs description override', () => {
+  let tempDir;
+  let mockPropertiesFile;
+
+  beforeAll(() => {
+    tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'property-docs-test-'));
+
+    // Create mock property data that includes admin property
+    const mockProperties = {
+      properties: {
+        admin: {
+          config_scope: "broker",
+          default: [{ address: "127.0.0.1", port: 9644 }],
+          defined_in: "src/v/config/node_config.cc",
+          description: "Default description for admin",
+          name: "admin",
+          needs_restart: true,
+          nullable: false,
+          type: "array",
+          visibility: "user"
+        },
+        kafka_api: {
+          config_scope: "broker", 
+          default: [{ address: "127.0.0.1", port: 9092 }],
+          defined_in: "src/v/config/node_config.cc",
+          description: "IP address and port of the Kafka API endpoint that handles requests.",
+          name: "kafka_api",
+          needs_restart: true,
+          nullable: false,
+          type: "array", 
+          visibility: "user"
+        }
+      }
+    };
+
+    // Write mock properties to a temp file
+    mockPropertiesFile = path.join(tempDir, 'mock-properties.json');
+    fs.writeFileSync(mockPropertiesFile, JSON.stringify(mockProperties, null, 2));
+  });
+
+  afterAll(() => {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  });
+
+  it('applies the override description for admin property', () => {
+    const overrides = JSON.parse(fs.readFileSync(overridesFile, 'utf8'));
+    const mockProperties = JSON.parse(fs.readFileSync(mockPropertiesFile, 'utf8'));
+
+    // Test 1: Verify the override file structure
+    const adminOverride = overrides.properties.admin;
+    expect(adminOverride).toBeTruthy();
+    expect(adminOverride.description).toBeTruthy();
+    expect(adminOverride.version).toBe('v23.1.0');
+    expect(adminOverride.description).toBe('Network addresses for Admin API servers with version info.');
+
+    // Test 2: Verify our mock data has the correct structure (no artificial name field)
+    const adminProperty = mockProperties.properties.admin;
+    expect(adminProperty.default).toEqual([{ address: "127.0.0.1", port: 9644 }]);
+
+    const adminDefault = adminProperty.default[0];
+    expect(adminDefault).toHaveProperty('address', '127.0.0.1');
+    expect(adminDefault).toHaveProperty('port', 9644);
+
+    // Test 3: Simulate applying overrides (this is what the Python script would do)
+    const adminWithOverrides = {
+      ...adminProperty,
+      description: adminOverride.description,
+      version: adminOverride.version
+    };
+
+    expect(adminWithOverrides.description).toBe('Network addresses for Admin API servers with version info.');
+    expect(adminWithOverrides.version).toBe('v23.1.0');
+    expect(adminWithOverrides.default).toEqual([{ address: "127.0.0.1", port: 9644 }]);
+
+    // Test 4: Verify that kafka_api (without overrides) keeps its original description
+    const kafkaProperty = mockProperties.properties.kafka_api;
+    expect(kafkaProperty.description).toBe('IP address and port of the Kafka API endpoint that handles requests.');
+    expect(kafkaProperty.default).toEqual([{ address: "127.0.0.1", port: 9092 }]);
+  });
+});

bin/doc-tools.js

@@ -98,7 +98,7 @@ function requireCmd(cmd, help, versionFlag = '--version') {
  * @param {number} [minMinor=10] - Minimum required minor version of Python.
  */
 function requirePython(minMajor = 3, minMinor = 10) {
-  const candidates = ['python3', 'python'];
+  const candidates = ['python3', 'python', 'python3.12', 'python3.11', 'python3.10'];
   for (const p of candidates) {
     try {
       const out = execSync(`${p} --version`, { encoding: 'utf8' }).trim();
@@ -234,12 +234,16 @@ For more details, visit: https://github.com/norwoodj/helm-docs
 /**
  * Ensures all dependencies required for generating property documentation are installed.
  *
- * Checks for the presence of `make`, Python 3.10 or newer, C++ compiler, and C++ standard library headers.
+ * Checks for the presence of `make`, Python 3.10 or newer, Node.js, C++ compiler, and C++ standard library headers.
  * Exits the process with an error message if any dependency is missing.
  */
 function verifyPropertyDependencies() {
   requireCmd('make', 'Your OS package manager');
   requirePython();
+
+  // Check for Node.js (required for Handlebars templates)
+  requireCmd('node', 'https://nodejs.org/en/download/ or use your package manager (e.g., brew install node)');
+  requireCmd('npm', 'Usually installed with Node.js');
 
   // Check for C++ compiler
   let cppCompiler = null;
@@ -463,32 +467,34 @@ function runClusterDocs(mode, tag, options) {
   if (r.status !== 0) process.exit(r.status);
 }
 
-// helper to diff two autogenerated directories
-function diffDirs(kind, oldTag, newTag) {
-  const oldDir = path.join('autogenerated', oldTag, kind);
-  const newDir = path.join('autogenerated', newTag, kind);
-  const diffDir = path.join('autogenerated', 'diffs', kind, `${oldTag}_to_${newTag}`);
+// helper to diff two temporary directories
+function diffDirs(kind, oldTag, newTag, oldTempDir, newTempDir) {
+  const diffDir = path.join('tmp', 'diffs', kind, `${oldTag}_to_${newTag}`);
   const patch = path.join(diffDir, 'changes.patch');
 
-  if (!fs.existsSync(oldDir)) {
-    console.error(`❌ Cannot diff: missing ${oldDir}`);
+  if (!fs.existsSync(oldTempDir)) {
+    console.error(`❌ Cannot diff: missing ${oldTempDir}`);
     process.exit(1);
   }
-  if (!fs.existsSync(newDir)) {
-    console.error(`❌ Cannot diff: missing ${newDir}`);
+  if (!fs.existsSync(newTempDir)) {
+    console.error(`❌ Cannot diff: missing ${newTempDir}`);
     process.exit(1);
   }
 
   fs.mkdirSync(diffDir, { recursive: true });
 
-  const cmd = `diff -ru "${oldDir}" "${newDir}" > "${patch}" || true`;
+  const cmd = `diff -ru "${oldTempDir}" "${newTempDir}" > "${patch}" || true`;
   const res = spawnSync(cmd, { stdio: 'inherit', shell: true });
 
   if (res.error) {
     console.error(`❌ diff failed: ${res.error.message}`);
     process.exit(1);
   }
   console.log(`✅ Wrote patch: ${patch}`);
+
+  // Clean up temporary directories
+  fs.rmSync(oldTempDir, { recursive: true, force: true });
+  fs.rmSync(newTempDir, { recursive: true, force: true });
 }
 
 automation
@@ -747,31 +753,95 @@ automation
   .description('Generate JSON and AsciiDoc documentation for Redpanda configuration properties')
   .option('--tag <tag>', 'Git tag or branch to extract from', 'dev')
   .option('--diff <oldTag>', 'Also diff autogenerated properties from <oldTag> → <tag>')
+  .option('--overrides <path>', 'Optional JSON file with property description overrides')
+  .option('--output-dir <dir>', 'Where to write all generated files', 'modules/reference')
+  .option('--template-property-page <path>', 'Custom Handlebars template for property page layout')
+  .option('--template-property <path>', 'Custom Handlebars template for individual property sections')
+  .option('--template-deprecated <path>', 'Custom Handlebars template for deprecated properties page')
+  .option('--template-deprecated-property <path>', 'Custom Handlebars template for individual deprecated property sections')
   .action((options) => {
     verifyPropertyDependencies();
 
     const newTag = options.tag;
     const oldTag = options.diff;
+    const overridesPath = options.overrides;
+    const outputDir = options.outputDir;
     const cwd = path.resolve(__dirname, '../tools/property-extractor');
-    const make = (tag) => {
-      console.log(`⏳ Building property docs for ${tag}…`);
-      const r = spawnSync('make', ['build', `TAG=${tag}`], { cwd, stdio: 'inherit' });
+
+    const make = (tag, overrides, templates = {}, outputDir = 'modules/reference/', tempDir = null) => {
+      console.log(`⏳ Building property docs for ${tag}${tempDir ? ' (for diff)' : ''}…`);
+      const args = ['build', `TAG=${tag}`];
+
+      // Pass all paths as environment variables for consistency
+      const env = { ...process.env };
+      if (overrides) {
+        env.OVERRIDES = path.resolve(overrides);
+      }
+      if (templates.propertyPage) {
+        env.TEMPLATE_PROPERTY_PAGE = path.resolve(templates.propertyPage);
+      }
+      if (templates.property) {
+        env.TEMPLATE_PROPERTY = path.resolve(templates.property);
+      }
+      if (templates.deprecated) {
+        env.TEMPLATE_DEPRECATED = path.resolve(templates.deprecated);
+      }
+      if (templates.deprecatedProperty) {
+        env.TEMPLATE_DEPRECATED_PROPERTY = path.resolve(templates.deprecatedProperty);
+      }
+
+      if (tempDir) {
+        // For diff purposes, generate to temporary directory
+        env.OUTPUT_ASCIIDOC_DIR = path.resolve(tempDir);
+        env.OUTPUT_JSON_DIR = path.resolve(tempDir, 'examples');
+        env.OUTPUT_AUTOGENERATED_DIR = path.resolve(tempDir);
+      } else {
+        // Normal generation - go directly to final destination
+        // Let Makefile calculate OUTPUT_ASCIIDOC_DIR as OUTPUT_AUTOGENERATED_DIR/pages
+        env.OUTPUT_JSON_DIR = path.resolve(outputDir, 'examples');
+        env.OUTPUT_AUTOGENERATED_DIR = path.resolve(outputDir);
+      }
+
+      const r = spawnSync('make', args, { cwd, stdio: 'inherit', env });
       if (r.error) {
         console.error(`❌ ${r.error.message}`);
         process.exit(1);
       }
       if (r.status !== 0) process.exit(r.status);
     };
 
+    // Collect template options
+    const templates = {
+      propertyPage: options.templatePropertyPage,
+      property: options.templateProperty,
+      deprecated: options.templateDeprecated,
+      deprecatedProperty: options.templateDeprecatedProperty
+    };
+
+    let oldTempDir = null;
+    let newTempDir = null;
+
     if (oldTag) {
-      const oldDir = path.join('autogenerated', oldTag, 'properties');
-      if (!fs.existsSync(oldDir)) make(oldTag);
+      // Generate old version to temporary directory for diff
+      oldTempDir = path.join('tmp', 'diff', `${oldTag}-properties`);
+      fs.mkdirSync(oldTempDir, { recursive: true });
+      make(oldTag, overridesPath, templates, outputDir, oldTempDir);
     }
 
-    make(newTag);
-
     if (oldTag) {
-      diffDirs('properties', oldTag, newTag);
+      // Generate new version to temporary directory for diff
+      newTempDir = path.join('tmp', 'diff', `${newTag}-properties`);
+      fs.mkdirSync(newTempDir, { recursive: true });
+      make(newTag, overridesPath, templates, outputDir, newTempDir);
+
+      // Then generate new version to final destination
+      make(newTag, overridesPath, templates, outputDir);
+
+      // Compare the temporary directories
+      diffDirs('properties', oldTag, newTag, oldTempDir, newTempDir);
+    } else {
+      // No diff requested, just generate to final destination
+      make(newTag, overridesPath, templates, outputDir);
     }
 
     process.exit(0);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.7.4",
+  "version": "4.8.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",