Add support for excluding properties from docs (using Asciidoc tags) (#142)

* Add support for excluding properties from docs (using Asciidoc tags)

* Set env var

* Apply suggestion

* Apply suggestions

Author: JakeSCahill

__tests__/docs-data/property-overrides-exclude-test.json

@@ -0,0 +1,13 @@
+{
+  "properties": {
+    "admin": {
+      "description": "Network addresses for Admin API servers (excluded test)",
+      "version": "v25.2.8",
+      "exclude_from_docs": true
+    },
+    "cloud_storage_access_key": {
+      "description": "Access key for cloud storage authentication (included test)",
+      "version": "v25.2.8"
+    }
+  }
+}

bin/doc-tools.js

@@ -502,11 +502,11 @@ function generatePropertyComparisonReport(oldTag, newTag, outputDir) {
   try {
     console.log(`\n📊 Generating detailed property comparison report...`);
 
-    // Look for the property JSON files in modules/reference/examples
-    const repoRoot = findRepoRoot();
-    const examplesDir = path.join(repoRoot, 'modules', 'reference', 'examples');
-    const oldJsonPath = path.join(examplesDir, `${oldTag}-properties.json`);
-    const newJsonPath = path.join(examplesDir, `${newTag}-properties.json`);
+  // Look for the property JSON files in outputDir/examples
+  const repoRoot = findRepoRoot();
+  const examplesDir = path.join(repoRoot, outputDir, 'examples');
+  const oldJsonPath = path.join(examplesDir, `${oldTag}-properties.json`);
+  const newJsonPath = path.join(examplesDir, `${newTag}-properties.json`);
 
     // Check if JSON files exist
     if (!fs.existsSync(oldJsonPath)) {
@@ -1031,28 +1031,31 @@ automation
   });
 
 automation
-  .command('property-docs')
-  .description('Generate JSON and AsciiDoc documentation for Redpanda configuration properties. By default, only extracts properties to JSON. Use --generate-partials to create consolidated AsciiDoc partials (including deprecated properties). Use --generate-pages to create complete property pages that include the partials using AsciiDoc includes.')
+    .command('property-docs')
+  .description(
+      'Generate JSON and consolidated AsciiDoc partials for Redpanda configuration properties. ' +
+      'By default, only extracts properties to JSON. Use --generate-partials to create consolidated ' +
+      'AsciiDoc partials (including deprecated properties).'
+    )
   .option('--tag <tag>', 'Git tag or branch to extract from', 'dev')
   .option('--diff <oldTag>', 'Also diff autogenerated properties from <oldTag> to <tag>')
   .option('--overrides <path>', 'Optional JSON file with property description overrides')
   .option('--output-dir <dir>', 'Where to write all generated files', 'modules/reference')
   .option('--cloud-support', 'Add AsciiDoc tags to generated property docs to indicate which ones are supported in Redpanda Cloud. This data is fetched from the cloudv2 repository so requires a GitHub token with repo permissions. Set the token as an environment variable using GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN')
-  .option('--template-property-page <path>', 'Custom Handlebars template for property page layout')
   .option('--template-property <path>', 'Custom Handlebars template for individual property sections')
   .option('--template-topic-property <path>', 'Custom Handlebars template for individual topic property sections')
+  .option('--template-topic-property-mappings <path>', 'Custom Handlebars template for topic property mappings table')
   .option('--template-deprecated <path>', 'Custom Handlebars template for deprecated properties page')
   .option('--template-deprecated-property <path>', 'Custom Handlebars template for individual deprecated property sections')
   .option('--generate-partials', 'Generate consolidated property partials (cluster-properties.adoc, topic-properties.adoc, etc.) in the partials directory')
   .option('--partials-dir <path>', 'Directory for property partials (relative to output-dir)', 'partials')
-  .option('--generate-pages', 'Generate complete property pages that include the partials using AsciiDoc includes')
-  .action((options) => {
-    verifyPropertyDependencies();
+    .action((options) => {
+      verifyPropertyDependencies();
 
     // Validate cloud support dependencies if requested
     if (options.cloudSupport) {
       console.log('🔍 Validating cloud support dependencies...');
-      
+
       // Check for GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN
       const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN || process.env.REDPANDA_GITHUB_TOKEN;
       if (!token) {
@@ -1065,7 +1068,7 @@ automation
         console.error('   Or: export REDPANDA_GITHUB_TOKEN=your_token_here');
         process.exit(1);
       }
-      
+
       console.log('📦 Cloud support enabled - Python dependencies will be validated during execution');
       if (process.env.VIRTUAL_ENV) {
         console.log(`   Using virtual environment: ${process.env.VIRTUAL_ENV}`);
@@ -1074,97 +1077,87 @@ automation
       console.log('✅ GitHub token validated');
     }
 
-    const newTag = options.tag;
-    const oldTag = options.diff;
-    const overridesPath = options.overrides;
-    const outputDir = options.outputDir;
-    const cwd = path.resolve(__dirname, '../tools/property-extractor');
-    
-    const make = (tag, overrides, templates = {}, outputDir = 'modules/reference/', tempDir = null, cloudSupport = false) => {
-      console.log(`⏳ Building property docs for ${tag}${tempDir ? ' (for diff)' : ''}…`);
-      const args = ['build', `TAG=${tag}`];
-      
-      // Pass all paths as environment variables for consistency
-      const env = { ...process.env };
-      if (overrides) {
-        env.OVERRIDES = path.resolve(overrides);
-      }
-      if (cloudSupport) {
-        env.CLOUD_SUPPORT = '1';
-      }
-      if (templates.propertyPage) {
-        env.TEMPLATE_PROPERTY_PAGE = path.resolve(templates.propertyPage);
-        env.TEMPLATE_PROPERTY_PAGE_WITH_INCLUDES = env.TEMPLATE_PROPERTY_PAGE;
-      }
-      if (templates.property) {
-        env.TEMPLATE_PROPERTY = path.resolve(templates.property);
-      }
-      if (templates.topicProperty) {
-        env.TEMPLATE_TOPIC_PROPERTY = path.resolve(templates.topicProperty);
-      }
-      if (templates.deprecated) {
-        env.TEMPLATE_DEPRECATED = path.resolve(templates.deprecated);
-      }
-      if (templates.deprecatedProperty) {
-        env.TEMPLATE_DEPRECATED_PROPERTY = path.resolve(templates.deprecatedProperty);
-      }
-      
-      if (tempDir) {
-        env.OUTPUT_JSON_DIR = path.resolve(tempDir, 'examples');
-        env.OUTPUT_AUTOGENERATED_DIR = path.resolve(tempDir);
-      } else {
-        env.OUTPUT_JSON_DIR = path.resolve(outputDir, 'examples');
-        env.OUTPUT_AUTOGENERATED_DIR = path.resolve(outputDir);
-        // Set property files to go to properties subdirectory
-        env.OUTPUT_ASCIIDOC_DIR = path.resolve(outputDir, 'pages', 'properties');
-      }
-      
-      // Set partials generation options
-      if (options.generatePartials) {
-        env.GENERATE_PARTIALS = '1';
-        env.OUTPUT_PARTIALS_DIR = path.resolve(outputDir, options.partialsDir || 'partials');
-      }
-      
-      // Set page generation options
-      if (options.generatePages) {
-        env.GENERATE_PAGES = '1';
-        if (templates.propertyPage) {
-          env.TEMPLATE_PROPERTY_PAGE_WITH_INCLUDES = env.TEMPLATE_PROPERTY_PAGE;
+      const newTag = options.tag;
+      const oldTag = options.diff;
+      const overridesPath = options.overrides;
+      const outputDir = options.outputDir;
+      const cwd = path.resolve(__dirname, '../tools/property-extractor');
+
+      const make = (tag, overrides, templates = {}, outDir = 'modules/reference/', tempDir = null) => {
+        console.log(`⏳ Building property docs for ${tag}${tempDir ? ' (for diff)' : ''}…`);
+        const args = ['build', `TAG=${tag}`];
+
+        // Pass all paths as environment variables for consistency
+        const env = { ...process.env };
+
+        if (overrides) {
+          env.OVERRIDES = path.resolve(overrides);
+        }
+        if (options.cloudSupport) {
+          env.CLOUD_SUPPORT = '1';
+        }
+        if (templates.property) {
+          env.TEMPLATE_PROPERTY = path.resolve(templates.property);
+        }
+        if (templates.topicProperty) {
+          env.TEMPLATE_TOPIC_PROPERTY = path.resolve(templates.topicProperty);
+        }
+        if (templates.topicPropertyMappings) {
+          env.TEMPLATE_TOPIC_PROPERTY_MAPPINGS = path.resolve(templates.topicPropertyMappings);
+        }
+        if (templates.deprecated) {
+          env.TEMPLATE_DEPRECATED = path.resolve(templates.deprecated);
+        }
+        if (templates.deprecatedProperty) {
+          env.TEMPLATE_DEPRECATED_PROPERTY = path.resolve(templates.deprecatedProperty);
+        }
+
+        if (tempDir) {
+          env.OUTPUT_JSON_DIR = path.resolve(tempDir, 'examples');
+          env.OUTPUT_AUTOGENERATED_DIR = path.resolve(tempDir);
+        } else {
+          env.OUTPUT_JSON_DIR = path.resolve(outDir, 'examples');
+          env.OUTPUT_AUTOGENERATED_DIR = path.resolve(outDir);
         }
-      }
-      
-      const r = spawnSync('make', args, { cwd, stdio: 'inherit', env });
-      if (r.error) {
-        console.error(`❌ ${r.error.message}`);
-        process.exit(1);
-      }
-      if (r.status !== 0) process.exit(r.status);
-    };
 
-    // Collect template options
-    const templates = {
-      propertyPage: options.templatePropertyPage,
-      property: options.templateProperty,
-      topicProperty: options.templateTopicProperty,
-      deprecated: options.templateDeprecated,
-      deprecatedProperty: options.templateDeprecatedProperty
-    };
+        // Partials generation options
+        if (options.generatePartials) {
+          env.GENERATE_PARTIALS = '1';
+          env.OUTPUT_PARTIALS_DIR = path.resolve(outDir, options.partialsDir || 'partials');
+        }
 
-    if (oldTag) {
-      // Generate old version directly to final destination so its JSON is available for comparison
-      make(oldTag, overridesPath, templates, outputDir, null, options.cloudSupport);
-    }
+        const r = spawnSync('make', args, { cwd, stdio: 'inherit', env });
+        if (r.error) {
+          console.error(`❌ ${r.error.message}`);
+          process.exit(1);
+        }
+        if (r.status !== 0) process.exit(r.status);
+      };
+
+      // Collect template options
+      const templates = {
+        property: options.templateProperty,
+        topicProperty: options.templateTopicProperty,
+        topicPropertyMappings: options.templateTopicPropertyMappings,
+        deprecated: options.templateDeprecated,
+        deprecatedProperty: options.templateDeprecatedProperty,
+      };
+
+      if (oldTag) {
+        // Build old version first so its JSON exists for the diff step
+        make(oldTag, overridesPath, templates, outputDir, null);
+      }
 
-    // Generate new version to final destination
-    make(newTag, overridesPath, templates, outputDir, null, options.cloudSupport);
+      // Build new version
+      make(newTag, overridesPath, templates, outputDir, null);
 
-    if (oldTag) {
-      // Generate property comparison report using the JSON files now in modules/reference/examples
-      generatePropertyComparisonReport(oldTag, newTag, 'modules/reference');
-    }
+      if (oldTag) {
+  // Generate property comparison report using the JSON now in outputDir/examples
+  generatePropertyComparisonReport(oldTag, newTag, outputDir);
+      }
 
-    process.exit(0);
-  });
+      process.exit(0);
+    });
 
 automation
   .command('rpk-docs')
@@ -1563,7 +1556,7 @@ automation
 automation
   .command('bundle-openapi')
   .description('Bundle Redpanda OpenAPI fragments for admin and connect APIs into complete OpenAPI 3.1 documents')
-  .requiredOption('-t, --tag <tag>', 'Git tag to check out (e.g., v24.3.2 or 24.3.2 or dev)')
+  .requiredOption('-t, --tag <tag>', 'Branch or tag to clone from the repository (for example v24.3.2 or 24.3.2 or dev)')
   .option('--repo <url>', 'Repository URL', 'https://github.com/redpanda-data/redpanda.git')
   .addOption(new Option('-s, --surface <surface>', 'Which API surface(s) to bundle').choices(['admin', 'connect', 'both']).makeOptionMandatory())
   .option('--out-admin <path>', 'Output path for admin API', 'admin/redpanda-admin-api.yaml')

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.10.3",
+  "version": "4.10.4",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package.json

@@ -148,7 +148,7 @@ function compareProperties(oldData, newData, oldVersion, newVersion) {
         name,
         type: prop.type,
         default: prop.default,
-        description: prop.description ? prop.description.substring(0, 100) + '...' : 'No description'
+        description: prop.description || 'No description'
       });
     }
   }
@@ -185,8 +185,8 @@ function compareProperties(oldData, newData, oldVersion, newVersion) {
       if (oldProp.description !== newProp.description) {
         report.changedDescriptions.push({
           name,
-          oldDescription: oldProp.description ? oldProp.description.substring(0, 50) + '...' : 'No description',
-          newDescription: newProp.description ? newProp.description.substring(0, 50) + '...' : 'No description'
+          oldDescription: oldProp.description || 'No description',
+          newDescription: newProp.description || 'No description'
         });
       }
 
@@ -203,7 +203,7 @@ function compareProperties(oldData, newData, oldVersion, newVersion) {
       report.removedProperties.push({
         name,
         type: oldProp.type,
-        description: oldProp.description ? oldProp.description.substring(0, 100) + '...' : 'No description'
+        description: oldProp.description || 'No description'
       });
     }
   }