diff --git a/__tests__/docs-data/property-overrides.json b/__tests__/docs-data/property-overrides.json index 2b7d88e..0404db5 100644 --- a/__tests__/docs-data/property-overrides.json +++ b/__tests__/docs-data/property-overrides.json @@ -47,7 +47,12 @@ "example": "67108864" }, "cloud_storage_access_key": { - "description": "Access key for cloud storage authentication. Used to authenticate with S3-compatible object storage services." + "description": "Access key for cloud storage authentication. Used to authenticate with S3-compatible object storage services.", + "related_topics": [ + "xref:manage:tiered-storage.adoc[Tiered Storage]", + "xref:reference:object-storage-credentials.adoc[Object Storage Credentials]", + "xref:deploy:deployment-option/self-hosted/manual/tiered-storage.adoc[Configure Tiered Storage]" + ] }, "cleanup.policy": { "description": "Determines how log segments are cleaned. **delete** removes old segments based on time or size. **compact** retains only the latest value for each key. **compact,delete** enables both strategies.", @@ -76,6 +81,28 @@ "rpk topic alter-config my-topic --set compression.type=zstd", "----" ] + }, + "custom.topic.property": { + "description": "A custom topic property that isn't extracted from source code but needs documentation.", + "type": "string", + "default": "default_value", + "config_scope": "topic", + "version": "v25.1.0", + "example": [ + ".Example: Setting custom topic property", + "[,bash]", + "----", + "rpk topic alter-config my-topic --set custom.topic.property=custom_value", + "----" + ], + "related_topics": ["topic-management", "configuration"] + }, + "new.cluster.setting": { + "description": "A new cluster-level configuration that needs manual documentation.", + "type": "integer", + "default": 100, + "config_scope": "cluster", + "version": "v25.2.0" } } } diff --git a/bin/doc-tools.js b/bin/doc-tools.js index 38211ad..6b9e5dd 100755 --- a/bin/doc-tools.js +++ b/bin/doc-tools.js @@ -459,18 +459,120 @@ const commonOptions = { consoleDockerRepo: 'console', }; +/** + * Run the cluster documentation generator script for a specific release/tag. + * + * Invokes the external `generate-cluster-docs.sh` script with the provided mode, tag, + * and Docker-related options. The script's stdout/stderr are forwarded to the current + * process; if the script exits with a non-zero status, this function will terminate + * the Node.js process with that status code. + * + * @param {string} mode - Operation mode passed to the script (e.g., "generate" or "clean"). + * @param {string} tag - Release tag or version to generate docs for. + * @param {Object} options - Runtime options. + * @param {string} options.dockerRepo - Docker repository used by the script. + * @param {string} options.consoleTag - Console image tag passed to the script. + * @param {string} options.consoleDockerRepo - Console Docker repository used by the script. + */ function runClusterDocs(mode, tag, options) { const script = path.join(__dirname, '../cli-utils/generate-cluster-docs.sh'); const args = [mode, tag, options.dockerRepo, options.consoleTag, options.consoleDockerRepo]; console.log(`โณ Running ${script} with arguments: ${args.join(' ')}`); - const r = spawnSync('bash', [script, ...args], { stdio: 'inherit', shell: true }); + const r = spawnSync('bash', [script, ...args], { stdio: 'inherit' }); if (r.status !== 0) process.exit(r.status); } -// helper to diff two temporary directories +/** + * Generate a detailed JSON report describing property changes between two releases. + * + * Looks for `-properties.json` and `-properties.json` in + * `modules/reference/examples`. If both files exist, invokes the external + * property comparison tool to produce `property-changes--to-.json` + * in the provided output directory. + * + * If either input JSON is missing the function logs a message and returns without + * error. Any errors from the comparison tool are logged; the function does not + * throw. + * + * @param {string} oldTag - Release tag or identifier for the "old" properties set. + * @param {string} newTag - Release tag or identifier for the "new" properties set. + * @param {string} outputDir - Directory where the comparison report will be written. + */ +function generatePropertyComparisonReport(oldTag, newTag, outputDir) { + try { + console.log(`\n๐Ÿ“Š Generating detailed property comparison report...`); + + // Look for the property JSON files in modules/reference/examples + const repoRoot = findRepoRoot(); + const examplesDir = path.join(repoRoot, 'modules', 'reference', 'examples'); + const oldJsonPath = path.join(examplesDir, `${oldTag}-properties.json`); + const newJsonPath = path.join(examplesDir, `${newTag}-properties.json`); + + // Check if JSON files exist + if (!fs.existsSync(oldJsonPath)) { + console.log(`โš ๏ธ Old properties JSON not found at: ${oldJsonPath}`); + console.log(` Skipping detailed property comparison.`); + return; + } + + if (!fs.existsSync(newJsonPath)) { + console.log(`โš ๏ธ New properties JSON not found at: ${newJsonPath}`); + console.log(` Skipping detailed property comparison.`); + return; + } + + // Ensure output directory exists (use absolute path) + const absoluteOutputDir = path.resolve(outputDir); + fs.mkdirSync(absoluteOutputDir, { recursive: true }); + + // Run the property comparison tool with descriptive filename + const propertyExtractorDir = path.resolve(__dirname, '../tools/property-extractor'); + const compareScript = path.join(propertyExtractorDir, 'compare-properties.js'); + const reportFilename = `property-changes-${oldTag}-to-${newTag}.json`; + const reportPath = path.join(absoluteOutputDir, reportFilename); + const args = [compareScript, oldJsonPath, newJsonPath, oldTag, newTag, absoluteOutputDir, reportFilename]; + + const result = spawnSync('node', args, { + stdio: 'inherit', + cwd: propertyExtractorDir + }); + + if (result.error) { + console.error(`โŒ Property comparison failed: ${result.error.message}`); + } else if (result.status !== 0) { + console.error(`โŒ Property comparison exited with code: ${result.status}`); + } else { + console.log(`โœ… Property comparison report saved to: ${reportPath}`); + } + } catch (error) { + console.error(`โŒ Error generating property comparison: ${error.message}`); + } +} + +/** + * Create a unified diff patch between two temporary directories and clean them up. + * + * Ensures both source directories exist, writes a recursive unified diff + * (changes.patch) to tmp/diffs//_to_/, and removes the + * provided temporary directories. On missing inputs or if the diff subprocess + * fails to spawn, the process exits with a non-zero status. + * + * @param {string} kind - Logical category for the diff (e.g., "metrics" or "rpk"); used in the output path. + * @param {string} oldTag - Identifier for the "old" version (used in the output path). + * @param {string} newTag - Identifier for the "new" version (used in the output path). + * @param {string} oldTempDir - Path to the existing temporary directory containing the old output; must exist. + * @param {string} newTempDir - Path to the existing temporary directory containing the new output; must exist. + */ function diffDirs(kind, oldTag, newTag, oldTempDir, newTempDir) { + // Backwards compatibility: if temp directories not provided, use autogenerated paths + if (!oldTempDir) { + oldTempDir = path.join('autogenerated', oldTag, kind); + } + if (!newTempDir) { + newTempDir = path.join('autogenerated', newTag, kind); + } + const diffDir = path.join('tmp', 'diffs', kind, `${oldTag}_to_${newTag}`); - const patch = path.join(diffDir, 'changes.patch'); if (!fs.existsSync(oldTempDir)) { console.error(`โŒ Cannot diff: missing ${oldTempDir}`); @@ -483,6 +585,8 @@ function diffDirs(kind, oldTag, newTag, oldTempDir, newTempDir) { fs.mkdirSync(diffDir, { recursive: true }); + // Generate traditional patch for metrics and rpk + const patch = path.join(diffDir, 'changes.patch'); const cmd = `diff -ru "${oldTempDir}" "${newTempDir}" > "${patch}" || true`; const res = spawnSync(cmd, { stdio: 'inherit', shell: true }); @@ -492,9 +596,35 @@ function diffDirs(kind, oldTag, newTag, oldTempDir, newTempDir) { } console.log(`โœ… Wrote patch: ${patch}`); - // Clean up temporary directories - fs.rmSync(oldTempDir, { recursive: true, force: true }); - fs.rmSync(newTempDir, { recursive: true, force: true }); + // Safety guard: only clean up directories that are explicitly passed as temp directories + // For backwards compatibility with autogenerated paths, don't clean up automatically + const tmpRoot = path.resolve('tmp') + path.sep; + const workspaceRoot = path.resolve('.') + path.sep; + + // Only clean up if directories were explicitly provided as temp directories + // (indicated by having all 5 parameters) and they're in the tmp/ directory + const explicitTempDirs = arguments.length >= 5; + + if (explicitTempDirs) { + [oldTempDir, newTempDir].forEach(dirPath => { + const resolvedPath = path.resolve(dirPath) + path.sep; + const isInTmp = resolvedPath.startsWith(tmpRoot); + const isInWorkspace = resolvedPath.startsWith(workspaceRoot); + + if (isInWorkspace && isInTmp) { + try { + fs.rmSync(dirPath, { recursive: true, force: true }); + console.log(`๐Ÿงน Cleaned up temporary directory: ${dirPath}`); + } catch (err) { + console.warn(`โš ๏ธ Warning: Could not clean up directory ${dirPath}: ${err.message}`); + } + } else { + console.log(`โ„น๏ธ Skipping cleanup of directory outside tmp/: ${dirPath}`); + } + }); + } else { + console.log(`โ„น๏ธ Using autogenerated directories - skipping cleanup for safety`); + } } automation @@ -755,20 +885,47 @@ automation .option('--diff ', 'Also diff autogenerated properties from โ†’ ') .option('--overrides ', 'Optional JSON file with property description overrides') .option('--output-dir ', 'Where to write all generated files', 'modules/reference') + .option('--cloud-support', 'Enable cloud support metadata by fetching configuration from the cloudv2 repository (requires GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN)') .option('--template-property-page ', 'Custom Handlebars template for property page layout') .option('--template-property ', 'Custom Handlebars template for individual property sections') + .option('--template-topic-property ', 'Custom Handlebars template for individual topic property sections') .option('--template-deprecated ', 'Custom Handlebars template for deprecated properties page') .option('--template-deprecated-property ', 'Custom Handlebars template for individual deprecated property sections') .action((options) => { verifyPropertyDependencies(); + // Validate cloud support dependencies if requested + if (options.cloudSupport) { + console.log('๐Ÿ” Validating cloud support dependencies...'); + + // Check for GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN + const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN || process.env.REDPANDA_GITHUB_TOKEN; + if (!token) { + console.error('โŒ Cloud support requires GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN environment variable'); + console.error(' Set up GitHub token:'); + console.error(' 1. Go to https://github.com/settings/tokens'); + console.error(' 2. Generate token with "repo" scope'); + console.error(' 3. Set: export GITHUB_TOKEN=your_token_here'); + console.error(' Or: export GH_TOKEN=your_token_here'); + console.error(' Or: export REDPANDA_GITHUB_TOKEN=your_token_here'); + process.exit(1); + } + + console.log('๐Ÿ“ฆ Cloud support enabled - Python dependencies will be validated during execution'); + if (process.env.VIRTUAL_ENV) { + console.log(` Using virtual environment: ${process.env.VIRTUAL_ENV}`); + } + console.log(' Required packages: pyyaml, requests'); + console.log('โœ… GitHub token validated'); + } + const newTag = options.tag; const oldTag = options.diff; const overridesPath = options.overrides; const outputDir = options.outputDir; const cwd = path.resolve(__dirname, '../tools/property-extractor'); - const make = (tag, overrides, templates = {}, outputDir = 'modules/reference/', tempDir = null) => { + const make = (tag, overrides, templates = {}, outputDir = 'modules/reference/', tempDir = null, cloudSupport = false) => { console.log(`โณ Building property docs for ${tag}${tempDir ? ' (for diff)' : ''}โ€ฆ`); const args = ['build', `TAG=${tag}`]; @@ -777,12 +934,18 @@ automation if (overrides) { env.OVERRIDES = path.resolve(overrides); } + if (cloudSupport) { + env.CLOUD_SUPPORT = '1'; + } if (templates.propertyPage) { env.TEMPLATE_PROPERTY_PAGE = path.resolve(templates.propertyPage); } if (templates.property) { env.TEMPLATE_PROPERTY = path.resolve(templates.property); } + if (templates.topicProperty) { + env.TEMPLATE_TOPIC_PROPERTY = path.resolve(templates.topicProperty); + } if (templates.deprecated) { env.TEMPLATE_DEPRECATED = path.resolve(templates.deprecated); } @@ -791,13 +954,9 @@ automation } if (tempDir) { - // For diff purposes, generate to temporary directory - env.OUTPUT_ASCIIDOC_DIR = path.resolve(tempDir); env.OUTPUT_JSON_DIR = path.resolve(tempDir, 'examples'); env.OUTPUT_AUTOGENERATED_DIR = path.resolve(tempDir); } else { - // Normal generation - go directly to final destination - // Let Makefile calculate OUTPUT_ASCIIDOC_DIR as OUTPUT_AUTOGENERATED_DIR/pages env.OUTPUT_JSON_DIR = path.resolve(outputDir, 'examples'); env.OUTPUT_AUTOGENERATED_DIR = path.resolve(outputDir); } @@ -814,34 +973,22 @@ automation const templates = { propertyPage: options.templatePropertyPage, property: options.templateProperty, + topicProperty: options.templateTopicProperty, deprecated: options.templateDeprecated, deprecatedProperty: options.templateDeprecatedProperty }; - let oldTempDir = null; - let newTempDir = null; - if (oldTag) { - // Generate old version to temporary directory for diff - oldTempDir = path.join('tmp', 'diff', `${oldTag}-properties`); - fs.mkdirSync(oldTempDir, { recursive: true }); - make(oldTag, overridesPath, templates, outputDir, oldTempDir); + // Generate old version directly to final destination so its JSON is available for comparison + make(oldTag, overridesPath, templates, outputDir, null, options.cloudSupport); } + // Generate new version to final destination + make(newTag, overridesPath, templates, outputDir, null, options.cloudSupport); + if (oldTag) { - // Generate new version to temporary directory for diff - newTempDir = path.join('tmp', 'diff', `${newTag}-properties`); - fs.mkdirSync(newTempDir, { recursive: true }); - make(newTag, overridesPath, templates, outputDir, newTempDir); - - // Then generate new version to final destination - make(newTag, overridesPath, templates, outputDir); - - // Compare the temporary directories - diffDirs('properties', oldTag, newTag, oldTempDir, newTempDir); - } else { - // No diff requested, just generate to final destination - make(newTag, overridesPath, templates, outputDir); + // Generate property comparison report using the JSON files now in modules/reference/examples + generatePropertyComparisonReport(oldTag, newTag, 'modules/reference'); } process.exit(0); @@ -1050,9 +1197,9 @@ automation const { generateCloudRegions } = require('../tools/cloud-regions/generate-cloud-regions.js'); try { - const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN; + const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN || process.env.REDPANDA_GITHUB_TOKEN; if (!token) { - throw new Error('GITHUB_TOKEN environment variable is required to fetch from private cloudv2-infra repo.'); + throw new Error('GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN environment variable is required to fetch from private cloudv2-infra repo.'); } const fmt = (options.format || 'md').toLowerCase(); let templatePath = undefined; diff --git a/package-lock.json b/package-lock.json index 78f264e..c6c3364 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@redpanda-data/docs-extensions-and-macros", - "version": "4.8.1", + "version": "4.9.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@redpanda-data/docs-extensions-and-macros", - "version": "4.8.1", + "version": "4.9.0", "license": "ISC", "dependencies": { "@asciidoctor/tabs": "^1.0.0-beta.6", diff --git a/package.json b/package.json index df41027..3e9a621 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@redpanda-data/docs-extensions-and-macros", - "version": "4.8.1", + "version": "4.9.0", "description": "Antora extensions and macros developed for Redpanda documentation.", "keywords": [ "antora", diff --git a/tools/property-extractor/Makefile b/tools/property-extractor/Makefile index 79b3c50..65018d7 100644 --- a/tools/property-extractor/Makefile +++ b/tools/property-extractor/Makefile @@ -1,17 +1,5 @@ .PHONY: build venv clean redpanda-git treesitter generate-docs check -# --- Main build: venv, fetch code, build parser, extract & docgen --- -build: venv redpanda-git treesitter - @echo "๐Ÿ”ง Building with Redpanda tag: $(TAG)" - @mkdir -p $(TOOL_ROOT)/gen - @cd $(TOOL_ROOT) && \ - $(PYTHON) -W ignore::FutureWarning property_extractor.py \ - --recursive \ - --path $(REDPANDA_SRC) \ - --output gen/properties-output.json - @echo "โœ… Cluster properties JSON generated at $(TOOL_ROOT)/gen/properties-output.json" - @$(MAKE) generate-docs - # Default tag (can be overridden via `make TAG=v25.1.1`) TAG ?= dev @@ -51,12 +39,13 @@ build: venv redpanda-git treesitter exit 1; \ fi @cd $(TOOL_ROOT) && \ - $(PYTHON) -W ignore::FutureWarning property_extractor.py \ + $(PYTHON) -W ignore property_extractor.py \ --recursive \ --path $(REDPANDA_SRC) \ --output gen/properties-output.json \ --enhanced-output gen/$(TAG)-properties.json \ - $(if $(OVERRIDES),$(if $(shell [ -f "$(OVERRIDES)" ] && echo exists),--overrides $(OVERRIDES),),) + $(if $(OVERRIDES),$(if $(shell [ -f "$(OVERRIDES)" ] && echo exists),--overrides $(OVERRIDES),),) \ + $(if $(CLOUD_SUPPORT),--cloud-support,) @echo "โœ… JSON generated at $(TOOL_ROOT)/gen/properties-output.json" @echo "โœ… Enhanced JSON (with overrides) generated at $(TOOL_ROOT)/gen/$(TAG)-properties.json" @$(MAKE) generate-docs @@ -66,11 +55,12 @@ venv: $(TOOL_ROOT)/requirements.txt @if [ ! -d "$(VENV)" ]; then \ echo "๐Ÿ Creating virtual environment in $(VENV)..."; \ python3 -m venv $(VENV); \ - $(VENV)/bin/pip install --upgrade pip --quiet; \ - $(VENV)/bin/pip install --no-cache-dir -r $<; \ else \ echo "๐Ÿ Virtual environment already exists at $(VENV)"; \ - fi + fi; \ + echo "๐Ÿ”„ Upgrading pip and installing requirements..."; \ + $(VENV)/bin/pip install --upgrade pip --quiet; \ + $(VENV)/bin/pip install --no-cache-dir -r $<; # --- Clean out all generated state --- clean: @@ -106,7 +96,7 @@ treesitter: git fetch --tags -q && \ git checkout -q v0.20.5 @echo "๐Ÿ”ง Generating parser in $(TREESITTER_DIR)โ€ฆ" - @cd "$(TREESITTER_DIR)" && npm install --silent && $(TREE_SITTER) generate + @cd "$(TREESITTER_DIR)" && export CFLAGS="-Wno-unused-but-set-variable" && npm install --silent && export CFLAGS="-Wno-unused-but-set-variable" && $(TREE_SITTER) generate # --- Install Node.js dependencies for Handlebars --- node-deps: diff --git a/tools/property-extractor/cloud_config.py b/tools/property-extractor/cloud_config.py new file mode 100644 index 0000000..9693a2f --- /dev/null +++ b/tools/property-extractor/cloud_config.py @@ -0,0 +1,594 @@ +#!/usr/bin/env python3 +""" +Cloud configuration integration for property documentation generation. + +This module fetches cloud configuration from the cloudv2 repository to determine +which Redpanda properties are supported, editable, or readonly in cloud deployments. + +Prerequisites: + - GITHUB_TOKEN environment variable set with appropriate permissions + - Internet connection to access GitHub API + +Usage: + from cloud_config import fetch_cloud_config, add_cloud_support_metadata + + # Fetch cloud configuration + config = fetch_cloud_config() + if config: + properties = add_cloud_support_metadata(properties, config) + +Error Handling: + - Network errors: Logged with retry suggestions + - Authentication errors: Clear instructions for token setup + - Parsing errors: Specific file and line information + - Missing dependencies: Installation instructions provided +""" + +import os +import json +import logging +from dataclasses import dataclass +from typing import Dict, Set, Optional, List + +# Check for required dependencies early +try: + import requests +except ImportError as e: + raise ImportError("Missing required dependency 'requests': install with pip install requests") from e + +try: + import yaml +except ImportError as e: + raise ImportError("Missing required dependency 'PyYAML': install with pip install pyyaml") from e + +# Set up logging with production-ready configuration +logger = logging.getLogger(__name__) + +class CloudConfigError(Exception): + """Base exception for cloud configuration errors.""" + pass + +class GitHubAuthError(CloudConfigError): + """Raised when GitHub authentication fails.""" + pass + +class CloudConfigParsingError(CloudConfigError): + """Raised when cloud configuration parsing fails.""" + pass + +class NetworkError(CloudConfigError): + """Raised when network operations fail.""" + pass + +@dataclass +class CloudConfig: + """Cloud configuration data for a specific version.""" + version: str + customer_managed_configs: List[Dict] + readonly_cluster_config: List[str] + + def get_editable_properties(self) -> Set[str]: + """Get set of property names that customers can edit.""" + return {config.get('name') for config in self.customer_managed_configs if config.get('name')} + + def get_readonly_properties(self) -> Set[str]: + """Get set of property names that are read-only for customers.""" + return set(self.readonly_cluster_config) + + def get_all_cloud_properties(self) -> Set[str]: + """ + Return the set of all property names present in the cloud configuration (union of editable and readonly). + + Returns: + Set[str]: Property names that are either customer-editable or readonly in cloud deployments. + """ + return self.get_editable_properties() | self.get_readonly_properties() + + def is_byoc_only(self, property_name: str) -> bool: + """ + Return True if the given property is defined in customer_managed_configs and its `cluster_types` list is exactly ['byoc']. + + Parameters: + property_name (str): Name of the property to check. + + Returns: + bool: True when a matching config entry exists and its `cluster_types` equals ['byoc']; False otherwise. + """ + for config in self.customer_managed_configs: + if config.get('name') == property_name: + cluster_types = config.get('cluster_types', []) + return cluster_types == ['byoc'] + return False + + +def fetch_cloud_config(github_token: Optional[str] = None) -> CloudConfig: + """ + Fetch the latest cloud configuration from the redpanda-data/cloudv2 repository and return it as a CloudConfig. + + This function uses a GitHub personal access token for authentication. If `github_token` is not provided, it will read GITHUB_TOKEN or REDPANDA_GITHUB_TOKEN from the environment. It downloads the most recent versioned YAML from the repository's install-pack directory, validates expected sections (`customer_managed_configs` and `readonly_cluster_config`), and constructs a CloudConfig instance. + + Parameters: + github_token (Optional[str]): Personal access token for GitHub API. If omitted, the function will try environment variables GITHUB_TOKEN or REDPANDA_GITHUB_TOKEN. + + Returns: + CloudConfig: Parsed cloud configuration for the latest available version. + + Raises: + GitHubAuthError: Authentication or access problems with the GitHub API (including 401/403 responses). + NetworkError: Network connectivity or timeout failures when contacting the GitHub API. + CloudConfigParsingError: Failure to parse or validate the repository YAML files or their expected structure. + CloudConfigError: Generic configuration error (e.g., missing token) or unexpected internal failures. + """ + if not github_token: + github_token = os.environ.get('GITHUB_TOKEN') or os.environ.get('REDPANDA_GITHUB_TOKEN') + + if not github_token: + error_msg = ( + "No GitHub token provided.\n" + "Cloud configuration requires authentication to access private repositories.\n" + "To fix this:\n" + "1. Go to https://github.com/settings/tokens\n" + "2. Generate a personal access token with 'repo' scope\n" + "3. Set the token: export GITHUB_TOKEN=your_token_here\n" + "4. Re-run the command with --cloud-support flag" + ) + logger.error(error_msg) + raise GitHubAuthError(error_msg) + + headers = { + 'Authorization': f'token {github_token}', + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'Redpanda-Docs-Property-Extractor/1.0' + } + + try: + # First, list all YAML files in the install-pack directory + logger.info("Fetching install-pack directory listing from cloudv2 repository...") + url = 'https://api.github.com/repos/redpanda-data/cloudv2/contents/install-pack' + + response = requests.get(url, headers=headers, timeout=30) + + # Handle common HTTP error responses with detailed guidance + if response.status_code in [401, 403, 404]: + status_messages = { + 401: { + "title": "GitHub authentication failed (HTTP 401)", + "causes": [ + "Invalid or expired GitHub token", + "Token lacks 'repo' scope for private repositories", + "Token user doesn't have access to redpanda-data/cloudv2" + ], + "fixes": [ + "Verify token at: https://github.com/settings/tokens", + "Ensure 'repo' scope is enabled", + "Contact team lead if access is needed to cloudv2 repository" + ], + "exception": GitHubAuthError + }, + 403: { + "title": f"GitHub API access denied (HTTP 403)", + "causes": [ + "API rate limit exceeded (5000 requests/hour for authenticated users)", + "Repository access denied" + ], + "fixes": [ + "Wait for rate limit reset if exceeded", + "Verify repository access permissions", + "Contact team lead if repository access is needed" + ], + "extra": f"Rate limit remaining: {response.headers.get('X-RateLimit-Remaining', 'unknown')}\n" + f"Rate limit resets at: {response.headers.get('X-RateLimit-Reset', 'unknown')}", + "exception": GitHubAuthError + }, + 404: { + "title": "Install-pack directory not found (HTTP 404)", + "causes": [ + "Directory 'install-pack' doesn't exist in cloudv2 repository", + "Repository 'redpanda-data/cloudv2' not accessible", + "Directory path has changed" + ], + "fixes": [ + "Verify directory exists in repository", + "Check if directory path has changed", + "Contact cloud team for current configuration location" + ], + "exception": NetworkError + } + } + + msg_config = status_messages[response.status_code] + error_msg = f"{msg_config['title']}.\n" + error_msg += f"Possible causes:\n" + for i, cause in enumerate(msg_config['causes'], 1): + error_msg += f"{i}. {cause}\n" + if 'extra' in msg_config: + error_msg += f"{msg_config['extra']}\n" + error_msg += f"\nTo fix:\n" + for i, fix in enumerate(msg_config['fixes'], 1): + error_msg += f"{i}. {fix}\n" + + logger.error(error_msg) + raise msg_config['exception'](error_msg) + + response.raise_for_status() + + try: + files = response.json() + except ValueError as e: + error_msg = ( + f"Invalid JSON response from GitHub API.\n" + "This indicates an API format change or server error.\n" + "Contact development team to update integration." + ) + logger.exception(error_msg) + raise CloudConfigParsingError(error_msg) from e + + if not isinstance(files, list): + error_msg = ( + f"Expected list of files, got {type(files)}: {files}\n" + "This indicates an API format change.\n" + "Contact development team to update integration." + ) + logger.error(error_msg) + raise CloudConfigParsingError(error_msg) + + # Find YAML files with version numbers + version_files = [] + for file in files: + if not isinstance(file, dict): + logger.warning(f"Skipping non-dictionary file entry: {file}") + continue + + file_name = file.get('name', '') + download_url = file.get('download_url', '') + + if not file_name or not download_url: + logger.warning(f"Skipping file with missing name/url: {file}") + continue + + # Look for version YAML files (e.g., "25.1.yml", "25.2.yml") + if file_name.endswith('.yml'): + version_part = file_name.replace('.yml', '') + # Check if it looks like a version number (e.g., "25.1", "25.2.1") + if version_part.replace('.', '').isdigit(): + version_files.append((version_part, download_url)) + logger.debug(f"Found version file: {file_name} -> {version_part}") + + if not version_files: + error_msg = ( + "No version YAML files found in cloudv2/install-pack directory.\n" + "Expected files like '25.1.yml', '25.2.yml', etc.\n" + "Available files: " + ", ".join([f.get('name', 'unknown') for f in files]) + "\n" + "Contact cloud team to verify configuration file naming convention." + ) + logger.error(error_msg) + raise CloudConfigParsingError(error_msg) + + # Parse and filter valid version entries before sorting + valid_versions = [] + for version_str, download_url in version_files: + try: + # Parse version string into tuple of integers + version_tuple = tuple(int(part) for part in version_str.split('.')) + valid_versions.append((version_tuple, version_str, download_url)) + logger.debug(f"Valid version parsed: {version_str} -> {version_tuple}") + except ValueError as e: + logger.warning(f"Skipping invalid version format: {version_str} (error: {e})") + continue + + # Check if we have any valid versions + if not valid_versions: + error_msg = ( + "No valid version files found in cloudv2/install-pack directory.\n" + f"Found {len(version_files)} files but none had valid version formats.\n" + f"Available files: {[v[0] for v in version_files]}\n" + "Expected version format: 'X.Y' or 'X.Y.Z' (e.g., '25.1', '25.2.1')\n" + "Contact cloud team to verify configuration file naming convention." + ) + logger.error(error_msg) + raise CloudConfigParsingError(error_msg) + + # Sort by parsed version tuple and get the latest + valid_versions.sort(key=lambda x: x[0]) # Sort by version tuple + latest_version_tuple, latest_version, download_url = valid_versions[-1] + + logger.info(f"Found {len(valid_versions)} valid version files, using latest: {latest_version}") + logger.info(f"Valid versions: {[v[1] for v in valid_versions]}") + if len(version_files) > len(valid_versions): + logger.info(f"Skipped {len(version_files) - len(valid_versions)} invalid version files") + + # Download the latest version file + logger.info(f"Downloading configuration file for version {latest_version}...") + response = requests.get(download_url, headers=headers, timeout=60) + response.raise_for_status() + + # Parse YAML content + try: + config_data = yaml.safe_load(response.text) + except yaml.YAMLError as e: + error_msg = ( + f"Failed to parse cloud configuration YAML for version {latest_version}: {e}\n" + f"File URL: {download_url}\n" + "The configuration file contains invalid YAML syntax.\n" + "Contact cloud team to fix configuration file.\n" + f"Parse error details: {str(e)}" + ) + logger.error(error_msg) + raise CloudConfigParsingError(error_msg) + + if not isinstance(config_data, dict): + error_msg = ( + f"Cloud configuration root is not a dictionary: {type(config_data)}\n" + f"Version: {latest_version}\n" + "Expected YAML file to contain a dictionary at root level.\n" + "Contact cloud team to verify configuration file format." + ) + logger.error(error_msg) + raise CloudConfigParsingError(error_msg) + + # Extract and validate the relevant sections + customer_managed = config_data.get('customer_managed_configs', []) + readonly_config = config_data.get('readonly_cluster_config', []) + + if not isinstance(customer_managed, list): + error_msg = ( + f"'customer_managed_configs' section is not a list: {type(customer_managed)}\n" + f"Version: {latest_version}\n" + "Expected format:\n" + "customer_managed_configs:\n" + " - name: property_name\n" + " ...\n" + "Contact cloud team to verify configuration file format." + ) + logger.error(error_msg) + raise CloudConfigParsingError(error_msg) + + if not isinstance(readonly_config, list): + error_msg = ( + f"'readonly_cluster_config' section is not a list: {type(readonly_config)}\n" + f"Version: {latest_version}\n" + "Expected format:\n" + "readonly_cluster_config:\n" + " - property_name_1\n" + " - property_name_2\n" + "Contact cloud team to verify configuration file format." + ) + logger.error(error_msg) + raise CloudConfigParsingError(error_msg) + + # Validate customer_managed_configs structure + for i, config in enumerate(customer_managed): + if not isinstance(config, dict): + logger.warning(f"customer_managed_configs[{i}] is not a dictionary: {config}, skipping") + continue + if 'name' not in config: + logger.warning(f"customer_managed_configs[{i}] missing 'name' field: {config}, skipping") + + # Validate readonly_cluster_config structure + for i, prop_name in enumerate(readonly_config): + if not isinstance(prop_name, str): + logger.warning(f"readonly_cluster_config[{i}] is not a string: {prop_name} ({type(prop_name)}), converting") + readonly_config[i] = str(prop_name) + + config = CloudConfig( + version=latest_version, + customer_managed_configs=customer_managed, + readonly_cluster_config=readonly_config + ) + + # Log summary statistics + editable_count = len(config.get_editable_properties()) + readonly_count = len(config.get_readonly_properties()) + total_count = len(config.get_all_cloud_properties()) + + logger.info(f"Cloud configuration loaded successfully:") + logger.info(f" Version: {latest_version}") + logger.info(f" Editable properties: {editable_count}") + logger.info(f" Readonly properties: {readonly_count}") + logger.info(f" Total cloud properties: {total_count}") + + return config + + except requests.exceptions.ConnectionError as e: + error_msg = ( + "Network connection failed.\n" + "Possible causes:\n" + "1. No internet connection\n" + "2. Corporate firewall blocking GitHub\n" + "3. DNS resolution issues\n" + "\nTo fix:\n" + "1. Check internet connectivity\n" + "2. Try: curl -I https://api.github.com\n" + "3. Contact IT if behind corporate firewall" + ) + logger.exception(error_msg) + raise NetworkError(error_msg) from e + + except requests.exceptions.Timeout as e: + error_msg = ( + "Request timeout after 30 seconds.\n" + "GitHub API may be experiencing issues.\n" + "To fix:\n" + "1. Check GitHub status: https://status.github.com/\n" + "2. Try again in a few minutes\n" + "3. Check network connectivity" + ) + logger.exception(error_msg) + raise NetworkError(error_msg) from e + + except (GitHubAuthError, NetworkError, CloudConfigParsingError): + # Re-raise our custom exceptions + raise + + except Exception as e: + error_msg = ( + "Unexpected error fetching cloud configuration.\n" + "This is likely a bug in the cloud configuration integration.\n" + "Please report this error to the development team with:\n" + "1. Full error message above\n" + "2. Command that triggered the error\n" + "3. Environment details (OS, Python version)\n" + "4. GitHub token permissions (without revealing the token)" + ) + logger.exception(error_msg) + raise CloudConfigError(error_msg) from e + + +def add_cloud_support_metadata(properties: Dict, cloud_config: CloudConfig) -> Dict: + """ + Annotate property definitions with cloud-support metadata derived from a CloudConfig. + + Mutates the provided properties dictionary in place by adding the boolean fields + 'cloud_editable', 'cloud_readonly', 'cloud_supported', and 'cloud_byoc_only' for + each property. Only entries whose value is a dict and whose 'config_scope' is + one of 'cluster', 'broker', or 'topic' are processed; other entries are skipped. + + Returns: + The same properties dictionary, updated with cloud metadata. + + Raises: + CloudConfigError: If `properties` is not a dict or if required data cannot be + extracted from the provided CloudConfig. + """ + if not isinstance(properties, dict): + error_msg = f"Properties argument must be a dictionary, got {type(properties)}" + logger.error(error_msg) + raise CloudConfigError(error_msg) + + try: + editable_props = cloud_config.get_editable_properties() + readonly_props = cloud_config.get_readonly_properties() + except (AttributeError, KeyError) as e: + error_msg = f"Failed to extract property sets from cloud configuration: {e}" + logger.error(error_msg) + raise CloudConfigError(error_msg) from e + + logger.info(f"Applying cloud metadata using configuration version {cloud_config.version}") + logger.info(f"Cloud properties: {len(editable_props)} editable, {len(readonly_props)} readonly") + + # Counters for reporting + processed_count = 0 + cloud_supported_count = 0 + editable_count = 0 + readonly_count = 0 + byoc_only_count = 0 + skipped_count = 0 + errors = [] + + for prop_name, prop_data in properties.items(): + try: + if not isinstance(prop_data, dict): + error_msg = f"Property '{prop_name}' data is not a dictionary: {type(prop_data)}" + logger.warning(error_msg) + errors.append(error_msg) + skipped_count += 1 + continue + + # Only process cluster, broker, and topic properties for cloud support + config_scope = prop_data.get('config_scope', '') + if config_scope not in ['cluster', 'broker', 'topic']: + # Skip node config properties and others without cloud relevance + skipped_count += 1 + continue + + processed_count += 1 + + # Initialize cloud metadata with defaults + prop_data['cloud_editable'] = False + prop_data['cloud_readonly'] = False + prop_data['cloud_supported'] = False + prop_data['cloud_byoc_only'] = False + + # Determine cloud support status + if prop_name in editable_props: + prop_data['cloud_editable'] = True + prop_data['cloud_readonly'] = False + prop_data['cloud_supported'] = True + cloud_supported_count += 1 + editable_count += 1 + + # Check if BYOC only + if cloud_config.is_byoc_only(prop_name): + prop_data['cloud_byoc_only'] = True + byoc_only_count += 1 + else: + prop_data['cloud_byoc_only'] = False + + elif prop_name in readonly_props: + prop_data['cloud_editable'] = False + prop_data['cloud_readonly'] = True + prop_data['cloud_supported'] = True + prop_data['cloud_byoc_only'] = False + cloud_supported_count += 1 + readonly_count += 1 + + else: + # Property not supported in cloud + prop_data['cloud_editable'] = False + prop_data['cloud_readonly'] = False + prop_data['cloud_supported'] = False + prop_data['cloud_byoc_only'] = False + + except Exception as e: + error_msg = f"Error processing property '{prop_name}': {e}" + logger.warning(error_msg) + errors.append(error_msg) + continue + + # Log comprehensive summary + logger.info(f"Cloud metadata application completed:") + logger.info(f" Properties processed: {processed_count}") + logger.info(f" Properties skipped (non-cloud scope): {skipped_count}") + logger.info(f" Cloud-supported properties: {cloud_supported_count}") + logger.info(f" - Editable: {editable_count}") + logger.info(f" - Readonly: {readonly_count}") + logger.info(f" - BYOC-only: {byoc_only_count}") + logger.info(f" Self-managed only: {processed_count - cloud_supported_count}") + + if errors: + logger.warning(f"Encountered {len(errors)} errors during processing:") + for error in errors[:10]: # Log first 10 errors + logger.warning(f" - {error}") + if len(errors) > 10: + logger.warning(f" ... and {len(errors) - 10} more errors") + + # Validation checks + if processed_count == 0: + logger.warning("No properties were processed for cloud metadata. This may indicate:") + logger.warning(" 1. All properties are node-scoped (not cluster/broker/topic)") + logger.warning(" 2. Properties dictionary is empty") + logger.warning(" 3. Properties missing 'config_scope' field") + + if cloud_supported_count == 0: + logger.warning("No cloud-supported properties found. This may indicate:") + logger.warning(" 1. Cloud configuration is empty or invalid") + logger.warning(" 2. Property names don't match between sources") + logger.warning(" 3. All properties are self-managed only") + + # Check for potential mismatches + unmatched_cloud_props = (editable_props | readonly_props) - { + name for name, data in properties.items() + if isinstance(data, dict) and data.get('config_scope') in ['cluster', 'broker', 'topic'] + } + + if unmatched_cloud_props: + logger.info(f"Cloud configuration contains {len(unmatched_cloud_props)} properties not found in extracted properties:") + for prop in sorted(list(unmatched_cloud_props)[:10]): # Show first 10 + logger.info(f" - {prop}") + if len(unmatched_cloud_props) > 10: + logger.info(f" ... and {len(unmatched_cloud_props) - 10} more") + logger.info("This is normal if cloud config includes deprecated or future properties.") + + return properties + + +if __name__ == "__main__": + # Test the cloud config fetcher + logging.basicConfig(level=logging.INFO) + config = fetch_cloud_config() + if config: + print(f"Successfully fetched cloud config for version {config.version}") + print(f"Editable properties: {len(config.get_editable_properties())}") + print(f"Readonly properties: {len(config.get_readonly_properties())}") + else: + print("Failed to fetch cloud configuration") diff --git a/tools/property-extractor/compare-properties.js b/tools/property-extractor/compare-properties.js new file mode 100644 index 0000000..682e2ab --- /dev/null +++ b/tools/property-extractor/compare-properties.js @@ -0,0 +1,378 @@ +#!/usr/bin/env node + +/** + * Property Comparison Tool + * + * Compares two property JSON files and generates a detailed report of: + * - New properties added + * - Properties with changed defaults + * - Properties with changed descriptions + * - Properties with changed types + * - Deprecated properties + */ + +const fs = require('fs'); +const path = require('path'); + +/** + * Recursively compares two values for structural deep equality. + * + * - Returns true if values are strictly equal (`===`). + * - Returns false if types differ or either is `null`/`undefined` while the other is not. + * - Arrays: ensures same length and recursively compares each element. + * - Objects: compares own enumerable keys (order-insensitive) and recursively compares corresponding values. + * + * @param {*} a - First value to compare. + * @param {*} b - Second value to compare. + * @returns {boolean} True if the two values are deeply equal. + */ +function deepEqual(a, b) { + if (a === b) return true; + if (a == null || b == null) return false; + if (typeof a !== typeof b) return false; + + if (Array.isArray(a) && Array.isArray(b)) { + if (a.length !== b.length) return false; + return a.every((val, i) => deepEqual(val, b[i])); + } + + if (typeof a === 'object' && typeof b === 'object') { + const keysA = Object.keys(a); + const keysB = Object.keys(b); + if (keysA.length !== keysB.length) return false; + return keysA.every(key => keysB.includes(key) && deepEqual(a[key], b[key])); + } + + return false; +} + +/** + * Format a value for concise human-readable display in comparison reports. + * + * Converts various JavaScript values into short string representations used + * in the report output: + * - null/undefined โ†’ `'null'` + * - Array: + * - empty โ†’ `'[]'` + * - single item โ†’ `[]` (recursively formatted) + * - multiple items โ†’ `'[ items]'` + * - Object โ†’ JSON string via `JSON.stringify` + * - String โ†’ quoted; truncated with `...` if longer than 50 characters + * - Other primitives โ†’ `String(value)` + * + * @param {*} value - The value to format for display. + * @return {string} A concise string suitable for report output. + */ +function formatValue(value) { + if (value === null || value === undefined) { + return 'null'; + } + if (Array.isArray(value)) { + if (value.length === 0) return '[]'; + if (value.length === 1) return `[${formatValue(value[0])}]`; + return `[${value.length} items]`; + } + if (typeof value === 'object') { + return JSON.stringify(value); + } + if (typeof value === 'string') { + return value.length > 50 ? `"${value.substring(0, 50)}..."` : `"${value}"`; + } + return String(value); +} + +/** + * Extracts a flat map of property definitions from a parsed JSON schema or similar object. + * + * If the input contains a top-level `properties` object, that object is returned directly. + * Otherwise, the function scans the root keys (excluding `definitions`) and returns any + * entries that look like property definitions (an object with at least one of `type`, + * `description`, or `default`). + * + * @param {Object} data - Parsed JSON data to scan for property definitions. + * @returns {Object} A map of property definitions (key โ†’ property object). Returns an empty object if none found. + */ +function extractProperties(data) { + // Properties are nested under a 'properties' key in the JSON structure + if (data.properties && typeof data.properties === 'object') { + return data.properties; + } + + // Fallback: look for properties at root level + const properties = {}; + for (const [key, value] of Object.entries(data)) { + if (key !== 'definitions' && typeof value === 'object' && value !== null) { + // Check if this looks like a property definition + if (value.hasOwnProperty('type') || value.hasOwnProperty('description') || value.hasOwnProperty('default')) { + properties[key] = value; + } + } + } + + return properties; +} + +/** + * Compare two property JSON structures and produce a detailed change report. + * + * Compares properties extracted from oldData and newData and classifies differences + * into newProperties, changedDefaults, changedDescriptions, changedTypes, + * deprecatedProperties (newly deprecated in newData), and removedProperties. + * Description fields in the report are truncated for brevity; default equality + * is determined by a deep structural comparison. + * + * @param {Object} oldData - Parsed JSON of the older property file. + * @param {Object} newData - Parsed JSON of the newer property file. + * @param {string} oldVersion - Version string corresponding to oldData. + * @param {string} newVersion - Version string corresponding to newData. + * @return {Object} Report object with arrays: newProperties, changedDefaults, + * changedDescriptions, changedTypes, deprecatedProperties, removedProperties. + */ +function compareProperties(oldData, newData, oldVersion, newVersion) { + const oldProps = extractProperties(oldData); + const newProps = extractProperties(newData); + + const report = { + newProperties: [], + changedDefaults: [], + changedDescriptions: [], + changedTypes: [], + deprecatedProperties: [], + removedProperties: [] + }; + + // Find new properties + for (const [name, prop] of Object.entries(newProps)) { + if (!oldProps.hasOwnProperty(name)) { + report.newProperties.push({ + name, + type: prop.type, + default: prop.default, + description: prop.description ? prop.description.substring(0, 100) + '...' : 'No description' + }); + } + } + + // Find changed properties + for (const [name, oldProp] of Object.entries(oldProps)) { + if (newProps.hasOwnProperty(name)) { + const newProp = newProps[name]; + + // Check for deprecation first (using is_deprecated field only) + const isNewlyDeprecated = newProp.is_deprecated === true && + oldProp.is_deprecated !== true; + + if (isNewlyDeprecated) { + report.deprecatedProperties.push({ + name, + reason: newProp.deprecatedReason || 'Property marked as deprecated' + }); + // Skip other change detection for deprecated properties + continue; + } + + // Only check other changes if property is not newly deprecated + // Check for default value changes + if (!deepEqual(oldProp.default, newProp.default)) { + report.changedDefaults.push({ + name, + oldDefault: oldProp.default, + newDefault: newProp.default + }); + } + + // Check for description changes + if (oldProp.description !== newProp.description) { + report.changedDescriptions.push({ + name, + oldDescription: oldProp.description ? oldProp.description.substring(0, 50) + '...' : 'No description', + newDescription: newProp.description ? newProp.description.substring(0, 50) + '...' : 'No description' + }); + } + + // Check for type changes + if (oldProp.type !== newProp.type) { + report.changedTypes.push({ + name, + oldType: oldProp.type, + newType: newProp.type + }); + } + } else { + // Property was removed + report.removedProperties.push({ + name, + type: oldProp.type, + description: oldProp.description ? oldProp.description.substring(0, 100) + '...' : 'No description' + }); + } + } + + return report; +} + +/** + * Print a human-readable console report summarizing property differences between two versions. + * + * The report includes sections for new properties, properties with changed defaults, + * changed types, updated descriptions, newly deprecated properties (with reason), and removed properties. + * + * @param {Object} report - Comparison report object returned by compareProperties(). + * @param {string} oldVersion - Label for the old version (displayed as the "from" version). + * @param {string} newVersion - Label for the new version (displayed as the "to" version). + */ +function generateConsoleReport(report, oldVersion, newVersion) { + console.log('\n' + '='.repeat(60)); + console.log(`๐Ÿ“‹ Property Changes Report (${oldVersion} โ†’ ${newVersion})`); + console.log('='.repeat(60)); + + if (report.newProperties.length > 0) { + console.log(`\nโžค New properties (${report.newProperties.length}):`); + report.newProperties.forEach(prop => { + console.log(` โ€ข ${prop.name} (${prop.type}) โ€” default: ${formatValue(prop.default)}`); + }); + } else { + console.log('\nโžค No new properties.'); + } + + if (report.changedDefaults.length > 0) { + console.log(`\nโžค Properties with changed defaults (${report.changedDefaults.length}):`); + report.changedDefaults.forEach(prop => { + console.log(` โ€ข ${prop.name}:`); + console.log(` - Old: ${formatValue(prop.oldDefault)}`); + console.log(` - New: ${formatValue(prop.newDefault)}`); + }); + } else { + console.log('\nโžค No default value changes.'); + } + + if (report.changedTypes.length > 0) { + console.log(`\nโžค Properties with changed types (${report.changedTypes.length}):`); + report.changedTypes.forEach(prop => { + console.log(` โ€ข ${prop.name}: ${prop.oldType} โ†’ ${prop.newType}`); + }); + } + + if (report.changedDescriptions.length > 0) { + console.log(`\nโžค Properties with updated descriptions (${report.changedDescriptions.length}):`); + report.changedDescriptions.forEach(prop => { + console.log(` โ€ข ${prop.name} โ€” description updated`); + }); + } + + if (report.deprecatedProperties.length > 0) { + console.log(`\nโžค Newly deprecated properties (${report.deprecatedProperties.length}):`); + report.deprecatedProperties.forEach(prop => { + console.log(` โ€ข ${prop.name} โ€” ${prop.reason}`); + }); + } + + if (report.removedProperties.length > 0) { + console.log(`\nโžค Removed properties (${report.removedProperties.length}):`); + report.removedProperties.forEach(prop => { + console.log(` โ€ข ${prop.name} (${prop.type})`); + }); + } + + console.log('\n' + '='.repeat(60)); +} + +/** + * Write a structured JSON comparison report to disk. + * + * Produces a JSON file containing a comparison header (old/new versions and timestamp), + * a summary with counts for each change category, and the full details object passed as `report`. + * + * @param {Object} report - Comparison details object produced by compareProperties; expected to contain arrays: `newProperties`, `changedDefaults`, `changedDescriptions`, `changedTypes`, `deprecatedProperties`, and `removedProperties`. + * @param {string} oldVersion - The previous version identifier included in the comparison header. + * @param {string} newVersion - The new version identifier included in the comparison header. + * @param {string} outputPath - Filesystem path where the JSON report will be written. + */ +function generateJsonReport(report, oldVersion, newVersion, outputPath) { + const jsonReport = { + comparison: { + oldVersion, + newVersion, + timestamp: new Date().toISOString() + }, + summary: { + newProperties: report.newProperties.length, + changedDefaults: report.changedDefaults.length, + changedDescriptions: report.changedDescriptions.length, + changedTypes: report.changedTypes.length, + deprecatedProperties: report.deprecatedProperties.length, + removedProperties: report.removedProperties.length + }, + details: report + }; + + fs.writeFileSync(outputPath, JSON.stringify(jsonReport, null, 2)); + console.log(`๐Ÿ“„ Detailed JSON report saved to: ${outputPath}`); +} + +/** + * Compare two property JSON files and produce a change report. + * + * Reads and parses the two JSON files at oldFilePath and newFilePath, compares their properties + * using compareProperties, prints a human-readable console report, and optionally writes a + * structured JSON report to outputDir/filename. + * + * Side effects: + * - Synchronously reads the two input files. + * - Writes a JSON report file when outputDir is provided (creates the directory if needed). + * - Logs progress and results to the console. + * - On error, logs the error and exits the process with code 1. + * + * @param {string} oldFilePath - Path to the old property JSON file. + * @param {string} newFilePath - Path to the new property JSON file. + * @param {string} oldVersion - Version label for the old file (used in reports). + * @param {string} newVersion - Version label for the new file (used in reports). + * @param {string|undefined} outputDir - Optional directory to write the JSON report; if falsy, no file is written. + * @param {string} [filename='property-changes.json'] - Name of the JSON report file to write inside outputDir. + * @returns {Object} The comparison report object produced by compareProperties. + */ +function comparePropertyFiles(oldFilePath, newFilePath, oldVersion, newVersion, outputDir, filename = 'property-changes.json') { + try { + console.log(`๐Ÿ“Š Comparing property files:`); + console.log(` Old: ${oldFilePath}`); + console.log(` New: ${newFilePath}`); + + const oldData = JSON.parse(fs.readFileSync(oldFilePath, 'utf8')); + const newData = JSON.parse(fs.readFileSync(newFilePath, 'utf8')); + + const report = compareProperties(oldData, newData, oldVersion, newVersion); + + // Generate console report + generateConsoleReport(report, oldVersion, newVersion); + + // Generate JSON report if output directory provided + if (outputDir) { + fs.mkdirSync(outputDir, { recursive: true }); + const jsonReportPath = path.join(outputDir, filename); + generateJsonReport(report, oldVersion, newVersion, jsonReportPath); + } + + return report; + } catch (error) { + console.error(`โŒ Error comparing properties: ${error.message}`); + process.exit(1); + } +} + +// CLI usage +if (require.main === module) { + const args = process.argv.slice(2); + + if (args.length < 4) { + console.log('Usage: node compare-properties.js [output-dir] [filename]'); + console.log(''); + console.log('Example:'); + console.log(' node compare-properties.js gen/v25.1.1-properties.json gen/v25.2.2-properties.json v25.1.1 v25.2.2 modules/reference property-changes-v25.1.1-to-v25.2.2.json'); + process.exit(1); + } + + const [oldFile, newFile, oldVersion, newVersion, outputDir, filename] = args; + comparePropertyFiles(oldFile, newFile, oldVersion, newVersion, outputDir, filename); +} + +module.exports = { comparePropertyFiles, compareProperties }; diff --git a/tools/property-extractor/generate-handlebars-docs.js b/tools/property-extractor/generate-handlebars-docs.js index eb348b4..a63294a 100644 --- a/tools/property-extractor/generate-handlebars-docs.js +++ b/tools/property-extractor/generate-handlebars-docs.js @@ -5,6 +5,19 @@ const path = require('path'); const handlebars = require('handlebars'); const helpers = require('./helpers'); +/** + * Handlebars documentation generator for Redpanda configuration properties. + * + * Supports custom template overrides using environment variables: + * - TEMPLATE_PROPERTY_PAGE: Main property page template + * - TEMPLATE_PROPERTY: Individual property section template + * - TEMPLATE_TOPIC_PROPERTY: Individual topic property section template + * - TEMPLATE_DEPRECATED_PROPERTY: Individual deprecated property section template + * - TEMPLATE_DEPRECATED: Deprecated properties page template + * + * CLI Usage: node generate-handlebars-docs.js + */ + // Register all helpers Object.entries(helpers).forEach(([name, fn]) => { if (typeof fn !== 'function') { @@ -110,34 +123,74 @@ function getTemplatePath(defaultPath, envVar) { } /** - * Registers Handlebars partials from template files + * Register Handlebars partials used to render property documentation. + * + * Loads templates from the local templates directory (overridable via environment + * variables handled by getTemplatePath) and registers three partials: + * - "property" (uses cloud-aware `property-cloud.hbs` when enabled) + * - "topic-property" (uses cloud-aware `topic-property-cloud.hbs` when enabled) + * - "deprecated-property" + * + * @param {boolean} [hasCloudSupport=false] - If true, select cloud-aware templates for the `property` and `topic-property` partials. + * @throws {Error} If any required template file is missing or cannot be read; errors are rethrown after logging. */ -function registerPartials() { +function registerPartials(hasCloudSupport = false) { const templatesDir = path.join(__dirname, 'templates'); - // Register property partial - const propertyTemplatePath = getTemplatePath( - path.join(templatesDir, 'property.hbs'), - 'TEMPLATE_PROPERTY' - ); - const propertyTemplate = fs.readFileSync(propertyTemplatePath, 'utf8'); - handlebars.registerPartial('property', propertyTemplate); - - // Register topic property partial - const topicPropertyTemplatePath = getTemplatePath( - path.join(templatesDir, 'topic-property.hbs'), - 'TEMPLATE_TOPIC_PROPERTY' - ); - const topicPropertyTemplate = fs.readFileSync(topicPropertyTemplatePath, 'utf8'); - handlebars.registerPartial('topic-property', topicPropertyTemplate); - - // Register deprecated property partial - const deprecatedPropertyTemplatePath = getTemplatePath( - path.join(templatesDir, 'deprecated-property.hbs'), - 'TEMPLATE_DEPRECATED_PROPERTY' - ); - const deprecatedPropertyTemplate = fs.readFileSync(deprecatedPropertyTemplatePath, 'utf8'); - handlebars.registerPartial('deprecated-property', deprecatedPropertyTemplate); + try { + console.log(`๐Ÿ“ Registering Handlebars templates (cloud support: ${hasCloudSupport ? 'enabled' : 'disabled'})`); + + // Register property partial (choose cloud or regular version) + const propertyTemplateFile = hasCloudSupport ? 'property-cloud.hbs' : 'property.hbs'; + const propertyTemplatePath = getTemplatePath( + path.join(templatesDir, propertyTemplateFile), + 'TEMPLATE_PROPERTY' + ); + + if (!fs.existsSync(propertyTemplatePath)) { + throw new Error(`Property template not found: ${propertyTemplatePath}`); + } + + const propertyTemplate = fs.readFileSync(propertyTemplatePath, 'utf8'); + handlebars.registerPartial('property', propertyTemplate); + console.log(`โœ… Registered property template: ${propertyTemplateFile}`); + + // Register topic property partial (choose cloud or regular version) + const topicPropertyTemplateFile = hasCloudSupport ? 'topic-property-cloud.hbs' : 'topic-property.hbs'; + const topicPropertyTemplatePath = getTemplatePath( + path.join(templatesDir, topicPropertyTemplateFile), + 'TEMPLATE_TOPIC_PROPERTY' + ); + + if (!fs.existsSync(topicPropertyTemplatePath)) { + throw new Error(`Topic property template not found: ${topicPropertyTemplatePath}`); + } + + const topicPropertyTemplate = fs.readFileSync(topicPropertyTemplatePath, 'utf8'); + handlebars.registerPartial('topic-property', topicPropertyTemplate); + console.log(`โœ… Registered topic property template: ${topicPropertyTemplateFile}`); + + // Register deprecated property partial + const deprecatedPropertyTemplatePath = getTemplatePath( + path.join(templatesDir, 'deprecated-property.hbs'), + 'TEMPLATE_DEPRECATED_PROPERTY' + ); + + if (!fs.existsSync(deprecatedPropertyTemplatePath)) { + throw new Error(`Deprecated property template not found: ${deprecatedPropertyTemplatePath}`); + } + + const deprecatedPropertyTemplate = fs.readFileSync(deprecatedPropertyTemplatePath, 'utf8'); + handlebars.registerPartial('deprecated-property', deprecatedPropertyTemplate); + console.log(`โœ… Registered deprecated property template`); + + } catch (error) { + console.error('โŒ Failed to register Handlebars templates:'); + console.error(` Error: ${error.message}`); + console.error(' This indicates missing or corrupted template files.'); + console.error(' Check that all .hbs files exist in tools/property-extractor/templates/'); + throw error; + } } /** @@ -180,7 +233,17 @@ function generatePropertyDocs(properties, config, outputDir) { } /** - * Generates deprecated properties documentation + * Generate an AsciiDoc fragment listing deprecated properties and write it to disk. + * + * Scans the provided properties map for entries with `is_deprecated === true`, groups + * them by `config_scope` ("broker" and "cluster"), sorts each group by property name, + * renders the `deprecated-properties` Handlebars template, and writes the output to + * `/deprecated/partials/deprecated-properties.adoc`. + * + * @param {Object.} properties - Map of property objects keyed by property name. + * Each property object may contain `is_deprecated`, `config_scope`, and `name` fields. + * @param {string} outputDir - Destination directory where the deprecated fragment will be written. + * @returns {number} The total number of deprecated properties found and written. */ function generateDeprecatedDocs(properties, outputDir) { const templatePath = getTemplatePath( @@ -216,16 +279,51 @@ function generateDeprecatedDocs(properties, outputDir) { } /** - * Main function to generate all property documentation + * Determine whether any property includes cloud support metadata. + * + * Checks the provided map of properties and returns true if at least one + * property object has a `cloud_supported` own property (regardless of its value). + * + * @param {Object} properties - Map from property name to its metadata object. + * @return {boolean} True if any property has a `cloud_supported` attribute; otherwise false. */ -function generateAllDocs(inputFile, outputDir) { - // Register partials - registerPartials(); +function hasCloudSupportMetadata(properties) { + return Object.values(properties).some(prop => + Object.prototype.hasOwnProperty.call(prop, 'cloud_supported') + ); +} +/** + * Generate all property documentation and write output files to disk. + * + * Reads properties from the provided JSON file, detects whether any property + * includes cloud support metadata to select cloud-aware templates, registers + * Handlebars partials accordingly, renders per-type property pages and a + * deprecated-properties partial, writes a flat list of all property names, and + * produces error reports. + * + * Generated artifacts are written under the given output directory (e.g.: + * pages//*.adoc, pages/deprecated/partials/deprecated-properties.adoc, + * all_properties.txt, and files under outputDir/error). + * + * @param {string} inputFile - Filesystem path to the input JSON containing a top-level `properties` object. + * @param {string} outputDir - Destination directory where generated pages and reports will be written. + * @returns {{totalProperties: number, brokerProperties: number, clusterProperties: number, objectStorageProperties: number, topicProperties: number, deprecatedProperties: number}} Summary counts for all properties and per-type totals. + */ +function generateAllDocs(inputFile, outputDir) { // Read input JSON const data = JSON.parse(fs.readFileSync(inputFile, 'utf8')); const properties = data.properties || {}; + // Check if cloud support is enabled + const hasCloudSupport = hasCloudSupportMetadata(properties); + if (hasCloudSupport) { + console.log('๐ŸŒค๏ธ Cloud support metadata detected, using cloud-aware templates'); + } + + // Register partials with cloud support detection + registerPartials(hasCloudSupport); + let totalProperties = 0; let totalBrokerProperties = 0; let totalClusterProperties = 0; @@ -291,13 +389,15 @@ function generateErrorReports(properties, outputDir) { }); // Write error reports + const totalProperties = Object.keys(properties).length; + if (emptyDescriptions.length > 0) { fs.writeFileSync( path.join(errorDir, 'empty_description.txt'), emptyDescriptions.join('\n'), 'utf8' ); - const percentage = ((emptyDescriptions.length / Object.keys(properties).length) * 100).toFixed(2); + const percentage = totalProperties > 0 ? ((emptyDescriptions.length / totalProperties) * 100).toFixed(2) : '0.00'; console.log(`You have ${emptyDescriptions.length} properties with empty description. Percentage of errors: ${percentage}%. Data written in 'empty_description.txt'.`); } @@ -307,7 +407,7 @@ function generateErrorReports(properties, outputDir) { deprecatedProperties.join('\n'), 'utf8' ); - const percentage = ((deprecatedProperties.length / Object.keys(properties).length) * 100).toFixed(2); + const percentage = totalProperties > 0 ? ((deprecatedProperties.length / totalProperties) * 100).toFixed(2) : '0.00'; console.log(`You have ${deprecatedProperties.length} deprecated properties. Percentage of errors: ${percentage}%. Data written in 'deprecated_properties.txt'.`); } } diff --git a/tools/property-extractor/parser.py b/tools/property-extractor/parser.py index 21e645a..44348ad 100644 --- a/tools/property-extractor/parser.py +++ b/tools/property-extractor/parser.py @@ -14,11 +14,37 @@ ) @declaration """ +# Tree-sitter query for extracting C++ property constructor arguments and enterprise values +# +# - Enhanced to capture all expression types including: +# * call_expression: Handles function calls like model::kafka_audit_logging_topic() +# * template_instantiation: Handles template syntax like std::vector{...} +# * concatenated_string: Handles C++ string concatenation with + +# * qualified_identifier: Handles namespaced identifiers like model::partition_autobalancing_mode::continuous +# * (_) @argument: Fallback to capture any other expression types +# +# This ensures enterprise values are captured in their complete form for proper +# processing by the process_enterprise_value function. SOURCE_QUERY = """ (field_initializer_list (field_initializer (field_identifier) @field - (argument_list (_) @argument)? @arguments + (argument_list + [ + (call_expression) @argument + (initializer_list) @argument + (template_instantiation) @argument + (concatenated_string) @argument + (string_literal) @argument + (raw_string_literal) @argument + (identifier) @argument + (qualified_identifier) @argument + (number_literal) @argument + (true) @argument + (false) @argument + (_) @argument + ] + )? @arguments ) @field ) """ diff --git a/tools/property-extractor/property_extractor.py b/tools/property-extractor/property_extractor.py old mode 100755 new mode 100644 index 5e95ca7..20edb70 --- a/tools/property-extractor/property_extractor.py +++ b/tools/property-extractor/property_extractor.py @@ -59,6 +59,7 @@ import json import re import yaml +import ast from copy import deepcopy from pathlib import Path @@ -76,18 +77,138 @@ # TopicPropertyExtractor not available, will skip topic property extraction TopicPropertyExtractor = None +# Import cloud configuration support +try: + from cloud_config import fetch_cloud_config, add_cloud_support_metadata + # Configure cloud_config logger to suppress INFO logs by default + import logging + logging.getLogger('cloud_config').setLevel(logging.WARNING) +except ImportError as e: + # Cloud configuration support not available due to missing dependencies + logging.warning(f"Cloud configuration support not available: {e}") + fetch_cloud_config = None + add_cloud_support_metadata = None + logger = logging.getLogger("viewer") +def process_enterprise_value(enterprise_str): + """ + Convert a raw C++ "enterprise" expression into a JSON-friendly value. + + Accepts a string extracted from C++ sources and returns a value suitable for JSON + serialization: a Python list for initializer-lists, a simple string for enum-like + tokens, a boolean-like or quoted string unchanged, or a human-readable hint for + lambda-based expressions. + + The function applies pattern matching in the following order (order is significant): + 1. std::vector<...>{...} initializer lists โ†’ Python list (quoted strings are unescaped, + unqualified enum tokens are reduced to their final identifier). + 2. C++ scoped enum-like tokens (foo::bar::BAZ) โ†’ "BAZ". + 3. Lambda expressions (strings starting with "[](" and ending with "}") โ†’ a short + human-readable hint such as "Enterprise feature enabled" or context-specific text. + 4. Simple literal values (e.g., "true", "false", "OIDC", or quoted strings) โ†’ returned as-is. + + Parameters: + enterprise_str (str): Raw C++ expression text to be converted. + + Returns: + Union[str, bool, list]: A JSON-serializable representation of the input. + """ + enterprise_str = enterprise_str.strip() + + # FIRST: Handle std::vector initialization patterns (highest priority) + # This must come before enum processing because vectors can contain enums + # Tolerate optional whitespace around braces + vector_match = re.match(r'std::vector<[^>]+>\s*\{\s*([^}]*)\s*\}', enterprise_str) + if vector_match: + content = vector_match.group(1).strip() + if not content: + return [] + + # Parse the content as a list of values + values = [] + current_value = "" + in_quotes = False + + for char in content: + if char == '"' and (not current_value or current_value[-1] != '\\'): + in_quotes = not in_quotes + current_value += char + elif char == ',' and not in_quotes: + if current_value.strip(): + # Clean up the value + value = current_value.strip() + if value.startswith('"') and value.endswith('"'): + values.append(ast.literal_eval(value)) + else: + # Handle enum values in the vector + enum_match = re.match(r'[a-zA-Z0-9_:]+::([a-zA-Z0-9_]+)', value) + if enum_match: + values.append(enum_match.group(1)) + else: + values.append(value) + current_value = "" + else: + current_value += char + + # Add the last value + if current_value.strip(): + value = current_value.strip() + if value.startswith('"') and value.endswith('"'): + values.append(ast.literal_eval(value)) + else: + # Handle enum values in the vector + enum_match = re.match(r'[a-zA-Z0-9_:]+::([a-zA-Z0-9_]+)', value) + if enum_match: + values.append(enum_match.group(1)) + else: + values.append(value) + + return values + + # SECOND: Handle enum-like patterns (extract the last part after ::) + enum_match = re.match(r'[a-zA-Z0-9_:]+::([a-zA-Z0-9_]+)', enterprise_str) + if enum_match: + enum_value = enum_match.group(1) + return enum_value + + # THIRD: Handle C++ lambda expressions - these usually indicate "any non-default value" + if enterprise_str.startswith("[](") and enterprise_str.endswith("}"): + # For lambda expressions, try to extract meaningful info from the logic + if "leaders_preference" in enterprise_str: + return "Any rack preference (not `none`)" + else: + return "Enterprise feature enabled" + + # FOURTH: Handle simple values with proper JSON types + # Convert boolean literals to actual boolean values for JSON compatibility + if enterprise_str == "true": + return True + elif enterprise_str == "false": + return False + elif enterprise_str == "OIDC" or enterprise_str.startswith('"'): + return enterprise_str + + # Fallback: return the original value + return enterprise_str + + def resolve_cpp_function_call(function_name): """ - Dynamically resolve C++ function calls to their return values by searching the source code. + Resolve certain small, known C++ zero-argument functions to their literal return values by searching Redpanda source files. - Args: - function_name: The C++ function name (e.g., "model::kafka_audit_logging_topic") + This function looks up predefined search patterns for well-known functions (currently a small set under `model::*`), locates a local Redpanda source tree from several commonly used paths, and scans the listed files (and, if needed, the broader model directory) for a regex match that captures the string returned by the function. If a match is found the captured string is returned; if the source tree cannot be found or no match is located the function returns None. + + Parameters: + function_name (str): Fully-qualified C++ function name to resolve (e.g., "model::kafka_audit_logging_topic"). Returns: - The resolved string value or None if not found + str or None: The resolved literal string returned by the C++ function, or None when unresolved (source not found or no matching pattern). + + Notes: + - The function performs filesystem I/O and regex-based source searching; it does not raise on read errors but logs and continues. + - Only a small, hard-coded set of function names/patterns is supported; unknown names immediately return None. """ # Map function names to likely search patterns and file locations search_patterns = { @@ -327,12 +448,18 @@ def apply_property_overrides(properties, overrides, overrides_file_path=None): 2. version: Add version information showing when the property was introduced 3. example: Add AsciiDoc example sections with flexible input formats (see below) 4. default: Override the auto-extracted default value - + 5. related_topics: Add an array of related topic links for cross-referencing + 6. config_scope: Specify the scope for new properties ("topic", "cluster", "broker") + 7. type: Specify the type for new properties + + Properties that don't exist in the extracted source can be created from overrides. + This is useful for topic properties or other configurations that aren't auto-detected. + Multiple example input formats are supported for user convenience: - + 1. Direct AsciiDoc string: "example": ".Example\n[,yaml]\n----\nredpanda:\n property_name: value\n----" - + 2. Multi-line array (each element becomes a line): "example": [ ".Example", @@ -342,10 +469,10 @@ def apply_property_overrides(properties, overrides, overrides_file_path=None): " property_name: value", "----" ] - + 3. External file reference: "example_file": "examples/property_name.adoc" - + 4. Auto-formatted YAML with title and description: "example_yaml": { "title": "Example Configuration", @@ -356,37 +483,103 @@ def apply_property_overrides(properties, overrides, overrides_file_path=None): } } } - + Args: properties: Dictionary of extracted properties from C++ source overrides: Dictionary loaded from overrides JSON file overrides_file_path: Path to the overrides file (for resolving relative example_file paths) - + Returns: - Updated properties dictionary with overrides applied + Updated properties dictionary with overrides applied and new properties created """ if overrides and "properties" in overrides: for prop, override in overrides["properties"].items(): if prop in properties: - # Apply description override - if "description" in override: - properties[prop]["description"] = override["description"] - - # Apply version override (introduced in version) - if "version" in override: - properties[prop]["version"] = override["version"] - - # Apply example override with multiple input format support - example_content = _process_example_override(override, overrides_file_path) - if example_content: - properties[prop]["example"] = example_content - - # Apply default override - if "default" in override: - properties[prop]["default"] = override["default"] + # Apply overrides to existing properties + _apply_override_to_existing_property(properties[prop], override, overrides_file_path) + else: + # Create new property from override + logger.info(f"Creating new property from override: {prop}") + properties[prop] = _create_property_from_override(prop, override, overrides_file_path) return properties +def _apply_override_to_existing_property(property_dict, override, overrides_file_path): + """Apply overrides to an existing property.""" + # Apply description override + if "description" in override: + property_dict["description"] = override["description"] + + # Apply version override (introduced in version) + if "version" in override: + property_dict["version"] = override["version"] + + # Apply example override with multiple input format support + example_content = _process_example_override(override, overrides_file_path) + if example_content: + property_dict["example"] = example_content + + # Apply default override + if "default" in override: + property_dict["default"] = override["default"] + + # Apply type override + if "type" in override: + property_dict["type"] = override["type"] + + # Apply config_scope override + if "config_scope" in override: + property_dict["config_scope"] = override["config_scope"] + + # Apply related_topics override + if "related_topics" in override: + if isinstance(override["related_topics"], list): + property_dict["related_topics"] = override["related_topics"] + else: + logger.warning(f"related_topics for property must be an array") + + +def _create_property_from_override(prop_name, override, overrides_file_path): + """Create a new property from override specification.""" + # Create base property structure + new_property = { + "name": prop_name, + "description": override.get("description", f"Configuration property: {prop_name}"), + "type": override.get("type", "string"), + "default": override.get("default", None), + "defined_in": "override", # Mark as override-created + "config_scope": override.get("config_scope", "topic"), # Default to topic for new properties + "is_topic_property": override.get("config_scope", "topic") == "topic", + "is_deprecated": override.get("is_deprecated", False), + "visibility": override.get("visibility", "user") + } + + # Add version if specified + if "version" in override: + new_property["version"] = override["version"] + + # Add example if specified + example_content = _process_example_override(override, overrides_file_path) + if example_content: + new_property["example"] = example_content + + # Add related_topics if specified + if "related_topics" in override: + if isinstance(override["related_topics"], list): + new_property["related_topics"] = override["related_topics"] + else: + logger.warning(f"related_topics for property '{prop_name}' must be an array") + + # Add any other custom fields from override + for key, value in override.items(): + if key not in ["description", "type", "default", "config_scope", "version", + "example", "example_file", "example_yaml", "related_topics", + "is_deprecated", "visibility"]: + new_property[key] = value + + return new_property + + def _process_example_override(override, overrides_file_path=None): """ Process example overrides in various user-friendly formats. @@ -444,15 +637,15 @@ def _process_example_override(override, overrides_file_path=None): if found_path: file_path = found_path else: - print(f"Warning: Example file not found: {override['example_file']}") - print(f"Searched in: {', '.join(search_paths)}") + logger.warning(f"Example file not found: {override['example_file']}") + logger.warning(f"Searched in: {', '.join(search_paths)}") return None try: with open(file_path, 'r', encoding='utf-8') as f: return f.read().strip() except Exception as e: - print(f"Error reading example file {file_path}: {e}") + logger.error(f"Error reading example file {file_path}: {e}") return None # Format 4: Auto-formatted YAML configuration @@ -495,44 +688,46 @@ def add_config_scope(properties): 'cluster' if defined_in == src/v/config/configuration.cc 'broker' if defined_in == src/v/config/node_config.cc 'topic' if is_topic_property == True + + For override-created properties, preserve existing config_scope if already set. """ for prop in properties.values(): # Check if this is a topic property first if prop.get("is_topic_property", False): prop["config_scope"] = "topic" else: - defined_in = prop.get("defined_in", "") - if defined_in == "src/v/config/configuration.cc": - prop["config_scope"] = "cluster" - elif defined_in == "src/v/config/node_config.cc": - prop["config_scope"] = "broker" + # For override-created properties, preserve existing config_scope if set + if prop.get("defined_in") == "override" and prop.get("config_scope") is not None: + # Keep the existing config_scope from override + pass else: - prop["config_scope"] = None + defined_in = prop.get("defined_in", "") + if defined_in == "src/v/config/configuration.cc": + prop["config_scope"] = "cluster" + elif defined_in == "src/v/config/node_config.cc": + prop["config_scope"] = "broker" + else: + prop["config_scope"] = None return properties def resolve_type_and_default(properties, definitions): """ - Resolve type references and expand default values for all properties. - - This function performs several critical transformations: + Resolve JSON Schema types and expand C++-style default values for all properties. - 1. **Type Resolution**: Converts C++ type names to JSON schema types - - model::broker_endpoint -> "object" - - std::string -> "string" - - Handles both direct type names and JSON pointer references (#/definitions/...) + This function: + - Resolves type references found in `properties` against `definitions` (supports "$ref" and direct type names) and normalizes property "type" to a JSON Schema primitive ("object", "string", "integer", "boolean", "array", "number") with sensible fallbacks. + - Expands C++ constructor/initializer syntax and common C++ patterns appearing in default values into JSON-compatible Python values (e.g., nested constructor calls -> dicts, initializer lists -> lists, `std::nullopt` -> None, enum-like tokens -> strings). + - Ensures array-typed properties (including one_or_many_property cases) have array defaults: single-object defaults are wrapped into a one-element list and "{}" string defaults become []. + - Updates array item type information when item types reference definitions. + - Applies a final pass to convert any remaining C++-patterned defaults and to transform any `enterprise_value` strings via process_enterprise_value. - 2. **Default Value Expansion**: Transforms C++ constructor syntax to JSON objects - - model::broker_endpoint(net::unresolved_address("127.0.0.1", 9644)) - -> {address: "127.0.0.1", port: 9644} + Parameters: + properties (dict): Mapping of property names to property metadata dictionaries. Each property may include keys like "type", "default", "items", and "enterprise_value". + definitions (dict): Mapping of type names to JSON Schema definition dictionaries used to resolve $ref targets and to infer property shapes when expanding constructors. - 3. **Array Default Handling**: Ensures one_or_many_property defaults are arrays - - For properties with type="array", wraps single object defaults in arrays - - Converts empty object strings "{}" to empty arrays [] - - This is essential for one_or_many_property types like 'admin' which should show: - - Type: array - - Default: [{address: "127.0.0.1", port: 9644}] (not just {address: ...}) + Returns: + dict: The same `properties` mapping after in-place normalization and expansion of types and defaults. """ import ast import re @@ -1167,6 +1362,14 @@ def expand_default(type_name, default_str): prop["type"] = "boolean" else: prop["type"] = "string" + + # Final pass: process enterprise values + for prop in properties.values(): + if "enterprise_value" in prop: + enterprise_value = prop["enterprise_value"] + if isinstance(enterprise_value, str): + processed_enterprise = process_enterprise_value(enterprise_value) + prop["enterprise_value"] = processed_enterprise return properties @@ -1218,53 +1421,62 @@ def extract_topic_properties(source_path): def main(): + """ + CLI entry point that extracts Redpanda configuration properties from C++ sources and emits JSON outputs. + + Runs a full extraction and transformation pipeline: + - Parses command-line options (required: --path). Optional flags include --recursive, --output, --enhanced-output, --definitions, --overrides, --cloud-support, and --verbose. + - Validates input paths and collects header/.cc file pairs. + - Initializes Tree-sitter C++ parser and extracts configuration properties from source files (optionally augmented with topic properties). + - Produces two outputs: + - Original properties JSON: resolved types, expanded C++ defaults, added config_scope, and optional cloud metadata. + - Enhanced properties JSON: same as original but with overrides applied before final resolution. + - If --cloud-support is requested, attempts to fetch cloud configuration and add cloud metadata; this requires the cloud_config integration and network access (also requires GITHUB_TOKEN for private access). If cloud support is requested but dependencies are missing, the process will exit with an error. + - Writes JSON to files when --output and/or --enhanced-output are provided; otherwise prints the original JSON to stdout. + - Exits with non-zero status on fatal errors (missing files, parse errors, missing Tree-sitter parser, I/O failures, or missing cloud dependencies when requested). + + Side effects: + - Reads and writes files, may call external cloud config fetchers, logs to the configured logger, and may call sys.exit() on fatal conditions. + """ import argparse def generate_options(): + """ + Create and return an argparse.ArgumentParser preconfigured for the property extractor CLI. + + The parser understands the following options: + - --path (required): path to the Redpanda source directory to scan. + - --recursive: scan the path recursively. + - --output: file path to write the JSON output (stdout if omitted). + - --enhanced-output: file path to write the enhanced JSON output with overrides applied. + - --definitions: JSON file containing type definitions (defaults to a definitions.json co-located with this module). + - --overrides: optional JSON file with property description/metadata overrides. + - --cloud-support: enable fetching cloud metadata from the cloudv2 repository (requires GITHUB_TOKEN and external dependencies such as pyyaml and requests). + - -v / --verbose: enable verbose (DEBUG-level) logging. + + Returns: + argparse.ArgumentParser: Parser configured with the above options. + """ arg_parser = argparse.ArgumentParser( - description="Extract all properties from the Redpanda's source code and generate a JSON output with their definitions" - ) - arg_parser.add_argument( - "--path", - type=str, - required=True, - help="Path to the Redpanda's source dir to extract the properties", - ) - - arg_parser.add_argument( - "--recursive", action="store_true", help="Scan the path recursively" - ) - - arg_parser.add_argument( - "--output", - type=str, - required=False, - help="File to store the JSON output. If no file is provided, the JSON will be printed to the standard output", - ) - - arg_parser.add_argument( - "--enhanced-output", - type=str, - required=False, - help="File to store the enhanced JSON output with overrides applied (such as 'dev-properties.json')", + description="Internal property extraction tool - use doc-tools.js for user interface" ) - - arg_parser.add_argument( - "--definitions", - type=str, - required=False, - default=os.path.dirname(os.path.realpath(__file__)) + "/definitions.json", - help='JSON file with the type definitions. This file will be merged in the output under the "definitions" field', - ) - - arg_parser.add_argument( - "--overrides", - type=str, - required=False, - help='Optional JSON file with property description overrides', - ) - - arg_parser.add_argument("-v", "--verbose", action="store_true") + # Core required parameters + arg_parser.add_argument("--path", type=str, required=True, help="Path to Redpanda source directory") + arg_parser.add_argument("--recursive", action="store_true", help="Scan path recursively") + + # Output options + arg_parser.add_argument("--output", type=str, help="JSON output file path") + arg_parser.add_argument("--enhanced-output", type=str, help="Enhanced JSON output file path") + + # Data sources + arg_parser.add_argument("--definitions", type=str, + default=os.path.dirname(os.path.realpath(__file__)) + "/definitions.json", + help="Type definitions JSON file") + arg_parser.add_argument("--overrides", type=str, help="Property overrides JSON file") + + # Feature flags (set by Makefile from environment variables) + arg_parser.add_argument("--cloud-support", action="store_true", help="Enable cloud metadata") + arg_parser.add_argument("-v", "--verbose", action="store_true", help="Verbose logging") return arg_parser @@ -1273,8 +1485,10 @@ def generate_options(): if options.verbose: logging.basicConfig(level="DEBUG") + # Also enable INFO logging for cloud_config in verbose mode + logging.getLogger('cloud_config').setLevel(logging.INFO) else: - logging.basicConfig(level="INFO") + logging.basicConfig(level="WARNING") # Suppress INFO logs by default validate_paths(options) @@ -1331,7 +1545,24 @@ def generate_options(): # 1. Add config_scope field based on which source file defines the property original_properties = add_config_scope(deepcopy(properties)) - # 2. Resolve type references and expand default values for original properties + # 2. Fetch cloud configuration and add cloud support metadata if requested + # Check both CLI flag and environment variable (CLOUD_SUPPORT=1 from Makefile) + cloud_support_enabled = options.cloud_support or os.environ.get('CLOUD_SUPPORT') == '1' + cloud_config = None + if cloud_support_enabled: + if fetch_cloud_config and add_cloud_support_metadata: + logging.info("Cloud support enabled, fetching cloud configuration...") + cloud_config = fetch_cloud_config() # This will raise an exception if it fails + original_properties = add_cloud_support_metadata(original_properties, cloud_config) + logging.info(f"โœ… Cloud support metadata applied successfully using configuration version {cloud_config.version}") + else: + logging.error("โŒ Cloud support requested but cloud_config module not available") + logging.error("This indicates missing Python dependencies for cloud configuration") + logging.error("Install required packages: pip install pyyaml requests") + logging.error("Or if using a virtual environment, activate it first") + sys.exit(1) + + # 3. Resolve type references and expand default values for original properties original_properties = resolve_type_and_default(original_properties, definitions) # Generate original properties JSON (without overrides) @@ -1347,7 +1578,12 @@ def generate_options(): # 2. Add config_scope field based on which source file defines the property enhanced_properties = add_config_scope(enhanced_properties) - # 3. Resolve type references and expand default values + # 3. Add cloud support metadata if requested + if cloud_config: + enhanced_properties = add_cloud_support_metadata(enhanced_properties, cloud_config) + logging.info("โœ… Cloud support metadata applied to enhanced properties") + + # 4. Resolve type references and expand default values # This step converts: # - C++ type names (model::broker_endpoint) to JSON schema types (object) # - C++ constructor defaults to structured JSON objects diff --git a/tools/property-extractor/requirements.txt b/tools/property-extractor/requirements.txt index 9627bb1..6ad6baf 100644 --- a/tools/property-extractor/requirements.txt +++ b/tools/property-extractor/requirements.txt @@ -1,3 +1,4 @@ tree_sitter==0.21.1 setuptools>=42.0.0 pyyaml>=6.0 +requests>=2.32.5 diff --git a/tools/property-extractor/templates/property-cloud.hbs b/tools/property-extractor/templates/property-cloud.hbs new file mode 100644 index 0000000..3609cd8 --- /dev/null +++ b/tools/property-extractor/templates/property-cloud.hbs @@ -0,0 +1,105 @@ +{{#if cloud_supported}} +// tag::redpanda-cloud[] +{{/if}} +=== {{name}} +{{#if version}} + +*Introduced in {{version}}* +{{/if}} +{{#if description}} + +{{{description}}} +{{else}} + +No description available. +{{/if}} +{{#if is_enterprise}} + +ifndef::env-cloud[] +*Enterprise license required*: `{{enterprise_value}}` (for license details, see xref:get-started:licensing/index.adoc[Redpanda Licensing]) +endif::[] +{{/if}} +{{#if cloud_byoc_only}} + +NOTE: This property is available only in Redpanda Cloud BYOC deployments. +{{/if}} +{{#if units}} + +*Unit:* {{units}} +{{else}} +{{#if (formatUnits name)}} + +*Unit:* {{formatUnits name}} +{{/if}} +{{/if}} +{{#if (ne defined_in "src/v/config/node_config.cc")}} +{{#if (ne needs_restart undefined)}} + +*Requires restart:* {{#if needs_restart}}Yes{{else}}No{{/if}} +{{/if}} +{{/if}} +{{#if visibility}} + +// tag::self-managed-only[] +*Visibility:* `{{visibility}}` +// end::self-managed-only[] +{{/if}} +{{#if type}} + +*Type:* {{type}} +{{/if}} +{{#if (and minimum maximum)}} + +*Accepted values:* [`{{minimum}}`, `{{maximum}}`] +{{else}} +{{#if minimum}} + +*Minimum value:* `{{minimum}}` +{{/if}} +{{#if maximum}} + +*Maximum value:* `{{maximum}}` +{{/if}} +{{/if}} +{{#if (ne default undefined)}} + +ifdef::env-cloud[] +*Default:* Available in the Redpanda Cloud Console +endif::[] +ifndef::env-cloud[] +*Default:* `{{formatPropertyValue default type}}` +endif::[] +{{/if}} + +// tag::self-managed-only[] +*Nullable:* {{#if nullable}}Yes{{else}}No{{/if}} +// end::self-managed-only[] +{{#if example}} + +{{{renderPropertyExample this}}} +{{/if}} +{{#if related_topics}} + +*Related topics:* + +{{#each related_topics}} +* {{{this}}} +{{/each}} +{{/if}} +{{#if aliases}} + +// tag::self-managed-only[] +*Aliases:* {{join aliases ", "}} +// end::self-managed-only[] +{{/if}} +{{#if is_deprecated}} + +[WARNING] +==== +This property is deprecated. +==== +{{/if}} +--- +{{#if cloud_supported}} +// end::redpanda-cloud[] +{{/if}} diff --git a/tools/property-extractor/templates/property.hbs b/tools/property-extractor/templates/property.hbs index 05f06f7..f15fdb3 100644 --- a/tools/property-extractor/templates/property.hbs +++ b/tools/property-extractor/templates/property.hbs @@ -2,14 +2,21 @@ {{#if version}} *Introduced in {{version}}* - {{/if}} + {{#if description}} {{{description}}} {{else}} No description available. {{/if}} +{{#if is_enterprise}} +ifndef::env-cloud[] +*Enterprise license required*: `{{enterprise_value}}` (for license details, see xref:get-started:licensing/index.adoc[Redpanda Licensing]) +endif::[] + +{{/if}} + {{#if units}} *Unit:* {{units}} @@ -56,6 +63,14 @@ No description available. {{{renderPropertyExample this}}} {{/if}} +{{#if related_topics}} +*Related topics:* + +{{#each related_topics}} +* {{{this}}} +{{/each}} +{{/if}} + {{#if aliases}} *Aliases:* {{join aliases ", "}} diff --git a/tools/property-extractor/templates/topic-property-cloud.hbs b/tools/property-extractor/templates/topic-property-cloud.hbs new file mode 100644 index 0000000..e111dd0 --- /dev/null +++ b/tools/property-extractor/templates/topic-property-cloud.hbs @@ -0,0 +1,97 @@ +{{#if cloud_supported}} +// tag::redpanda-cloud[] +{{/if}} +=== {{name}} +{{#if version}} + +*Introduced in {{version}}* +{{/if}} +{{#if description}} + +{{{description}}} +{{else}} + +No description available. +{{/if}} +{{#if is_enterprise}} + +ifndef::env-cloud[] +*Enterprise license required*: `{{enterprise_value}}` (for license details, see xref:get-started:licensing/index.adoc[Redpanda Licensing]) +endif::[] +{{/if}} +{{#if cloud_byoc_only}} + +NOTE: This property is only available in Redpanda Cloud BYOC deployments. +{{/if}} +{{#if type}} + +*Type:* {{type}} +{{/if}} +{{#if acceptable_values}} + +*Accepted values:* {{{acceptable_values}}} +{{/if}} +{{#if corresponding_cluster_property}} + +*Related cluster property:* xref:reference:cluster-properties.adoc#{{corresponding_cluster_property}}[{{corresponding_cluster_property}}] +{{/if}} +{{#if (and minimum maximum)}} + +*Accepted values:* [`{{minimum}}`, `{{maximum}}`] +{{else}} +{{#if minimum}} + +*Minimum value:* `{{minimum}}` +{{/if}} +{{#if maximum}} + +*Maximum value:* `{{maximum}}` +{{/if}} +{{/if}} +{{#if (ne default undefined)}} +{{#if cloud_supported}} + +ifdef::env-cloud[] +*Default:* Available in the Redpanda Cloud Console +endif::[] +ifndef::env-cloud[] +*Default:* `{{formatPropertyValue default type}}` +endif::[] +{{else}} + +*Default:* `{{formatPropertyValue default type}}` +{{/if}} +{{/if}} + +// tag::self-managed-only[] +*Nullable:* {{#if nullable}}Yes{{else}}No{{/if}} +// end::self-managed-only[] +{{#if example}} + +{{{renderPropertyExample this}}} +{{/if}} +{{#if related_topics}} + +*Related topics:* + +{{#each related_topics}} +* {{{this}}} +{{/each}} +{{/if}} +{{#if aliases}} + +// tag::self-managed-only[] +*Aliases:* {{join aliases ", "}} +// end::self-managed-only[] +{{/if}} +{{#if is_deprecated}} + +[WARNING] +==== +This property is deprecated. +==== +{{/if}} +--- +{{#if cloud_supported}} +// end::redpanda-cloud[] +{{/if}} diff --git a/tools/property-extractor/templates/topic-property.hbs b/tools/property-extractor/templates/topic-property.hbs index 7e3eaf7..4e12e34 100644 --- a/tools/property-extractor/templates/topic-property.hbs +++ b/tools/property-extractor/templates/topic-property.hbs @@ -1,59 +1,73 @@ === {{name}} - {{#if version}} -*Introduced in {{version}}* +*Introduced in {{version}}* {{/if}} {{#if description}} + {{{description}}} {{else}} + No description available. {{/if}} +{{#if is_enterprise}} +ifndef::env-cloud[] +*Enterprise license required*: `{{enterprise_value}}` (for license details, see xref:get-started:licensing/index.adoc[Redpanda Licensing]) +endif::[] +{{/if}} {{#if type}} -*Type:* {{type}} +*Type:* {{type}} {{/if}} {{#if acceptable_values}} -*Accepted values:* {{{acceptable_values}}} +*Accepted values:* {{{acceptable_values}}} {{/if}} {{#if corresponding_cluster_property}} -*Related cluster property:* xref:reference:cluster-properties.adoc#{{corresponding_cluster_property}}[{{corresponding_cluster_property}}] +*Related cluster property:* xref:reference:cluster-properties.adoc#{{corresponding_cluster_property}}[{{corresponding_cluster_property}}] {{/if}} {{#if (and minimum maximum)}} -*Accepted values:* [`{{minimum}}`, `{{maximum}}`] +*Accepted values:* [`{{minimum}}`, `{{maximum}}`] {{else}} {{#if minimum}} -*Minimum value:* `{{minimum}}` +*Minimum value:* `{{minimum}}` {{/if}} {{#if maximum}} -*Maximum value:* `{{maximum}}` +*Maximum value:* `{{maximum}}` {{/if}} {{/if}} {{#if (ne default undefined)}} -*Default:* `{{formatPropertyValue default type}}` +*Default:* `{{formatPropertyValue default type}}` {{/if}} -*Nullable:* {{#if nullable}}Yes{{else}}No{{/if}} +*Nullable:* {{#if nullable}}Yes{{else}}No{{/if}} {{#if example}} + {{{renderPropertyExample this}}} {{/if}} +{{#if related_topics}} + +*Related topics:* +{{#each related_topics}} +* {{{this}}} +{{/each}} +{{/if}} {{#if aliases}} -*Aliases:* {{join aliases ", "}} +*Aliases:* {{join aliases ", "}} {{/if}} {{#if is_deprecated}} + [WARNING] ==== This property is deprecated. ==== - {{/if}} --- diff --git a/tools/property-extractor/transformers.py b/tools/property-extractor/transformers.py index 56c7ef4..182b658 100644 --- a/tools/property-extractor/transformers.py +++ b/tools/property-extractor/transformers.py @@ -1,13 +1,50 @@ import re +import logging from property_bag import PropertyBag from parser import normalize_string +# Get logger for this module +logger = logging.getLogger(__name__) + +# Import the process_enterprise_value function from property_extractor +# Note: We import at function level to avoid circular imports since property_extractor +# imports transformers.py. This pattern allows the EnterpriseTransformer to access +# the centralized enterprise value processing logic without creating import cycles. +def get_process_enterprise_value(): + """ + Lazily import and return the centralized `process_enterprise_value` function from `property_extractor`. + + Attempts to import `process_enterprise_value` and return it to avoid circular-import issues. If the import fails an error message is printed and None is returned. + + Returns: + Callable or None: The `process_enterprise_value` callable when available, otherwise `None`. + """ + try: + from property_extractor import process_enterprise_value + return process_enterprise_value + except ImportError as e: + logger.error("Cannot import process_enterprise_value from property_extractor: %s", e) + return None + class BasicInfoTransformer: def accepts(self, info, file_pair): + """ + Always accepts the provided info and file_pair. + + Parameters: + info (dict): Parsed metadata for a property (annotation/params/declaration). + file_pair (object): Pair of source/implementation file metadata used by transformers. + + Returns: + bool: Always returns True, indicating this transformer should be applied. + """ return True def parse(self, property, info, file_pair): + if not info.get("params") or len(info["params"]) == 0: + return property + property["name"] = info["params"][0]["value"] property["defined_in"] = re.sub( r"^.*src/", "src/", str(file_pair.implementation) @@ -427,13 +464,65 @@ def parse(self, property, info, file_pair): class EnterpriseTransformer: + """ + Transforms enterprise property values from C++ expressions to user-friendly JSON. + + This transformer processes enterprise values by delegating to the + centralized process_enterprise_value function which handles the full range of + C++ expression types found in enterprise property definitions. + """ def accepts(self, info, file_pair): + """ + Return True if the provided info indicates an enterprise-only property. + + Parameters: + info (dict): The metadata dictionary for a property. This function checks for a 'type' key whose string contains 'enterprise'. + file_pair: Unused; present for transformer interface compatibility. + + Returns: + bool: True when info contains a 'type' that includes 'enterprise', otherwise False. + """ return bool(info.get('type') and 'enterprise' in info['type']) def parse(self, property, info, file_pair): + """ + Mark a property as enterprise-only and attach its enterprise value. + + If an enterprise value is present in info['params'][0]['value'], this method attempts to process it using the shared + process_enterprise_value helper (loaded via get_process_enterprise_value()). If the processor is unavailable or raises + an exception, the raw enterprise value is used. + + Side effects: + - Sets property["enterprise_value"] to the processed or raw value. + - Sets property["is_enterprise"] = True. + - Removes the first element from info['params']. + + Parameters: + property (dict): Property bag to modify and return. + info (dict): Parsed metadata; must have a non-None 'params' list for processing. + file_pair: Unused here but accepted for transformer API compatibility. + + Returns: + dict: The updated property bag. + """ if info['params'] is not None: enterpriseValue = info['params'][0]['value'] - property['enterprise_value'] = enterpriseValue + + # Get the processing function + process_enterprise_value = get_process_enterprise_value() + if process_enterprise_value is None: + property["enterprise_value"] = enterpriseValue + property['is_enterprise'] = True + del info['params'][0] + return property + + try: + processed_value = process_enterprise_value(enterpriseValue) + property["enterprise_value"] = processed_value + except Exception: + # Fallback to raw value if processing fails + property["enterprise_value"] = enterpriseValue + property['is_enterprise'] = True del info['params'][0] return property @@ -462,7 +551,14 @@ def parse(self, property, info, file_pair): iterable_params = info['params'] for param in iterable_params: if isinstance(param['value'], str) and param['value'].startswith("meta{"): - meta_content = param['value'].strip("meta{ }").strip() + # Extract content between meta{ and } using explicit slicing + param_value = param['value'] + if param_value.endswith('}'): + meta_content = param_value[5:-1].strip() # Remove "meta{" and "}" + else: + # Handle malformed meta{ without closing } + meta_content = param_value[5:].strip() # Remove "meta{" only + meta_dict = {} for item in meta_content.split(','): item = item.strip()