Skip to content

Commit 7f834e7

Browse files
JakeSCahillJakeSCahill
committed
Integrate topic property automation
1 parent f08e9ba commit 7f834e7

File tree

7 files changed

+188
-61
lines changed

7 files changed

+188
-61
lines changed

‎__tests__/docs-data/property-overrides.json‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,34 @@
4848
},
4949
"cloud_storage_access_key": {
5050
"description": "Access key for cloud storage authentication. Used to authenticate with S3-compatible object storage services."
51+
},
52+
"cleanup.policy": {
53+
"description": "Determines how log segments are cleaned. **delete** removes old segments based on time or size. **compact** retains only the latest value for each key. **compact,delete** enables both strategies.",
54+
"version": "v23.1.0",
55+
"example": [
56+
".Example: Setting cleanup policy",
57+
"[,bash]",
58+
"----",
59+
"rpk topic alter-config my-topic --set cleanup.policy=compact",
60+
"----",
61+
"",
62+
"For topics that require both compaction and deletion:",
63+
"",
64+
"[,bash]",
65+
"----",
66+
"rpk topic alter-config my-topic --set cleanup.policy=compact,delete",
67+
"----"
68+
]
69+
},
70+
"compression.type": {
71+
"description": "Compression algorithm used for compressing message batches. Options include **none** (no compression), **gzip**, **snappy**, **lz4**, and **zstd**.",
72+
"example": [
73+
".Example: Setting compression type",
74+
"[,bash]",
75+
"----",
76+
"rpk topic alter-config my-topic --set compression.type=zstd",
77+
"----"
78+
]
5179
}
5280
}
5381
}

‎bin/doc-tools.js‎

Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -847,41 +847,6 @@ automation
847847
process.exit(0);
848848
});
849849

850-
automation
851-
.command('topic-property-docs')
852-
.description('Generate JSON and AsciiDoc documentation for Redpanda topic configuration properties')
853-
.option('--tag <tag>', 'Git tag or branch to extract from', 'dev')
854-
.option('--diff <oldTag>', 'Also diff autogenerated topic properties from <oldTag> → <tag>')
855-
.action((options) => {
856-
verifyPropertyDependencies();
857-
858-
const newTag = options.tag;
859-
const oldTag = options.diff;
860-
const cwd = path.resolve(__dirname, '../tools/property-extractor');
861-
const make = (tag) => {
862-
console.log(`⏳ Building topic property docs for ${tag}…`);
863-
const r = spawnSync('make', ['topic-properties', `TAG=${tag}`], { cwd, stdio: 'inherit' });
864-
if (r.error) {
865-
console.error(`❌ ${r.error.message}`);
866-
process.exit(1);
867-
}
868-
if (r.status !== 0) process.exit(r.status);
869-
};
870-
871-
if (oldTag) {
872-
const oldDir = path.join('autogenerated', oldTag, 'properties');
873-
if (!fs.existsSync(oldDir)) make(oldTag);
874-
}
875-
876-
make(newTag);
877-
878-
if (oldTag) {
879-
diffDirs('properties', oldTag, newTag);
880-
}
881-
882-
process.exit(0);
883-
});
884-
885850
automation
886851
.command('rpk-docs')
887852
.description('Generate AsciiDoc documentation for rpk CLI commands')

‎tools/property-extractor/Makefile‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
.PHONY: build venv clean redpanda-git treesitter topic-properties generate-docs check
1+
.PHONY: build venv clean redpanda-git treesitter generate-docs check
22

33
# --- Main build: venv, fetch code, build parser, extract & docgen ---
44
build: venv redpanda-git treesitter
@@ -142,16 +142,4 @@ check:
142142
@echo "PYTHON: $(PYTHON)"
143143
@echo "OUTPUT_ASCIIDOC_DIR: $(OUTPUT_ASCIIDOC_DIR)"
144144
@echo "OUTPUT_JSON_DIR: $(OUTPUT_JSON_DIR)"
145-
@echo "OUTPUT_AUTOGENERATED_DIR: $(OUTPUT_AUTOGENERATED_DIR)"
146-
147-
# --- Extract topic properties ---
148-
topic-properties: venv redpanda-git treesitter
149-
@echo "🔧 Extracting topic properties with Redpanda tag: $(TAG)"
150-
@mkdir -p $(TOOL_ROOT)/gen
151-
@mkdir -p "$(OUTPUT_DIR)"
152-
@cd $(TOOL_ROOT) && \
153-
$(PYTHON) topic_property_extractor.py \
154-
--source-path $(REDPANDA_SRC) \
155-
--output-json "$(OUTPUT_DIR)/topic-properties-output.json" \
156-
--output-adoc "$(OUTPUT_DIR)/topic-properties.adoc"
157-
@echo "✅ Topic properties extracted"
145+
@echo "OUTPUT_AUTOGENERATED_DIR: $(OUTPUT_AUTOGENERATED_DIR)"

‎tools/property-extractor/generate-handlebars-docs.js‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,19 @@ NOTE: Some object storage properties require that you restart the cluster for an
7979
pageTitle: 'Topic Configuration Properties',
8080
pageAliases: ['reference:topic-properties.adoc'],
8181
description: 'Reference of topic configuration properties.',
82-
intro: `A topic-level property sets a Redpanda or Kafka configuration for a particular topic.\n\nMany topic-level properties have corresponding xref:manage:cluster-maintenance/cluster-property-configuration.adoc[cluster properties] that set a default value for all topics of a cluster. To customize the value for a topic, you can set a topic-level property that overrides the value of the corresponding cluster property.\n\n"
83-
"NOTE: All topic properties take effect immediately after being set.\n\n`,
84-
sectionTitle: 'Topic configuration'
82+
intro: `A topic-level property sets a Redpanda or Kafka configuration for a particular topic.
83+
84+
Many topic-level properties have corresponding xref:manage:cluster-maintenance/cluster-property-configuration.adoc[cluster properties] that set a default value for all topics of a cluster. To customize the value for a topic, you can set a topic-level property that overrides the value of the corresponding cluster property.
85+
86+
NOTE: All topic properties take effect immediately after being set.`,
87+
sectionTitle: 'Topic configuration',
88+
groups: [
89+
{
90+
filter: (prop) => prop.config_scope === 'topic' && !prop.is_deprecated,
91+
template: 'topic-property'
92+
}
93+
],
94+
filename: 'topic-properties.adoc'
8595
}
8696
};
8797

@@ -113,6 +123,14 @@ function registerPartials() {
113123
const propertyTemplate = fs.readFileSync(propertyTemplatePath, 'utf8');
114124
handlebars.registerPartial('property', propertyTemplate);
115125

126+
// Register topic property partial
127+
const topicPropertyTemplatePath = getTemplatePath(
128+
path.join(templatesDir, 'topic-property.hbs'),
129+
'TEMPLATE_TOPIC_PROPERTY'
130+
);
131+
const topicPropertyTemplate = fs.readFileSync(topicPropertyTemplatePath, 'utf8');
132+
handlebars.registerPartial('topic-property', topicPropertyTemplate);
133+
116134
// Register deprecated property partial
117135
const deprecatedPropertyTemplatePath = getTemplatePath(
118136
path.join(templatesDir, 'deprecated-property.hbs'),
@@ -141,7 +159,8 @@ function generatePropertyDocs(properties, config, outputDir) {
141159
return {
142160
title: group.title,
143161
intro: group.intro,
144-
properties: filteredProperties
162+
properties: filteredProperties,
163+
template: group.template || 'property' // Default to 'property' template
145164
};
146165
}).filter(group => group.properties.length > 0);
147166

@@ -211,6 +230,7 @@ function generateAllDocs(inputFile, outputDir) {
211230
let totalBrokerProperties = 0;
212231
let totalClusterProperties = 0;
213232
let totalObjectStorageProperties = 0;
233+
let totalTopicProperties = 0;
214234

215235
// Generate each type of documentation
216236
for (const [type, config] of Object.entries(PROPERTY_CONFIG)) {
@@ -220,6 +240,7 @@ function generateAllDocs(inputFile, outputDir) {
220240
if (type === 'broker') totalBrokerProperties = count;
221241
else if (type === 'cluster') totalClusterProperties = count;
222242
else if (type === 'object-storage') totalObjectStorageProperties = count;
243+
else if (type === 'topic') totalTopicProperties = count;
223244
}
224245

225246
// Generate deprecated properties documentation
@@ -237,13 +258,15 @@ function generateAllDocs(inputFile, outputDir) {
237258
console.log(` Total Broker properties: ${totalBrokerProperties}`);
238259
console.log(` Total Cluster properties: ${totalClusterProperties}`);
239260
console.log(` Total Object Storage properties: ${totalObjectStorageProperties}`);
261+
console.log(` Total Topic properties: ${totalTopicProperties}`);
240262
console.log(` Total Deprecated properties: ${deprecatedCount}`);
241263

242264
return {
243265
totalProperties: Object.keys(properties).length,
244266
brokerProperties: totalBrokerProperties,
245267
clusterProperties: totalClusterProperties,
246268
objectStorageProperties: totalObjectStorageProperties,
269+
topicProperties: totalTopicProperties,
247270
deprecatedProperties: deprecatedCount
248271
};
249272
}

‎tools/property-extractor/property_extractor.py‎

Lines changed: 71 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,13 @@
6969
from property_bag import PropertyBag
7070
from transformers import *
7171

72+
# Import topic property extractor
73+
try:
74+
from topic_property_extractor import TopicPropertyExtractor
75+
except ImportError:
76+
# TopicPropertyExtractor not available, will skip topic property extraction
77+
TopicPropertyExtractor = None
78+
7279
logger = logging.getLogger("viewer")
7380

7481

@@ -484,18 +491,23 @@ def _process_example_override(override, overrides_file_path=None):
484491

485492
def add_config_scope(properties):
486493
"""
487-
Add a config_scope field to each property based on its defined_in value.
494+
Add a config_scope field to each property based on its defined_in value or property type.
488495
'cluster' if defined_in == src/v/config/configuration.cc
489496
'broker' if defined_in == src/v/config/node_config.cc
497+
'topic' if is_topic_property == True
490498
"""
491499
for prop in properties.values():
492-
defined_in = prop.get("defined_in", "")
493-
if defined_in == "src/v/config/configuration.cc":
494-
prop["config_scope"] = "cluster"
495-
elif defined_in == "src/v/config/node_config.cc":
496-
prop["config_scope"] = "broker"
500+
# Check if this is a topic property first
501+
if prop.get("is_topic_property", False):
502+
prop["config_scope"] = "topic"
497503
else:
498-
prop["config_scope"] = None
504+
defined_in = prop.get("defined_in", "")
505+
if defined_in == "src/v/config/configuration.cc":
506+
prop["config_scope"] = "cluster"
507+
elif defined_in == "src/v/config/node_config.cc":
508+
prop["config_scope"] = "broker"
509+
else:
510+
prop["config_scope"] = None
499511
return properties
500512

501513

@@ -1159,6 +1171,52 @@ def expand_default(type_name, default_str):
11591171
return properties
11601172

11611173

1174+
def extract_topic_properties(source_path):
1175+
"""
1176+
Extract topic properties and convert them to the standard properties format.
1177+
1178+
Args:
1179+
source_path: Path to the Redpanda source code
1180+
1181+
Returns:
1182+
Dictionary of topic properties in the standard format with config_scope: "topic"
1183+
"""
1184+
if TopicPropertyExtractor is None:
1185+
logging.warning("TopicPropertyExtractor not available, skipping topic property extraction")
1186+
return {}
1187+
1188+
try:
1189+
extractor = TopicPropertyExtractor(source_path)
1190+
topic_data = extractor.extract_topic_properties()
1191+
topic_properties = topic_data.get("topic_properties", {})
1192+
1193+
# Convert topic properties to the standard properties format
1194+
converted_properties = {}
1195+
for prop_name, prop_data in topic_properties.items():
1196+
# Skip no-op properties
1197+
if prop_data.get("is_noop", False):
1198+
continue
1199+
1200+
converted_properties[prop_name] = {
1201+
"name": prop_name,
1202+
"description": prop_data.get("description", ""),
1203+
"type": prop_data.get("type", "string"),
1204+
"config_scope": "topic",
1205+
"source_file": prop_data.get("source_file", ""),
1206+
"corresponding_cluster_property": prop_data.get("corresponding_cluster_property", ""),
1207+
"acceptable_values": prop_data.get("acceptable_values", ""),
1208+
"is_deprecated": False,
1209+
"is_topic_property": True
1210+
}
1211+
1212+
logging.info(f"Extracted {len(converted_properties)} topic properties (excluding {len([p for p in topic_properties.values() if p.get('is_noop', False)])} no-op properties)")
1213+
return converted_properties
1214+
1215+
except Exception as e:
1216+
logging.error(f"Failed to extract topic properties: {e}")
1217+
return {}
1218+
1219+
11621220
def main():
11631221
import argparse
11641222

@@ -1263,6 +1321,12 @@ def generate_options():
12631321
)
12641322
properties = transform_files_with_properties(files_with_properties)
12651323

1324+
# Extract topic properties and add them to the main properties dictionary
1325+
topic_properties = extract_topic_properties(options.path)
1326+
if topic_properties:
1327+
properties.update(topic_properties)
1328+
logging.info(f"Added {len(topic_properties)} topic properties to the main properties collection")
1329+
12661330
# First, create the original properties without overrides for the base JSON output
12671331
# 1. Add config_scope field based on which source file defines the property
12681332
original_properties = add_config_scope(deepcopy(properties))

‎tools/property-extractor/templates/property-page.hbs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
{{#each groups}}
1818
{{#each this.properties}}
19-
{{> property}}
19+
{{> (lookup ../this "template")}}
2020

2121
{{/each}}
2222
{{/each}}

‎tools/property-extractor/templates/topic-property.hbs‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
=== {{name}}
2+
3+
{{#if version}}
4+
*Introduced in {{version}}*
5+
6+
{{/if}}
7+
{{#if description}}
8+
{{{description}}}
9+
{{else}}
10+
No description available.
11+
{{/if}}
12+
13+
{{#if type}}
14+
*Type:* {{type}}
15+
16+
{{/if}}
17+
{{#if acceptable_values}}
18+
*Accepted values:* {{{acceptable_values}}}
19+
20+
{{/if}}
21+
{{#if corresponding_cluster_property}}
22+
*Related cluster property:* xref:reference:cluster-properties.adoc#{{corresponding_cluster_property}}[{{corresponding_cluster_property}}]
23+
24+
{{/if}}
25+
{{#if (and minimum maximum)}}
26+
*Accepted values:* [`{{minimum}}`, `{{maximum}}`]
27+
28+
{{else}}
29+
{{#if minimum}}
30+
*Minimum value:* `{{minimum}}`
31+
32+
{{/if}}
33+
{{#if maximum}}
34+
*Maximum value:* `{{maximum}}`
35+
36+
{{/if}}
37+
{{/if}}
38+
{{#if (ne default undefined)}}
39+
*Default:* `{{formatPropertyValue default type}}`
40+
41+
{{/if}}
42+
*Nullable:* {{#if nullable}}Yes{{else}}No{{/if}}
43+
44+
{{#if example}}
45+
{{{renderPropertyExample this}}}
46+
{{/if}}
47+
48+
{{#if aliases}}
49+
*Aliases:* {{join aliases ", "}}
50+
51+
{{/if}}
52+
{{#if is_deprecated}}
53+
[WARNING]
54+
====
55+
This property is deprecated.
56+
====
57+
58+
{{/if}}
59+
---

0 commit comments

Comments
 (0)