From 4b215e350a7d4f797eeff1f5c5be263996b69907 Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Tue, 15 Apr 2025 18:35:39 +0530 Subject: [PATCH 1/8] feat: centralized code for supporting csc --- splunk_add_on_ucc_framework/commands/build.py | 2 +- splunk_add_on_ucc_framework/const.py | 175 +++++++++++ splunk_add_on_ucc_framework/global_config.py | 7 + .../global_config_validator.py | 57 +++- .../schema/schema.json | 279 ++++++++++++++++-- .../globalConfig.json | 69 ++++- 6 files changed, 565 insertions(+), 24 deletions(-) create mode 100644 splunk_add_on_ucc_framework/const.py diff --git a/splunk_add_on_ucc_framework/commands/build.py b/splunk_add_on_ucc_framework/commands/build.py index fdf5a7e31a..c8d94695bc 100644 --- a/splunk_add_on_ucc_framework/commands/build.py +++ b/splunk_add_on_ucc_framework/commands/build.py @@ -489,7 +489,7 @@ def generate( global_config_update.handle_global_config_update(global_config, gc_path) try: validator = global_config_validator.GlobalConfigValidator( - internal_root_dir, global_config + internal_root_dir, global_config, source=source ) validator.validate() logger.info("globalConfig file is valid") diff --git a/splunk_add_on_ucc_framework/const.py b/splunk_add_on_ucc_framework/const.py new file mode 100644 index 0000000000..47bd2fe743 --- /dev/null +++ b/splunk_add_on_ucc_framework/const.py @@ -0,0 +1,175 @@ +# +# Copyright 2025 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# TODO: Update the list as and when Splunk introduces new commands. +# Links to use: https://docs.splunk.com/Documentation/SplunkCloud/latest/SearchReference +# https://docs.splunk.com/Documentation/Splunk/latest/SearchReference +SPLUNK_COMMANDS = [ + "abstract", + "accum", + "addcoltotals", + "addinfo", + "addtotals", + "analyzefields", + "anomalies", + "anomalousvalue", + "anomalydetection", + "append", + "appendcols", + "appendpipe", + "arules", + "associate", + "autoregress", + "awssnsalert", + "bin", + "bucket", + "bucketdir", + "chart", + "cluster", + "cofilter", + "collect", + "concurrency", + "contingency", + "convert", + "correlate", + "ctable", + "datamodel", + "datamodelsimple", + "dbinspect", + "dbxquery", + "dedup", + "delete", + "delta", + "diff", + "entitymerge", + "erex", + "eval", + "eventcount", + "eventstats", + "extract", + "fieldformat", + "fields", + "fieldsummary", + "filldown", + "fillnull", + "findtypes", + "folderize", + "foreach", + "format", + "from", + "fromjson", + "gauge", + "gentimes", + "geom", + "geomfilter", + "geostats", + "head", + "highlight", + "history", + "iconify", + "inputcsv", + "inputintelligence", + "inputlookup", + "iplocation", + "join", + "kmeans", + "kvform", + "loadjob", + "localize", + "localop", + "lookup", + "makecontinuous", + "makemv", + "makeresults", + "map", + "mcollect", + "metadata", + "metasearch", + "meventcollect", + "mpreview", + "msearch", + "mstats", + "multikv", + "multisearch", + "mvcombine", + "mvexpand", + "nomv", + "outlier", + "outputcsv", + "outputlookup", + "outputtext", + "overlap", + "pivot", + "predict", + "rangemap", + "rare", + "regex", + "reltime", + "rename", + "replace", + "require", + "rest", + "return", + "reverse", + "rex", + "rtorder", + "run", + "savedsearch", + "script", + "scrub", + "search", + "searchtxn", + "selfjoin", + "sendalert", + "sendemail", + "set", + "setfields", + "sichart", + "sirare", + "sistats", + "sitimechart", + "sitop", + "sort", + "spath", + "stats", + "strcat", + "streamstats", + "table", + "tags", + "tail", + "timechart", + "timewrap", + "tojson", + "top", + "transaction", + "transpose", + "trendline", + "tscollect", + "tstats", + "typeahead", + "typelearner", + "typer", + "union", + "uniq", + "untable", + "walklex", + "where", + "x11", + "xmlkv", + "xmlunescape", + "xpath", + "xyseries", +] diff --git a/splunk_add_on_ucc_framework/global_config.py b/splunk_add_on_ucc_framework/global_config.py index 1f54d8d17d..967cfbddff 100644 --- a/splunk_add_on_ucc_framework/global_config.py +++ b/splunk_add_on_ucc_framework/global_config.py @@ -209,6 +209,10 @@ def configs(self) -> List[Any]: def alerts(self) -> List[Dict[str, Any]]: return self._content.get("alerts", []) + @property + def custom_search_commands(self) -> List[Dict[str, Any]]: + return self._content.get("customSearchCommand", []) + @property def meta(self) -> Dict[str, Any]: return self._content["meta"] @@ -271,6 +275,9 @@ def has_configuration(self) -> bool: def has_alerts(self) -> bool: return bool(self.alerts) + def has_custom_search_commands(self) -> bool: + return bool(self.custom_search_commands) + def has_dashboard(self) -> bool: return self.dashboard is not None diff --git a/splunk_add_on_ucc_framework/global_config_validator.py b/splunk_add_on_ucc_framework/global_config_validator.py index 6f4f9b5ad4..ff8c847aeb 100644 --- a/splunk_add_on_ucc_framework/global_config_validator.py +++ b/splunk_add_on_ucc_framework/global_config_validator.py @@ -20,6 +20,7 @@ from typing import Any, Dict, List import logging import itertools +from splunk_add_on_ucc_framework.const import SPLUNK_COMMANDS import jsonschema @@ -47,8 +48,14 @@ class GlobalConfigValidator: Custom validation should be implemented here. """ - def __init__(self, source_dir: str, global_config: global_config_lib.GlobalConfig): - self._source_dir = source_dir + def __init__( + self, + internal_root_dir: str, + global_config: global_config_lib.GlobalConfig, + **kwargs: Any, + ): + self._internal_root_dir = internal_root_dir + self._source_dir = kwargs.get("source", "") self._global_config = global_config self._config = global_config.content self.resolved_configuration = global_config.resolved_configuration @@ -58,7 +65,7 @@ def _validate_config_against_schema(self) -> None: Validates config against JSON schema. Raises jsonschema.ValidationError if config is not valid. """ - schema_path = os.path.join(self._source_dir, "schema", "schema.json") + schema_path = os.path.join(self._internal_root_dir, "schema", "schema.json") with open(schema_path, encoding="utf-8") as f_schema: schema_raw = f_schema.read() schema = json.loads(schema_raw) @@ -710,6 +717,49 @@ def _validate_meta_default_view(self) -> None: 'meta.defaultView == "dashboard" but there is no dashboard defined in globalConfig' ) + def _validate_custom_search_commands(self) -> None: + for command in self._global_config.custom_search_commands: + file_path = os.path.join(self._source_dir, "bin", command["fileName"]) + if not os.path.isfile(file_path): + raise GlobalConfigValidatorException( + f"{command['fileName']} is not present in `{os.path.join(self._source_dir, 'bin')}` directory. " + "Please ensure the file exists." + ) + + if (command.get("requiredSearchAssistant", False) is False) and ( + command.get("description") + or command.get("usage") + or command.get("syntax") + ): + logger.warning( + "requiredSearchAssistant is set to false " + "but attributes required for 'searchbnf.conf' is defined which is not required." + ) + if (command.get("requiredSearchAssistant", False) is True) and not ( + command.get("description") + and command.get("usage") + and command.get("syntax") + ): + raise GlobalConfigValidatorException( + "One of the attributes among `description`, `usage`, `syntax`" + " is not been defined in globalConfig. Define them as requiredSearchAssistant is set to True." + ) + + if command["commandName"] in SPLUNK_COMMANDS: + raise GlobalConfigValidatorException( + f"CommandName: {command['commandName']}" + " cannot have the same name as Splunk built-in command." + ) + + fileName_without_extension = command["fileName"].replace(".py", "") + if command["commandName"] == fileName_without_extension: + # Here we are generating file based on commandName therefore + # the core logic should not have the same name as commandName + raise GlobalConfigValidatorException( + f"Filename: {fileName_without_extension} and CommandName: {command['commandName']}" + " should not be same for custom search command." + ) + def validate(self) -> None: self._validate_config_against_schema() if self._global_config.has_pages(): @@ -723,6 +773,7 @@ def validate(self) -> None: self._validate_checkbox_group() self._validate_groups() self._validate_field_modifications() + self._validate_custom_search_commands() self._validate_alerts() self._validate_meta_default_view() diff --git a/splunk_add_on_ucc_framework/schema/schema.json b/splunk_add_on_ucc_framework/schema/schema.json index e3da4df112..c5c65876ce 100644 --- a/splunk_add_on_ucc_framework/schema/schema.json +++ b/splunk_add_on_ucc_framework/schema/schema.json @@ -427,6 +427,211 @@ ], "additionalProperties": false }, + "customSearchCommand": { + "type": "object", + "description": "Support for custom Search Command", + "properties": { + "commandName": { + "type": "string", + "pattern": "^[a-z0-9]+$", + "maxLength": 100, + "description": "Name of the custom Search Command" + }, + "fileName": { + "type": "string", + "pattern": "^\\w+\\.py$", + "description": "Name of the file which contains logic for custom search command. The file should be a python file only." + }, + "commandType": { + "type": "string", + "description": "Type of the custom search command. There are 4 types of command i.e generating, streaming, transforming and dataset processing", + "enum": [ + "generating", + "streaming", + "dataset processing" + ] + }, + "requiredSearchAssistant": { + "type": "boolean", + "default": false, + "description": "Specifies if search assistant is required or not. If yes then searchbnf.conf will be generated." + }, + "description": { + "type": "string", + "description": "description of the custom search command.It is an required attribute for searchbnf.conf." + }, + "syntax": { + "type": "string", + "maxLength": 100, + "description": "syntax for the custom search command, It is an required attribute for searchbnf.conf." + }, + "usage": { + "type": "string", + "description": "Specifies what will be the usage of custom search command. It is an required attribute for searchbnf.conf.", + "enum": [ + "public", + "private", + "deprecated" + ] + }, + "arguments": { + "type": "array", + "items": { + "$ref": "#/definitions/arguments" + }, + "minItems": 1 + } + }, + "required": [ + "commandName", + "fileName", + "commandType", + "arguments" + ], + "additionalProperties": false + }, + "arguments": { + "type": "object", + "description": "Arguments used for custom search command", + "properties": { + "name": { + "type": "string", + "description": "Name of the argument" + }, + "required": { + "type": "boolean", + "default": true, + "description": "Specifies if the argument is required or not" + }, + "validate": { + "$ref": "#/definitions/CustomSearchCommandValidator" + }, + "defaultValue": { + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, + "description": "Provide default value to the arguments passed for custom search command" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + "CustomSearchCommandValidator": { + "type": "object", + "description": "It is used to validate the values of arguments for custom search command", + "oneOf": [ + { + "$ref": "#/definitions/CustomIntegerValidator" + }, + { + "$ref": "#/definitions/CustomFloatValidator" + }, + { + "$ref": "#/definitions/CustomRegularExpressionValidator" + }, + { + "$ref": "#/definitions/CustomFieldnameValidator" + }, + { + "$ref": "#/definitions/CustomBooleanValidator" + } + ] + }, + "CustomIntegerValidator": { + "type": "object", + "properties": { + "minimum": { + "type": "number", + "description": "Minimum value used for validation" + }, + "maximum": { + "type": "number", + "description": "Maximum value used for validation" + }, + "type": { + "const": "Integer", + "type": "string", + "description": "This is integer Validator for custom search command, provide at least one of `minimumValue` or `maximumValue`" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "CustomFloatValidator": { + "type": "object", + "properties": { + "minimumValue": { + "type": "number", + "description": "Minimum value used for validation" + }, + "maximumValue": { + "type": "number", + "description": "Maximum value used for validation" + }, + "type": { + "const": "Float", + "type": "string", + "description": "This is Float Validator for custom search command, provide at least one of `minimumValue` or `maximumValue` for validation purpose." + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "CustomRegularExpressionValidator": { + "type": "object", + "properties": { + "type": { + "const": "RegularExpression", + "type": "string", + "description": "This is RegularExpression Validator which validates if the input value is a valid regular expression pattern or not." + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "CustomFieldnameValidator": { + "type": "object", + "properties": { + "type": { + "const": "Fieldname", + "type": "string", + "description": "Validates field name option values." + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "CustomBooleanValidator": { + "type": "object", + "properties": { + "type": { + "const": "Boolean", + "type": "string", + "description": "Validates Boolean option values." + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, "ConfigurationPage": { "type": "object", "properties": { @@ -480,7 +685,7 @@ "field": { "type": "string", "pattern": "^\\w+$", - "description": "Used to dispaly the data in a column." + "description": "Used to display the data in a column." }, "label": { "type": "string", @@ -1509,7 +1714,10 @@ "uniqueItems": true } }, - "required": ["label", "fields"], + "required": [ + "label", + "fields" + ], "additionalProperties": false } }, @@ -1540,7 +1748,9 @@ "additionalProperties": false } }, - "required": ["field"], + "required": [ + "field" + ], "additionalProperties": false }, "minItems": 1 @@ -1549,7 +1759,9 @@ "$ref": "#/definitions/disableonEdit" } }, - "required": ["rows"], + "required": [ + "rows" + ], "additionalProperties": false }, "validators": { @@ -1566,7 +1778,12 @@ "type": "boolean" } }, - "required": ["field", "label", "type", "options"], + "required": [ + "field", + "label", + "type", + "options" + ], "additionalProperties": false }, "RadioEntity": { @@ -1843,12 +2060,12 @@ "access_token_endpoint": { "type": "string", "maxLength": 350, - "description": "Must be present and its value should be the endpoint value for getting the ccess_token using the auth_code received." + "description": "Must be present and its value should be the endpoint value for getting the access_token using the auth_code received." }, "auth_endpoint_token_access_type": { "type": "string", "maxLength": 350, - "description": "Optional parameter that is mapped into the value of the token_access_type query param in the authorisation url." + "description": "Optional parameter that is mapped into the value of the token_access_type query param in the authorization url." }, "oauth_state_enabled": { "type": "boolean", @@ -2450,7 +2667,12 @@ "properties": { "type": { "type": "string", - "enum": ["string", "number", "integer", "boolean"] + "enum": [ + "string", + "number", + "integer", + "boolean" + ] }, "nullable": { "type": "boolean", @@ -2514,7 +2736,7 @@ "anyOf": { "type": "array", "items": { - "$ref": "#/definitions/OpenApiType" + "$ref": "#/definitions/OpenApiType" } }, "nullable": { @@ -2532,7 +2754,7 @@ "oneOf": { "type": "array", "items": { - "$ref": "#/definitions/OpenApiType" + "$ref": "#/definitions/OpenApiType" } }, "nullable": { @@ -2564,7 +2786,9 @@ } }, "additionalProperties": false, - "required": ["schema"] + "required": [ + "schema" + ] } } }, @@ -2604,7 +2828,9 @@ "handlerType": { "type": "string", "description": "The type of the handler.", - "enum": ["EAI"] + "enum": [ + "EAI" + ] }, "registerHandler": { "type": "object", @@ -2613,7 +2839,7 @@ "file": { "type": "string", "description": "The file where the custom rest handler is located.", - "pattern": "^[a-zA-Z0-9_]+\\.py$" + "pattern": "^[a-zA-Z0-9_]+\\.py$" }, "actions": { "type": "array", @@ -2621,7 +2847,12 @@ "items": { "type": "string", "description": "The action that the custom rest handler supports.", - "enum": ["list", "edit", "create", "remove"], + "enum": [ + "list", + "edit", + "create", + "remove" + ], "minItems": 1, "uniqueItems": true } @@ -2996,7 +3227,9 @@ "$ref": "#/definitions/links" } }, - "required": ["text"] + "required": [ + "text" + ] }, "singleFieldToModify": { "type": "object", @@ -3784,7 +4017,8 @@ "anyOf": [ { "type": "string", - "description": "Help text gives context about a fields input, such as how the input will be used. It is displayed directly below an input field." }, + "description": "Help text gives context about a fields input, such as how the input will be used. It is displayed directly below an input field." + }, { "$ref": "#/definitions/helpWithLinks" } @@ -3805,7 +4039,9 @@ "type": "string" } }, - "required": ["text"] + "required": [ + "text" + ] }, "tooltip": { "type": "string", @@ -3909,7 +4145,7 @@ } ] }, - "inputsUniqueAcrossSingleService":{ + "inputsUniqueAcrossSingleService": { "description": "Defines if inputs in different services can share same name.", "type": "boolean", "default": false @@ -3976,6 +4212,13 @@ }, "minItems": 1 }, + "customSearchCommand": { + "type": "array", + "items": { + "$ref": "#/definitions/customSearchCommand" + }, + "minItems": 1 + }, "options": { "$ref": "#/definitions/Options" } diff --git a/tests/testdata/test_addons/package_global_config_everything/globalConfig.json b/tests/testdata/test_addons/package_global_config_everything/globalConfig.json index b8bb0fa55f..0fdf6ef57b 100644 --- a/tests/testdata/test_addons/package_global_config_everything/globalConfig.json +++ b/tests/testdata/test_addons/package_global_config_everything/globalConfig.json @@ -2090,7 +2090,7 @@ "meta": { "name": "Splunk_TA_UCCExample", "restRoot": "splunk_ta_uccexample", - "version": "5.60.0+34ad0cb7a", + "version": "5.61.0+ea96ce003", "displayName": "Splunk UCC test Add-on", "schemaVersion": "0.0.9", "supportedThemes": [ @@ -2098,5 +2098,70 @@ "dark" ], "isVisible": true - } + }, + "customSearchCommand": [ + { + "commandName": "generatetextcommand", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": true, + "description": " This command generates COUNT occurrences of a TEXT string.", + "syntax": "generatetextcommand count= text=", + "usage": "public", + "arguments": [ + { + "name": "count", + "required": true, + "validate": { + "type": "Integer", + "minimum": 5, + "maximum": 10 + } + }, + { + "name": "text", + "required": true + } + ] + }, + { + "commandName": "filtercommand", + "fileName": "filter.py", + "commandType": "dataset processing", + "requiredSearchAssistant": true, + "description": "It filters records from the events stream returning only those which has :code:`contains` in them and replaces :code:`replace_array[0]` with :code:`replace_array[1]`.", + "syntax": "| filtercommand contains='value1' replace='value to be replaced,value to replace with'", + "usage": "public", + "arguments": [ + { + "name": "contains" + }, + { + "name": "replace_array" + } + ] + }, + { + "commandName": "countmatchescommand", + "fileName": "countmatches.py", + "commandType": "streaming", + "requiredSearchAssistant": false, + "arguments": [ + { + "name": "fieldname", + "validate": { + "type": "Fieldname" + }, + "required": true + }, + { + "name": "pattern", + "validate": { + "type": "RegularExpression" + }, + "required": true + } + ] + } + ] } From 5a87671c679eec044f2bc42b9af1556c6651c17e Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Tue, 15 Apr 2025 18:37:47 +0530 Subject: [PATCH 2/8] tests: added respective unit test cases --- tests/unit/test_const.py | 33 +++++ tests/unit/test_global_config.py | 25 ++++ tests/unit/test_global_config_validator.py | 120 +++++++++++++++++- ...alid_config_for_custom_search_command.json | 79 ++++++++++++ tests/unit/testdata/valid_config.json | 28 +++- 5 files changed, 283 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_const.py create mode 100644 tests/unit/testdata/invalid_config_for_custom_search_command.json diff --git a/tests/unit/test_const.py b/tests/unit/test_const.py new file mode 100644 index 0000000000..d296474620 --- /dev/null +++ b/tests/unit/test_const.py @@ -0,0 +1,33 @@ +import re +import urllib.request + +from splunk_add_on_ucc_framework.const import SPLUNK_COMMANDS + + +def test_command_list_up_to_date(): + with urllib.request.urlopen( + "https://docs.splunk.com/Documentation/Splunk/latest/SearchReference" + ) as resp: + content = resp.read().decode() + + match = re.search(r"Search\s+Commands.+?(.+?)", content, re.S) + if match: + search_commands_ul = match.group(1) + search_commands = re.findall( + r"]*>.*?]*>\s*([^\s<]+)\s+?", search_commands_ul, re.S + ) + else: + search_commands_ul = None + search_commands = [] + + # These are the search commands for the serviceNow add-on. They are not present by default in Splunk instance + not_global_commands = [ + "snowincidentstream", + "snoweventstream", + "snowincident", + "snowevent", + ] + for command in not_global_commands: + search_commands.remove(command) + + assert set(search_commands) == set(SPLUNK_COMMANDS) diff --git a/tests/unit/test_global_config.py b/tests/unit/test_global_config.py index 6b18784f5b..f1529dbb45 100644 --- a/tests/unit/test_global_config.py +++ b/tests/unit/test_global_config.py @@ -66,6 +66,31 @@ def test_global_config_equal(): assert global_config_1 == global_config_2 +def test_global_config_custom_search_commands(global_config_all_json): + custom_search_commands = global_config_all_json.custom_search_commands + expected_result = [ + { + "commandName": "generatetextcommand", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": True, + "description": " This command generates COUNT occurrences of a TEXT string.", + "syntax": "mycommand count= text=", + "usage": "public", + "arguments": [ + { + "name": "count", + "required": True, + "validate": {"type": "Integer", "minimum": 5, "maximum": 10}, + }, + {"name": "text", "required": True}, + ], + } + ] + assert expected_result == custom_search_commands + assert global_config_all_json.has_custom_search_commands() is True + + def test_global_config_when_no_equal(): global_config_1 = global_config_lib.GlobalConfig.from_file( helpers.get_testdata_file_path("valid_config.json") diff --git a/tests/unit/test_global_config_validator.py b/tests/unit/test_global_config_validator.py index a69a6c0106..2a57d6bd8f 100644 --- a/tests/unit/test_global_config_validator.py +++ b/tests/unit/test_global_config_validator.py @@ -15,6 +15,7 @@ should_warn_on_empty_validators, ) from splunk_add_on_ucc_framework import global_config as global_config_lib +from unittest.mock import patch @pytest.mark.parametrize( @@ -25,7 +26,8 @@ "valid_config_only_logging.json", ], ) -def test_config_validation_when_valid(filename): +@patch("os.path.isfile", return_value=True) +def test_config_validation_when_valid(mock_isFile, filename): global_config_path = helpers.get_testdata_file_path(filename) global_config = global_config_lib.GlobalConfig.from_file(global_config_path) @@ -75,6 +77,10 @@ def test_autocompletefields_children_support_integer_values(): @pytest.mark.parametrize( "filename,exception_message", [ + ( + "invalid_config_for_custom_search_command.json", + "generatetext.py is not present in `bin` directory. Please ensure the file exists.", + ), ( "invalid_config_no_configuration_tabs.json", "[] is too short", @@ -355,6 +361,118 @@ def test_config_validation_when_error(filename, exception_message): assert msg == exception_message +@pytest.mark.parametrize( + "filename,invalid_custom_search_command,exception_message", + [ + ( + "invalid_config_for_custom_search_command.json", + [ + { + "commandName": "generatetextcommand", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": True, + "description": " This command generates COUNT occurrences of a TEXT string.", + "arguments": [ + { + "name": "text", + } + ], + } + ], + "One of the attributes among `description`, `usage`, `syntax` " + "is not been defined in globalConfig. Define them as requiredSearchAssistant is set to True.", + ), + ( + "invalid_config_for_custom_search_command.json", + [ + { + "commandName": "generatetext", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": False, + "arguments": [ + { + "name": "text", + } + ], + } + ], + "Filename: generatetext and CommandName: generatetext should not be same for custom search command.", + ), + ( + "invalid_config_for_custom_search_command.json", + [ + { + "commandName": "abstract", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": False, + "arguments": [ + { + "name": "text", + } + ], + } + ], + "CommandName: abstract cannot have the same name as Splunk built-in command.", + ), + ], +) +@patch("os.path.isfile", return_value=True) +def test_validate_custom_search_command( + mock_isFile, filename, invalid_custom_search_command, exception_message +): + global_config_path = helpers.get_testdata_file_path(filename) + global_config = global_config_lib.GlobalConfig.from_file(global_config_path) + + validator = GlobalConfigValidator(helpers.get_path_to_source_dir(), global_config) + global_config._content["customSearchCommand"] = invalid_custom_search_command + with pytest.raises(GlobalConfigValidatorException) as exc_info: + validator.validate() + + (msg,) = exc_info.value.args + assert msg == exception_message + + +@pytest.mark.parametrize( + "filename,valid_custom_search_command,warning_message", + [ + ( + "invalid_config_for_custom_search_command.json", + [ + { + "commandName": "generatetextcommand", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": False, + "description": " This command generates COUNT occurrences of a TEXT string.", + "arguments": [ + { + "name": "text", + } + ], + } + ], + "requiredSearchAssistant is set to false " + "but attributes required for 'searchbnf.conf' is defined which is not required.", + ), + ], +) +@patch("os.path.isfile", return_value=True) +def test_validate_custom_search_command_warning_msg( + mock_isFile, filename, valid_custom_search_command, warning_message, caplog +): + global_config_path = helpers.get_testdata_file_path(filename) + global_config = global_config_lib.GlobalConfig.from_file(global_config_path) + + validator = GlobalConfigValidator(helpers.get_path_to_source_dir(), global_config) + global_config._content["customSearchCommand"] = valid_custom_search_command + validator.validate() + + assert warning_message in caplog.text + + def test_config_validation_modifications_on_change(): global_config_path = helpers.get_testdata_file_path( "valid_config_with_modification_on_value_change.json" diff --git a/tests/unit/testdata/invalid_config_for_custom_search_command.json b/tests/unit/testdata/invalid_config_for_custom_search_command.json new file mode 100644 index 0000000000..e3f1f7a301 --- /dev/null +++ b/tests/unit/testdata/invalid_config_for_custom_search_command.json @@ -0,0 +1,79 @@ +{ + "pages": { + "configuration": { + "tabs": [ + { + "name": "logging", + "entity": [ + { + "type": "singleSelect", + "label": "Log level", + "options": { + "disableSearch": true, + "autoCompleteFields": [ + { + "value": "DEBUG", + "label": "DEBUG" + }, + { + "value": "INFO", + "label": "INFO" + }, + { + "value": "WARNING", + "label": "WARNING" + }, + { + "value": "ERROR", + "label": "ERROR" + }, + { + "value": "CRITICAL", + "label": "CRITICAL" + } + ] + }, + "defaultValue": "INFO", + "field": "loglevel", + "required": true + } + ], + "title": "Logging" + } + ], + "title": "Configuration", + "description": "Set up your add-on" + } + }, + "meta": { + "name": "Splunk_TA_UCCExample", + "restRoot": "splunk_ta_uccexample", + "version": "1.0.0", + "displayName": "Splunk UCC test Add-on", + "schemaVersion": "0.0.3" + }, + "customSearchCommand": [ + { + "commandName": "generatetextcommand", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": true, + "description": " This command generates COUNT occurrences of a TEXT string.", + "arguments": [ + { + "name": "count", + "required": true, + "validate": { + "type": "Integer", + "minimum": 5, + "maximum": 10 + } + }, + { + "name": "text", + "required": true + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/unit/testdata/valid_config.json b/tests/unit/testdata/valid_config.json index 1ebb30b219..37fff6704d 100644 --- a/tests/unit/testdata/valid_config.json +++ b/tests/unit/testdata/valid_config.json @@ -1356,5 +1356,31 @@ "target": "3rdparty/linux" } ] - } + }, + "customSearchCommand": [ + { + "commandName": "generatetextcommand", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": true, + "description": " This command generates COUNT occurrences of a TEXT string.", + "syntax": "mycommand count= text=", + "usage": "public", + "arguments": [ + { + "name": "count", + "required": true, + "validate": { + "type": "Integer", + "minimum": 5, + "maximum": 10 + } + }, + { + "name": "text", + "required": true + } + ] + } + ] } \ No newline at end of file From 65007d432fad53d0c1a4277c90a28b3338ad7395 Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Tue, 15 Apr 2025 18:38:44 +0530 Subject: [PATCH 3/8] chore: added different csc logic files in _everything addon --- .../package/bin/countmatches.py | 4 +++ .../package/bin/filter.py | 36 +++++++++++++++++++ .../package/bin/generatetext.py | 8 +++++ 3 files changed, 48 insertions(+) create mode 100644 tests/testdata/test_addons/package_global_config_everything/package/bin/countmatches.py create mode 100644 tests/testdata/test_addons/package_global_config_everything/package/bin/filter.py create mode 100644 tests/testdata/test_addons/package_global_config_everything/package/bin/generatetext.py diff --git a/tests/testdata/test_addons/package_global_config_everything/package/bin/countmatches.py b/tests/testdata/test_addons/package_global_config_everything/package/bin/countmatches.py new file mode 100644 index 0000000000..da79b16420 --- /dev/null +++ b/tests/testdata/test_addons/package_global_config_everything/package/bin/countmatches.py @@ -0,0 +1,4 @@ +def stream(self, records): + for record in records: + # write custom logic for the search command + yield record \ No newline at end of file diff --git a/tests/testdata/test_addons/package_global_config_everything/package/bin/filter.py b/tests/testdata/test_addons/package_global_config_everything/package/bin/filter.py new file mode 100644 index 0000000000..51d476833f --- /dev/null +++ b/tests/testdata/test_addons/package_global_config_everything/package/bin/filter.py @@ -0,0 +1,36 @@ +def transform(self, records): + contains = self.contains + replace_array = self.replace_array + + if contains and replace_array: + arr = replace_array.split(",") + if len(arr) != 2: + raise ValueError("Please provide only two arguments, separated by comma for 'replace'") + + for record in records: + _raw = record.get("_raw") + if contains in _raw: + record["_raw"] = _raw.replace(arr[0], arr[1]) + yield record + return + + if contains: + for record in records: + _raw = record.get("_raw") + if contains in _raw: + yield record + return + + if replace_array: + arr = replace_array.split(",") + if len(arr) != 2: + raise ValueError("Please provide only two arguments, separated by comma for 'replace'") + + for record in records: + _raw = record.get("_raw") + record["_raw"] = _raw.replace(arr[0], arr[1]) + yield record + return + + for record in records: + yield record \ No newline at end of file diff --git a/tests/testdata/test_addons/package_global_config_everything/package/bin/generatetext.py b/tests/testdata/test_addons/package_global_config_everything/package/bin/generatetext.py new file mode 100644 index 0000000000..897779284c --- /dev/null +++ b/tests/testdata/test_addons/package_global_config_everything/package/bin/generatetext.py @@ -0,0 +1,8 @@ +import time +import logging + + +def generate(self): + logging.debug("Generating %d events with text %s" % (self.count, self.text)) + for i in range(1, self.count + 1): + yield {'_serial': i, '_time': time.time(), '_raw': str(i) + '. ' + self.text} From 0816ddd084ed09cfd755ed1111e3b5d1e773f331 Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Wed, 16 Apr 2025 12:29:08 +0530 Subject: [PATCH 4/8] chore: updated syntax for csc in valid_config.json --- tests/unit/test_global_config.py | 4 ++-- tests/unit/testdata/valid_config.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_global_config.py b/tests/unit/test_global_config.py index f1529dbb45..9b00ad8ede 100644 --- a/tests/unit/test_global_config.py +++ b/tests/unit/test_global_config.py @@ -74,8 +74,8 @@ def test_global_config_custom_search_commands(global_config_all_json): "fileName": "generatetext.py", "commandType": "generating", "requiredSearchAssistant": True, - "description": " This command generates COUNT occurrences of a TEXT string.", - "syntax": "mycommand count= text=", + "description": "This command generates COUNT occurrences of a TEXT string.", + "syntax": "generatetextcommand count= text=", "usage": "public", "arguments": [ { diff --git a/tests/unit/testdata/valid_config.json b/tests/unit/testdata/valid_config.json index 37fff6704d..7019c6d5b4 100644 --- a/tests/unit/testdata/valid_config.json +++ b/tests/unit/testdata/valid_config.json @@ -1363,8 +1363,8 @@ "fileName": "generatetext.py", "commandType": "generating", "requiredSearchAssistant": true, - "description": " This command generates COUNT occurrences of a TEXT string.", - "syntax": "mycommand count= text=", + "description": "This command generates COUNT occurrences of a TEXT string.", + "syntax": "generatetextcommand count= text=", "usage": "public", "arguments": [ { From 2a4c32b4c4f9ea3a95e9d35f52e8207b91974f95 Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Fri, 18 Apr 2025 15:24:59 +0530 Subject: [PATCH 5/8] chore: removed keyword arguments --- splunk_add_on_ucc_framework/commands/build.py | 2 +- splunk_add_on_ucc_framework/global_config_validator.py | 4 ++-- splunk_add_on_ucc_framework/schema/schema.json | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/splunk_add_on_ucc_framework/commands/build.py b/splunk_add_on_ucc_framework/commands/build.py index c8d94695bc..c3ff923dc2 100644 --- a/splunk_add_on_ucc_framework/commands/build.py +++ b/splunk_add_on_ucc_framework/commands/build.py @@ -489,7 +489,7 @@ def generate( global_config_update.handle_global_config_update(global_config, gc_path) try: validator = global_config_validator.GlobalConfigValidator( - internal_root_dir, global_config, source=source + internal_root_dir, global_config, source ) validator.validate() logger.info("globalConfig file is valid") diff --git a/splunk_add_on_ucc_framework/global_config_validator.py b/splunk_add_on_ucc_framework/global_config_validator.py index ff8c847aeb..f6cde2f3f7 100644 --- a/splunk_add_on_ucc_framework/global_config_validator.py +++ b/splunk_add_on_ucc_framework/global_config_validator.py @@ -52,10 +52,10 @@ def __init__( self, internal_root_dir: str, global_config: global_config_lib.GlobalConfig, - **kwargs: Any, + source: str = "", ): self._internal_root_dir = internal_root_dir - self._source_dir = kwargs.get("source", "") + self._source_dir = source self._global_config = global_config self._config = global_config.content self.resolved_configuration = global_config.resolved_configuration diff --git a/splunk_add_on_ucc_framework/schema/schema.json b/splunk_add_on_ucc_framework/schema/schema.json index bba81a3dad..532b8fd83d 100644 --- a/splunk_add_on_ucc_framework/schema/schema.json +++ b/splunk_add_on_ucc_framework/schema/schema.json @@ -429,13 +429,13 @@ }, "customSearchCommand": { "type": "object", - "description": "Support for custom Search Command", + "description": "Support for custom Search Command.", "properties": { "commandName": { "type": "string", "pattern": "^[a-z0-9]+$", "maxLength": 100, - "description": "Name of the custom Search Command" + "description": "Name of the custom Search Command." }, "fileName": { "type": "string", @@ -458,12 +458,12 @@ }, "description": { "type": "string", - "description": "description of the custom search command.It is an required attribute for searchbnf.conf." + "description": "Description of the custom search command. It is an required attribute for searchbnf.conf." }, "syntax": { "type": "string", "maxLength": 100, - "description": "syntax for the custom search command, It is an required attribute for searchbnf.conf." + "description": "Syntax for the custom search command. It is an required attribute for searchbnf.conf." }, "usage": { "type": "string", From 863d5a0e756ce89a9bd4794f02fd8609f80c1c08 Mon Sep 17 00:00:00 2001 From: Hetang Modi <62056057+hetangmodi-crest@users.noreply.github.com> Date: Fri, 9 May 2025 15:34:05 +0530 Subject: [PATCH 6/8] feat: generation of `searchbnf.conf` (#1695) **Issue number:** ADDON-76780 ### PR Type **What kind of change does this PR introduce?** * [x] Feature * [ ] Bug Fix * [ ] Refactoring (no functional or API changes) * [x] Documentation Update * [ ] Maintenance (dependency updates, CI, etc.) ## Summary ### Changes This PR includes the generation of the `searchbnf.conf` file for the custom search command. ### User experience After defining a valid configuration for the custom search command in `globalConfig.json`, the `searchbnf.conf` file will be generated. ## Checklist If an item doesn't apply to your changes, leave it unchecked. ### Review * [x] self-review - I have performed a self-review of this change according to the [development guidelines](https://splunk.github.io/addonfactory-ucc-generator/contributing/#development-guidelines) * [x] Changes are documented. The documentation is understandable, examples work [(more info)](https://splunk.github.io/addonfactory-ucc-generator/contributing/#documentation-guidelines) * [x] PR title and description follows the [contributing principles](https://splunk.github.io/addonfactory-ucc-generator/contributing/#pull-requests) * [ ] meeting - I have scheduled a meeting or recorded a demo to explain these changes (if there is a video, put a link below and in the ticket) ### Tests See [the testing doc](https://splunk.github.io/addonfactory-ucc-generator/contributing/#build-and-test). * [x] Unit - tests have been added/modified to cover the changes * [x] Smoke - tests have been added/modified to cover the changes * [ ] UI - tests have been added/modified to cover the changes * [x] coverage - I have checked the code coverage of my changes [(see more)](https://splunk.github.io/addonfactory-ucc-generator/contributing/#checking-the-code-coverage) **Demo/meeting:** *Reviewers are encouraged to request meetings or demos if any part of the change is unclear* --- docs/generated_files.md | 1 + .../generators/conf_files/__init__.py | 2 + .../conf_files/create_searchbnf_conf.py | 54 ++++++++ .../generators/file_const.py | 2 + .../conf_files/searchbnf_conf.template | 6 + tests/smoke/test_ucc_build.py | 1 + .../default/searchbnf.conf | 9 ++ .../conf_files/test_conf_files_init.py | 2 + .../conf_files/test_create_searchbnf_conf.py | 117 ++++++++++++++++++ tests/unit/generators/test_doc_generator.py | 1 + tests/unit/test_utils.py | 1 + 11 files changed, 196 insertions(+) create mode 100644 splunk_add_on_ucc_framework/generators/conf_files/create_searchbnf_conf.py create mode 100644 splunk_add_on_ucc_framework/templates/conf_files/searchbnf_conf.template create mode 100644 tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/searchbnf.conf create mode 100644 tests/unit/generators/conf_files/test_create_searchbnf_conf.py diff --git a/docs/generated_files.md b/docs/generated_files.md index 09a27cc892..40e7eea2c7 100644 --- a/docs/generated_files.md +++ b/docs/generated_files.md @@ -14,6 +14,7 @@ The following table describes the files generated by UCC framework. | alert_actions.conf | output/<YOUR_ADDON_NAME>/default | Generates `alert_actions.conf` and `alert_actions.conf.spec` file for the custom alert actions defined in globalConfig | | eventtypes.conf | output/<YOUR_ADDON_NAME>/default | Generates `eventtypes.conf` file if the sourcetype is mentioned in Adaptive Response of custom alert action in globalConfig | | tags.conf | output/<YOUR_ADDON_NAME>/default | Generates `tags.conf` file based on the `eventtypes.conf` created for custom alert actions. | +| searchbnf.conf | output/<YOUR_ADDON_NAME>/default | Generates `searchbnf.conf` for custom search commands provided in the globalConfig. | | _account.conf | output/<YOUR_ADDON_NAME>/README | Generates `_account.conf.spec` file for the configuration mentioned in globalConfig | | _settings.conf | output/<YOUR_ADDON_NAME>/README | Generates `_settings.conf.spec` file for the Proxy, Logging or Custom Tab mentioned in globalConfig | | configuration.xml | output/<YOUR_ADDON_NAME>/default/data/ui/views | Generates configuration.xml file in `default/data/ui/views/` folder if configuration is defined in globalConfig. | diff --git a/splunk_add_on_ucc_framework/generators/conf_files/__init__.py b/splunk_add_on_ucc_framework/generators/conf_files/__init__.py index d6159cf629..f9d562730c 100644 --- a/splunk_add_on_ucc_framework/generators/conf_files/__init__.py +++ b/splunk_add_on_ucc_framework/generators/conf_files/__init__.py @@ -24,6 +24,7 @@ from .create_web_conf import WebConf from .create_account_conf import AccountConf from .create_settings_conf import SettingsConf +from .create_searchbnf_conf import SearchbnfConf __all__ = [ "FileGenerator", @@ -37,4 +38,5 @@ "InputsConf", "AccountConf", "SettingsConf", + "SearchbnfConf", ] diff --git a/splunk_add_on_ucc_framework/generators/conf_files/create_searchbnf_conf.py b/splunk_add_on_ucc_framework/generators/conf_files/create_searchbnf_conf.py new file mode 100644 index 0000000000..f102644eb2 --- /dev/null +++ b/splunk_add_on_ucc_framework/generators/conf_files/create_searchbnf_conf.py @@ -0,0 +1,54 @@ +# +# Copyright 2025 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Any, Dict + +from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator + + +class SearchbnfConf(FileGenerator): + __description__ = "Generates `searchbnf.conf` for custom search commands provided in the globalConfig." + + def _set_attributes(self, **kwargs: Any) -> None: + self.conf_file = "searchbnf.conf" + self.searchbnf_info = [] + if self._global_config.has_custom_search_commands(): + for command in self._global_config.custom_search_commands: + if command.get("requiredSearchAssistant", False): + searchbnf_dict = { + "command_name": command["commandName"], + "description": command["description"], + "syntax": command["syntax"], + "usage": command["usage"], + } + self.searchbnf_info.append(searchbnf_dict) + + def generate(self) -> Dict[str, str]: + if not self.searchbnf_info: + return {} + + file_path = self.get_file_output_path(["default", self.conf_file]) + self.set_template_and_render( + template_file_path=["conf_files"], file_name="searchbnf_conf.template" + ) + rendered_content = self._template.render( + searchbnf_info=self.searchbnf_info, + ) + self.writer( + file_name=self.conf_file, + file_path=file_path, + content=rendered_content, + ) + return {self.conf_file: file_path} diff --git a/splunk_add_on_ucc_framework/generators/file_const.py b/splunk_add_on_ucc_framework/generators/file_const.py index 0ebb7e65c2..34c4b67734 100644 --- a/splunk_add_on_ucc_framework/generators/file_const.py +++ b/splunk_add_on_ucc_framework/generators/file_const.py @@ -35,6 +35,7 @@ WebConf, AccountConf, SettingsConf, + SearchbnfConf, ) __all__ = ["FileClass", "GEN_FILE_LIST"] @@ -59,6 +60,7 @@ class FileClass(NamedTuple): ), FileClass("eventtypes.conf", EventtypesConf, ["default"]), FileClass("tags.conf", TagsConf, ["default"]), + FileClass("searchbnf.conf", SearchbnfConf, ["default"]), FileClass("_account.conf", AccountConf, ["README"]), FileClass("_settings.conf", SettingsConf, ["README"]), FileClass( diff --git a/splunk_add_on_ucc_framework/templates/conf_files/searchbnf_conf.template b/splunk_add_on_ucc_framework/templates/conf_files/searchbnf_conf.template new file mode 100644 index 0000000000..c7227070ca --- /dev/null +++ b/splunk_add_on_ucc_framework/templates/conf_files/searchbnf_conf.template @@ -0,0 +1,6 @@ +{% for info in searchbnf_info -%} +[{{info["command_name"]}}] +syntax = {{info["syntax"]}} +description = {{info["description"]}} +usage = {{info["usage"]}} +{% endfor -%} \ No newline at end of file diff --git a/tests/smoke/test_ucc_build.py b/tests/smoke/test_ucc_build.py index 59919fff6f..f358a08455 100644 --- a/tests/smoke/test_ucc_build.py +++ b/tests/smoke/test_ucc_build.py @@ -153,6 +153,7 @@ def test_ucc_generate_with_everything(caplog): ("default", "splunk_ta_uccexample_settings.conf"), ("default", "web.conf"), ("default", "server.conf"), + ("default", "searchbnf.conf"), ("default", "data", "ui", "alerts", "test_alert.html"), ("default", "data", "ui", "nav", "default.xml"), ("default", "data", "ui", "views", "configuration.xml"), diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/searchbnf.conf b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/searchbnf.conf new file mode 100644 index 0000000000..16d6021c1f --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/searchbnf.conf @@ -0,0 +1,9 @@ +[generatetextcommand] +syntax = generatetextcommand count= text= +description = This command generates COUNT occurrences of a TEXT string. +usage = public + +[filtercommand] +syntax = | filtercommand contains='value1' replace='value to be replaced,value to replace with' +description = It filters records from the events stream returning only those which has :code:`contains` in them and replaces :code:`replace_array[0]` with :code:`replace_array[1]`. +usage = public diff --git a/tests/unit/generators/conf_files/test_conf_files_init.py b/tests/unit/generators/conf_files/test_conf_files_init.py index 18e764f396..a242a9a4ed 100644 --- a/tests/unit/generators/conf_files/test_conf_files_init.py +++ b/tests/unit/generators/conf_files/test_conf_files_init.py @@ -11,6 +11,7 @@ def test___init__conf(): "InputsConf", "AccountConf", "SettingsConf", + "SearchbnfConf", ] expected_modules = [ "file_generator", @@ -24,6 +25,7 @@ def test___init__conf(): "create_web_conf", "create_account_conf", "create_settings_conf", + "create_searchbnf_conf", ] import splunk_add_on_ucc_framework.generators.conf_files as conf diff --git a/tests/unit/generators/conf_files/test_create_searchbnf_conf.py b/tests/unit/generators/conf_files/test_create_searchbnf_conf.py new file mode 100644 index 0000000000..0b43f5d96a --- /dev/null +++ b/tests/unit/generators/conf_files/test_create_searchbnf_conf.py @@ -0,0 +1,117 @@ +from pytest import fixture +from splunk_add_on_ucc_framework.generators.conf_files import SearchbnfConf +from splunk_add_on_ucc_framework import __file__ as ucc_framework_file +import os.path +from textwrap import dedent + +UCC_DIR = os.path.dirname(ucc_framework_file) + + +@fixture +def custom_search_command_without_search_assistance(): + return [ + { + "commandName": "testcommand2", + "commandType": "streaming", + "fileName": "test2.py", + } + ] + + +def test_set_attributes_without_custom_command( + global_config_only_configuration, input_dir, output_dir, ucc_dir, ta_name +): + searchbnf_conf = SearchbnfConf( + global_config_only_configuration, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert searchbnf_conf.searchbnf_info == [] + + +def test_set_attributes( + global_config_all_json, input_dir, output_dir, ucc_dir, ta_name +): + searchbnf_conf = SearchbnfConf( + global_config_all_json, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert searchbnf_conf.conf_file == "searchbnf.conf" + assert searchbnf_conf.searchbnf_info == [ + { + "command_name": "generatetextcommand", + "description": "This command generates COUNT occurrences of a TEXT string.", + "syntax": "generatetextcommand count= text=", + "usage": "public", + } + ] + + +def test_set_attributes_without_search_assistance( + global_config_all_json, + input_dir, + output_dir, + ucc_dir, + ta_name, + custom_search_command_without_search_assistance, +): + global_config_all_json._content[ + "customSearchCommand" + ] = custom_search_command_without_search_assistance + searchbnf_conf = SearchbnfConf( + global_config_all_json, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert searchbnf_conf.searchbnf_info == [] + + +def test_generate_conf_without_custom_command( + global_config_only_configuration, input_dir, output_dir, ucc_dir, ta_name +): + searchbnf_conf = SearchbnfConf( + global_config_only_configuration, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + file_paths = searchbnf_conf.generate() + + # Assert that no files are returned since no custom command is configured + assert file_paths == {} + + +def test_generate_conf(global_config_all_json, input_dir, output_dir, ta_name): + searchbnf_conf = SearchbnfConf( + global_config_all_json, + input_dir, + output_dir, + ucc_dir=UCC_DIR, + addon_name=ta_name, + ) + file_paths = searchbnf_conf.generate() + exp_fname = "searchbnf.conf" + + assert file_paths == {exp_fname: f"{output_dir}/{ta_name}/default/{exp_fname}"} + + with open(file_paths["searchbnf.conf"]) as fp: + content = fp.read() + + expected_content = dedent( + """ + [generatetextcommand] + syntax = generatetextcommand count= text= + description = This command generates COUNT occurrences of a TEXT string. + usage = public + """ + ).lstrip() + + assert content == expected_content diff --git a/tests/unit/generators/test_doc_generator.py b/tests/unit/generators/test_doc_generator.py index 08b103c3f9..491bd75711 100644 --- a/tests/unit/generators/test_doc_generator.py +++ b/tests/unit/generators/test_doc_generator.py @@ -21,6 +21,7 @@ def test_generate_docs(): | alert_actions.conf | output/<YOUR_ADDON_NAME>/default | Generates `alert_actions.conf` and `alert_actions.conf.spec` file for the custom alert actions defined in globalConfig | | eventtypes.conf | output/<YOUR_ADDON_NAME>/default | Generates `eventtypes.conf` file if the sourcetype is mentioned in Adaptive Response of custom alert action in globalConfig | | tags.conf | output/<YOUR_ADDON_NAME>/default | Generates `tags.conf` file based on the `eventtypes.conf` created for custom alert actions. | +| searchbnf.conf | output/<YOUR_ADDON_NAME>/default | Generates `searchbnf.conf` for custom search commands provided in the globalConfig. | | _account.conf | output/<YOUR_ADDON_NAME>/README | Generates `_account.conf.spec` file for the configuration mentioned in globalConfig | | _settings.conf | output/<YOUR_ADDON_NAME>/README | Generates `_settings.conf.spec` file for the Proxy, Logging or Custom Tab mentioned in globalConfig | | configuration.xml | output/<YOUR_ADDON_NAME>/default/data/ui/views | Generates configuration.xml file in `default/data/ui/views/` folder if configuration is defined in globalConfig. | diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 864ff6747b..6112c633e8 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -33,6 +33,7 @@ def test_get_j2_env(): "conf_files/settings_conf.template", "conf_files/tags_conf.template", "web_conf.template", + "conf_files/searchbnf_conf.template", ] assert sorted(expected_list_of_templates) == sorted(list_of_templates) From c8291797fc4e4c816111a28115a965ddabd03979 Mon Sep 17 00:00:00 2001 From: Hetang Modi <62056057+hetangmodi-crest@users.noreply.github.com> Date: Fri, 9 May 2025 16:04:20 +0530 Subject: [PATCH 7/8] feat: generation of `commands.conf` (#1694) **Issue number:** ADDON-76780, ADDON-79647 ### PR Type **What kind of change does this PR introduce?** * [x] Feature * [ ] Bug Fix * [ ] Refactoring (no functional or API changes) * [x] Documentation Update * [ ] Maintenance (dependency updates, CI, etc.) ## Summary ### Changes This PR includes the generation of the `commands.conf` file for the custom search command. ### User experience After defining a valid configuration for the custom search command in `globalConfig.json`, the `commands.conf` file will be generated. ## Checklist If an item doesn't apply to your changes, leave it unchecked. ### Review * [x] self-review - I have performed a self-review of this change according to the [development guidelines](https://splunk.github.io/addonfactory-ucc-generator/contributing/#development-guidelines) * [x] Changes are documented. The documentation is understandable, examples work [(more info)](https://splunk.github.io/addonfactory-ucc-generator/contributing/#documentation-guidelines) * [x] PR title and description follows the [contributing principles](https://splunk.github.io/addonfactory-ucc-generator/contributing/#pull-requests) * [ ] meeting - I have scheduled a meeting or recorded a demo to explain these changes (if there is a video, put a link below and in the ticket) ### Tests See [the testing doc](https://splunk.github.io/addonfactory-ucc-generator/contributing/#build-and-test). * [x] Unit - tests have been added/modified to cover the changes * [x] Smoke - tests have been added/modified to cover the changes * [ ] UI - tests have been added/modified to cover the changes * [x] coverage - I have checked the code coverage of my changes [(see more)](https://splunk.github.io/addonfactory-ucc-generator/contributing/#checking-the-code-coverage) **Demo/meeting:** *Reviewers are encouraged to request meetings or demos if any part of the change is unclear* --- docs/generated_files.md | 1 + .../generators/conf_files/__init__.py | 2 + .../conf_files/create_commands_conf.py | 49 ++++++++++++ .../generators/file_const.py | 2 + .../conf_files/commands_conf.template | 6 ++ tests/smoke/test_ucc_build.py | 1 + .../default/commands.conf | 14 ++++ .../conf_files/test_conf_files_init.py | 2 + .../conf_files/test_create_commands_conf.py | 79 +++++++++++++++++++ tests/unit/generators/test_doc_generator.py | 1 + tests/unit/test_utils.py | 1 + 11 files changed, 158 insertions(+) create mode 100644 splunk_add_on_ucc_framework/generators/conf_files/create_commands_conf.py create mode 100644 splunk_add_on_ucc_framework/templates/conf_files/commands_conf.template create mode 100644 tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/commands.conf create mode 100644 tests/unit/generators/conf_files/test_create_commands_conf.py diff --git a/docs/generated_files.md b/docs/generated_files.md index 40e7eea2c7..581a485c8f 100644 --- a/docs/generated_files.md +++ b/docs/generated_files.md @@ -14,6 +14,7 @@ The following table describes the files generated by UCC framework. | alert_actions.conf | output/<YOUR_ADDON_NAME>/default | Generates `alert_actions.conf` and `alert_actions.conf.spec` file for the custom alert actions defined in globalConfig | | eventtypes.conf | output/<YOUR_ADDON_NAME>/default | Generates `eventtypes.conf` file if the sourcetype is mentioned in Adaptive Response of custom alert action in globalConfig | | tags.conf | output/<YOUR_ADDON_NAME>/default | Generates `tags.conf` file based on the `eventtypes.conf` created for custom alert actions. | +| commands.conf | output/<YOUR_ADDON_NAME>/default | Generates `commands.conf` for custom commands provided in the globalConfig. | | searchbnf.conf | output/<YOUR_ADDON_NAME>/default | Generates `searchbnf.conf` for custom search commands provided in the globalConfig. | | _account.conf | output/<YOUR_ADDON_NAME>/README | Generates `_account.conf.spec` file for the configuration mentioned in globalConfig | | _settings.conf | output/<YOUR_ADDON_NAME>/README | Generates `_settings.conf.spec` file for the Proxy, Logging or Custom Tab mentioned in globalConfig | diff --git a/splunk_add_on_ucc_framework/generators/conf_files/__init__.py b/splunk_add_on_ucc_framework/generators/conf_files/__init__.py index f9d562730c..b94ee3a87b 100644 --- a/splunk_add_on_ucc_framework/generators/conf_files/__init__.py +++ b/splunk_add_on_ucc_framework/generators/conf_files/__init__.py @@ -24,6 +24,7 @@ from .create_web_conf import WebConf from .create_account_conf import AccountConf from .create_settings_conf import SettingsConf +from .create_commands_conf import CommandsConf from .create_searchbnf_conf import SearchbnfConf __all__ = [ @@ -38,5 +39,6 @@ "InputsConf", "AccountConf", "SettingsConf", + "CommandsConf", "SearchbnfConf", ] diff --git a/splunk_add_on_ucc_framework/generators/conf_files/create_commands_conf.py b/splunk_add_on_ucc_framework/generators/conf_files/create_commands_conf.py new file mode 100644 index 0000000000..cf6360dbd6 --- /dev/null +++ b/splunk_add_on_ucc_framework/generators/conf_files/create_commands_conf.py @@ -0,0 +1,49 @@ +# +# Copyright 2025 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Any, Dict + +from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator + + +class CommandsConf(FileGenerator): + __description__ = ( + "Generates `commands.conf` for custom commands provided in the globalConfig." + ) + + def _set_attributes(self, **kwargs: Any) -> None: + self.conf_file = "commands.conf" + if self._global_config.has_custom_search_commands(): + self.command_names = [] + for command in self._global_config.custom_search_commands: + self.command_names.append(command["commandName"]) + + def generate(self) -> Dict[str, str]: + if not self._global_config.has_custom_search_commands(): + return {} + + file_path = self.get_file_output_path(["default", self.conf_file]) + self.set_template_and_render( + template_file_path=["conf_files"], file_name="commands_conf.template" + ) + rendered_content = self._template.render( + command_names=self.command_names, + ) + self.writer( + file_name=self.conf_file, + file_path=file_path, + content=rendered_content, + ) + return {self.conf_file: file_path} diff --git a/splunk_add_on_ucc_framework/generators/file_const.py b/splunk_add_on_ucc_framework/generators/file_const.py index 34c4b67734..2477aa8fc9 100644 --- a/splunk_add_on_ucc_framework/generators/file_const.py +++ b/splunk_add_on_ucc_framework/generators/file_const.py @@ -35,6 +35,7 @@ WebConf, AccountConf, SettingsConf, + CommandsConf, SearchbnfConf, ) @@ -60,6 +61,7 @@ class FileClass(NamedTuple): ), FileClass("eventtypes.conf", EventtypesConf, ["default"]), FileClass("tags.conf", TagsConf, ["default"]), + FileClass("commands.conf", CommandsConf, ["default"]), FileClass("searchbnf.conf", SearchbnfConf, ["default"]), FileClass("_account.conf", AccountConf, ["README"]), FileClass("_settings.conf", SettingsConf, ["README"]), diff --git a/splunk_add_on_ucc_framework/templates/conf_files/commands_conf.template b/splunk_add_on_ucc_framework/templates/conf_files/commands_conf.template new file mode 100644 index 0000000000..1aa32b9aa1 --- /dev/null +++ b/splunk_add_on_ucc_framework/templates/conf_files/commands_conf.template @@ -0,0 +1,6 @@ +{% for name in command_names -%} +[{{name}}] +filename = {{name}}.py +chunked = true +python.version = python3 +{% endfor -%} \ No newline at end of file diff --git a/tests/smoke/test_ucc_build.py b/tests/smoke/test_ucc_build.py index f358a08455..4f14093532 100644 --- a/tests/smoke/test_ucc_build.py +++ b/tests/smoke/test_ucc_build.py @@ -153,6 +153,7 @@ def test_ucc_generate_with_everything(caplog): ("default", "splunk_ta_uccexample_settings.conf"), ("default", "web.conf"), ("default", "server.conf"), + ("default", "commands.conf"), ("default", "searchbnf.conf"), ("default", "data", "ui", "alerts", "test_alert.html"), ("default", "data", "ui", "nav", "default.xml"), diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/commands.conf b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/commands.conf new file mode 100644 index 0000000000..ee84578dbf --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/commands.conf @@ -0,0 +1,14 @@ +[generatetextcommand] +filename = generatetextcommand.py +chunked = true +python.version = python3 + +[filtercommand] +filename = filtercommand.py +chunked = true +python.version = python3 + +[countmatchescommand] +filename = countmatchescommand.py +chunked = true +python.version = python3 diff --git a/tests/unit/generators/conf_files/test_conf_files_init.py b/tests/unit/generators/conf_files/test_conf_files_init.py index a242a9a4ed..71904f7450 100644 --- a/tests/unit/generators/conf_files/test_conf_files_init.py +++ b/tests/unit/generators/conf_files/test_conf_files_init.py @@ -11,6 +11,7 @@ def test___init__conf(): "InputsConf", "AccountConf", "SettingsConf", + "CommandsConf", "SearchbnfConf", ] expected_modules = [ @@ -25,6 +26,7 @@ def test___init__conf(): "create_web_conf", "create_account_conf", "create_settings_conf", + "create_commands_conf", "create_searchbnf_conf", ] import splunk_add_on_ucc_framework.generators.conf_files as conf diff --git a/tests/unit/generators/conf_files/test_create_commands_conf.py b/tests/unit/generators/conf_files/test_create_commands_conf.py new file mode 100644 index 0000000000..babaa2151b --- /dev/null +++ b/tests/unit/generators/conf_files/test_create_commands_conf.py @@ -0,0 +1,79 @@ +from splunk_add_on_ucc_framework.generators.conf_files import CommandsConf +from splunk_add_on_ucc_framework import __file__ as ucc_framework_file +import os.path +from textwrap import dedent + +UCC_DIR = os.path.dirname(ucc_framework_file) + + +def test_set_attributes_without_custom_command( + global_config_only_configuration, input_dir, output_dir, ucc_dir, ta_name +): + commands_conf = CommandsConf( + global_config_only_configuration, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert not hasattr(commands_conf, "command_names") + + +def test_set_attributes( + global_config_all_json, input_dir, output_dir, ucc_dir, ta_name +): + commands_conf = CommandsConf( + global_config_all_json, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert commands_conf.conf_file == "commands.conf" + assert commands_conf.command_names == ["generatetextcommand"] + + +def test_generate_conf_without_custom_command( + global_config_only_configuration, input_dir, output_dir, ucc_dir, ta_name +): + commands_conf = CommandsConf( + global_config_only_configuration, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + file_paths = commands_conf.generate() + + # Assert that no files are returned since no custom command is configured + assert file_paths == {} + + +def test_commands_conf_generation( + global_config_all_json, input_dir, output_dir, ta_name +): + commands_conf = CommandsConf( + global_config_all_json, + input_dir, + output_dir, + addon_name=ta_name, + ucc_dir=UCC_DIR, + ) + file_paths = commands_conf.generate() + + assert file_paths is not None + assert file_paths.keys() == {"commands.conf"} + assert file_paths["commands.conf"].endswith("test_addon/default/commands.conf") + + with open(file_paths["commands.conf"]) as fp: + content = fp.read() + + expected_content = dedent( + """ + [generatetextcommand] + filename = generatetextcommand.py + chunked = true + python.version = python3 + """ + ).lstrip() + assert content == expected_content diff --git a/tests/unit/generators/test_doc_generator.py b/tests/unit/generators/test_doc_generator.py index 491bd75711..623f2dfa7b 100644 --- a/tests/unit/generators/test_doc_generator.py +++ b/tests/unit/generators/test_doc_generator.py @@ -21,6 +21,7 @@ def test_generate_docs(): | alert_actions.conf | output/<YOUR_ADDON_NAME>/default | Generates `alert_actions.conf` and `alert_actions.conf.spec` file for the custom alert actions defined in globalConfig | | eventtypes.conf | output/<YOUR_ADDON_NAME>/default | Generates `eventtypes.conf` file if the sourcetype is mentioned in Adaptive Response of custom alert action in globalConfig | | tags.conf | output/<YOUR_ADDON_NAME>/default | Generates `tags.conf` file based on the `eventtypes.conf` created for custom alert actions. | +| commands.conf | output/<YOUR_ADDON_NAME>/default | Generates `commands.conf` for custom commands provided in the globalConfig. | | searchbnf.conf | output/<YOUR_ADDON_NAME>/default | Generates `searchbnf.conf` for custom search commands provided in the globalConfig. | | _account.conf | output/<YOUR_ADDON_NAME>/README | Generates `_account.conf.spec` file for the configuration mentioned in globalConfig | | _settings.conf | output/<YOUR_ADDON_NAME>/README | Generates `_settings.conf.spec` file for the Proxy, Logging or Custom Tab mentioned in globalConfig | diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 6112c633e8..620169b590 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -33,6 +33,7 @@ def test_get_j2_env(): "conf_files/settings_conf.template", "conf_files/tags_conf.template", "web_conf.template", + "conf_files/commands_conf.template", "conf_files/searchbnf_conf.template", ] assert sorted(expected_list_of_templates) == sorted(list_of_templates) From 3776f0ecb70dd1b782dda1cb77f974965da47571 Mon Sep 17 00:00:00 2001 From: Hetang Modi <62056057+hetangmodi-crest@users.noreply.github.com> Date: Fri, 9 May 2025 18:05:05 +0530 Subject: [PATCH 8/8] feat: generation of python files for custom search command (#1697) **Issue number:** ADDON-76780 ### PR Type **What kind of change does this PR introduce?** * [x] Feature * [ ] Bug Fix * [ ] Refactoring (no functional or API changes) * [x] Documentation Update * [ ] Maintenance (dependency updates, CI, etc.) ## Summary ### Changes This PR contains generation of template `python files` (which will import logic from the files provided by the users) for the custom search command. For now, we are not supporting generation of python files for `Transforming/Reporting` command. ### User experience After defining required and valid configuration for custom search command in globalconfig.json, required template python files (based on the command type specified by the user) will be generated for the users, which will import logic from the files provided by the users. ## Checklist If an item doesn't apply to your changes, leave it unchecked. ### Review * [x] self-review - I have performed a self-review of this change according to the [development guidelines](https://splunk.github.io/addonfactory-ucc-generator/contributing/#development-guidelines) * [x] Changes are documented. The documentation is understandable, examples work [(more info)](https://splunk.github.io/addonfactory-ucc-generator/contributing/#documentation-guidelines) * [x] PR title and description follows the [contributing principles](https://splunk.github.io/addonfactory-ucc-generator/contributing/#pull-requests) * [ ] meeting - I have scheduled a meeting or recorded a demo to explain these changes (if there is a video, put a link below and in the ticket) ### Tests See [the testing doc](https://splunk.github.io/addonfactory-ucc-generator/contributing/#build-and-test). * [x] Unit - tests have been added/modified to cover the changes * [x] Smoke - tests have been added/modified to cover the changes * [ ] UI - tests have been added/modified to cover the changes * [x] coverage - I have checked the code coverage of my changes [(see more)](https://splunk.github.io/addonfactory-ucc-generator/contributing/#checking-the-code-coverage) **Demo/meeting:** *Reviewers are encouraged to request meetings or demos if any part of the change is unclear* --- docs/generated_files.md | 1 + .../generators/file_const.py | 6 + .../generators/python_files/__init__.py | 19 +++ .../create_custom_command_python.py | 115 ++++++++++++++++++ .../dataset_processing.template | 33 +++++ .../custom_command/generating.template | 32 +++++ .../custom_command/streaming.template | 33 +++++ tests/smoke/test_ucc_build.py | 6 + .../Splunk_TA_UCCExample/bin/countmatches.py | 4 + .../bin/countmatchescommand.py | 17 +++ .../Splunk_TA_UCCExample/bin/filter.py | 36 ++++++ .../Splunk_TA_UCCExample/bin/filtercommand.py | 26 ++++ .../Splunk_TA_UCCExample/bin/generatetext.py | 8 ++ .../bin/generatetextcommand.py | 25 ++++ .../test_create_custom_command_python.py | 114 +++++++++++++++++ .../python_files/test_python_files_init.py | 12 ++ tests/unit/generators/test_doc_generator.py | 1 + tests/unit/generators/test_init_.py | 8 +- tests/unit/test_utils.py | 3 + 19 files changed, 498 insertions(+), 1 deletion(-) create mode 100644 splunk_add_on_ucc_framework/generators/python_files/__init__.py create mode 100644 splunk_add_on_ucc_framework/generators/python_files/create_custom_command_python.py create mode 100644 splunk_add_on_ucc_framework/templates/custom_command/dataset_processing.template create mode 100644 splunk_add_on_ucc_framework/templates/custom_command/generating.template create mode 100644 splunk_add_on_ucc_framework/templates/custom_command/streaming.template create mode 100644 tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatches.py create mode 100644 tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatchescommand.py create mode 100644 tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filter.py create mode 100644 tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filtercommand.py create mode 100644 tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetext.py create mode 100644 tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetextcommand.py create mode 100644 tests/unit/generators/python_files/test_create_custom_command_python.py create mode 100644 tests/unit/generators/python_files/test_python_files_init.py diff --git a/docs/generated_files.md b/docs/generated_files.md index 581a485c8f..2960666e2c 100644 --- a/docs/generated_files.md +++ b/docs/generated_files.md @@ -24,5 +24,6 @@ The following table describes the files generated by UCC framework. | inputs.xml | output/<YOUR_ADDON_NAME>/default/data/ui/views | Generates inputs.xml based on inputs configuration present in globalConfig, in `default/data/ui/views/inputs.xml` folder | | _redirect.xml | output/<YOUR_ADDON_NAME>/default/data/ui/views | Generates ta_name_redirect.xml file, if oauth is mentioned in globalConfig, in `default/data/ui/views/` folder. | | _.html | output/<YOUR_ADDON_NAME>/default/data/ui/alerts | Generates `alert_name.html` file based on alerts configuration present in globalConfig, in `default/data/ui/alerts` folder. | +| _.py | output/<YOUR_ADDON_NAME>/bin | Generates Python files for custom search commands provided in the globalConfig. | | globalConfig.json | <source_dir> | Generates globalConfig.json file in the source code if globalConfig is not present in source directory at build time. | diff --git a/splunk_add_on_ucc_framework/generators/file_const.py b/splunk_add_on_ucc_framework/generators/file_const.py index 2477aa8fc9..356499e516 100644 --- a/splunk_add_on_ucc_framework/generators/file_const.py +++ b/splunk_add_on_ucc_framework/generators/file_const.py @@ -24,6 +24,7 @@ RedirectXml, ) from splunk_add_on_ucc_framework.generators.html_files import AlertActionsHtml +from splunk_add_on_ucc_framework.generators.python_files import CustomCommandPy from splunk_add_on_ucc_framework.generators.conf_files import ( AlertActionsConf, AppConf, @@ -95,4 +96,9 @@ class FileClass(NamedTuple): AlertActionsHtml, ["default", "data", "ui", "alerts"], ), + FileClass( + "_.py", + CustomCommandPy, + ["bin"], + ), ] diff --git a/splunk_add_on_ucc_framework/generators/python_files/__init__.py b/splunk_add_on_ucc_framework/generators/python_files/__init__.py new file mode 100644 index 0000000000..19496df238 --- /dev/null +++ b/splunk_add_on_ucc_framework/generators/python_files/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright 2025 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from ..file_generator import FileGenerator +from .create_custom_command_python import CustomCommandPy + +__all__ = ["FileGenerator", "CustomCommandPy"] diff --git a/splunk_add_on_ucc_framework/generators/python_files/create_custom_command_python.py b/splunk_add_on_ucc_framework/generators/python_files/create_custom_command_python.py new file mode 100644 index 0000000000..b159b8e530 --- /dev/null +++ b/splunk_add_on_ucc_framework/generators/python_files/create_custom_command_python.py @@ -0,0 +1,115 @@ +# +# Copyright 2025 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Any, Dict, List + +from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator + + +class CustomCommandPy(FileGenerator): + __description__ = "Generates Python files for custom search commands provided in the globalConfig." + + def argument_generator( + self, argument_list: List[str], arg: Dict[str, Any] + ) -> List[str]: + validate_str = "" + validate = arg.get("validate", {}) + if validate: + validate_type = validate["type"] + if validate_type in ("Integer", "Float"): + min_val = validate.get("minimum") + max_val = validate.get("maximum") + args = [] + if min_val is not None: + args.append(f"minimum={min_val}") + if max_val is not None: + args.append(f"maximum={max_val}") + validate_args = ", ".join(args) + validate_str = ( + f", validate=validators.{validate_type}({validate_args})" + if args + else f", validate=validators.{validate_type}()" + ) + elif validate_type: + validate_str = f", validate=validators.{validate_type}()" + + if arg["default"] is None: + arg_str = ( + f"{arg['name']} = Option(name='{arg['name']}', " + f"require={arg.get('require')}" + f"{validate_str})" + ) + else: + arg_str = ( + f"{arg['name']} = Option(name='{arg['name']}', " + f"require={arg.get('require')}" + f"{validate_str}, " + f"default='{arg.get('default', '')}')" + ) + argument_list.append(arg_str) + return argument_list + + def _set_attributes(self, **kwargs: Any) -> None: + self.commands_info = [] + for command in self._global_config.custom_search_commands: + argument_list: List[str] = [] + imported_file_name = command["fileName"].replace(".py", "") + template = command["commandType"].replace(" ", "_") + ".template" + for argument in command["arguments"]: + argument_dict = { + "name": argument["name"], + "require": argument.get("required", False), + "validate": argument.get("validate"), + "default": argument.get("defaultValue"), + } + self.argument_generator(argument_list, argument_dict) + self.commands_info.append( + { + "imported_file_name": imported_file_name, + "file_name": command["commandName"], + "class_name": command["commandName"].title(), + "description": command.get("description"), + "syntax": command.get("syntax"), + "template": template, + "list_arg": argument_list, + } + ) + + def generate(self) -> Dict[str, str]: + if not self.commands_info: + return {} + + generated_files = {} + for command_info in self.commands_info: + file_name = command_info["file_name"] + ".py" + file_path = self.get_file_output_path(["bin", file_name]) + self.set_template_and_render( + template_file_path=["custom_command"], + file_name=command_info["template"], + ) + rendered_content = self._template.render( + imported_file_name=command_info["imported_file_name"], + class_name=command_info["class_name"], + description=command_info["description"], + syntax=command_info["syntax"], + list_arg=command_info["list_arg"], + ) + self.writer( + file_name=file_name, + file_path=file_path, + content=rendered_content, + ) + generated_files.update({file_name: file_path}) + return generated_files diff --git a/splunk_add_on_ucc_framework/templates/custom_command/dataset_processing.template b/splunk_add_on_ucc_framework/templates/custom_command/dataset_processing.template new file mode 100644 index 0000000000..e8cb58dc87 --- /dev/null +++ b/splunk_add_on_ucc_framework/templates/custom_command/dataset_processing.template @@ -0,0 +1,33 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, EventingCommand, Configuration, Option, validators +from {{imported_file_name}} import transform + +@Configuration() +class {{class_name}}Command(EventingCommand): + {% if syntax or description%} + """ + + {% if syntax %} + ##Syntax + {{syntax}} + {% endif %} + + {% if description %} + ##Description + {{description}} + {% endif %} + + """ + {% endif %} + + {% for arg in list_arg %} + {{arg}} + {% endfor %} + + def transform(self, events): + return transform(self, events) + +dispatch({{class_name}}Command, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/splunk_add_on_ucc_framework/templates/custom_command/generating.template b/splunk_add_on_ucc_framework/templates/custom_command/generating.template new file mode 100644 index 0000000000..4256e62d15 --- /dev/null +++ b/splunk_add_on_ucc_framework/templates/custom_command/generating.template @@ -0,0 +1,32 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, GeneratingCommand, Configuration, Option, validators +from {{imported_file_name}} import generate + +@Configuration() +class {{class_name}}Command(GeneratingCommand): + {% if syntax or description%} + """ + + {% if syntax %} + ##Syntax + {{syntax}} + {% endif %} + + {% if description %} + ##Description + {{description}} + {% endif %} + + """ + {% endif %} + {% for arg in list_arg %} + {{arg}} + {% endfor %} + + def generate(self): + return generate(self) + +dispatch({{class_name}}Command, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/splunk_add_on_ucc_framework/templates/custom_command/streaming.template b/splunk_add_on_ucc_framework/templates/custom_command/streaming.template new file mode 100644 index 0000000000..53754b84bb --- /dev/null +++ b/splunk_add_on_ucc_framework/templates/custom_command/streaming.template @@ -0,0 +1,33 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, StreamingCommand, Configuration, Option, validators +from {{imported_file_name}} import stream + +@Configuration() +class {{class_name}}Command(StreamingCommand): + {% if syntax or description%} + """ + + {% if syntax %} + ##Syntax + {{syntax}} + {% endif %} + + {% if description %} + ##Description + {{description}} + {% endif %} + + """ + {% endif %} + + {% for arg in list_arg %} + {{arg}} + {% endfor %} + + def stream(self, events): + return stream(self, events) + +dispatch({{class_name}}Command, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/tests/smoke/test_ucc_build.py b/tests/smoke/test_ucc_build.py index 4f14093532..8d0cebb68f 100644 --- a/tests/smoke/test_ucc_build.py +++ b/tests/smoke/test_ucc_build.py @@ -168,6 +168,12 @@ def test_ucc_generate_with_everything(caplog): ("bin", "example_input_three.py"), ("bin", "example_input_four.py"), ("bin", "import_declare_test.py"), + ("bin", "countmatchescommand.py"), + ("bin", "countmatches.py"), + ("bin", "filter.py"), + ("bin", "filtercommand.py"), + ("bin", "generatetext.py"), + ("bin", "generatetextcommand.py"), ("bin", "splunk_ta_uccexample_rh_account.py"), ("bin", "splunk_ta_uccexample_rh_example_input_one.py"), ("bin", "splunk_ta_uccexample_rh_example_input_two.py"), diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatches.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatches.py new file mode 100644 index 0000000000..da79b16420 --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatches.py @@ -0,0 +1,4 @@ +def stream(self, records): + for record in records: + # write custom logic for the search command + yield record \ No newline at end of file diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatchescommand.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatchescommand.py new file mode 100644 index 0000000000..e2faa5da18 --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatchescommand.py @@ -0,0 +1,17 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, StreamingCommand, Configuration, Option, validators +from countmatches import stream + +@Configuration() +class CountmatchescommandCommand(StreamingCommand): + + fieldname = Option(name='fieldname', require=True, validate=validators.Fieldname()) + pattern = Option(name='pattern', require=True, validate=validators.RegularExpression()) + + def stream(self, events): + return stream(self, events) + +dispatch(CountmatchescommandCommand, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filter.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filter.py new file mode 100644 index 0000000000..51d476833f --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filter.py @@ -0,0 +1,36 @@ +def transform(self, records): + contains = self.contains + replace_array = self.replace_array + + if contains and replace_array: + arr = replace_array.split(",") + if len(arr) != 2: + raise ValueError("Please provide only two arguments, separated by comma for 'replace'") + + for record in records: + _raw = record.get("_raw") + if contains in _raw: + record["_raw"] = _raw.replace(arr[0], arr[1]) + yield record + return + + if contains: + for record in records: + _raw = record.get("_raw") + if contains in _raw: + yield record + return + + if replace_array: + arr = replace_array.split(",") + if len(arr) != 2: + raise ValueError("Please provide only two arguments, separated by comma for 'replace'") + + for record in records: + _raw = record.get("_raw") + record["_raw"] = _raw.replace(arr[0], arr[1]) + yield record + return + + for record in records: + yield record \ No newline at end of file diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filtercommand.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filtercommand.py new file mode 100644 index 0000000000..8ab7cb354d --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filtercommand.py @@ -0,0 +1,26 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, EventingCommand, Configuration, Option, validators +from filter import transform + +@Configuration() +class FiltercommandCommand(EventingCommand): + """ + + ##Syntax + | filtercommand contains='value1' replace='value to be replaced,value to replace with' + + ##Description + It filters records from the events stream returning only those which has :code:`contains` in them and replaces :code:`replace_array[0]` with :code:`replace_array[1]`. + + """ + + contains = Option(name='contains', require=False) + replace_array = Option(name='replace_array', require=False) + + def transform(self, events): + return transform(self, events) + +dispatch(FiltercommandCommand, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetext.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetext.py new file mode 100644 index 0000000000..897779284c --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetext.py @@ -0,0 +1,8 @@ +import time +import logging + + +def generate(self): + logging.debug("Generating %d events with text %s" % (self.count, self.text)) + for i in range(1, self.count + 1): + yield {'_serial': i, '_time': time.time(), '_raw': str(i) + '. ' + self.text} diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetextcommand.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetextcommand.py new file mode 100644 index 0000000000..89459faa90 --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetextcommand.py @@ -0,0 +1,25 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, GeneratingCommand, Configuration, Option, validators +from generatetext import generate + +@Configuration() +class GeneratetextcommandCommand(GeneratingCommand): + """ + + ##Syntax + generatetextcommand count= text= + + ##Description + This command generates COUNT occurrences of a TEXT string. + + """ + count = Option(name='count', require=True, validate=validators.Integer(minimum=5, maximum=10)) + text = Option(name='text', require=True) + + def generate(self): + return generate(self) + +dispatch(GeneratetextcommandCommand, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/tests/unit/generators/python_files/test_create_custom_command_python.py b/tests/unit/generators/python_files/test_create_custom_command_python.py new file mode 100644 index 0000000000..67ec2eae72 --- /dev/null +++ b/tests/unit/generators/python_files/test_create_custom_command_python.py @@ -0,0 +1,114 @@ +from pytest import fixture +from splunk_add_on_ucc_framework.generators.python_files import CustomCommandPy +from splunk_add_on_ucc_framework import __file__ as ucc_framework_file +import os.path + +UCC_DIR = os.path.dirname(ucc_framework_file) + + +@fixture +def custom_search_commands(): + return [ + { + "commandName": "testcommand", + "commandType": "generating", + "fileName": "test.py", + "requiredSearchAssistant": True, + "description": "This is test command", + "syntax": "testcommand count= text=", + "usage": "public", + "arguments": [ + { + "name": "count", + "required": True, + "validate": {"type": "Integer", "minimum": 5, "maximum": 10}, + }, + { + "name": "age", + "validate": {"type": "Integer", "minimum": 18}, + }, + {"name": "text", "required": True, "defaultValue": "test_text"}, + {"name": "contains"}, + {"name": "fieldname", "validate": {"type": "Fieldname"}}, + ], + } + ] + + +def test_set_attributes_without_custom_command( + global_config_only_configuration, input_dir, output_dir, ucc_dir, ta_name +): + custom_command = CustomCommandPy( + global_config_only_configuration, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert custom_command.commands_info == [] + + +def test_set_attributes( + global_config_all_json, + input_dir, + output_dir, + ucc_dir, + ta_name, + custom_search_commands, +): + global_config_all_json._content["customSearchCommand"] = custom_search_commands + custom_command_py = CustomCommandPy( + global_config_all_json, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert custom_command_py.commands_info == [ + { + "imported_file_name": "test", + "file_name": "testcommand", + "class_name": "Testcommand", + "description": "This is test command", + "syntax": "testcommand count= text=", + "template": "generating.template", + "list_arg": [ + "count = Option(name='count', require=True, " + "validate=validators.Integer(minimum=5, maximum=10))", + "age = Option(name='age', require=False, validate=validators.Integer(minimum=18))", + "text = Option(name='text', require=True, default='test_text')", + "contains = Option(name='contains', require=False)", + "fieldname = Option(name='fieldname', require=False, validate=validators.Fieldname())", + ], + } + ] + + +def test_generate_python_without_custom_command( + global_config_only_configuration, input_dir, output_dir, ucc_dir, ta_name +): + custom_command = CustomCommandPy( + global_config_only_configuration, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + file_paths = custom_command.generate() + + # Assert that no files are returned since no custom command is configured + assert file_paths == {} + + +def test_generate_python(global_config_all_json, input_dir, output_dir, ta_name): + exp_fname = "generatetextcommand.py" + + custom_command_py = CustomCommandPy( + global_config_all_json, + input_dir, + output_dir, + ucc_dir=UCC_DIR, + addon_name=ta_name, + ) + file_paths = custom_command_py.generate() + assert file_paths == {exp_fname: f"{output_dir}/{ta_name}/bin/{exp_fname}"} diff --git a/tests/unit/generators/python_files/test_python_files_init.py b/tests/unit/generators/python_files/test_python_files_init.py new file mode 100644 index 0000000000..b5f5d2b771 --- /dev/null +++ b/tests/unit/generators/python_files/test_python_files_init.py @@ -0,0 +1,12 @@ +def test___init__Python(): + expected_classes = ["FileGenerator", "CustomCommandPy"] + expected_modules = ["file_generator", "create_custom_command_python"] + import splunk_add_on_ucc_framework.generators.python_files as py + + assert py.__all__ == expected_classes + + for attrib in dir(py): + if attrib.startswith("__") and attrib.endswith("__"): + # ignore the builtin modules + continue + assert attrib in expected_classes or attrib in expected_modules diff --git a/tests/unit/generators/test_doc_generator.py b/tests/unit/generators/test_doc_generator.py index 623f2dfa7b..3ed7d38a85 100644 --- a/tests/unit/generators/test_doc_generator.py +++ b/tests/unit/generators/test_doc_generator.py @@ -31,6 +31,7 @@ def test_generate_docs(): | inputs.xml | output/<YOUR_ADDON_NAME>/default/data/ui/views | Generates inputs.xml based on inputs configuration present in globalConfig, in `default/data/ui/views/inputs.xml` folder | | _redirect.xml | output/<YOUR_ADDON_NAME>/default/data/ui/views | Generates ta_name_redirect.xml file, if oauth is mentioned in globalConfig, in `default/data/ui/views/` folder. | | _.html | output/<YOUR_ADDON_NAME>/default/data/ui/alerts | Generates `alert_name.html` file based on alerts configuration present in globalConfig, in `default/data/ui/alerts` folder. | +| _.py | output/<YOUR_ADDON_NAME>/bin | Generates Python files for custom search commands provided in the globalConfig. | | globalConfig.json | <source_dir> | Generates globalConfig.json file in the source code if globalConfig is not present in source directory at build time. | """ diff --git a/tests/unit/generators/test_init_.py b/tests/unit/generators/test_init_.py index 01b48a6931..69576a7cdd 100644 --- a/tests/unit/generators/test_init_.py +++ b/tests/unit/generators/test_init_.py @@ -12,7 +12,13 @@ def test___init__gen(): assert gen.__all__ == expected_classes - not_allowed = ["conf_files", "xml_files", "html_files", "doc_generator"] + not_allowed = [ + "conf_files", + "xml_files", + "html_files", + "python_files", + "doc_generator", + ] for attrib in dir(gen): if attrib.startswith("__") and attrib.endswith("__") or attrib in not_allowed: # ignore the builtin modules diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 620169b590..8904ab600c 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -33,6 +33,9 @@ def test_get_j2_env(): "conf_files/settings_conf.template", "conf_files/tags_conf.template", "web_conf.template", + "custom_command/dataset_processing.template", + "custom_command/generating.template", + "custom_command/streaming.template", "conf_files/commands_conf.template", "conf_files/searchbnf_conf.template", ]