diff --git a/docs/generated_files.md b/docs/generated_files.md index 09a27cc892..2960666e2c 100644 --- a/docs/generated_files.md +++ b/docs/generated_files.md @@ -14,6 +14,8 @@ The following table describes the files generated by UCC framework. | alert_actions.conf | output/<YOUR_ADDON_NAME>/default | Generates `alert_actions.conf` and `alert_actions.conf.spec` file for the custom alert actions defined in globalConfig | | eventtypes.conf | output/<YOUR_ADDON_NAME>/default | Generates `eventtypes.conf` file if the sourcetype is mentioned in Adaptive Response of custom alert action in globalConfig | | tags.conf | output/<YOUR_ADDON_NAME>/default | Generates `tags.conf` file based on the `eventtypes.conf` created for custom alert actions. | +| commands.conf | output/<YOUR_ADDON_NAME>/default | Generates `commands.conf` for custom commands provided in the globalConfig. | +| searchbnf.conf | output/<YOUR_ADDON_NAME>/default | Generates `searchbnf.conf` for custom search commands provided in the globalConfig. | | _account.conf | output/<YOUR_ADDON_NAME>/README | Generates `_account.conf.spec` file for the configuration mentioned in globalConfig | | _settings.conf | output/<YOUR_ADDON_NAME>/README | Generates `_settings.conf.spec` file for the Proxy, Logging or Custom Tab mentioned in globalConfig | | configuration.xml | output/<YOUR_ADDON_NAME>/default/data/ui/views | Generates configuration.xml file in `default/data/ui/views/` folder if configuration is defined in globalConfig. | @@ -22,5 +24,6 @@ The following table describes the files generated by UCC framework. | inputs.xml | output/<YOUR_ADDON_NAME>/default/data/ui/views | Generates inputs.xml based on inputs configuration present in globalConfig, in `default/data/ui/views/inputs.xml` folder | | _redirect.xml | output/<YOUR_ADDON_NAME>/default/data/ui/views | Generates ta_name_redirect.xml file, if oauth is mentioned in globalConfig, in `default/data/ui/views/` folder. | | _.html | output/<YOUR_ADDON_NAME>/default/data/ui/alerts | Generates `alert_name.html` file based on alerts configuration present in globalConfig, in `default/data/ui/alerts` folder. | +| _.py | output/<YOUR_ADDON_NAME>/bin | Generates Python files for custom search commands provided in the globalConfig. | | globalConfig.json | <source_dir> | Generates globalConfig.json file in the source code if globalConfig is not present in source directory at build time. | diff --git a/splunk_add_on_ucc_framework/commands/build.py b/splunk_add_on_ucc_framework/commands/build.py index fdf5a7e31a..c3ff923dc2 100644 --- a/splunk_add_on_ucc_framework/commands/build.py +++ b/splunk_add_on_ucc_framework/commands/build.py @@ -489,7 +489,7 @@ def generate( global_config_update.handle_global_config_update(global_config, gc_path) try: validator = global_config_validator.GlobalConfigValidator( - internal_root_dir, global_config + internal_root_dir, global_config, source ) validator.validate() logger.info("globalConfig file is valid") diff --git a/splunk_add_on_ucc_framework/const.py b/splunk_add_on_ucc_framework/const.py new file mode 100644 index 0000000000..47bd2fe743 --- /dev/null +++ b/splunk_add_on_ucc_framework/const.py @@ -0,0 +1,175 @@ +# +# Copyright 2025 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# TODO: Update the list as and when Splunk introduces new commands. +# Links to use: https://docs.splunk.com/Documentation/SplunkCloud/latest/SearchReference +# https://docs.splunk.com/Documentation/Splunk/latest/SearchReference +SPLUNK_COMMANDS = [ + "abstract", + "accum", + "addcoltotals", + "addinfo", + "addtotals", + "analyzefields", + "anomalies", + "anomalousvalue", + "anomalydetection", + "append", + "appendcols", + "appendpipe", + "arules", + "associate", + "autoregress", + "awssnsalert", + "bin", + "bucket", + "bucketdir", + "chart", + "cluster", + "cofilter", + "collect", + "concurrency", + "contingency", + "convert", + "correlate", + "ctable", + "datamodel", + "datamodelsimple", + "dbinspect", + "dbxquery", + "dedup", + "delete", + "delta", + "diff", + "entitymerge", + "erex", + "eval", + "eventcount", + "eventstats", + "extract", + "fieldformat", + "fields", + "fieldsummary", + "filldown", + "fillnull", + "findtypes", + "folderize", + "foreach", + "format", + "from", + "fromjson", + "gauge", + "gentimes", + "geom", + "geomfilter", + "geostats", + "head", + "highlight", + "history", + "iconify", + "inputcsv", + "inputintelligence", + "inputlookup", + "iplocation", + "join", + "kmeans", + "kvform", + "loadjob", + "localize", + "localop", + "lookup", + "makecontinuous", + "makemv", + "makeresults", + "map", + "mcollect", + "metadata", + "metasearch", + "meventcollect", + "mpreview", + "msearch", + "mstats", + "multikv", + "multisearch", + "mvcombine", + "mvexpand", + "nomv", + "outlier", + "outputcsv", + "outputlookup", + "outputtext", + "overlap", + "pivot", + "predict", + "rangemap", + "rare", + "regex", + "reltime", + "rename", + "replace", + "require", + "rest", + "return", + "reverse", + "rex", + "rtorder", + "run", + "savedsearch", + "script", + "scrub", + "search", + "searchtxn", + "selfjoin", + "sendalert", + "sendemail", + "set", + "setfields", + "sichart", + "sirare", + "sistats", + "sitimechart", + "sitop", + "sort", + "spath", + "stats", + "strcat", + "streamstats", + "table", + "tags", + "tail", + "timechart", + "timewrap", + "tojson", + "top", + "transaction", + "transpose", + "trendline", + "tscollect", + "tstats", + "typeahead", + "typelearner", + "typer", + "union", + "uniq", + "untable", + "walklex", + "where", + "x11", + "xmlkv", + "xmlunescape", + "xpath", + "xyseries", +] diff --git a/splunk_add_on_ucc_framework/generators/conf_files/__init__.py b/splunk_add_on_ucc_framework/generators/conf_files/__init__.py index d6159cf629..b94ee3a87b 100644 --- a/splunk_add_on_ucc_framework/generators/conf_files/__init__.py +++ b/splunk_add_on_ucc_framework/generators/conf_files/__init__.py @@ -24,6 +24,8 @@ from .create_web_conf import WebConf from .create_account_conf import AccountConf from .create_settings_conf import SettingsConf +from .create_commands_conf import CommandsConf +from .create_searchbnf_conf import SearchbnfConf __all__ = [ "FileGenerator", @@ -37,4 +39,6 @@ "InputsConf", "AccountConf", "SettingsConf", + "CommandsConf", + "SearchbnfConf", ] diff --git a/splunk_add_on_ucc_framework/generators/conf_files/create_commands_conf.py b/splunk_add_on_ucc_framework/generators/conf_files/create_commands_conf.py new file mode 100644 index 0000000000..cf6360dbd6 --- /dev/null +++ b/splunk_add_on_ucc_framework/generators/conf_files/create_commands_conf.py @@ -0,0 +1,49 @@ +# +# Copyright 2025 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Any, Dict + +from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator + + +class CommandsConf(FileGenerator): + __description__ = ( + "Generates `commands.conf` for custom commands provided in the globalConfig." + ) + + def _set_attributes(self, **kwargs: Any) -> None: + self.conf_file = "commands.conf" + if self._global_config.has_custom_search_commands(): + self.command_names = [] + for command in self._global_config.custom_search_commands: + self.command_names.append(command["commandName"]) + + def generate(self) -> Dict[str, str]: + if not self._global_config.has_custom_search_commands(): + return {} + + file_path = self.get_file_output_path(["default", self.conf_file]) + self.set_template_and_render( + template_file_path=["conf_files"], file_name="commands_conf.template" + ) + rendered_content = self._template.render( + command_names=self.command_names, + ) + self.writer( + file_name=self.conf_file, + file_path=file_path, + content=rendered_content, + ) + return {self.conf_file: file_path} diff --git a/splunk_add_on_ucc_framework/generators/conf_files/create_searchbnf_conf.py b/splunk_add_on_ucc_framework/generators/conf_files/create_searchbnf_conf.py new file mode 100644 index 0000000000..f102644eb2 --- /dev/null +++ b/splunk_add_on_ucc_framework/generators/conf_files/create_searchbnf_conf.py @@ -0,0 +1,54 @@ +# +# Copyright 2025 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Any, Dict + +from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator + + +class SearchbnfConf(FileGenerator): + __description__ = "Generates `searchbnf.conf` for custom search commands provided in the globalConfig." + + def _set_attributes(self, **kwargs: Any) -> None: + self.conf_file = "searchbnf.conf" + self.searchbnf_info = [] + if self._global_config.has_custom_search_commands(): + for command in self._global_config.custom_search_commands: + if command.get("requiredSearchAssistant", False): + searchbnf_dict = { + "command_name": command["commandName"], + "description": command["description"], + "syntax": command["syntax"], + "usage": command["usage"], + } + self.searchbnf_info.append(searchbnf_dict) + + def generate(self) -> Dict[str, str]: + if not self.searchbnf_info: + return {} + + file_path = self.get_file_output_path(["default", self.conf_file]) + self.set_template_and_render( + template_file_path=["conf_files"], file_name="searchbnf_conf.template" + ) + rendered_content = self._template.render( + searchbnf_info=self.searchbnf_info, + ) + self.writer( + file_name=self.conf_file, + file_path=file_path, + content=rendered_content, + ) + return {self.conf_file: file_path} diff --git a/splunk_add_on_ucc_framework/generators/file_const.py b/splunk_add_on_ucc_framework/generators/file_const.py index 0ebb7e65c2..356499e516 100644 --- a/splunk_add_on_ucc_framework/generators/file_const.py +++ b/splunk_add_on_ucc_framework/generators/file_const.py @@ -24,6 +24,7 @@ RedirectXml, ) from splunk_add_on_ucc_framework.generators.html_files import AlertActionsHtml +from splunk_add_on_ucc_framework.generators.python_files import CustomCommandPy from splunk_add_on_ucc_framework.generators.conf_files import ( AlertActionsConf, AppConf, @@ -35,6 +36,8 @@ WebConf, AccountConf, SettingsConf, + CommandsConf, + SearchbnfConf, ) __all__ = ["FileClass", "GEN_FILE_LIST"] @@ -59,6 +62,8 @@ class FileClass(NamedTuple): ), FileClass("eventtypes.conf", EventtypesConf, ["default"]), FileClass("tags.conf", TagsConf, ["default"]), + FileClass("commands.conf", CommandsConf, ["default"]), + FileClass("searchbnf.conf", SearchbnfConf, ["default"]), FileClass("_account.conf", AccountConf, ["README"]), FileClass("_settings.conf", SettingsConf, ["README"]), FileClass( @@ -91,4 +96,9 @@ class FileClass(NamedTuple): AlertActionsHtml, ["default", "data", "ui", "alerts"], ), + FileClass( + "_.py", + CustomCommandPy, + ["bin"], + ), ] diff --git a/splunk_add_on_ucc_framework/generators/python_files/__init__.py b/splunk_add_on_ucc_framework/generators/python_files/__init__.py new file mode 100644 index 0000000000..19496df238 --- /dev/null +++ b/splunk_add_on_ucc_framework/generators/python_files/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright 2025 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from ..file_generator import FileGenerator +from .create_custom_command_python import CustomCommandPy + +__all__ = ["FileGenerator", "CustomCommandPy"] diff --git a/splunk_add_on_ucc_framework/generators/python_files/create_custom_command_python.py b/splunk_add_on_ucc_framework/generators/python_files/create_custom_command_python.py new file mode 100644 index 0000000000..b159b8e530 --- /dev/null +++ b/splunk_add_on_ucc_framework/generators/python_files/create_custom_command_python.py @@ -0,0 +1,115 @@ +# +# Copyright 2025 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Any, Dict, List + +from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator + + +class CustomCommandPy(FileGenerator): + __description__ = "Generates Python files for custom search commands provided in the globalConfig." + + def argument_generator( + self, argument_list: List[str], arg: Dict[str, Any] + ) -> List[str]: + validate_str = "" + validate = arg.get("validate", {}) + if validate: + validate_type = validate["type"] + if validate_type in ("Integer", "Float"): + min_val = validate.get("minimum") + max_val = validate.get("maximum") + args = [] + if min_val is not None: + args.append(f"minimum={min_val}") + if max_val is not None: + args.append(f"maximum={max_val}") + validate_args = ", ".join(args) + validate_str = ( + f", validate=validators.{validate_type}({validate_args})" + if args + else f", validate=validators.{validate_type}()" + ) + elif validate_type: + validate_str = f", validate=validators.{validate_type}()" + + if arg["default"] is None: + arg_str = ( + f"{arg['name']} = Option(name='{arg['name']}', " + f"require={arg.get('require')}" + f"{validate_str})" + ) + else: + arg_str = ( + f"{arg['name']} = Option(name='{arg['name']}', " + f"require={arg.get('require')}" + f"{validate_str}, " + f"default='{arg.get('default', '')}')" + ) + argument_list.append(arg_str) + return argument_list + + def _set_attributes(self, **kwargs: Any) -> None: + self.commands_info = [] + for command in self._global_config.custom_search_commands: + argument_list: List[str] = [] + imported_file_name = command["fileName"].replace(".py", "") + template = command["commandType"].replace(" ", "_") + ".template" + for argument in command["arguments"]: + argument_dict = { + "name": argument["name"], + "require": argument.get("required", False), + "validate": argument.get("validate"), + "default": argument.get("defaultValue"), + } + self.argument_generator(argument_list, argument_dict) + self.commands_info.append( + { + "imported_file_name": imported_file_name, + "file_name": command["commandName"], + "class_name": command["commandName"].title(), + "description": command.get("description"), + "syntax": command.get("syntax"), + "template": template, + "list_arg": argument_list, + } + ) + + def generate(self) -> Dict[str, str]: + if not self.commands_info: + return {} + + generated_files = {} + for command_info in self.commands_info: + file_name = command_info["file_name"] + ".py" + file_path = self.get_file_output_path(["bin", file_name]) + self.set_template_and_render( + template_file_path=["custom_command"], + file_name=command_info["template"], + ) + rendered_content = self._template.render( + imported_file_name=command_info["imported_file_name"], + class_name=command_info["class_name"], + description=command_info["description"], + syntax=command_info["syntax"], + list_arg=command_info["list_arg"], + ) + self.writer( + file_name=file_name, + file_path=file_path, + content=rendered_content, + ) + generated_files.update({file_name: file_path}) + return generated_files diff --git a/splunk_add_on_ucc_framework/global_config.py b/splunk_add_on_ucc_framework/global_config.py index 1f54d8d17d..967cfbddff 100644 --- a/splunk_add_on_ucc_framework/global_config.py +++ b/splunk_add_on_ucc_framework/global_config.py @@ -209,6 +209,10 @@ def configs(self) -> List[Any]: def alerts(self) -> List[Dict[str, Any]]: return self._content.get("alerts", []) + @property + def custom_search_commands(self) -> List[Dict[str, Any]]: + return self._content.get("customSearchCommand", []) + @property def meta(self) -> Dict[str, Any]: return self._content["meta"] @@ -271,6 +275,9 @@ def has_configuration(self) -> bool: def has_alerts(self) -> bool: return bool(self.alerts) + def has_custom_search_commands(self) -> bool: + return bool(self.custom_search_commands) + def has_dashboard(self) -> bool: return self.dashboard is not None diff --git a/splunk_add_on_ucc_framework/global_config_validator.py b/splunk_add_on_ucc_framework/global_config_validator.py index 6f4f9b5ad4..f6cde2f3f7 100644 --- a/splunk_add_on_ucc_framework/global_config_validator.py +++ b/splunk_add_on_ucc_framework/global_config_validator.py @@ -20,6 +20,7 @@ from typing import Any, Dict, List import logging import itertools +from splunk_add_on_ucc_framework.const import SPLUNK_COMMANDS import jsonschema @@ -47,8 +48,14 @@ class GlobalConfigValidator: Custom validation should be implemented here. """ - def __init__(self, source_dir: str, global_config: global_config_lib.GlobalConfig): - self._source_dir = source_dir + def __init__( + self, + internal_root_dir: str, + global_config: global_config_lib.GlobalConfig, + source: str = "", + ): + self._internal_root_dir = internal_root_dir + self._source_dir = source self._global_config = global_config self._config = global_config.content self.resolved_configuration = global_config.resolved_configuration @@ -58,7 +65,7 @@ def _validate_config_against_schema(self) -> None: Validates config against JSON schema. Raises jsonschema.ValidationError if config is not valid. """ - schema_path = os.path.join(self._source_dir, "schema", "schema.json") + schema_path = os.path.join(self._internal_root_dir, "schema", "schema.json") with open(schema_path, encoding="utf-8") as f_schema: schema_raw = f_schema.read() schema = json.loads(schema_raw) @@ -710,6 +717,49 @@ def _validate_meta_default_view(self) -> None: 'meta.defaultView == "dashboard" but there is no dashboard defined in globalConfig' ) + def _validate_custom_search_commands(self) -> None: + for command in self._global_config.custom_search_commands: + file_path = os.path.join(self._source_dir, "bin", command["fileName"]) + if not os.path.isfile(file_path): + raise GlobalConfigValidatorException( + f"{command['fileName']} is not present in `{os.path.join(self._source_dir, 'bin')}` directory. " + "Please ensure the file exists." + ) + + if (command.get("requiredSearchAssistant", False) is False) and ( + command.get("description") + or command.get("usage") + or command.get("syntax") + ): + logger.warning( + "requiredSearchAssistant is set to false " + "but attributes required for 'searchbnf.conf' is defined which is not required." + ) + if (command.get("requiredSearchAssistant", False) is True) and not ( + command.get("description") + and command.get("usage") + and command.get("syntax") + ): + raise GlobalConfigValidatorException( + "One of the attributes among `description`, `usage`, `syntax`" + " is not been defined in globalConfig. Define them as requiredSearchAssistant is set to True." + ) + + if command["commandName"] in SPLUNK_COMMANDS: + raise GlobalConfigValidatorException( + f"CommandName: {command['commandName']}" + " cannot have the same name as Splunk built-in command." + ) + + fileName_without_extension = command["fileName"].replace(".py", "") + if command["commandName"] == fileName_without_extension: + # Here we are generating file based on commandName therefore + # the core logic should not have the same name as commandName + raise GlobalConfigValidatorException( + f"Filename: {fileName_without_extension} and CommandName: {command['commandName']}" + " should not be same for custom search command." + ) + def validate(self) -> None: self._validate_config_against_schema() if self._global_config.has_pages(): @@ -723,6 +773,7 @@ def validate(self) -> None: self._validate_checkbox_group() self._validate_groups() self._validate_field_modifications() + self._validate_custom_search_commands() self._validate_alerts() self._validate_meta_default_view() diff --git a/splunk_add_on_ucc_framework/schema/schema.json b/splunk_add_on_ucc_framework/schema/schema.json index 2bfeab2675..f0d7f6b3d1 100644 --- a/splunk_add_on_ucc_framework/schema/schema.json +++ b/splunk_add_on_ucc_framework/schema/schema.json @@ -427,6 +427,211 @@ ], "additionalProperties": false }, + "customSearchCommand": { + "type": "object", + "description": "Support for custom Search Command.", + "properties": { + "commandName": { + "type": "string", + "pattern": "^[a-z0-9]+$", + "maxLength": 100, + "description": "Name of the custom Search Command." + }, + "fileName": { + "type": "string", + "pattern": "^\\w+\\.py$", + "description": "Name of the file which contains logic for custom search command. The file should be a python file only." + }, + "commandType": { + "type": "string", + "description": "Type of the custom search command. There are 4 types of command i.e generating, streaming, transforming and dataset processing", + "enum": [ + "generating", + "streaming", + "dataset processing" + ] + }, + "requiredSearchAssistant": { + "type": "boolean", + "default": false, + "description": "Specifies if search assistant is required or not. If yes then searchbnf.conf will be generated." + }, + "description": { + "type": "string", + "description": "Description of the custom search command. It is an required attribute for searchbnf.conf." + }, + "syntax": { + "type": "string", + "maxLength": 100, + "description": "Syntax for the custom search command. It is an required attribute for searchbnf.conf." + }, + "usage": { + "type": "string", + "description": "Specifies what will be the usage of custom search command. It is an required attribute for searchbnf.conf.", + "enum": [ + "public", + "private", + "deprecated" + ] + }, + "arguments": { + "type": "array", + "items": { + "$ref": "#/definitions/arguments" + }, + "minItems": 1 + } + }, + "required": [ + "commandName", + "fileName", + "commandType", + "arguments" + ], + "additionalProperties": false + }, + "arguments": { + "type": "object", + "description": "Arguments used for custom search command", + "properties": { + "name": { + "type": "string", + "description": "Name of the argument" + }, + "required": { + "type": "boolean", + "default": true, + "description": "Specifies if the argument is required or not" + }, + "validate": { + "$ref": "#/definitions/CustomSearchCommandValidator" + }, + "defaultValue": { + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, + "description": "Provide default value to the arguments passed for custom search command" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + "CustomSearchCommandValidator": { + "type": "object", + "description": "It is used to validate the values of arguments for custom search command", + "oneOf": [ + { + "$ref": "#/definitions/CustomIntegerValidator" + }, + { + "$ref": "#/definitions/CustomFloatValidator" + }, + { + "$ref": "#/definitions/CustomRegularExpressionValidator" + }, + { + "$ref": "#/definitions/CustomFieldnameValidator" + }, + { + "$ref": "#/definitions/CustomBooleanValidator" + } + ] + }, + "CustomIntegerValidator": { + "type": "object", + "properties": { + "minimum": { + "type": "number", + "description": "Minimum value used for validation" + }, + "maximum": { + "type": "number", + "description": "Maximum value used for validation" + }, + "type": { + "const": "Integer", + "type": "string", + "description": "This is integer Validator for custom search command, provide at least one of `minimumValue` or `maximumValue`" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "CustomFloatValidator": { + "type": "object", + "properties": { + "minimumValue": { + "type": "number", + "description": "Minimum value used for validation" + }, + "maximumValue": { + "type": "number", + "description": "Maximum value used for validation" + }, + "type": { + "const": "Float", + "type": "string", + "description": "This is Float Validator for custom search command, provide at least one of `minimumValue` or `maximumValue` for validation purpose." + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "CustomRegularExpressionValidator": { + "type": "object", + "properties": { + "type": { + "const": "RegularExpression", + "type": "string", + "description": "This is RegularExpression Validator which validates if the input value is a valid regular expression pattern or not." + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "CustomFieldnameValidator": { + "type": "object", + "properties": { + "type": { + "const": "Fieldname", + "type": "string", + "description": "Validates field name option values." + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "CustomBooleanValidator": { + "type": "object", + "properties": { + "type": { + "const": "Boolean", + "type": "string", + "description": "Validates Boolean option values." + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, "ConfigurationPage": { "type": "object", "properties": { @@ -480,7 +685,7 @@ "field": { "type": "string", "pattern": "^\\w+$", - "description": "Used to dispaly the data in a column." + "description": "Used to display the data in a column." }, "label": { "type": "string", @@ -1520,7 +1725,10 @@ "uniqueItems": true } }, - "required": ["label", "fields"], + "required": [ + "label", + "fields" + ], "additionalProperties": false } }, @@ -1551,7 +1759,9 @@ "additionalProperties": false } }, - "required": ["field"], + "required": [ + "field" + ], "additionalProperties": false }, "minItems": 1 @@ -1560,7 +1770,9 @@ "$ref": "#/definitions/disableonEdit" } }, - "required": ["rows"], + "required": [ + "rows" + ], "additionalProperties": false }, "validators": { @@ -1577,7 +1789,12 @@ "type": "boolean" } }, - "required": ["field", "label", "type", "options"], + "required": [ + "field", + "label", + "type", + "options" + ], "additionalProperties": false }, "RadioEntity": { @@ -1871,7 +2088,7 @@ "auth_endpoint_token_access_type": { "type": "string", "maxLength": 350, - "description": "Optional parameter that is mapped into the value of the token_access_type query param in the authorisation url." + "description": "Optional parameter that is mapped into the value of the token_access_type query param in the authorization url." }, "oauth_state_enabled": { "type": "boolean", @@ -2473,7 +2690,12 @@ "properties": { "type": { "type": "string", - "enum": ["string", "number", "integer", "boolean"] + "enum": [ + "string", + "number", + "integer", + "boolean" + ] }, "nullable": { "type": "boolean", @@ -2537,7 +2759,7 @@ "anyOf": { "type": "array", "items": { - "$ref": "#/definitions/OpenApiType" + "$ref": "#/definitions/OpenApiType" } }, "nullable": { @@ -2555,7 +2777,7 @@ "oneOf": { "type": "array", "items": { - "$ref": "#/definitions/OpenApiType" + "$ref": "#/definitions/OpenApiType" } }, "nullable": { @@ -2587,7 +2809,9 @@ } }, "additionalProperties": false, - "required": ["schema"] + "required": [ + "schema" + ] } } }, @@ -2627,7 +2851,9 @@ "handlerType": { "type": "string", "description": "The type of the handler.", - "enum": ["EAI"] + "enum": [ + "EAI" + ] }, "registerHandler": { "type": "object", @@ -2636,7 +2862,7 @@ "file": { "type": "string", "description": "The file where the custom rest handler is located.", - "pattern": "^[a-zA-Z0-9_]+\\.py$" + "pattern": "^[a-zA-Z0-9_]+\\.py$" }, "actions": { "type": "array", @@ -2644,7 +2870,12 @@ "items": { "type": "string", "description": "The action that the custom rest handler supports.", - "enum": ["list", "edit", "create", "remove"], + "enum": [ + "list", + "edit", + "create", + "remove" + ], "minItems": 1, "uniqueItems": true } @@ -3019,7 +3250,9 @@ "$ref": "#/definitions/links" } }, - "required": ["text"] + "required": [ + "text" + ] }, "singleFieldToModify": { "type": "object", @@ -3807,7 +4040,8 @@ "anyOf": [ { "type": "string", - "description": "Help text gives context about a fields input, such as how the input will be used. It is displayed directly below an input field." }, + "description": "Help text gives context about a fields input, such as how the input will be used. It is displayed directly below an input field." + }, { "$ref": "#/definitions/helpWithLinks" } @@ -3828,7 +4062,9 @@ "type": "string" } }, - "required": ["text"] + "required": [ + "text" + ] }, "tooltip": { "type": "string", @@ -3932,7 +4168,7 @@ } ] }, - "inputsUniqueAcrossSingleService":{ + "inputsUniqueAcrossSingleService": { "description": "Defines if inputs in different services can share same name.", "type": "boolean", "default": false @@ -3999,6 +4235,13 @@ }, "minItems": 1 }, + "customSearchCommand": { + "type": "array", + "items": { + "$ref": "#/definitions/customSearchCommand" + }, + "minItems": 1 + }, "options": { "$ref": "#/definitions/Options" } diff --git a/splunk_add_on_ucc_framework/templates/conf_files/commands_conf.template b/splunk_add_on_ucc_framework/templates/conf_files/commands_conf.template new file mode 100644 index 0000000000..1aa32b9aa1 --- /dev/null +++ b/splunk_add_on_ucc_framework/templates/conf_files/commands_conf.template @@ -0,0 +1,6 @@ +{% for name in command_names -%} +[{{name}}] +filename = {{name}}.py +chunked = true +python.version = python3 +{% endfor -%} \ No newline at end of file diff --git a/splunk_add_on_ucc_framework/templates/conf_files/searchbnf_conf.template b/splunk_add_on_ucc_framework/templates/conf_files/searchbnf_conf.template new file mode 100644 index 0000000000..c7227070ca --- /dev/null +++ b/splunk_add_on_ucc_framework/templates/conf_files/searchbnf_conf.template @@ -0,0 +1,6 @@ +{% for info in searchbnf_info -%} +[{{info["command_name"]}}] +syntax = {{info["syntax"]}} +description = {{info["description"]}} +usage = {{info["usage"]}} +{% endfor -%} \ No newline at end of file diff --git a/splunk_add_on_ucc_framework/templates/custom_command/dataset_processing.template b/splunk_add_on_ucc_framework/templates/custom_command/dataset_processing.template new file mode 100644 index 0000000000..e8cb58dc87 --- /dev/null +++ b/splunk_add_on_ucc_framework/templates/custom_command/dataset_processing.template @@ -0,0 +1,33 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, EventingCommand, Configuration, Option, validators +from {{imported_file_name}} import transform + +@Configuration() +class {{class_name}}Command(EventingCommand): + {% if syntax or description%} + """ + + {% if syntax %} + ##Syntax + {{syntax}} + {% endif %} + + {% if description %} + ##Description + {{description}} + {% endif %} + + """ + {% endif %} + + {% for arg in list_arg %} + {{arg}} + {% endfor %} + + def transform(self, events): + return transform(self, events) + +dispatch({{class_name}}Command, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/splunk_add_on_ucc_framework/templates/custom_command/generating.template b/splunk_add_on_ucc_framework/templates/custom_command/generating.template new file mode 100644 index 0000000000..4256e62d15 --- /dev/null +++ b/splunk_add_on_ucc_framework/templates/custom_command/generating.template @@ -0,0 +1,32 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, GeneratingCommand, Configuration, Option, validators +from {{imported_file_name}} import generate + +@Configuration() +class {{class_name}}Command(GeneratingCommand): + {% if syntax or description%} + """ + + {% if syntax %} + ##Syntax + {{syntax}} + {% endif %} + + {% if description %} + ##Description + {{description}} + {% endif %} + + """ + {% endif %} + {% for arg in list_arg %} + {{arg}} + {% endfor %} + + def generate(self): + return generate(self) + +dispatch({{class_name}}Command, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/splunk_add_on_ucc_framework/templates/custom_command/streaming.template b/splunk_add_on_ucc_framework/templates/custom_command/streaming.template new file mode 100644 index 0000000000..53754b84bb --- /dev/null +++ b/splunk_add_on_ucc_framework/templates/custom_command/streaming.template @@ -0,0 +1,33 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, StreamingCommand, Configuration, Option, validators +from {{imported_file_name}} import stream + +@Configuration() +class {{class_name}}Command(StreamingCommand): + {% if syntax or description%} + """ + + {% if syntax %} + ##Syntax + {{syntax}} + {% endif %} + + {% if description %} + ##Description + {{description}} + {% endif %} + + """ + {% endif %} + + {% for arg in list_arg %} + {{arg}} + {% endfor %} + + def stream(self, events): + return stream(self, events) + +dispatch({{class_name}}Command, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/tests/smoke/test_ucc_build.py b/tests/smoke/test_ucc_build.py index 2cb177288c..f2a9cefc52 100644 --- a/tests/smoke/test_ucc_build.py +++ b/tests/smoke/test_ucc_build.py @@ -153,6 +153,8 @@ def test_ucc_generate_with_everything(caplog): ("default", "splunk_ta_uccexample_settings.conf"), ("default", "web.conf"), ("default", "server.conf"), + ("default", "commands.conf"), + ("default", "searchbnf.conf"), ("default", "data", "ui", "alerts", "test_alert.html"), ("default", "data", "ui", "nav", "default.xml"), ("default", "data", "ui", "views", "configuration.xml"), @@ -167,6 +169,12 @@ def test_ucc_generate_with_everything(caplog): ("bin", "example_input_three.py"), ("bin", "example_input_four.py"), ("bin", "import_declare_test.py"), + ("bin", "countmatchescommand.py"), + ("bin", "countmatches.py"), + ("bin", "filter.py"), + ("bin", "filtercommand.py"), + ("bin", "generatetext.py"), + ("bin", "generatetextcommand.py"), ("bin", "splunk_ta_uccexample_rh_account.py"), ("bin", "splunk_ta_uccexample_rh_example_input_one.py"), ("bin", "splunk_ta_uccexample_rh_example_input_two.py"), diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatches.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatches.py new file mode 100644 index 0000000000..da79b16420 --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatches.py @@ -0,0 +1,4 @@ +def stream(self, records): + for record in records: + # write custom logic for the search command + yield record \ No newline at end of file diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatchescommand.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatchescommand.py new file mode 100644 index 0000000000..e2faa5da18 --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/countmatchescommand.py @@ -0,0 +1,17 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, StreamingCommand, Configuration, Option, validators +from countmatches import stream + +@Configuration() +class CountmatchescommandCommand(StreamingCommand): + + fieldname = Option(name='fieldname', require=True, validate=validators.Fieldname()) + pattern = Option(name='pattern', require=True, validate=validators.RegularExpression()) + + def stream(self, events): + return stream(self, events) + +dispatch(CountmatchescommandCommand, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filter.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filter.py new file mode 100644 index 0000000000..51d476833f --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filter.py @@ -0,0 +1,36 @@ +def transform(self, records): + contains = self.contains + replace_array = self.replace_array + + if contains and replace_array: + arr = replace_array.split(",") + if len(arr) != 2: + raise ValueError("Please provide only two arguments, separated by comma for 'replace'") + + for record in records: + _raw = record.get("_raw") + if contains in _raw: + record["_raw"] = _raw.replace(arr[0], arr[1]) + yield record + return + + if contains: + for record in records: + _raw = record.get("_raw") + if contains in _raw: + yield record + return + + if replace_array: + arr = replace_array.split(",") + if len(arr) != 2: + raise ValueError("Please provide only two arguments, separated by comma for 'replace'") + + for record in records: + _raw = record.get("_raw") + record["_raw"] = _raw.replace(arr[0], arr[1]) + yield record + return + + for record in records: + yield record \ No newline at end of file diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filtercommand.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filtercommand.py new file mode 100644 index 0000000000..8ab7cb354d --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/filtercommand.py @@ -0,0 +1,26 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, EventingCommand, Configuration, Option, validators +from filter import transform + +@Configuration() +class FiltercommandCommand(EventingCommand): + """ + + ##Syntax + | filtercommand contains='value1' replace='value to be replaced,value to replace with' + + ##Description + It filters records from the events stream returning only those which has :code:`contains` in them and replaces :code:`replace_array[0]` with :code:`replace_array[1]`. + + """ + + contains = Option(name='contains', require=False) + replace_array = Option(name='replace_array', require=False) + + def transform(self, events): + return transform(self, events) + +dispatch(FiltercommandCommand, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetext.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetext.py new file mode 100644 index 0000000000..897779284c --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetext.py @@ -0,0 +1,8 @@ +import time +import logging + + +def generate(self): + logging.debug("Generating %d events with text %s" % (self.count, self.text)) + for i in range(1, self.count + 1): + yield {'_serial': i, '_time': time.time(), '_raw': str(i) + '. ' + self.text} diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetextcommand.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetextcommand.py new file mode 100644 index 0000000000..89459faa90 --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/generatetextcommand.py @@ -0,0 +1,25 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, GeneratingCommand, Configuration, Option, validators +from generatetext import generate + +@Configuration() +class GeneratetextcommandCommand(GeneratingCommand): + """ + + ##Syntax + generatetextcommand count= text= + + ##Description + This command generates COUNT occurrences of a TEXT string. + + """ + count = Option(name='count', require=True, validate=validators.Integer(minimum=5, maximum=10)) + text = Option(name='text', require=True) + + def generate(self): + return generate(self) + +dispatch(GeneratetextcommandCommand, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/commands.conf b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/commands.conf new file mode 100644 index 0000000000..ee84578dbf --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/commands.conf @@ -0,0 +1,14 @@ +[generatetextcommand] +filename = generatetextcommand.py +chunked = true +python.version = python3 + +[filtercommand] +filename = filtercommand.py +chunked = true +python.version = python3 + +[countmatchescommand] +filename = countmatchescommand.py +chunked = true +python.version = python3 diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/searchbnf.conf b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/searchbnf.conf new file mode 100644 index 0000000000..16d6021c1f --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/searchbnf.conf @@ -0,0 +1,9 @@ +[generatetextcommand] +syntax = generatetextcommand count= text= +description = This command generates COUNT occurrences of a TEXT string. +usage = public + +[filtercommand] +syntax = | filtercommand contains='value1' replace='value to be replaced,value to replace with' +description = It filters records from the events stream returning only those which has :code:`contains` in them and replaces :code:`replace_array[0]` with :code:`replace_array[1]`. +usage = public diff --git a/tests/testdata/test_addons/package_global_config_everything/globalConfig.json b/tests/testdata/test_addons/package_global_config_everything/globalConfig.json index 95aa1e4855..d8060a082c 100644 --- a/tests/testdata/test_addons/package_global_config_everything/globalConfig.json +++ b/tests/testdata/test_addons/package_global_config_everything/globalConfig.json @@ -2198,5 +2198,70 @@ "target": "3rdparty/linux" } ] - } + }, + "customSearchCommand": [ + { + "commandName": "generatetextcommand", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": true, + "description": " This command generates COUNT occurrences of a TEXT string.", + "syntax": "generatetextcommand count= text=", + "usage": "public", + "arguments": [ + { + "name": "count", + "required": true, + "validate": { + "type": "Integer", + "minimum": 5, + "maximum": 10 + } + }, + { + "name": "text", + "required": true + } + ] + }, + { + "commandName": "filtercommand", + "fileName": "filter.py", + "commandType": "dataset processing", + "requiredSearchAssistant": true, + "description": "It filters records from the events stream returning only those which has :code:`contains` in them and replaces :code:`replace_array[0]` with :code:`replace_array[1]`.", + "syntax": "| filtercommand contains='value1' replace='value to be replaced,value to replace with'", + "usage": "public", + "arguments": [ + { + "name": "contains" + }, + { + "name": "replace_array" + } + ] + }, + { + "commandName": "countmatchescommand", + "fileName": "countmatches.py", + "commandType": "streaming", + "requiredSearchAssistant": false, + "arguments": [ + { + "name": "fieldname", + "validate": { + "type": "Fieldname" + }, + "required": true + }, + { + "name": "pattern", + "validate": { + "type": "RegularExpression" + }, + "required": true + } + ] + } + ] } diff --git a/tests/testdata/test_addons/package_global_config_everything/package/bin/countmatches.py b/tests/testdata/test_addons/package_global_config_everything/package/bin/countmatches.py new file mode 100644 index 0000000000..da79b16420 --- /dev/null +++ b/tests/testdata/test_addons/package_global_config_everything/package/bin/countmatches.py @@ -0,0 +1,4 @@ +def stream(self, records): + for record in records: + # write custom logic for the search command + yield record \ No newline at end of file diff --git a/tests/testdata/test_addons/package_global_config_everything/package/bin/filter.py b/tests/testdata/test_addons/package_global_config_everything/package/bin/filter.py new file mode 100644 index 0000000000..51d476833f --- /dev/null +++ b/tests/testdata/test_addons/package_global_config_everything/package/bin/filter.py @@ -0,0 +1,36 @@ +def transform(self, records): + contains = self.contains + replace_array = self.replace_array + + if contains and replace_array: + arr = replace_array.split(",") + if len(arr) != 2: + raise ValueError("Please provide only two arguments, separated by comma for 'replace'") + + for record in records: + _raw = record.get("_raw") + if contains in _raw: + record["_raw"] = _raw.replace(arr[0], arr[1]) + yield record + return + + if contains: + for record in records: + _raw = record.get("_raw") + if contains in _raw: + yield record + return + + if replace_array: + arr = replace_array.split(",") + if len(arr) != 2: + raise ValueError("Please provide only two arguments, separated by comma for 'replace'") + + for record in records: + _raw = record.get("_raw") + record["_raw"] = _raw.replace(arr[0], arr[1]) + yield record + return + + for record in records: + yield record \ No newline at end of file diff --git a/tests/testdata/test_addons/package_global_config_everything/package/bin/generatetext.py b/tests/testdata/test_addons/package_global_config_everything/package/bin/generatetext.py new file mode 100644 index 0000000000..897779284c --- /dev/null +++ b/tests/testdata/test_addons/package_global_config_everything/package/bin/generatetext.py @@ -0,0 +1,8 @@ +import time +import logging + + +def generate(self): + logging.debug("Generating %d events with text %s" % (self.count, self.text)) + for i in range(1, self.count + 1): + yield {'_serial': i, '_time': time.time(), '_raw': str(i) + '. ' + self.text} diff --git a/tests/unit/generators/conf_files/test_conf_files_init.py b/tests/unit/generators/conf_files/test_conf_files_init.py index 18e764f396..71904f7450 100644 --- a/tests/unit/generators/conf_files/test_conf_files_init.py +++ b/tests/unit/generators/conf_files/test_conf_files_init.py @@ -11,6 +11,8 @@ def test___init__conf(): "InputsConf", "AccountConf", "SettingsConf", + "CommandsConf", + "SearchbnfConf", ] expected_modules = [ "file_generator", @@ -24,6 +26,8 @@ def test___init__conf(): "create_web_conf", "create_account_conf", "create_settings_conf", + "create_commands_conf", + "create_searchbnf_conf", ] import splunk_add_on_ucc_framework.generators.conf_files as conf diff --git a/tests/unit/generators/conf_files/test_create_commands_conf.py b/tests/unit/generators/conf_files/test_create_commands_conf.py new file mode 100644 index 0000000000..babaa2151b --- /dev/null +++ b/tests/unit/generators/conf_files/test_create_commands_conf.py @@ -0,0 +1,79 @@ +from splunk_add_on_ucc_framework.generators.conf_files import CommandsConf +from splunk_add_on_ucc_framework import __file__ as ucc_framework_file +import os.path +from textwrap import dedent + +UCC_DIR = os.path.dirname(ucc_framework_file) + + +def test_set_attributes_without_custom_command( + global_config_only_configuration, input_dir, output_dir, ucc_dir, ta_name +): + commands_conf = CommandsConf( + global_config_only_configuration, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert not hasattr(commands_conf, "command_names") + + +def test_set_attributes( + global_config_all_json, input_dir, output_dir, ucc_dir, ta_name +): + commands_conf = CommandsConf( + global_config_all_json, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert commands_conf.conf_file == "commands.conf" + assert commands_conf.command_names == ["generatetextcommand"] + + +def test_generate_conf_without_custom_command( + global_config_only_configuration, input_dir, output_dir, ucc_dir, ta_name +): + commands_conf = CommandsConf( + global_config_only_configuration, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + file_paths = commands_conf.generate() + + # Assert that no files are returned since no custom command is configured + assert file_paths == {} + + +def test_commands_conf_generation( + global_config_all_json, input_dir, output_dir, ta_name +): + commands_conf = CommandsConf( + global_config_all_json, + input_dir, + output_dir, + addon_name=ta_name, + ucc_dir=UCC_DIR, + ) + file_paths = commands_conf.generate() + + assert file_paths is not None + assert file_paths.keys() == {"commands.conf"} + assert file_paths["commands.conf"].endswith("test_addon/default/commands.conf") + + with open(file_paths["commands.conf"]) as fp: + content = fp.read() + + expected_content = dedent( + """ + [generatetextcommand] + filename = generatetextcommand.py + chunked = true + python.version = python3 + """ + ).lstrip() + assert content == expected_content diff --git a/tests/unit/generators/conf_files/test_create_searchbnf_conf.py b/tests/unit/generators/conf_files/test_create_searchbnf_conf.py new file mode 100644 index 0000000000..0b43f5d96a --- /dev/null +++ b/tests/unit/generators/conf_files/test_create_searchbnf_conf.py @@ -0,0 +1,117 @@ +from pytest import fixture +from splunk_add_on_ucc_framework.generators.conf_files import SearchbnfConf +from splunk_add_on_ucc_framework import __file__ as ucc_framework_file +import os.path +from textwrap import dedent + +UCC_DIR = os.path.dirname(ucc_framework_file) + + +@fixture +def custom_search_command_without_search_assistance(): + return [ + { + "commandName": "testcommand2", + "commandType": "streaming", + "fileName": "test2.py", + } + ] + + +def test_set_attributes_without_custom_command( + global_config_only_configuration, input_dir, output_dir, ucc_dir, ta_name +): + searchbnf_conf = SearchbnfConf( + global_config_only_configuration, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert searchbnf_conf.searchbnf_info == [] + + +def test_set_attributes( + global_config_all_json, input_dir, output_dir, ucc_dir, ta_name +): + searchbnf_conf = SearchbnfConf( + global_config_all_json, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert searchbnf_conf.conf_file == "searchbnf.conf" + assert searchbnf_conf.searchbnf_info == [ + { + "command_name": "generatetextcommand", + "description": "This command generates COUNT occurrences of a TEXT string.", + "syntax": "generatetextcommand count= text=", + "usage": "public", + } + ] + + +def test_set_attributes_without_search_assistance( + global_config_all_json, + input_dir, + output_dir, + ucc_dir, + ta_name, + custom_search_command_without_search_assistance, +): + global_config_all_json._content[ + "customSearchCommand" + ] = custom_search_command_without_search_assistance + searchbnf_conf = SearchbnfConf( + global_config_all_json, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert searchbnf_conf.searchbnf_info == [] + + +def test_generate_conf_without_custom_command( + global_config_only_configuration, input_dir, output_dir, ucc_dir, ta_name +): + searchbnf_conf = SearchbnfConf( + global_config_only_configuration, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + file_paths = searchbnf_conf.generate() + + # Assert that no files are returned since no custom command is configured + assert file_paths == {} + + +def test_generate_conf(global_config_all_json, input_dir, output_dir, ta_name): + searchbnf_conf = SearchbnfConf( + global_config_all_json, + input_dir, + output_dir, + ucc_dir=UCC_DIR, + addon_name=ta_name, + ) + file_paths = searchbnf_conf.generate() + exp_fname = "searchbnf.conf" + + assert file_paths == {exp_fname: f"{output_dir}/{ta_name}/default/{exp_fname}"} + + with open(file_paths["searchbnf.conf"]) as fp: + content = fp.read() + + expected_content = dedent( + """ + [generatetextcommand] + syntax = generatetextcommand count= text= + description = This command generates COUNT occurrences of a TEXT string. + usage = public + """ + ).lstrip() + + assert content == expected_content diff --git a/tests/unit/generators/python_files/test_create_custom_command_python.py b/tests/unit/generators/python_files/test_create_custom_command_python.py new file mode 100644 index 0000000000..67ec2eae72 --- /dev/null +++ b/tests/unit/generators/python_files/test_create_custom_command_python.py @@ -0,0 +1,114 @@ +from pytest import fixture +from splunk_add_on_ucc_framework.generators.python_files import CustomCommandPy +from splunk_add_on_ucc_framework import __file__ as ucc_framework_file +import os.path + +UCC_DIR = os.path.dirname(ucc_framework_file) + + +@fixture +def custom_search_commands(): + return [ + { + "commandName": "testcommand", + "commandType": "generating", + "fileName": "test.py", + "requiredSearchAssistant": True, + "description": "This is test command", + "syntax": "testcommand count= text=", + "usage": "public", + "arguments": [ + { + "name": "count", + "required": True, + "validate": {"type": "Integer", "minimum": 5, "maximum": 10}, + }, + { + "name": "age", + "validate": {"type": "Integer", "minimum": 18}, + }, + {"name": "text", "required": True, "defaultValue": "test_text"}, + {"name": "contains"}, + {"name": "fieldname", "validate": {"type": "Fieldname"}}, + ], + } + ] + + +def test_set_attributes_without_custom_command( + global_config_only_configuration, input_dir, output_dir, ucc_dir, ta_name +): + custom_command = CustomCommandPy( + global_config_only_configuration, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert custom_command.commands_info == [] + + +def test_set_attributes( + global_config_all_json, + input_dir, + output_dir, + ucc_dir, + ta_name, + custom_search_commands, +): + global_config_all_json._content["customSearchCommand"] = custom_search_commands + custom_command_py = CustomCommandPy( + global_config_all_json, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + assert custom_command_py.commands_info == [ + { + "imported_file_name": "test", + "file_name": "testcommand", + "class_name": "Testcommand", + "description": "This is test command", + "syntax": "testcommand count= text=", + "template": "generating.template", + "list_arg": [ + "count = Option(name='count', require=True, " + "validate=validators.Integer(minimum=5, maximum=10))", + "age = Option(name='age', require=False, validate=validators.Integer(minimum=18))", + "text = Option(name='text', require=True, default='test_text')", + "contains = Option(name='contains', require=False)", + "fieldname = Option(name='fieldname', require=False, validate=validators.Fieldname())", + ], + } + ] + + +def test_generate_python_without_custom_command( + global_config_only_configuration, input_dir, output_dir, ucc_dir, ta_name +): + custom_command = CustomCommandPy( + global_config_only_configuration, + input_dir, + output_dir, + ucc_dir=ucc_dir, + addon_name=ta_name, + ) + file_paths = custom_command.generate() + + # Assert that no files are returned since no custom command is configured + assert file_paths == {} + + +def test_generate_python(global_config_all_json, input_dir, output_dir, ta_name): + exp_fname = "generatetextcommand.py" + + custom_command_py = CustomCommandPy( + global_config_all_json, + input_dir, + output_dir, + ucc_dir=UCC_DIR, + addon_name=ta_name, + ) + file_paths = custom_command_py.generate() + assert file_paths == {exp_fname: f"{output_dir}/{ta_name}/bin/{exp_fname}"} diff --git a/tests/unit/generators/python_files/test_python_files_init.py b/tests/unit/generators/python_files/test_python_files_init.py new file mode 100644 index 0000000000..b5f5d2b771 --- /dev/null +++ b/tests/unit/generators/python_files/test_python_files_init.py @@ -0,0 +1,12 @@ +def test___init__Python(): + expected_classes = ["FileGenerator", "CustomCommandPy"] + expected_modules = ["file_generator", "create_custom_command_python"] + import splunk_add_on_ucc_framework.generators.python_files as py + + assert py.__all__ == expected_classes + + for attrib in dir(py): + if attrib.startswith("__") and attrib.endswith("__"): + # ignore the builtin modules + continue + assert attrib in expected_classes or attrib in expected_modules diff --git a/tests/unit/generators/test_doc_generator.py b/tests/unit/generators/test_doc_generator.py index 08b103c3f9..3ed7d38a85 100644 --- a/tests/unit/generators/test_doc_generator.py +++ b/tests/unit/generators/test_doc_generator.py @@ -21,6 +21,8 @@ def test_generate_docs(): | alert_actions.conf | output/<YOUR_ADDON_NAME>/default | Generates `alert_actions.conf` and `alert_actions.conf.spec` file for the custom alert actions defined in globalConfig | | eventtypes.conf | output/<YOUR_ADDON_NAME>/default | Generates `eventtypes.conf` file if the sourcetype is mentioned in Adaptive Response of custom alert action in globalConfig | | tags.conf | output/<YOUR_ADDON_NAME>/default | Generates `tags.conf` file based on the `eventtypes.conf` created for custom alert actions. | +| commands.conf | output/<YOUR_ADDON_NAME>/default | Generates `commands.conf` for custom commands provided in the globalConfig. | +| searchbnf.conf | output/<YOUR_ADDON_NAME>/default | Generates `searchbnf.conf` for custom search commands provided in the globalConfig. | | _account.conf | output/<YOUR_ADDON_NAME>/README | Generates `_account.conf.spec` file for the configuration mentioned in globalConfig | | _settings.conf | output/<YOUR_ADDON_NAME>/README | Generates `_settings.conf.spec` file for the Proxy, Logging or Custom Tab mentioned in globalConfig | | configuration.xml | output/<YOUR_ADDON_NAME>/default/data/ui/views | Generates configuration.xml file in `default/data/ui/views/` folder if configuration is defined in globalConfig. | @@ -29,6 +31,7 @@ def test_generate_docs(): | inputs.xml | output/<YOUR_ADDON_NAME>/default/data/ui/views | Generates inputs.xml based on inputs configuration present in globalConfig, in `default/data/ui/views/inputs.xml` folder | | _redirect.xml | output/<YOUR_ADDON_NAME>/default/data/ui/views | Generates ta_name_redirect.xml file, if oauth is mentioned in globalConfig, in `default/data/ui/views/` folder. | | _.html | output/<YOUR_ADDON_NAME>/default/data/ui/alerts | Generates `alert_name.html` file based on alerts configuration present in globalConfig, in `default/data/ui/alerts` folder. | +| _.py | output/<YOUR_ADDON_NAME>/bin | Generates Python files for custom search commands provided in the globalConfig. | | globalConfig.json | <source_dir> | Generates globalConfig.json file in the source code if globalConfig is not present in source directory at build time. | """ diff --git a/tests/unit/generators/test_init_.py b/tests/unit/generators/test_init_.py index 01b48a6931..69576a7cdd 100644 --- a/tests/unit/generators/test_init_.py +++ b/tests/unit/generators/test_init_.py @@ -12,7 +12,13 @@ def test___init__gen(): assert gen.__all__ == expected_classes - not_allowed = ["conf_files", "xml_files", "html_files", "doc_generator"] + not_allowed = [ + "conf_files", + "xml_files", + "html_files", + "python_files", + "doc_generator", + ] for attrib in dir(gen): if attrib.startswith("__") and attrib.endswith("__") or attrib in not_allowed: # ignore the builtin modules diff --git a/tests/unit/test_const.py b/tests/unit/test_const.py new file mode 100644 index 0000000000..d296474620 --- /dev/null +++ b/tests/unit/test_const.py @@ -0,0 +1,33 @@ +import re +import urllib.request + +from splunk_add_on_ucc_framework.const import SPLUNK_COMMANDS + + +def test_command_list_up_to_date(): + with urllib.request.urlopen( + "https://docs.splunk.com/Documentation/Splunk/latest/SearchReference" + ) as resp: + content = resp.read().decode() + + match = re.search(r"Search\s+Commands.+?(.+?)", content, re.S) + if match: + search_commands_ul = match.group(1) + search_commands = re.findall( + r"]*>.*?]*>\s*([^\s<]+)\s+?", search_commands_ul, re.S + ) + else: + search_commands_ul = None + search_commands = [] + + # These are the search commands for the serviceNow add-on. They are not present by default in Splunk instance + not_global_commands = [ + "snowincidentstream", + "snoweventstream", + "snowincident", + "snowevent", + ] + for command in not_global_commands: + search_commands.remove(command) + + assert set(search_commands) == set(SPLUNK_COMMANDS) diff --git a/tests/unit/test_global_config.py b/tests/unit/test_global_config.py index 6b18784f5b..9b00ad8ede 100644 --- a/tests/unit/test_global_config.py +++ b/tests/unit/test_global_config.py @@ -66,6 +66,31 @@ def test_global_config_equal(): assert global_config_1 == global_config_2 +def test_global_config_custom_search_commands(global_config_all_json): + custom_search_commands = global_config_all_json.custom_search_commands + expected_result = [ + { + "commandName": "generatetextcommand", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": True, + "description": "This command generates COUNT occurrences of a TEXT string.", + "syntax": "generatetextcommand count= text=", + "usage": "public", + "arguments": [ + { + "name": "count", + "required": True, + "validate": {"type": "Integer", "minimum": 5, "maximum": 10}, + }, + {"name": "text", "required": True}, + ], + } + ] + assert expected_result == custom_search_commands + assert global_config_all_json.has_custom_search_commands() is True + + def test_global_config_when_no_equal(): global_config_1 = global_config_lib.GlobalConfig.from_file( helpers.get_testdata_file_path("valid_config.json") diff --git a/tests/unit/test_global_config_validator.py b/tests/unit/test_global_config_validator.py index a69a6c0106..2a57d6bd8f 100644 --- a/tests/unit/test_global_config_validator.py +++ b/tests/unit/test_global_config_validator.py @@ -15,6 +15,7 @@ should_warn_on_empty_validators, ) from splunk_add_on_ucc_framework import global_config as global_config_lib +from unittest.mock import patch @pytest.mark.parametrize( @@ -25,7 +26,8 @@ "valid_config_only_logging.json", ], ) -def test_config_validation_when_valid(filename): +@patch("os.path.isfile", return_value=True) +def test_config_validation_when_valid(mock_isFile, filename): global_config_path = helpers.get_testdata_file_path(filename) global_config = global_config_lib.GlobalConfig.from_file(global_config_path) @@ -75,6 +77,10 @@ def test_autocompletefields_children_support_integer_values(): @pytest.mark.parametrize( "filename,exception_message", [ + ( + "invalid_config_for_custom_search_command.json", + "generatetext.py is not present in `bin` directory. Please ensure the file exists.", + ), ( "invalid_config_no_configuration_tabs.json", "[] is too short", @@ -355,6 +361,118 @@ def test_config_validation_when_error(filename, exception_message): assert msg == exception_message +@pytest.mark.parametrize( + "filename,invalid_custom_search_command,exception_message", + [ + ( + "invalid_config_for_custom_search_command.json", + [ + { + "commandName": "generatetextcommand", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": True, + "description": " This command generates COUNT occurrences of a TEXT string.", + "arguments": [ + { + "name": "text", + } + ], + } + ], + "One of the attributes among `description`, `usage`, `syntax` " + "is not been defined in globalConfig. Define them as requiredSearchAssistant is set to True.", + ), + ( + "invalid_config_for_custom_search_command.json", + [ + { + "commandName": "generatetext", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": False, + "arguments": [ + { + "name": "text", + } + ], + } + ], + "Filename: generatetext and CommandName: generatetext should not be same for custom search command.", + ), + ( + "invalid_config_for_custom_search_command.json", + [ + { + "commandName": "abstract", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": False, + "arguments": [ + { + "name": "text", + } + ], + } + ], + "CommandName: abstract cannot have the same name as Splunk built-in command.", + ), + ], +) +@patch("os.path.isfile", return_value=True) +def test_validate_custom_search_command( + mock_isFile, filename, invalid_custom_search_command, exception_message +): + global_config_path = helpers.get_testdata_file_path(filename) + global_config = global_config_lib.GlobalConfig.from_file(global_config_path) + + validator = GlobalConfigValidator(helpers.get_path_to_source_dir(), global_config) + global_config._content["customSearchCommand"] = invalid_custom_search_command + with pytest.raises(GlobalConfigValidatorException) as exc_info: + validator.validate() + + (msg,) = exc_info.value.args + assert msg == exception_message + + +@pytest.mark.parametrize( + "filename,valid_custom_search_command,warning_message", + [ + ( + "invalid_config_for_custom_search_command.json", + [ + { + "commandName": "generatetextcommand", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": False, + "description": " This command generates COUNT occurrences of a TEXT string.", + "arguments": [ + { + "name": "text", + } + ], + } + ], + "requiredSearchAssistant is set to false " + "but attributes required for 'searchbnf.conf' is defined which is not required.", + ), + ], +) +@patch("os.path.isfile", return_value=True) +def test_validate_custom_search_command_warning_msg( + mock_isFile, filename, valid_custom_search_command, warning_message, caplog +): + global_config_path = helpers.get_testdata_file_path(filename) + global_config = global_config_lib.GlobalConfig.from_file(global_config_path) + + validator = GlobalConfigValidator(helpers.get_path_to_source_dir(), global_config) + global_config._content["customSearchCommand"] = valid_custom_search_command + validator.validate() + + assert warning_message in caplog.text + + def test_config_validation_modifications_on_change(): global_config_path = helpers.get_testdata_file_path( "valid_config_with_modification_on_value_change.json" diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 864ff6747b..8904ab600c 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -33,6 +33,11 @@ def test_get_j2_env(): "conf_files/settings_conf.template", "conf_files/tags_conf.template", "web_conf.template", + "custom_command/dataset_processing.template", + "custom_command/generating.template", + "custom_command/streaming.template", + "conf_files/commands_conf.template", + "conf_files/searchbnf_conf.template", ] assert sorted(expected_list_of_templates) == sorted(list_of_templates) diff --git a/tests/unit/testdata/invalid_config_for_custom_search_command.json b/tests/unit/testdata/invalid_config_for_custom_search_command.json new file mode 100644 index 0000000000..e3f1f7a301 --- /dev/null +++ b/tests/unit/testdata/invalid_config_for_custom_search_command.json @@ -0,0 +1,79 @@ +{ + "pages": { + "configuration": { + "tabs": [ + { + "name": "logging", + "entity": [ + { + "type": "singleSelect", + "label": "Log level", + "options": { + "disableSearch": true, + "autoCompleteFields": [ + { + "value": "DEBUG", + "label": "DEBUG" + }, + { + "value": "INFO", + "label": "INFO" + }, + { + "value": "WARNING", + "label": "WARNING" + }, + { + "value": "ERROR", + "label": "ERROR" + }, + { + "value": "CRITICAL", + "label": "CRITICAL" + } + ] + }, + "defaultValue": "INFO", + "field": "loglevel", + "required": true + } + ], + "title": "Logging" + } + ], + "title": "Configuration", + "description": "Set up your add-on" + } + }, + "meta": { + "name": "Splunk_TA_UCCExample", + "restRoot": "splunk_ta_uccexample", + "version": "1.0.0", + "displayName": "Splunk UCC test Add-on", + "schemaVersion": "0.0.3" + }, + "customSearchCommand": [ + { + "commandName": "generatetextcommand", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": true, + "description": " This command generates COUNT occurrences of a TEXT string.", + "arguments": [ + { + "name": "count", + "required": true, + "validate": { + "type": "Integer", + "minimum": 5, + "maximum": 10 + } + }, + { + "name": "text", + "required": true + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/unit/testdata/valid_config.json b/tests/unit/testdata/valid_config.json index a406d07292..140f693d53 100644 --- a/tests/unit/testdata/valid_config.json +++ b/tests/unit/testdata/valid_config.json @@ -1363,5 +1363,31 @@ "target": "3rdparty/linux" } ] - } + }, + "customSearchCommand": [ + { + "commandName": "generatetextcommand", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": true, + "description": "This command generates COUNT occurrences of a TEXT string.", + "syntax": "generatetextcommand count= text=", + "usage": "public", + "arguments": [ + { + "name": "count", + "required": true, + "validate": { + "type": "Integer", + "minimum": 5, + "maximum": 10 + } + }, + { + "name": "text", + "required": true + } + ] + } + ] } \ No newline at end of file