From c6a19831ed1cfdfe2e0d1835528b7bae7c02a2a9 Mon Sep 17 00:00:00 2001 From: Raj Karkar Date: Fri, 29 Jan 2021 20:08:14 +0530 Subject: [PATCH 1/4] feat: Adds support to ingest data via file monitor --- Dockerfile.uf | 7 + docker-compose.yml | 22 +++- docs/api_reference/event_ingestion.rst | 6 + docs/sample_generator.rst | 6 +- pytest_splunk_addon/splunk.py | 104 ++++++++++++++- .../standard_lib/event_ingestors/__init__.py | 1 + .../event_ingestors/file_monitor_ingestor.py | 122 +++++++++++++++++ .../event_ingestors/hec_raw_ingestor.py | 2 +- .../event_ingestors/ingestor_helper.py | 4 + .../sample_generation/sample_stanza.py | 2 + .../TA_fiction_indextime/default/props.conf | 17 ++- .../default/pytest-splunk-addon-data.conf | 123 +++++++++++++++++- .../default/transforms.conf | 17 +++ .../samples/uf_file_monitor_host_event.sample | 2 + .../uf_file_monitor_host_plugin.sample | 2 + .../uf_file_monitor_indexed_extraction.sample | 4 + .../uf_file_monitor_rename_sourcetype.sample | 2 + tests/constants.py | 16 ++- tests/test_splunk_addon.py | 4 + 19 files changed, 452 insertions(+), 11 deletions(-) create mode 100644 Dockerfile.uf create mode 100644 pytest_splunk_addon/standard_lib/event_ingestors/file_monitor_ingestor.py create mode 100644 tests/addons/TA_fiction_indextime/samples/uf_file_monitor_host_event.sample create mode 100644 tests/addons/TA_fiction_indextime/samples/uf_file_monitor_host_plugin.sample create mode 100644 tests/addons/TA_fiction_indextime/samples/uf_file_monitor_indexed_extraction.sample create mode 100644 tests/addons/TA_fiction_indextime/samples/uf_file_monitor_rename_sourcetype.sample diff --git a/Dockerfile.uf b/Dockerfile.uf new file mode 100644 index 000000000..eedcd6823 --- /dev/null +++ b/Dockerfile.uf @@ -0,0 +1,7 @@ +ARG SPLUNK_VERSION=latest +FROM splunk/universalforwarder:$SPLUNK_VERSION +ARG SPLUNK_VERSION=latest +ARG SPLUNK_APP_ID=TA_UNKNOWN +ARG SPLUNK_APP_PACKAGE=package +RUN echo ${SPLUNK_VERSION} $SPLUNK_APP_PACKAGE +COPY ${SPLUNK_APP_PACKAGE} /opt/splunkforwarder/etc/apps/${SPLUNK_APP_ID} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index cc472689c..2cd1c49cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,27 @@ services: - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} - SPLUNK_START_ARGS=--accept-license - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - + + uf: + build: + context: . + dockerfile: Dockerfile.uf + args: + SPLUNK_APP_ID: ${SPLUNK_APP_ID} + SPLUNK_APP_PACKAGE: ${SPLUNK_APP_PACKAGE} + SPLUNK_VERSION: ${SPLUNK_VERSION} + hostname: uf + ports: + - "9997" + - "8089" + links: + - splunk + environment: + - SPLUNK_PASSWORD=Chang3d! + - SPLUNK_START_ARGS=--accept-license + volumes: + - ${CURRENT_DIR}/uf_files:${CURRENT_DIR}/uf_files + volumes: splunk-sc4s-var: external: false \ No newline at end of file diff --git a/docs/api_reference/event_ingestion.rst b/docs/api_reference/event_ingestion.rst index 43c341027..d7da62500 100644 --- a/docs/api_reference/event_ingestion.rst +++ b/docs/api_reference/event_ingestion.rst @@ -16,5 +16,11 @@ HEC Raw Ingestor SC4S Event Ingestor ~~~~~~~~~~~~~~~~~~~~ .. automodule:: standard_lib.event_ingestors.sc4s_event_ingestor + :members: + :show-inheritance: + +File Monitor Ingestor +~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: standard_lib.event_ingestors.file_monitor_ingestor :members: :show-inheritance: \ No newline at end of file diff --git a/docs/sample_generator.rst b/docs/sample_generator.rst index cd3914157..6000dd168 100644 --- a/docs/sample_generator.rst +++ b/docs/sample_generator.rst @@ -52,10 +52,14 @@ host_type = plugin | event * If the value is plugin, the plugin will generate host with format of "stanza_{count}" to uniquely identify the events. * If the value is event, the host field should be provided for a token using "token..field = host". -input_type = modinput | scripted_input | syslog_tcp | file_monitor | windows_input | default +input_type = modinput | scripted_input | syslog_tcp | file_monitor | windows_input | uf_file_monitor | default * The input_type used in addon to ingest data of a sourcetype used in stanza. * The way with which the sample data is ingested in Splunk depends on Splunk. The most similar ingesting approach is used for each input_type to get accurate index-time testing. + * In input_type=uf_file_monitor, universal forwarder will use file monitor to read event and then it will send data to indexer. * For example, in an Add-on, a sourcetype "alert" is ingested through syslog in live environment, provide input_type=syslog_tcp. + + .. warning:: + uf_file_monitor input_type will only work with splunk_type=docker. index = * The index used to ingest the data. diff --git a/pytest_splunk_addon/splunk.py b/pytest_splunk_addon/splunk.py index 1b4d4e52d..8489bb48d 100644 --- a/pytest_splunk_addon/splunk.py +++ b/pytest_splunk_addon/splunk.py @@ -8,6 +8,7 @@ import logging import os +import shutil from time import sleep import json import pytest @@ -244,6 +245,34 @@ def pytest_addoption(parser): dest="ignore_addon_errors", help=("Path to file where list of addon related errors are suppressed."), ) + group.addoption( + "--splunk-uf-host", + action="store", + dest="splunk_uf_host", + default="uf", + help="Address of Universal Forwarder Server.", + ) + group.addoption( + "--splunk-uf-port", + action="store", + dest="splunk_uf_port", + default="8089", + help="Universal Forwarder Management port. default is 8089.", + ) + group.addoption( + "--splunk-uf-user", + action="store", + dest="splunk_uf_user", + default="admin", + help="Universal Forwarder login user.", + ) + group.addoption( + "--splunk-uf-password", + action="store", + dest="splunk_uf_password", + default="Chang3d!", + help="Password of the Universal Forwarder user", + ) @pytest.fixture(scope="session") @@ -321,7 +350,7 @@ def ignore_internal_errors(request): @pytest.fixture(scope="session") -def splunk(request): +def splunk(request, file_system_prerequisite): """ This fixture based on the passed option will provide a real fixture for external or docker Splunk @@ -380,6 +409,61 @@ def sc4s(request): yield sc4s +@pytest.fixture(scope="session") +def uf(request): + """ + This fixture based on the passed option will provide a real fixture + for external or docker uf configuration + + Returns: + dict: Details of uf which includes host, port, username and password + """ + if request.config.getoption("splunk_type") == "external": + request.fixturenames.append("uf_external") + uf = request.getfixturevalue("uf_external") + elif request.config.getoption("splunk_type") == "docker": + request.fixturenames.append("uf_docker") + uf = request.getfixturevalue("uf_docker") + else: + raise Exception + uf["uf_username"] = request.config.getoption("splunk_uf_user") + uf["uf_password"] = request.config.getoption("splunk_uf_password") + for _ in range(RESPONSIVE_SPLUNK_TIMEOUT): + if is_responsive_splunk(uf): + break + sleep(1) + yield uf + +@pytest.fixture(scope="session") +def uf_docker(docker_services, tmp_path_factory, worker_id): + """ + Provides IP of the uf server and management port based on pytest-args(splunk_type) + """ + LOGGER.info("Starting docker_service=uf") + os.environ["CURRENT_DIR"] = os.getcwd() + if worker_id: + # get the temp directory shared by all workers + root_tmp_dir = tmp_path_factory.getbasetemp().parent + fn = root_tmp_dir / "pytest_docker" + with FileLock(str(fn) + ".lock"): + docker_services.start("uf") + uf_info = { + "uf_host": docker_services.docker_ip, + "uf_port": docker_services.port_for("uf", 8089), + } + return uf_info + +@pytest.fixture(scope="session") +def uf_external(request): + """ + Provides IP of the uf server and management port based on pytest-args(splunk_type) + """ + uf_info = { + "uf_host": request.config.getoption("splunk_uf_host"), + "uf_port": request.config.getoption("splunk_uf_port"), + } + return uf_info + @pytest.fixture(scope="session") def splunk_docker( @@ -544,7 +628,7 @@ def splunk_web_uri(request, splunk): @pytest.fixture(scope="session") -def splunk_ingest_data(request, splunk_hec_uri, sc4s, splunk_events_cleanup): +def splunk_ingest_data(request, splunk_hec_uri, sc4s, uf, splunk_events_cleanup): """ Generates events for the add-on and ingests into Splunk. The ingestion can be done using the following methods: @@ -570,6 +654,10 @@ def splunk_ingest_data(request, splunk_hec_uri, sc4s, splunk_events_cleanup): config_path = request.config.getoption("splunk_data_generator") ingest_meta_data = { + "uf_host": uf.get("uf_host"), + "uf_port": uf.get("uf_port"), + "uf_username": uf.get("uf_username"), + "uf_password": uf.get("uf_password"), "session_headers": splunk_hec_uri[0].headers, "splunk_hec_uri": splunk_hec_uri[1], "sc4s_host": sc4s[0], # for sc4s @@ -608,6 +696,18 @@ def splunk_events_cleanup(request, splunk_search_util): else: LOGGER.info("Events cleanup was disabled.") +@pytest.fixture(scope="session") +def file_system_prerequisite(): + """ + File system prerequisite before running tests. + Creating uf_files directory to write tokenized events for uf_file_monitor input. + """ + UF_FILE_MONTOR_DIR = "uf_files" + monitor_dir = os.path.join(os.getcwd(), UF_FILE_MONTOR_DIR) + if os.path.exists(monitor_dir): + shutil.rmtree(UF_FILE_MONTOR_DIR, ignore_errors=True) + os.mkdir(monitor_dir) + def is_responsive_splunk(splunk): """ Verify if the management port of Splunk is responsive or not diff --git a/pytest_splunk_addon/standard_lib/event_ingestors/__init__.py b/pytest_splunk_addon/standard_lib/event_ingestors/__init__.py index 2f1b32430..1a075a717 100644 --- a/pytest_splunk_addon/standard_lib/event_ingestors/__init__.py +++ b/pytest_splunk_addon/standard_lib/event_ingestors/__init__.py @@ -2,4 +2,5 @@ from .hec_metric_ingestor import HECMetricEventIngestor from .hec_raw_ingestor import HECRawEventIngestor from .sc4s_event_ingestor import SC4SEventIngestor +from .file_monitor_ingestor import FileMonitorEventIngestor from .ingestor_helper import IngestorHelper diff --git a/pytest_splunk_addon/standard_lib/event_ingestors/file_monitor_ingestor.py b/pytest_splunk_addon/standard_lib/event_ingestors/file_monitor_ingestor.py new file mode 100644 index 000000000..1837f6d11 --- /dev/null +++ b/pytest_splunk_addon/standard_lib/event_ingestors/file_monitor_ingestor.py @@ -0,0 +1,122 @@ +from .base_event_ingestor import EventIngestor +import requests +import logging +import os +from time import sleep +from requests.exceptions import ConnectionError + +LOGGER = logging.getLogger("pytest-splunk-addon") +MONITOR_DIR = "uf_files" + +class FileMonitorEventIngestor(EventIngestor): + """ + Class to ingest event via File monitor + This ingestor will only work if splunk_type is docker and container of universal forwarder is linked with container + of splunk instance as 'splunk' service. + + The format for required_configs is:: + + { + uf_host: Host of universal forwarder + uf_port: Management port of universal forwarder + uf_username: Name of user for universal forwarder + uf_password: Password of universal forwarder + } + + Args: + required_configs (dict): Dictionary containing information about universal forwarder + + """ + def __init__(self, required_configs): + self.uf_host = required_configs.get("uf_host") + self.uf_port = required_configs.get("uf_port") + self.uf_username = required_configs.get("uf_username") + self.uf_password = required_configs.get("uf_password") + # Container of universal forwarder is linked with splunk instance. + # So using splunk_host as splunk and port 9997 directly. + self.splunk_host = "splunk" + self.splunk_s2s_port = "9997" + self.uf_rest_uri = "https://{}:{}".format(self.uf_host, self.uf_port) + self.outputs_endpoint = "{}/services/data/outputs/tcp/group".format(self.uf_rest_uri) + self.inputs_endpoint = "{}/servicesNS/nobody/search/data/inputs/monitor".format(self.uf_rest_uri) + + def ingest(self, events, thread_count): + """ + Ingests data into splunk via file monitor. + Args: + events (list): List of events (SampleEvent) to be ingested + """ + self.create_output_conf() + for each_event in events: + self.create_event_file(each_event) + sleep(10) + self.create_inputs_stanza(each_event) + + def create_output_conf(self): + """ + Create stanza in outputs.conf file of universal forwarder to send on splunk(indexer). + """ + tcp_out_dict = {"name":"uf_monitor", "servers":"{}:{}".format(self.splunk_host, self.splunk_s2s_port)} + LOGGER.info("Making rest call to create stanza in outputs.conf file with following endpoint : {}".format(self.outputs_endpoint)) + LOGGER.debug("Creating following stanza in output.conf : {}".format(tcp_out_dict)) + try: + response = requests.post(self.outputs_endpoint, tcp_out_dict, auth=(self.uf_username, self.uf_password), verify=False) + if response.status_code not in (200, 201): + LOGGER.warning("Unable to create stanza in outputs.conf\nStatus code: {} \nReason: {} \ntext:{}".format(response.status_code, response.reason, response.text)) + except ConnectionError as e: + LOGGER.error("Unable to connect to Universal forwarder, {}".format(e)) + + def create_event_file(self, event): + """ + Write each tokenized event in files with host name as name of file. The host of all events will be unique. + + Args: + event (SampleEvent): Instance containing event info + """ + try: + with open(self.get_file_path(event), "w+") as fp: + LOGGER.info("Writing events file for host={}".format(event.metadata.get("host"))) + fp.write(event.event) + LOGGER.debug("Wrote tokenized events file on path : {}".format(self.get_file_path(event))) + except Exception as e: + LOGGER.warning("Unable to create event file for host : {}, Reason : {}".format(event.metadata.get("host"), e)) + + def create_inputs_stanza(self, event): + """ + Create stanza in inputs.conf on universal forwarder for each tokenized event. + + Args: + event (SampleEvent): Instance containing event info + """ + file_path = self.get_file_path(event) + sourcetype = event.metadata.get("sourcetype") + if not sourcetype: + sourcetype = event.metadata.get("sourcetype_to_search", "pytest_splunk_addon") + stanza = { + "name": file_path, + "sourcetype": sourcetype, + "index": event.metadata.get("index", "main"), + "disabled": False, + "crc-salt": "" + } + if event.metadata.get("host_type") in ("plugin"): + stanza["host"] = event.metadata.get("host") + if event.metadata.get("source"): + stanza["rename-source"] = event.metadata.get("source") + LOGGER.info("Making rest call to create stanza in inputs.conf file with following endpoint : {}".format(self.inputs_endpoint)) + LOGGER.debug("Creating following stanza in inputs.conf : {}".format(stanza)) + try: + response = requests.post(self.inputs_endpoint, stanza, auth=(self.uf_username, self.uf_password), verify=False) + if response.status_code not in (200, 201): + LOGGER.warning("Unable to add stanza in inputs.conf\nStatus code: {} \nReason: {} \ntext:{}".format(response.status_code, response.reason, response.text)) + except ConnectionError as e: + LOGGER.error("Unable to connect to Universal forwarder, {}".format(e)) + + def get_file_path(self, event): + """ + Returns absolute path for tokenized events. + + Args: + event (SampleEvent): Instance containing event info + """ + return "{}/{}/{}".format(os.getcwd(), MONITOR_DIR, event.metadata.get("host")) diff --git a/pytest_splunk_addon/standard_lib/event_ingestors/hec_raw_ingestor.py b/pytest_splunk_addon/standard_lib/event_ingestors/hec_raw_ingestor.py index 0dc49daee..118cfe445 100644 --- a/pytest_splunk_addon/standard_lib/event_ingestors/hec_raw_ingestor.py +++ b/pytest_splunk_addon/standard_lib/event_ingestors/hec_raw_ingestor.py @@ -86,7 +86,7 @@ def __ingest(self, event, params): try: LOGGER.info("Making a HEC raw endpoint request with the following params:\nhec_uri:{}\nheaders:{}".format( str(self.hec_uri), str(self.session_headers))) - LOGGER.debug("Creating the following sample event to be ingested via HEC RAW endpoint:\nEvents: {}\nParams:".format( + LOGGER.debug("Creating the following sample event to be ingested via HEC RAW endpoint:\nEvents: {}\nParams:{}".format( str(event),str(params))) response = requests.post( "{}/{}".format(self.hec_uri, "raw"), diff --git a/pytest_splunk_addon/standard_lib/event_ingestors/ingestor_helper.py b/pytest_splunk_addon/standard_lib/event_ingestors/ingestor_helper.py index 86e447193..c4f252f7d 100644 --- a/pytest_splunk_addon/standard_lib/event_ingestors/ingestor_helper.py +++ b/pytest_splunk_addon/standard_lib/event_ingestors/ingestor_helper.py @@ -3,6 +3,7 @@ HECRawEventIngestor, HECMetricEventIngestor, SC4SEventIngestor, + FileMonitorEventIngestor, ) import logging from ..sample_generation import SampleXdistGenerator @@ -21,6 +22,7 @@ def get_event_ingestor(cls, input_type, ingest_meta_data): "modinput": HECEventIngestor, "windows_input": HECEventIngestor, "file_monitor": HECRawEventIngestor, + "uf_file_monitor": FileMonitorEventIngestor, "scripted_input": HECRawEventIngestor, "hec_metric": HECMetricEventIngestor, "syslog_tcp": SC4SEventIngestor, @@ -51,6 +53,8 @@ def ingest_events(cls, ingest_meta_data, addon_path, config_path, thread_count, input_type = event.metadata.get("input_type") if input_type in ["modinput", "windows_input", "syslog_tcp", "syslog_udp"]: event.event = event.event.encode("utf-8").decode() + elif input_type in ["uf_file_monitor"]: + pass else: event.event = event.event.encode("utf-8") if input_type in ingestor_dict: diff --git a/pytest_splunk_addon/standard_lib/sample_generation/sample_stanza.py b/pytest_splunk_addon/standard_lib/sample_generation/sample_stanza.py index fc582668c..0fb6cb5a8 100644 --- a/pytest_splunk_addon/standard_lib/sample_generation/sample_stanza.py +++ b/pytest_splunk_addon/standard_lib/sample_generation/sample_stanza.py @@ -128,6 +128,7 @@ def _parse_meta(self, eventgen_params): "modinput", "windows_input", "file_monitor", + "uf_file_monitor", "scripted_input", "syslog_tcp", "syslog_udp", @@ -206,6 +207,7 @@ def _get_raw_sample(self): ) elif self.input_type in [ "file_monitor", + "uf_file_monitor", "scripted_input", "syslog_tcp", "syslog_udp", diff --git a/tests/addons/TA_fiction_indextime/default/props.conf b/tests/addons/TA_fiction_indextime/default/props.conf index 5b2a26f92..d0225d577 100644 --- a/tests/addons/TA_fiction_indextime/default/props.conf +++ b/tests/addons/TA_fiction_indextime/default/props.conf @@ -31,4 +31,19 @@ TRANSFORMS-update_host = update_host TRANSFORMS-add_host_prefix_modinput = add_host_prefix [test:indextime:file_monitor_host_prefix] -TRANSFORMS-add_host_prefix_file_monitor = add_host_prefix \ No newline at end of file +TRANSFORMS-add_host_prefix_file_monitor = add_host_prefix + +[test:indextime:uf_file_monitor_host_event] +TRANSFORMS-uf_file_monitor_host = uf_file_monitor_host + +[test:indextime:uf_file_monitor_rename_sourcetype] +TRANSFORMS-rename_sourcetype_using_source = rename_sourcetype_using_source + +[test:indextime:uf_file_monitor_indexed_extraction] +TIME_FORMAT = %s.%6N +TRANSFORMS-TrashComments = TrashComments +INDEXED_EXTRACTIONS = TSV +FIELD_HEADER_REGEX = ^#fields\t(.*) +FIELD_DELIMITER = \t +FIELD_QUOTE = \t +EVENT_BREAKER_ENABLE = true diff --git a/tests/addons/TA_fiction_indextime/default/pytest-splunk-addon-data.conf b/tests/addons/TA_fiction_indextime/default/pytest-splunk-addon-data.conf index 5098a3b41..ff7e934c0 100644 --- a/tests/addons/TA_fiction_indextime/default/pytest-splunk-addon-data.conf +++ b/tests/addons/TA_fiction_indextime/default/pytest-splunk-addon-data.conf @@ -1,5 +1,5 @@ -## Total test cases: 112 -## Index time Passed: 81 (Including untokenized event test case) +## Total test cases: 124 +## Index time Passed: 93 (Including untokenized event test case) ## Index time Failed: 00 ## Search Time Passed: 28 ## Search Time Failed: 00 @@ -1043,4 +1043,121 @@ index = test_pytest_splunk_addon_index_event token.0.token = ##static_value_1## token.0.replacementType = static -token.0.replacement = sample_value_1 \ No newline at end of file +token.0.replacement = sample_value_1 + +# Using uf file monitor to ingest data +# Total test cases: 1 +# 2 key_fields passed +# 1 line_breaker passed +[uf_file_monitor_host_event.sample] +sourcetype = test:indextime:uf_file_monitor_host_event +sourcetype_to_search = test:indextime:uf_file_monitor_host_event +host_type = event +input_type = uf_file_monitor +source = pytest-splunk-addon:uf_file_monitor +timestamp_type = plugin +sample_count = 2 + +token.0.token = (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+) +token.0.replacementType = timestamp +token.0.replacement = %Y-%m-%dT%H:%M:%S + +token.1.token = ##host_token## +token.1.replacementType = random +token.1.replacement = host['host'] +token.1.field = host + +token.2.token = ##token_list_all## +token.2.replacementType = all +token.2.replacement = list["a","b"] + +# Using uf file monitor to ingest data +# Host type is event, so file_monitor_ingestor will add host field in inputs stanza +# 2 _time passed +# 1 line_breaker passed +[uf_file_monitor_host_plugin.sample] +sourcetype = test:indextime:uf_file_monitor_host_plugin +sourcetype_to_search = test:indextime:uf_file_monitor_host_plugin +host_type = plugin +input_type = uf_file_monitor +source = pytest-splunk-addon:uf_file_monitor +timestamp_type = event +sample_count = 2 + +token.0.token = (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+) +token.0.replacementType = timestamp +token.0.replacement = %Y-%m-%dT%H:%M:%S +token.0.field = _time + +token.1.token = ##host_token## +token.1.replacementType = random +token.1.replacement = host['host'] + +token.2.token = ##token_list_all## +token.2.replacementType = all +token.2.replacement = list["a","b"] + +# Renaming sourcetype, adding source in sourcetype to test source value getting extracted properly +# 2 _time passed +# 1 line_breaker passed +[uf_file_monitor_rename_sourcetype.sample] +sourcetype = test:indextime:uf_file_monitor_rename_sourcetype +sourcetype_to_search = test:indextime:uf_file_monitor_rename_sourcetype:source +host_type = plugin +input_type = uf_file_monitor +source = source.log +timestamp_type = event +sample_count = 2 + +token.1.token = (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+) +token.1.replacementType = timestamp +token.1.replacement = %s +token.1.field = _time + +token.2.token = ##host_token## +token.2.replacementType = random +token.2.replacement = host['host'] + +token.3.token = ##token_list_all## +token.3.replacementType = all +token.3.replacement = list["a","b"] + +# Test INDEXED_EXTRATION field +# src, src_port, dest, dest_port fields should be extracted at indextime +# Total test cases: 3 +# 1 key_fields are Passed +# 1 _time Passed +# 1 line_breaker Passed +[uf_file_monitor_indexed_extraction.sample] +sourcetype = test:indextime:uf_file_monitor_indexed_extraction +sourcetype_to_search = test:indextime:uf_file_monitor_indexed_extraction +host_type = plugin +input_type = uf_file_monitor +source = pytest-splunk-addon:uf_file_monitor +timestamp_type = event +sample_count = 2 + +token.0.token = \d{10}.\d{6} +token.0.replacementType = timestamp +token.0.replacement = %s +token.0.field = _time + +token.1.token = (##SRC##) +token.1.replacementType = random +token.1.replacement = src['ipv4'] +token.1.field = src + +token.2.token = (##SRC_PORT##) +token.2.replacementType = random +token.2.replacement = src_port +token.2.field = src_port + +token.3.token = (##DEST##) +token.3.replacementType = random +token.3.replacement = dest['ipv4'] +token.3.field = dest + +token.4.token = (##DEST_PORT##) +token.4.replacementType = random +token.4.replacement = dest_port +token.4.field = dest_port \ No newline at end of file diff --git a/tests/addons/TA_fiction_indextime/default/transforms.conf b/tests/addons/TA_fiction_indextime/default/transforms.conf index 20474b4a3..01593fbd1 100644 --- a/tests/addons/TA_fiction_indextime/default/transforms.conf +++ b/tests/addons/TA_fiction_indextime/default/transforms.conf @@ -18,3 +18,20 @@ FORMAT = host::$1 REGEX = host=([^\s]+) FORMAT = host::$1 DEST_KEY = MetaData:Host + +[uf_file_monitor_host] +REGEX = host=([^\s]+) +FORMAT = host::$1 +DEST_KEY = MetaData:Host + +[rename_sourcetype_using_source] +DEST_KEY = MetaData:Sourcetype +SOURCE_KEY = MetaData:Source +REGEX = (?:[a-zA-Z0-9_]+\.)?([a-zA-Z0-9_]+)\.log +FORMAT = sourcetype::test:indextime:uf_file_monitor_rename_sourcetype:$1 +WRITE_META = true + +[TrashComments] +REGEX = ^\s*# +DEST_KEY = queue +FORMAT = nullQueue diff --git a/tests/addons/TA_fiction_indextime/samples/uf_file_monitor_host_event.sample b/tests/addons/TA_fiction_indextime/samples/uf_file_monitor_host_event.sample new file mode 100644 index 000000000..4d6fb13cd --- /dev/null +++ b/tests/addons/TA_fiction_indextime/samples/uf_file_monitor_host_event.sample @@ -0,0 +1,2 @@ +2020-06-23T00:00:00.000Z host=##host_token## ##token_list_all## Test for uf_file monitor host is event +2020-06-23T02:00:00.000Z host=##host_token## ##token_list_all## Test for uf_file monitor host is event \ No newline at end of file diff --git a/tests/addons/TA_fiction_indextime/samples/uf_file_monitor_host_plugin.sample b/tests/addons/TA_fiction_indextime/samples/uf_file_monitor_host_plugin.sample new file mode 100644 index 000000000..005c530f7 --- /dev/null +++ b/tests/addons/TA_fiction_indextime/samples/uf_file_monitor_host_plugin.sample @@ -0,0 +1,2 @@ +2020-06-23T00:00:00.000Z host=##host_token## ##token_list_all## Test for uf_file monitor host is plugin +2020-06-23T02:00:00.000Z host=##host_token## ##token_list_all## Test for uf_file monitor host is plugin \ No newline at end of file diff --git a/tests/addons/TA_fiction_indextime/samples/uf_file_monitor_indexed_extraction.sample b/tests/addons/TA_fiction_indextime/samples/uf_file_monitor_indexed_extraction.sample new file mode 100644 index 000000000..f9cc9b730 --- /dev/null +++ b/tests/addons/TA_fiction_indextime/samples/uf_file_monitor_indexed_extraction.sample @@ -0,0 +1,4 @@ +#fields ts src src_port dest dest_port +#types time addr port addr port +1331904609.110000 ##SRC## ##SRC_PORT## ##DEST## ##DEST_PORT## +1331904609.120000 ##SRC## ##SRC_PORT## ##DEST## ##DEST_PORT## \ No newline at end of file diff --git a/tests/addons/TA_fiction_indextime/samples/uf_file_monitor_rename_sourcetype.sample b/tests/addons/TA_fiction_indextime/samples/uf_file_monitor_rename_sourcetype.sample new file mode 100644 index 000000000..18db09b58 --- /dev/null +++ b/tests/addons/TA_fiction_indextime/samples/uf_file_monitor_rename_sourcetype.sample @@ -0,0 +1,2 @@ +2020-06-23T00:00:00.000Z host=##host_token## ##token_list_all## Test for uf_file monitor sourcetype rename +2020-06-23T02:00:00.000Z host=##host_token## ##token_list_all## Test for uf_file monitor sourcetype rename \ No newline at end of file diff --git a/tests/constants.py b/tests/constants.py index 336ba5e92..7fad46d72 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -607,6 +607,18 @@ '*test_splunk_fiction_indextime.py::Test_App::test_indextime_time*test:indextime:test_pytest_splunk_addon_index_raw::test-index-samples-64* PASSED*', '*test_splunk_fiction_indextime.py::Test_App::test_indextime_line_breaker*test:indextime:test_pytest_splunk_addon_index_raw::test_index.samples* PASSED*', '*test_splunk_fiction_indextime.py::Test_App::test_indextime_line_breaker*test:indextime:test_index_HECEventIngestor::test_index_HECEventIngestor.samples* PASSED*', + '*test_splunk_fiction_indextime.py::Test_App::test_indextime_key_fields*test:indextime:uf_file_monitor_host_event::host-sample_host387_to_host-sample_host388* PASSED*', + '*test_splunk_fiction_indextime.py::Test_App::test_indextime_key_fields*test:indextime:uf_file_monitor_host_event::host-sample_host389_to_host-sample_host390* PASSED*', + '*test_splunk_fiction_indextime.py::Test_App::test_indextime_time*test:indextime:uf_file_monitor_host_plugin::uf-file-monitor-host-plugin-sample-69* PASSED*', + '*test_splunk_fiction_indextime.py::Test_App::test_indextime_time*test:indextime:uf_file_monitor_host_plugin::uf-file-monitor-host-plugin-sample-70* PASSED*', + '*test_splunk_fiction_indextime.py::Test_App::test_indextime_time*test:indextime:uf_file_monitor_rename_sourcetype:source::uf-file-monitor-rename-sourcetype-sample-71* PASSED*', + '*test_splunk_fiction_indextime.py::Test_App::test_indextime_time*test:indextime:uf_file_monitor_rename_sourcetype:source::uf-file-monitor-rename-sourcetype-sample-72* PASSED*', + '*test_splunk_fiction_indextime.py::Test_App::test_indextime_line_breaker*test:indextime:uf_file_monitor_host_event::uf_file_monitor_host_event.sample* PASSED*', + '*test_splunk_fiction_indextime.py::Test_App::test_indextime_line_breaker*test:indextime:uf_file_monitor_host_plugin::uf_file_monitor_host_plugin.sample* PASSED*', + '*test_splunk_fiction_indextime.py::Test_App::test_indextime_line_breaker*test:indextime:uf_file_monitor_rename_sourcetype:source::uf_file_monitor_rename_sourcetype.sample* PASSED*', + '*test_splunk_fiction_indextime.py::Test_App::test_indextime_key_fields*test:indextime:uf_file_monitor_indexed_extraction::uf_file_monitor_indexed_extraction.sample* PASSED*', + '*test_splunk_fiction_indextime.py::Test_App::test_indextime_time*test:indextime:uf_file_monitor_indexed_extraction::uf_file_monitor_indexed_extraction.sample* PASSED*', + '*test_splunk_fiction_indextime.py::Test_App::test_indextime_line_breaker*test:indextime:uf_file_monitor_indexed_extraction::uf_file_monitor_indexed_extraction.sample* PASSED*', ] """ @@ -667,11 +679,11 @@ '*test_splunk_fiction_indextime_broken.py::Test_App::test_events_with_untokenised_values FAILED*', '*test_splunk_fiction_indextime_broken.py::Test_App::test_indextime_key_fields*test:indextime:failing::failing-samples-1* FAILED*', '*test_splunk_fiction_indextime_broken.py::Test_App::test_indextime_key_fields*test:indextime:failing::failing-samples-2* FAILED*', - '*test_splunk_fiction_indextime_broken.py::Test_App::test_indextime_key_fields*test:indextime:file_monitor_host_prefix::test1-host-wrong_prefix_filemonitor.sample-458_to_test1-host-wrong_prefix_filemonitor.sample-460* FAILED*', + '*test_splunk_fiction_indextime_broken.py::Test_App::test_indextime_key_fields*test:indextime:file_monitor_host_prefix::test1-host-wrong_prefix_filemonitor.sample-470_to_test1-host-wrong_prefix_filemonitor.sample-472* FAILED*', '*test_splunk_fiction_indextime_broken.py::Test_App::test_indextime_key_fields*test:indextime:scripted_input_key_fields::scripted_input_key_fields.samples* FAILED*', '*test_splunk_fiction_indextime_broken.py::Test_App::test_indextime_time*test:indextime:failing::failing-samples-1* FAILED*', '*test_splunk_fiction_indextime_broken.py::Test_App::test_indextime_time*test:indextime:failing::failing-samples-2* FAILED*', - '*test_splunk_fiction_indextime_broken.py::Test_App::test_indextime_time*test:indextime:file_monitor_host_prefix::test1-host-wrong_prefix_filemonitor.sample-458_to_test1-host-wrong_prefix_filemonitor.sample-460* FAILED*', + '*test_splunk_fiction_indextime_broken.py::Test_App::test_indextime_time*test:indextime:file_monitor_host_prefix::test1-host-wrong_prefix_filemonitor.sample-470_to_test1-host-wrong_prefix_filemonitor.sample-472* FAILED*', '*test_splunk_fiction_indextime_broken.py::Test_App::test_indextime_time*test:indextime:no_host_provided_when_type_event::no_host_provided_when_type_event.samples* FAILED*', '*test_splunk_fiction_indextime_broken.py::Test_App::test_indextime_time*test:indextime:no_prefix_stanza_modinput::no_prefix_stanza_modinput.sample_1* FAILED*', '*test_splunk_fiction_indextime_broken.py::Test_App::test_indextime_time*test:indextime:no_prefix_stanza_modinput::no_prefix_stanza_modinput.sample_2* FAILED*', diff --git a/tests/test_splunk_addon.py b/tests/test_splunk_addon.py index 81d260f06..e9808c61f 100644 --- a/tests/test_splunk_addon.py +++ b/tests/test_splunk_addon.py @@ -40,6 +40,10 @@ def setup_test_dir(testdir): os.path.join(testdir.request.config.invocation_dir, "Dockerfile.splunk"), testdir.tmpdir, ) + shutil.copy( + os.path.join(testdir.request.config.invocation_dir, "Dockerfile.uf"), + testdir.tmpdir, + ) shutil.copy( os.path.join(testdir.request.config.invocation_dir, "Dockerfile.tests"), testdir.tmpdir, From 1a4e8acaf4012b41cfa0c116c401aff843497c73 Mon Sep 17 00:00:00 2001 From: Raj Karkar Date: Mon, 1 Feb 2021 11:23:03 +0530 Subject: [PATCH 2/4] test: Updated docs and ingestor_helper --- docs/sample_generator.rst | 2 +- .../standard_lib/event_ingestors/ingestor_helper.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/sample_generator.rst b/docs/sample_generator.rst index 6000dd168..b7b156855 100644 --- a/docs/sample_generator.rst +++ b/docs/sample_generator.rst @@ -59,7 +59,7 @@ input_type = modinput | scripted_input | syslog_tcp | file_monitor | windows_inp * For example, in an Add-on, a sourcetype "alert" is ingested through syslog in live environment, provide input_type=syslog_tcp. .. warning:: - uf_file_monitor input_type will only work with splunk_type=docker. + uf_file_monitor input_type will only work with splunk-type=docker. index = * The index used to ingest the data. diff --git a/pytest_splunk_addon/standard_lib/event_ingestors/ingestor_helper.py b/pytest_splunk_addon/standard_lib/event_ingestors/ingestor_helper.py index c4f252f7d..e6106b430 100644 --- a/pytest_splunk_addon/standard_lib/event_ingestors/ingestor_helper.py +++ b/pytest_splunk_addon/standard_lib/event_ingestors/ingestor_helper.py @@ -51,10 +51,8 @@ def ingest_events(cls, ingest_meta_data, addon_path, config_path, thread_count, ingestor_dict = dict() for event in tokenized_events: input_type = event.metadata.get("input_type") - if input_type in ["modinput", "windows_input", "syslog_tcp", "syslog_udp"]: + if input_type in ["modinput", "windows_input", "syslog_tcp", "syslog_udp", "uf_file_monitor"]: event.event = event.event.encode("utf-8").decode() - elif input_type in ["uf_file_monitor"]: - pass else: event.event = event.event.encode("utf-8") if input_type in ingestor_dict: From 5148bcb8a685b10b1e2eb1d7627af54e055d8775 Mon Sep 17 00:00:00 2001 From: Raj Karkar Date: Mon, 1 Feb 2021 12:11:22 +0530 Subject: [PATCH 3/4] test: Updated doc and wait for uf method --- docs/release_history.rst | 15 +++++++++++++-- pytest_splunk_addon/splunk.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/docs/release_history.rst b/docs/release_history.rst index c556a5abb..0d1dc73a4 100644 --- a/docs/release_history.rst +++ b/docs/release_history.rst @@ -8,7 +8,18 @@ Release History The best way to track the development of pytest-splunk-addon is through `the GitHub Repo `_. -1.4.0 +1.5.0 +""""""""""""""""""""""""" + **Changes:** + + * Added support for ingestion of data via pytest-splunk-addon with a user-defined index ``index = ``. + + **Known Issues:** + + * Event ingestion through SC4S via UDP port + * Fields for modular regular expressions are not extracted in the plugin. + +1.4.0 (2021-01-05) """""""""""""""""""""""""" **Changes:** @@ -20,7 +31,7 @@ The best way to track the development of pytest-splunk-addon is through `the Git * Fields for modular regular expressions are not extracted in the plugin. -1.3.15 +1.3.15 (2020-12-16) """""""""""""""""""""""""" **Changes:** diff --git a/pytest_splunk_addon/splunk.py b/pytest_splunk_addon/splunk.py index 8489bb48d..7863a33d3 100644 --- a/pytest_splunk_addon/splunk.py +++ b/pytest_splunk_addon/splunk.py @@ -429,7 +429,7 @@ def uf(request): uf["uf_username"] = request.config.getoption("splunk_uf_user") uf["uf_password"] = request.config.getoption("splunk_uf_password") for _ in range(RESPONSIVE_SPLUNK_TIMEOUT): - if is_responsive_splunk(uf): + if is_responsive_uf(uf): break sleep(1) yield uf @@ -708,6 +708,35 @@ def file_system_prerequisite(): shutil.rmtree(UF_FILE_MONTOR_DIR, ignore_errors=True) os.mkdir(monitor_dir) +def is_responsive_uf(uf): + """ + Verify if the management port of Universal Forwarder is responsive or not + + Args: + uf (dict): details of the Universal Forwarder instance + + Returns: + bool: True if Universal Forwarder is responsive. False otherwise + """ + try: + LOGGER.info( + "Trying to connect Universal Forwarder instance... splunk=%s", json.dumps(uf), + ) + client.connect( + username=uf["uf_username"], + password=uf["uf_password"], + host=uf["uf_host"], + port=uf["uf_port"], + ) + LOGGER.info("Connected to Universal Forwarder instance.") + + return True + except Exception as e: + LOGGER.warning( + "Could not connect to Universal Forwarder Instance. Will try again. exception=%s", str(e), + ) + return False + def is_responsive_splunk(splunk): """ Verify if the management port of Splunk is responsive or not From ea4046a7a6205c709938aa854b20b692913b8951 Mon Sep 17 00:00:00 2001 From: Raj Karkar Date: Mon, 1 Feb 2021 12:31:30 +0530 Subject: [PATCH 4/4] test: Updated doc for release history --- docs/release_history.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release_history.rst b/docs/release_history.rst index 0d1dc73a4..4a0d64047 100644 --- a/docs/release_history.rst +++ b/docs/release_history.rst @@ -12,7 +12,7 @@ The best way to track the development of pytest-splunk-addon is through `the Git """"""""""""""""""""""""" **Changes:** - * Added support for ingestion of data via pytest-splunk-addon with a user-defined index ``index = ``. + * Added support for file monitoring which uses universal forwarder to monitor files and ingest data. **Known Issues:**