Merge pull request #253 from splunk/feature/Add-support-for-uf-file-monitor

feat: Adds support to ingest data via file monitor

Author: rajkarkar-crest

Dockerfile.uf

@@ -0,0 +1,7 @@
+ARG SPLUNK_VERSION=latest
+FROM splunk/universalforwarder:$SPLUNK_VERSION
+ARG SPLUNK_VERSION=latest
+ARG SPLUNK_APP_ID=TA_UNKNOWN
+ARG SPLUNK_APP_PACKAGE=package
+RUN echo ${SPLUNK_VERSION} $SPLUNK_APP_PACKAGE
+COPY ${SPLUNK_APP_PACKAGE} /opt/splunkforwarder/etc/apps/${SPLUNK_APP_ID}

docker-compose.yml

@@ -63,7 +63,27 @@ services:
       - SPLUNK_PASSWORD=${SPLUNK_PASSWORD}
       - SPLUNK_START_ARGS=--accept-license
       - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
-  
+
+  uf:
+    build:
+      context: .
+      dockerfile: Dockerfile.uf
+      args:
+        SPLUNK_APP_ID: ${SPLUNK_APP_ID}
+        SPLUNK_APP_PACKAGE: ${SPLUNK_APP_PACKAGE}
+        SPLUNK_VERSION: ${SPLUNK_VERSION}
+    hostname: uf
+    ports:
+      - "9997"
+      - "8089"
+    links:
+      - splunk
+    environment:
+      - SPLUNK_PASSWORD=Chang3d!
+      - SPLUNK_START_ARGS=--accept-license
+    volumes:
+      - ${CURRENT_DIR}/uf_files:${CURRENT_DIR}/uf_files
+
 volumes:
   splunk-sc4s-var:
     external: false

docs/api_reference/event_ingestion.rst

@@ -16,5 +16,11 @@ HEC Raw Ingestor
 SC4S Event Ingestor
 ~~~~~~~~~~~~~~~~~~~~
 .. automodule:: standard_lib.event_ingestors.sc4s_event_ingestor
+   :members:
+   :show-inheritance:
+
+File Monitor Ingestor
+~~~~~~~~~~~~~~~~~~~~~
+.. automodule:: standard_lib.event_ingestors.file_monitor_ingestor
    :members:
    :show-inheritance:

docs/release_history.rst

@@ -8,7 +8,18 @@ Release History
 
 The best way to track the development of pytest-splunk-addon is through `the GitHub Repo <https://github.com/splunk/pytest-splunk-addon/>`_.
 
-1.4.0
+1.5.0
+"""""""""""""""""""""""""
+    **Changes:**
+ 
+    * Added support for file monitoring  which uses universal forwarder to monitor files and ingest data.
+
+    **Known Issues:**
+
+    * Event ingestion through SC4S via UDP port
+    * Fields for modular regular expressions are not extracted in the plugin.
+
+1.4.0 (2021-01-05)
 """"""""""""""""""""""""""
     **Changes:**
 
@@ -20,7 +31,7 @@ The best way to track the development of pytest-splunk-addon is through `the Git
     * Fields for modular regular expressions are not extracted in the plugin.
 
 
-1.3.15
+1.3.15 (2020-12-16)
 """"""""""""""""""""""""""
     **Changes:**
 

docs/sample_generator.rst

@@ -52,10 +52,14 @@ host_type = plugin | event
     * If the value is plugin, the plugin will generate host with format of "stanza_{count}" to uniquely identify the events.
     * If the value is event, the host field should be provided for a token using "token.<n>.field = host". 
 
-input_type = modinput | scripted_input | syslog_tcp | file_monitor | windows_input | default
+input_type = modinput | scripted_input | syslog_tcp | file_monitor | windows_input | uf_file_monitor | default
     * The input_type used in addon to ingest data of a sourcetype used in stanza.
     * The way with which the sample data is ingested in Splunk depends on Splunk. The most similar ingesting approach is used for each input_type to get accurate index-time testing.
+    * In input_type=uf_file_monitor, universal forwarder will use file monitor to read event and then it will send data to indexer.
     * For example, in an Add-on, a sourcetype "alert" is ingested through syslog in live environment, provide input_type=syslog_tcp.
+    
+    .. warning::
+        uf_file_monitor input_type will only work with splunk-type=docker.
 
 index = <index>
     * The index used to ingest the data.

pytest_splunk_addon/splunk.py

@@ -8,6 +8,7 @@
 
 import logging
 import os
+import shutil
 from time import sleep
 import json
 import pytest
@@ -244,6 +245,34 @@ def pytest_addoption(parser):
         dest="ignore_addon_errors",
         help=("Path to file where list of addon related errors are suppressed."),
     )
+    group.addoption(
+        "--splunk-uf-host",
+        action="store",
+        dest="splunk_uf_host",
+        default="uf",
+        help="Address of Universal Forwarder Server.",
+    )
+    group.addoption(
+        "--splunk-uf-port",
+        action="store",
+        dest="splunk_uf_port",
+        default="8089",
+        help="Universal Forwarder Management port. default is 8089.",
+    )
+    group.addoption(
+        "--splunk-uf-user",
+        action="store",
+        dest="splunk_uf_user",
+        default="admin",
+        help="Universal Forwarder login user.",
+    )
+    group.addoption(
+        "--splunk-uf-password",
+        action="store",
+        dest="splunk_uf_password",
+        default="Chang3d!",
+        help="Password of the Universal Forwarder user",
+    )
 
 
 @pytest.fixture(scope="session")
@@ -321,7 +350,7 @@ def ignore_internal_errors(request):
 
 
 @pytest.fixture(scope="session")
-def splunk(request):
+def splunk(request, file_system_prerequisite):
     """
     This fixture based on the passed option will provide a real fixture
     for external or docker Splunk
@@ -380,6 +409,61 @@ def sc4s(request):
 
     yield sc4s
 
+@pytest.fixture(scope="session")
+def uf(request):
+    """
+    This fixture based on the passed option will provide a real fixture
+    for external or docker uf configuration
+
+    Returns:
+        dict: Details of uf which includes host, port, username and password
+    """
+    if request.config.getoption("splunk_type") == "external":
+        request.fixturenames.append("uf_external")
+        uf = request.getfixturevalue("uf_external")
+    elif request.config.getoption("splunk_type") == "docker":
+        request.fixturenames.append("uf_docker")
+        uf = request.getfixturevalue("uf_docker")
+    else:
+        raise Exception
+    uf["uf_username"] = request.config.getoption("splunk_uf_user")
+    uf["uf_password"] = request.config.getoption("splunk_uf_password")
+    for _ in range(RESPONSIVE_SPLUNK_TIMEOUT):
+        if is_responsive_uf(uf):
+            break
+        sleep(1)
+    yield uf
+
+@pytest.fixture(scope="session")
+def uf_docker(docker_services, tmp_path_factory, worker_id):
+    """
+    Provides IP of the uf server and management port based on pytest-args(splunk_type)
+    """
+    LOGGER.info("Starting docker_service=uf")
+    os.environ["CURRENT_DIR"] = os.getcwd()
+    if worker_id:
+        # get the temp directory shared by all workers
+        root_tmp_dir = tmp_path_factory.getbasetemp().parent
+        fn = root_tmp_dir / "pytest_docker"
+        with FileLock(str(fn) + ".lock"):
+            docker_services.start("uf")
+    uf_info = {
+        "uf_host": docker_services.docker_ip,
+        "uf_port": docker_services.port_for("uf", 8089),
+    }
+    return uf_info
+
+@pytest.fixture(scope="session")
+def uf_external(request):
+    """
+    Provides IP of the uf server and management port based on pytest-args(splunk_type)
+    """
+    uf_info = {
+        "uf_host": request.config.getoption("splunk_uf_host"),
+        "uf_port": request.config.getoption("splunk_uf_port"),
+    }
+    return uf_info
+
 
 @pytest.fixture(scope="session")
 def splunk_docker(
@@ -544,7 +628,7 @@ def splunk_web_uri(request, splunk):
 
 
 @pytest.fixture(scope="session")
-def splunk_ingest_data(request, splunk_hec_uri, sc4s, splunk_events_cleanup):
+def splunk_ingest_data(request, splunk_hec_uri, sc4s, uf, splunk_events_cleanup):
     """
     Generates events for the add-on and ingests into Splunk.
     The ingestion can be done using the following methods:
@@ -570,6 +654,10 @@ def splunk_ingest_data(request, splunk_hec_uri, sc4s, splunk_events_cleanup):
         config_path = request.config.getoption("splunk_data_generator")
 
         ingest_meta_data = {
+            "uf_host": uf.get("uf_host"),
+            "uf_port": uf.get("uf_port"),
+            "uf_username": uf.get("uf_username"),
+            "uf_password": uf.get("uf_password"),
             "session_headers": splunk_hec_uri[0].headers,
             "splunk_hec_uri": splunk_hec_uri[1],
             "sc4s_host": sc4s[0],  # for sc4s
@@ -608,6 +696,47 @@ def splunk_events_cleanup(request, splunk_search_util):
     else:
         LOGGER.info("Events cleanup was disabled.")
 
+@pytest.fixture(scope="session")
+def file_system_prerequisite():
+    """
+    File system prerequisite before running tests.
+    Creating uf_files directory to write tokenized events for uf_file_monitor input.
+    """
+    UF_FILE_MONTOR_DIR = "uf_files"
+    monitor_dir = os.path.join(os.getcwd(), UF_FILE_MONTOR_DIR)
+    if os.path.exists(monitor_dir):
+        shutil.rmtree(UF_FILE_MONTOR_DIR, ignore_errors=True)
+    os.mkdir(monitor_dir)
+
+def is_responsive_uf(uf):
+    """
+    Verify if the management port of Universal Forwarder is responsive or not
+
+    Args:
+        uf (dict): details of the Universal Forwarder instance
+
+    Returns:
+        bool: True if Universal Forwarder is responsive. False otherwise
+    """
+    try:
+        LOGGER.info(
+            "Trying to connect Universal Forwarder instance...  splunk=%s", json.dumps(uf),
+        )
+        client.connect(
+            username=uf["uf_username"],
+            password=uf["uf_password"],
+            host=uf["uf_host"],
+            port=uf["uf_port"],
+        )
+        LOGGER.info("Connected to Universal Forwarder instance.")
+
+        return True
+    except Exception as e:
+        LOGGER.warning(
+            "Could not connect to Universal Forwarder Instance. Will try again. exception=%s", str(e),
+        )
+        return False
+
 def is_responsive_splunk(splunk):
     """
     Verify if the management port of Splunk is responsive or not

pytest_splunk_addon/standard_lib/event_ingestors/__init__.py

@@ -2,4 +2,5 @@
 from .hec_metric_ingestor import HECMetricEventIngestor
 from .hec_raw_ingestor import HECRawEventIngestor
 from .sc4s_event_ingestor import SC4SEventIngestor
+from .file_monitor_ingestor import FileMonitorEventIngestor
 from .ingestor_helper import IngestorHelper