Ci/adding e2e tests

.github/workflows/build-test-release.yml

@@ -170,6 +170,7 @@ jobs:
           "splunk_setup_fixture",
           "splunk_app_req",
           "splunk_app_req_broken",
+          "splunk_cim_model_ipv6_regex",
         ]
     steps:
       - uses: actions/checkout@v4

Dockerfile.tests

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM ubuntu:latest
+FROM ubuntu:22.04
 
 RUN mkdir -p /work/tests
 RUN mkdir -p /work/test-results/functional

tests/e2e/addons/TA_cim_addon/app.manifest

@@ -0,0 +1,53 @@
+{
+  "schemaVersion": "2.0.0",
+  "info": {
+    "title": "TA_transition_from_req",
+    "id": {
+      "group": null,
+      "name": "TA_transition_from_req",
+      "version": "0.0.0-dev"
+    },
+    "author": [
+      {
+        "name": "Splunk Inc.",
+        "email": "[email protected]",
+        "company": "Splunk, Inc."
+      }
+    ],
+    "releaseDate": null,
+    "description": "TA IN DEV UNKNOWN",
+    "classification": {
+      "intendedAudience": null,
+      "categories": [],
+      "developmentStatus": null
+    },
+    "commonInformationModels": null,
+    "license": {
+      "name": "Splunk EULA",
+      "text": null,
+      "uri": "LICENSE"
+    },
+    "privacyPolicy": {
+      "name": null,
+      "text": null,
+      "uri": null
+    },
+    "releaseNotes": {
+      "name": null,
+      "text": null,
+      "uri": null
+    }
+  },
+  "dependencies": {
+  },
+  "tasks": [],
+  "inputGroups": {
+  },
+  "incompatibleApps": {
+  },
+  "platformRequirements": {
+    "splunk": {
+      "Enterprise": "*"
+    }
+  }
+}

tests/e2e/addons/TA_cim_addon/default/app.conf

@@ -0,0 +1,20 @@
+# this add-on is powered by splunk Add-on builder
+[install]
+state_change_requires_restart = true
+build = 2
+
+[launcher]
+author =Splunk Inc.x
+version =0.0.0-dev
+
+[ui]
+is_visible = 0
+label = TA_transition_from_req
+docs_section_override = AddOns:released
+
+[package]
+id =TA_transition_from_req
+
+[id]
+name =TA_transition_from_req
+version =0.0.0-dev

tests/e2e/addons/TA_cim_addon/default/eventtypes.conf

@@ -0,0 +1,11 @@
+##
+## SPDX-FileCopyrightText: 2021 Splunk, Inc. <[email protected]>
+## SPDX-License-Identifier: LicenseRef-Splunk-8-2021
+##
+##
+
+; [test_auth]
+; search = sourcetype=test:data:1 AND action IN ("success","failure","error")
+
+[test_ipv6]
+search = sourcetype=test:ipv6

tests/e2e/addons/TA_cim_addon/default/props.conf

@@ -0,0 +1,17 @@
+##
+## SPDX-FileCopyrightText: 2021 Splunk, Inc. <[email protected]>
+## SPDX-License-Identifier: LicenseRef-Splunk-8-2021
+##
+##
+
+; [test:data:1]
+; FIELDALIAS-dest = host AS dest
+; FIELDALIAS-action = result AS action
+; EVAL-app = "psa"
+; FIELDALIAS-user = tester AS user
+; FIELDALIAS-src = ip AS src
+; EVAL-status = case(action=="success", "PASS", action=="failure", "FAIL", 0==0, "OTHER")
+
+[test:ipv6]
+FIELDALIAS-client_ipAddress_as_src_ip = client.ipAddress AS src_ip
+FIELDALIAS-client_destadd_as_dest_ip = client.destadd AS dest_ip

tests/e2e/addons/TA_cim_addon/default/pytest-splunk-addon-data.conf

@@ -0,0 +1,44 @@
+; [sample_modinput.xml]
+; requirement_test_sample = 1
+; interval = 120
+; earliest = -2m
+; latest = now
+; source = test_data.1
+; sourcetype = test:data:1
+; count = 100
+; input_type = modinput
+; host_type = plugin
+; sourcetype_to_search = test:data:1
+; host = so1
+; timestamp_type = event
+;
+; token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+; token.0.replacementType = timestamp
+; token.0.replacement = %Y-%m-%d %H:%M:%S
+; token.0.field = _time
+;
+; token.1.token = ##dest_ipv4##
+; token.1.replacementType = random
+; token.1.replacement = dest["ipv4"]
+;
+; token.2.token = ##result##
+; token.2.replacementType = all
+; token.2.replacement = file[$SPLUNK_HOME/etc/apps/TA_transition_from_req/samples/result_mapping:1]
+;
+; token.3.token = ##result_mapping##
+; token.3.replacementType = all
+; token.3.replacement = file[$SPLUNK_HOME/etc/apps/TA_transition_from_req/samples/result_mapping:2]
+
+[sample_requirement.xml]
+requirement_test_sample = 1
+interval = 30
+earliest = -60m
+latest = now
+sourcetype = test:ipv6
+source = test_okta
+maxIntervalsBeforeFlush = 1
+input_type = modinput
+host_type = plugin
+sourcetype_to_search = test:ipv6
+timestamp_type = plugin
+expected_event_count = 3

tests/e2e/addons/TA_cim_addon/default/tags.conf

@@ -0,0 +1,12 @@
+##
+## SPDX-FileCopyrightText: 2021 Splunk, Inc. <[email protected]>
+## SPDX-License-Identifier: LicenseRef-Splunk-8-2021
+##
+##
+
+; [eventtype=test_auth]
+; authentication = enabled
+
+[eventtype=test_ipv6]
+network = enabled
+communicate = enabled

tests/e2e/addons/TA_cim_addon/metadata/default.meta

@@ -0,0 +1,10 @@
+
+# This is a TA, so export almost everything
+
+[]
+access = read : [ * ], write : [ admin, power ]
+export = system
+
+# Do not export commands 
+[commands]
+export = none

tests/e2e/addons/TA_cim_addon/samples/result_mapping

@@ -0,0 +1,3 @@
+success,PASS
+failure,FAIL
+error,OTHER

tests/e2e/addons/TA_cim_addon/samples/sample_requirement.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+<device>
+  <vendor>OKTA</vendor>
+  <product>OKTA Identity Cloud</product>
+  <version id="2022.03.01"/>
+  <version id="2023.10.0"/>
+  <event code="" name="security.request.blocked" format="json">
+    <version>2023.10.0</version>
+    <transport type="modinput" sourcetype="test:ipv6" source="test_okta" host="test_ipv6"/>
+    <source>
+      <jira id=""/>
+      <comment>generated, Generated from lab environment</comment>
+    </source>
+    <raw><![CDATA[{"actor": {"id": "spr6w8a2l2V88E3nk5d7", "type": "SystemPrincipal", "alternateId": "[email protected]", "displayName": "Okta System", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "Splunk Add-on for Okta Identity Cloud", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": null, "ipAddress": "68c6:7554::89",  "destadd": "0:0:0:0:0:0:0:1", "geographicalContext": null}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": null}, "displayMessage": "Blocked request from IP: 68c6:7554::89", "eventType": "security.request.blocked", "outcome": {"result": "SUCCESS", "reason": "NETWORK_ZONE_BLACKLIST"}, "published": "2023-10-11T10:28:27.114Z", "securityContext": {"asNumber": null, "asOrg": null, "isp": null, "domain": null, "isProxy": null}, "severity": "WARN", "debugContext": {"debugData": {"requestId": "ZSZ4y03BLKYbIGjkjkgRvgAADhk", "requestUri": "/api/v1/logs", "url": "/api/v1/logs?limit=1000&since=2023-10-11T09%3A01%3A17.417Z&until=2023-10-11T10%3A27%3A56.967Z"}}, "legacyEventType": "security.zone.request.blocked", "transaction": {"type": "WEB", "id": "ZSZ4y03BLKYbIGjkjkgRvgAADhk", "detail": {}}, "uuid": "ea1dfcef-6820-11ee-bb11-f3c3bf8f14d1", "version": "0", "request": {"ipChain": [{"ip": "68c6:7554::89", "geographicalContext": null, "version": "V4", "source": null}]}, "target": null}]]></raw>
+    <cim>
+      <models>
+        <model>Network_Traffic</model>
+      </models>
+      <cim_fields>
+        <field name="src_ip" value="68c6:7554::89"/>
+        <field name="dest_ip" value="0:0:0:0:0:0:0:1"/>
+      </cim_fields>
+      <missing_recommended_fields>
+        <field>bytes</field>
+        <field>bytes_in</field>
+        <field>bytes_out</field>
+        <field>dest_port</field>
+        <field>dest_zone</field>
+        <field>src_port</field>
+        <field>src_translated_ip</field>
+        <field>src_zone</field>
+      </missing_recommended_fields>
+    </cim>
+  </event>
+    <event code="" name="security.request.blocked" format="json">
+    <version>2023.10.0</version>
+    <transport type="modinput" sourcetype="test:ipv6" source="test_okta" host="test_ipv6"/>
+    <source>
+      <jira id=""/>
+      <comment>generated, Generated from lab environment</comment>
+    </source>
+    <raw><![CDATA[{"actor": {"id": "spr6w8a2l2V88E3nk5d7", "type": "SystemPrincipal", "alternateId": "[email protected]", "displayName": "Okta System", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "Splunk Add-on for Okta Identity Cloud", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": null, "ipAddress": "ab01::12aa",  "destadd": "::ffff:192.168.10.10", "geographicalContext": null}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": null}, "displayMessage": "Blocked request from IP: 68c6:7554::89", "eventType": "security.request.blocked", "outcome": {"result": "SUCCESS", "reason": "NETWORK_ZONE_BLACKLIST"}, "published": "2023-10-11T10:28:27.114Z", "securityContext": {"asNumber": null, "asOrg": null, "isp": null, "domain": null, "isProxy": null}, "severity": "WARN", "debugContext": {"debugData": {"requestId": "ZSZ4y03BLKYbIGjkjkgRvgAADhk", "requestUri": "/api/v1/logs", "url": "/api/v1/logs?limit=1000&since=2023-10-11T09%3A01%3A17.417Z&until=2023-10-11T10%3A27%3A56.967Z"}}, "legacyEventType": "security.zone.request.blocked", "transaction": {"type": "WEB", "id": "ZSZ4y03BLKYbIGjkjkgRvgAADhk", "detail": {}}, "uuid": "ea1dfcef-6820-11ee-bb11-f3c3bf8f14d1", "version": "0", "request": {"ipChain": [{"ip": "68c6:7554::89", "geographicalContext": null, "version": "V4", "source": null}]}, "target": null}]]></raw>
+    <cim>
+      <models>
+        <model>Network_Traffic</model>
+      </models>
+      <cim_fields>
+        <field name="src_ip" value="ab01::12aa"/>
+        <field name="dest_ip" value="::ffff:192.168.10.10"/>
+      </cim_fields>
+      <missing_recommended_fields>
+        <field>bytes</field>
+        <field>bytes_in</field>
+        <field>bytes_out</field>
+        <field>dest_port</field>
+        <field>dest_zone</field>
+        <field>src_port</field>
+        <field>src_translated_ip</field>
+        <field>src_zone</field>
+      </missing_recommended_fields>
+    </cim>
+  </event>
+    <event code="" name="security.request.blocked" format="json">
+    <version>2023.10.0</version>
+    <transport type="modinput" sourcetype="test:ipv6" source="test_okta" host="test_ipv6"/>
+    <source>
+      <jira id=""/>
+      <comment>generated, Generated from lab environment</comment>
+    </source>
+    <raw><![CDATA[{"actor": {"id": "spr6w8a2l2V88E3nk5d7", "type": "SystemPrincipal", "alternateId": "[email protected]", "displayName": "Okta System", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "Splunk Add-on for Okta Identity Cloud", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": null, "ipAddress": "2001:db80:1000:a000:0000:bc00:abcd:d0b0",  "destadd": "::", "geographicalContext": null}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": null}, "displayMessage": "Blocked request from IP: 68c6:7554::89", "eventType": "security.request.blocked", "outcome": {"result": "SUCCESS", "reason": "NETWORK_ZONE_BLACKLIST"}, "published": "2023-10-11T10:28:27.114Z", "securityContext": {"asNumber": null, "asOrg": null, "isp": null, "domain": null, "isProxy": null}, "severity": "WARN", "debugContext": {"debugData": {"requestId": "ZSZ4y03BLKYbIGjkjkgRvgAADhk", "requestUri": "/api/v1/logs", "url": "/api/v1/logs?limit=1000&since=2023-10-11T09%3A01%3A17.417Z&until=2023-10-11T10%3A27%3A56.967Z"}}, "legacyEventType": "security.zone.request.blocked", "transaction": {"type": "WEB", "id": "ZSZ4y03BLKYbIGjkjkgRvgAADhk", "detail": {}}, "uuid": "ea1dfcef-6820-11ee-bb11-f3c3bf8f14d1", "version": "0", "request": {"ipChain": [{"ip": "68c6:7554::89", "geographicalContext": null, "version": "V4", "source": null}]}, "target": null}]]></raw>
+    <cim>
+      <models>
+        <model>Network_Traffic</model>
+      </models>
+      <cim_fields>
+        <field name="src_ip" value="2001:db80:1000:a000:0000:bc00:abcd:d0b0"/>
+        <field name="dest_ip" value="::"/>
+      </cim_fields>
+      <missing_recommended_fields>
+        <field>bytes</field>
+        <field>bytes_in</field>
+        <field>bytes_out</field>
+        <field>dest_port</field>
+        <field>dest_zone</field>
+        <field>src_port</field>
+        <field>src_translated_ip</field>
+        <field>src_zone</field>
+      </missing_recommended_fields>
+    </cim>
+  </event>
+</device>

tests/e2e/constants.py

@@ -761,7 +761,6 @@
     "*test_splunk_fiction_indextime_broken.py::Test_App::test_requirements_fields[splunk_searchtime_fields_requirements0* SKIPPED*",
     "*test_splunk_fiction_indextime_broken.py::Test_App::test_cim_fields_recommended[splunk_searchtime_cim_fields_recommended0* SKIPPED*",
 ]
-
 TA_REQ_TRANSITION_PASSED = [
     "*test_splunk_app_req.py::Test_App::test_events_with_untokenised_values PASSED*",
     "*test_splunk_app_req.py::Test_App::test_indextime_time[test:data:1::* PASSED*",
@@ -922,3 +921,8 @@
     "*test_splunk_app_req_broken.py::Test_App::test_props_fields_no_dash_not_empty[splunk_searchtime_fields_negative0* SKIPPED*",
     "*test_splunk_app_req_broken.py::Test_App::test_savedsearches[splunk_searchtime_fields_savedsearches0* SKIPPED*",
 ]
+
+TA_CIM_MODEL_RESULT = [
+    '*test_splunk_cim_model_ipv6_regex.py::Test_App::test_cim_required_fields[eventtype="test_ipv6"::All_Traffic::dest_ip* PASSED*',
+    '*test_splunk_cim_model_ipv6_regex.py::Test_App::test_cim_required_fields[eventtype="test_ipv6"::All_Traffic::src_ip* PASSED*',
+]

tests/e2e/test_splunk_addon.py

@@ -741,6 +741,55 @@ def empty_method():
     assert result.ret == 0, "result not equal to 0"
 
 
+@pytest.mark.docker
+@pytest.mark.splunk_cim_model_ipv6_regex
+def test_splunk_cim_model_ipv6_regex(testdir, request):
+    """
+    In this test we are only checking if src_ip and dest_ip are extracted and are valid and tests are passing
+    Both these fields contains diff advanced form of ipv6 formats which would then be extracted via fields in data modles
+    """
+    testdir.makepyfile(
+        """
+        from pytest_splunk_addon.standard_lib.addon_basic import Basic
+        class Test_App(Basic):
+            def empty_method():
+                pass
+    """
+    )
+
+    shutil.copytree(
+        os.path.join(testdir.request.fspath.dirname, "addons/TA_cim_addon"),
+        os.path.join(testdir.tmpdir, "package"),
+    )
+
+    shutil.copytree(
+        os.path.join(testdir.request.fspath.dirname, "test_data_models"),
+        os.path.join(testdir.tmpdir, "tests/data_models"),
+    )
+
+    setup_test_dir(testdir)
+    SampleGenerator.clean_samples()
+    Rule.clean_rules()
+
+    # run pytest with the following cmd args
+    result = testdir.runpytest(
+        f"--splunk-version={request.config.getoption('splunk_version')}",
+        "--splunk-type=docker",
+        "-v",
+        "--search-interval=2",
+        "--search-retry=4",
+        "--search-index=*",
+        "--splunk-data-generator=tests/addons/TA_transition_from_req/default",
+        "-k test_cim_required_fields",
+    )
+    logger.info(result.outlines)
+
+    result.stdout.fnmatch_lines_random(constants.TA_CIM_MODEL_RESULT)
+
+    # make sure that we get a non '0' exit code for the testsuite as it contains failure
+    assert result.ret != 0, "result not equal to 0"
+
+
 @pytest.mark.test_infinite_loop_fixture
 @pytest.mark.external
 def test_infinite_loop_in_ingest_data_fixture(testdir, request):