[Cloud Security] add cloud_configuration_finding data stream to wiz

packages/wiz/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.2.0"
+  changes:
+    - description: Add cloud_configuration_finding data stream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9528/
 - version: "1.1.1"
   changes:
     - description: Add cloudsecurity_cdr sub category label

packages/wiz/data_stream/cloud_configuration_finding/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,4 @@
+fields:
+  tags:
+    - preserve_original_event
+    - preserve_duplicate_custom_fields

packages/wiz/data_stream/cloud_configuration_finding/_dev/test/pipeline/test-issue.log

@@ -0,0 +1 @@
+{"id":"bdeba988-f41b-55e6-9b99-96b8d3dc67d4","targetExternalId":"k8s/pod/da99fd668e64c2def251b1d48b7b69ad3129638787a0f9144a993fe30fd4554f/default/cluster-autoscaler-azure-cluster-autoscaler-8bc677d64-z2qfx","targetObjectProviderUniqueId":"cd971d74-92db-495c-8244-82da9a988fd0","firstSeenAt":"2023-06-12T11:38:07.900129Z","analyzedAt":"2023-06-12T11:38:07.900129Z","severity":"LOW","result":"FAIL","status":"OPEN","remediation":"Follow the step below to ensure that each [Pod](https://kubernetes.io/docs/concepts/workloads/pods) should runs containers with allowed additional capabilities: \r\n* The following capabilities are not allowed : {{removeUnnecessaryCapabilities}} .  \r\n* `securityContext.capabilities.drop` key is set to `ALL`. \r\n","resource":{"id":"0e814bb7-29e8-5c15-be9c-8da42c67ee99","providerId":"cd971d74-92db-495c-8244-82da9a988fd0","name":"cluster-autoscaler-azure-cluster-autoscaler-8bc677d64-z2qfx","nativeType":"Pod","type":"POD","region":null,"subscription":{"id":"a3a3cc43-1dfd-50f1-882e-692840d4a891","name":"Wiz - DEV Outpost","externalId":"cfd132be-3bc7-4f86-8efd-ed53ae498fec","cloudProvider":"Azure"},"projects":null,"tags":[{"key":"pod-template-hash","value":"8bc677d64"},{"key":"app.kubernetes.io/name","value":"azure-cluster-autoscaler"},{"key":"app.kubernetes.io/instance","value":"cluster-autoscaler"}]},"rule":{"id":"73553de7-f2ad-4ffb-b425-c69815033530","graphId":"99ffeef7-75df-5c88-9265-5ab50ffbc2b9","name":"Pod should run containers with authorized additional capabilities (PSS Restricted)","description":"This rule is part of the Kubernetes [Pod Security Standards (PSS) restricted policies](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted).   \nThis rule checks whether the pod is running containers with authorized additional capabilities.     \nThis rule fails if the `securityContext.capabilities.add` contains any capability beyond `NET_BIND_SERVICE` and if `securityContext.capabilities.drop` is not set to `ALL`.  \nBy default, if the `securityContext.capabilities.add` key is not set, the pod will not run with additional capabilities, and the rule will pass.   \nLinux capabilities allow granting certain privileges to a container without granting any unnecessary ones intended for the root user.","remediationInstructions":"Follow the step below to ensure that each [Pod](https://kubernetes.io/docs/concepts/workloads/pods) should runs containers with allowed additional capabilities: \r\n* The following capabilities are not allowed : {{removeUnnecessaryCapabilities}} .  \r\n* `securityContext.capabilities.drop` key is set to `ALL`. \r\n","functionAsControl":false},"securitySubCategories":[{"id":"wsct-id-5206","title":"Container Security","category":{"id":"wct-id-423","name":"9 Container Security","framework":{"id":"wf-id-1","name":"Wiz"}}},{"id":"wsct-id-8176","title":"5.1 Containers should not run with additional capabilities","category":{"id":"wct-id-1295","name":"5 Capabilities","framework":{"id":"wf-id-57","name":"Kubernetes Pod Security Standards (Restricted)"}}},{"id":"wsct-id-8344","title":"Cluster misconfiguration","category":{"id":"wct-id-1169","name":"2 Container & Kubernetes Security","framework":{"id":"wf-id-53","name":"Wiz Detailed"}}}]}

packages/wiz/data_stream/cloud_configuration_finding/_dev/test/pipeline/test-issue.log-expected.json

@@ -0,0 +1,41 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2023-06-12T11:38:07.900Z",
+            "cloud": {
+                "provider": "Azure"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": ["configuration"],
+                "created": "2023-06-12T11:38:07.900Z",
+                "id": "bdeba988-f41b-55e6-9b99-96b8d3dc67d4",
+                "kind": "event",
+                "original": "{\"id\":\"bdeba988-f41b-55e6-9b99-96b8d3dc67d4\",\"targetExternalId\":\"k8s/pod/da99fd668e64c2def251b1d48b7b69ad3129638787a0f9144a993fe30fd4554f/default/cluster-autoscaler-azure-cluster-autoscaler-8bc677d64-z2qfx\",\"targetObjectProviderUniqueId\":\"cd971d74-92db-495c-8244-82da9a988fd0\",\"firstSeenAt\":\"2023-06-12T11:38:07.900129Z\",\"analyzedAt\":\"2023-06-12T11:38:07.900129Z\",\"severity\":\"LOW\",\"result\":\"FAIL\",\"status\":\"OPEN\",\"remediation\":\"Follow the step below to ensure that each [Pod](https://kubernetes.io/docs/concepts/workloads/pods) should runs containers with allowed additional capabilities: \\r\\n* The following capabilities are not allowed : {{removeUnnecessaryCapabilities}} .  \\r\\n* `securityContext.capabilities.drop` key is set to `ALL`. \\r\\n\",\"resource\":{\"id\":\"0e814bb7-29e8-5c15-be9c-8da42c67ee99\",\"providerId\":\"cd971d74-92db-495c-8244-82da9a988fd0\",\"name\":\"cluster-autoscaler-azure-cluster-autoscaler-8bc677d64-z2qfx\",\"nativeType\":\"Pod\",\"type\":\"POD\",\"region\":null,\"subscription\":{\"id\":\"a3a3cc43-1dfd-50f1-882e-692840d4a891\",\"name\":\"Wiz - DEV Outpost\",\"externalId\":\"cfd132be-3bc7-4f86-8efd-ed53ae498fec\",\"cloudProvider\":\"Azure\"},\"projects\":null,\"tags\":[{\"key\":\"pod-template-hash\",\"value\":\"8bc677d64\"},{\"key\":\"app.kubernetes.io/name\",\"value\":\"azure-cluster-autoscaler\"},{\"key\":\"app.kubernetes.io/instance\",\"value\":\"cluster-autoscaler\"}]},\"rule\":{\"id\":\"73553de7-f2ad-4ffb-b425-c69815033530\",\"graphId\":\"99ffeef7-75df-5c88-9265-5ab50ffbc2b9\",\"name\":\"Pod should run containers with authorized additional capabilities (PSS Restricted)\",\"description\":\"This rule is part of the Kubernetes [Pod Security Standards (PSS) restricted policies](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted).   \\nThis rule checks whether the pod is running containers with authorized additional capabilities.     \\nThis rule fails if the `securityContext.capabilities.add` contains any capability beyond `NET_BIND_SERVICE` and if `securityContext.capabilities.drop` is not set to `ALL`.  \\nBy default, if the `securityContext.capabilities.add` key is not set, the pod will not run with additional capabilities, and the rule will pass.   \\nLinux capabilities allow granting certain privileges to a container without granting any unnecessary ones intended for the root user.\",\"remediationInstructions\":\"Follow the step below to ensure that each [Pod](https://kubernetes.io/docs/concepts/workloads/pods) should runs containers with allowed additional capabilities: \\r\\n* The following capabilities are not allowed : {{removeUnnecessaryCapabilities}} .  \\r\\n* `securityContext.capabilities.drop` key is set to `ALL`. \\r\\n\",\"functionAsControl\":false},\"securitySubCategories\":[{\"id\":\"wsct-id-5206\",\"title\":\"Container Security\",\"category\":{\"id\":\"wct-id-423\",\"name\":\"9 Container Security\",\"framework\":{\"id\":\"wf-id-1\",\"name\":\"Wiz\"}}},{\"id\":\"wsct-id-8176\",\"title\":\"5.1 Containers should not run with additional capabilities\",\"category\":{\"id\":\"wct-id-1295\",\"name\":\"5 Capabilities\",\"framework\":{\"id\":\"wf-id-57\",\"name\":\"Kubernetes Pod Security Standards (Restricted)\"}}},{\"id\":\"wsct-id-8344\",\"title\":\"Cluster misconfiguration\",\"category\":{\"id\":\"wct-id-1169\",\"name\":\"2 Container & Kubernetes Security\",\"framework\":{\"id\":\"wf-id-53\",\"name\":\"Wiz Detailed\"}}}]}",
+                "type": ["info"]
+            },
+            "message": "This rule is part of the Kubernetes [Pod Security Standards (PSS) restricted policies](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted).   \nThis rule checks whether the pod is running containers with authorized additional capabilities.     \nThis rule fails if the `securityContext.capabilities.add` contains any capability beyond `NET_BIND_SERVICE` and if `securityContext.capabilities.drop` is not set to `ALL`.  \nBy default, if the `securityContext.capabilities.add` key is not set, the pod will not run with additional capabilities, and the rule will pass.   \nLinux capabilities allow granting certain privileges to a container without granting any unnecessary ones intended for the root user.",
+            "tags": [
+                "preserve_original_event",
+                "preserve_duplicate_custom_fields"
+            ],
+            "wiz": {
+                "cloud_configuration_finding": {
+                    "analyzed_at": "2023-06-12T11:38:07.900Z",
+                    "first_seen_at": "2023-06-12T11:38:07.900Z",
+                    "id": "bdeba988-f41b-55e6-9b99-96b8d3dc67d4",
+                    "resource": {
+                        "subscription": {
+                            "cloud_provider": "Azure"
+                        }
+                    },
+                    "rule": {
+                        "description": "This rule is part of the Kubernetes [Pod Security Standards (PSS) restricted policies](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted).   \nThis rule checks whether the pod is running containers with authorized additional capabilities.     \nThis rule fails if the `securityContext.capabilities.add` contains any capability beyond `NET_BIND_SERVICE` and if `securityContext.capabilities.drop` is not set to `ALL`.  \nBy default, if the `securityContext.capabilities.add` key is not set, the pod will not run with additional capabilities, and the rule will pass.   \nLinux capabilities allow granting certain privileges to a container without granting any unnecessary ones intended for the root user."
+                    }
+                }
+            }
+        }
+    ]
+}

packages/wiz/data_stream/cloud_configuration_finding/_dev/test/system/test-default-config.yml

@@ -0,0 +1,15 @@
+input: cel
+service: wiz-cloud_configuration_finding
+vars:
+  url: http://{{Hostname}}:{{Port}}
+  client_id: xxxx
+  client_secret: xxxx
+  token_url: http://{{Hostname}}:{{Port}}/oauth/token
+data_stream:
+  vars:
+    interval: 10s
+    batch_size: 2
+    preserve_original_event: true
+    preserve_duplicate_custom_fields: true
+assert:
+  hit_count: 6

packages/wiz/data_stream/cloud_configuration_finding/agent/stream/cel.yml.hbs

@@ -0,0 +1,194 @@
+config_version: 2
+interval: {{interval}}
+{{#if enable_request_tracer}}
+resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson"
+resource.tracer.maxbackups: 5
+{{/if}}
+{{#if proxy_url}}
+resource.proxy_url: {{proxy_url}}
+{{/if}}
+{{#if ssl}}
+resource.ssl: {{ssl}}
+{{/if}}
+{{#if http_client_timeout}}
+resource.timeout: {{http_client_timeout}}
+{{/if}}
+resource.url: {{url}}
+auth.oauth2:
+  client.id: {{client_id}}
+  client.secret: {{client_secret}}
+  token_url: {{token_url}}
+  endpoint_params:
+    grant_type: client_credentials
+    audience: wiz-api
+state:
+  initial_interval: {{initial_interval}}
+  want_more: false
+  batch_size: {{batch_size}}
+  query: >-
+    query CloudConfigurationFindingsPage($filterBy: ConfigurationFindingFilters $first: Int $after: String $orderBy: ConfigurationFindingOrder){
+       configurationFindings(filterBy: $filterBy first: $first after: $after orderBy: $orderBy) {
+         nodes {
+          id
+          targetExternalId
+          targetObjectProviderUniqueId
+          analyzedAt
+          firstSeenAt
+          severity
+          result
+          status
+          remediation
+          resource {
+            id
+            providerId
+            name
+            nativeType
+            type
+            region
+            subscription {
+              id
+              name
+              externalId
+              cloudProvider
+            }
+            projects {
+              id
+              name
+              riskProfile {
+                businessImpact
+              }
+            }
+            tags {
+              key
+              value
+            }
+          }
+          rule {
+            id
+            graphId
+            name
+            description
+            remediationInstructions
+            functionAsControl
+          }
+          securitySubCategories {
+            id
+            title
+            category {
+              id
+              name
+              framework {
+                id
+                name
+              }
+            }
+          }
+          ignoreRules{
+            id
+            name
+            enabled
+            expiredAt
+          }
+        }
+        pageInfo {
+          hasNextPage
+          endCursor
+        }
+     }
+program: |
+  post_request(
+    state.url + "/graphql",
+    "application/json",
+    {
+      "query": state.query,
+      "variables": {
+          "first": state.batch_size,
+          "after": (has(state.end_cursor) && has(state.end_cursor.value) && state.end_cursor.value != null ? state.end_cursor.value : null),
+          "filterBy": {
+            "analyzedAt": {
+                "after":
+                (
+                    has(state.want_more) && !state.want_more
+                    ?
+                     (
+                         has(state.cursor) && has(state.cursor.last_timestamp) && state.cursor.last_timestamp != null
+                         ?
+                         state.cursor.last_timestamp
+                         :
+                         (now() - duration(state.initial_interval)).format(time_layout.RFC3339)
+                     )
+                    :
+                     (
+                       has(state.cursor) && has(state.cursor.first_timestamp) && state.cursor.first_timestamp != null
+                       ?
+                       state.cursor.first_timestamp
+                       :
+                       null
+                     )
+                ) }
+                }
+            }
+      }.encode_json()
+    ).do_request().as(resp, bytes(resp.Body).decode_json().as(body, {
+   "events": body.data.configurationFindings.nodes.map(e, {
+   "message": e.encode_json(),
+   }),
+    "cursor": {
+      "last_timestamp": (
+        has(body.data.configurationFindings.nodes) && body.data.configurationFindings.nodes.size() > 0
+        ?
+         (
+          has(state.cursor) && has(state.cursor.last_timestamp) && body.data.configurationFindings.nodes.map(e, e.analyzedAt).max() < state.cursor.last_timestamp
+          ?
+          state.cursor.last_timestamp
+          :
+          body.data.configurationFindings.nodes.map(e, e.analyzedAt).max()
+          )
+        :
+          (
+          has(state.cursor) && has(state.cursor.last_timestamp)
+          ?
+          state.cursor.last_timestamp
+          :
+          null
+          )
+        ),
+        "first_timestamp": (
+           has(state.cursor) && has(state.cursor.first_timestamp) && has(body.data) && state.cursor.first_timestamp != null
+            ?
+            ( body.data.configurationFindings.pageInfo.hasNextPage ? state.cursor.first_timestamp : state.cursor.last_timestamp )
+            :
+            (now() - duration(state.initial_interval)).format(time_layout.RFC3339)
+        ),
+      },
+    "end_cursor": {
+      "value": (
+      has(body.data) && has(body.data.configurationFindings) && has(body.data.configurationFindings.pageInfo) && has(body.data.configurationFindings.pageInfo.hasNextPage) && body.data.configurationFindings.pageInfo.hasNextPage
+       ?
+       body.data.configurationFindings.pageInfo.endCursor
+       :
+       null
+       )
+      },
+    "query": state.query,
+    "url": state.url,
+    "want_more": body.data.configurationFindings.pageInfo.hasNextPage,
+    "batch_size": state.batch_size,
+    }))
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#if preserve_duplicate_custom_fields}}
+  - preserve_duplicate_custom_fields
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}