More fixes after trying the simplified api

packages/m365_defender/_dev/deploy/docker/alert-http-mock-config.yml

@@ -14,14 +14,12 @@ rules:
         body: |-
           {"access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN","token_type": "Bearer","not_before": 1549647431,"expires_in": 3600}
   # Refresh token to get access token: https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-auth-code-flow#refresh-the-access-token
-  - path: /tenant_id/oauth2/token
+  - path: /tenant_id/oauth2/v2.0/token
     methods: [ POST ]
     query_params:
       refresh_token: refresh_token_123
       grant_type: refresh_token
     request_headers:
-      Authorization:
-        - "Basic dGVzdC1hcHAtaWQ6dGVzdC1zZWNyZXQ="
       Content-Type:
         - "application/x-www-form-urlencoded"
     responses:

packages/m365_defender/_dev/deploy/docker/incident-http-mock-config.yml

@@ -14,14 +14,12 @@ rules:
         body: |-
           {"access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN","token_type": "Bearer","not_before": 1549647431,"expires_in": 3600}
   # Refresh token to get access token: https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-auth-code-flow#refresh-the-access-token
-  - path: /tenant_id/oauth2/token
+  - path: /tenant_id/oauth2/v2.0/token
     methods: [ POST ]
     query_params:
       refresh_token: refresh_token_123
       grant_type: refresh_token
     request_headers:
-      Authorization:
-        - "Basic dGVzdC1hcHAtaWQ6dGVzdC1zZWNyZXQ="
       Content-Type:
         - "application/x-www-form-urlencoded"
     responses:

packages/microsoft_defender_cloud/_dev/deploy/docker/files/config.yml

@@ -22,17 +22,15 @@ rules:
             "expires_in": 3599
           }
   # Refresh token to get access token: https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-auth-code-flow#refresh-the-access-token
-  - path: /<tenant_id>/oauth2/token
-    methods: [ POST ]
+  - path: /<tenant_id>/oauth2/v2.0/token
+    methods: [POST]
     query_params:
       client_id: <client_id>
       client_secret: <client_secret>
       refresh_token: refresh_token_123
       grant_type: refresh_token
       scope: https://management.azure.com/.default
     request_headers:
-      Authorization:
-        - "Basic dGVzdC1hcHAtaWQ6dGVzdC1zZWNyZXQ="
       Content-Type:
         - "application/x-www-form-urlencoded"
     responses:
@@ -43,7 +41,7 @@ rules:
         body: |-
           {{ minify_json `
           {
-            "access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN",
+            "access_token": "eyJ0eabDVHDgdBDG59BRDG57DBDh56skjsbvDFBDge95sI6Ilg1ZVhrN",
             "token_type": "Bearer",
             "expires_in": 2,
             "scope": "https%3A%2F%2Fgraph.microsoft.com%2Fmail.read",

packages/microsoft_defender_cloud/data_stream/assessment/_dev/test/policy/test-oauth-params.expected

@@ -0,0 +1,200 @@
+inputs:
+    - data_stream:
+        namespace: ep
+      meta:
+        package:
+            name: microsoft_defender_cloud
+      name: test-oauth-params-microsoft_defender_cloud
+      streams:
+        - auth.oauth2:
+            client.id: ${SECRET_0}
+            client.secret: ${SECRET_1}
+            endpoint_params:
+                grant_type: refresh_token
+                refresh_token: refresh_token_123
+            scopes: https://management.azure.com/.default
+            token_url: https://login.microsoftonline.com/<tenant_id>/oauth2/v2.0/token
+          config_version: 2
+          data_stream:
+            dataset: microsoft_defender_cloud.assessment
+          interval: 24h
+          max_executions: 1000
+          processors:
+            - drop_event:
+                when:
+                    equals:
+                        message: retry
+          program: |-
+            (
+              state.?worklist[0].hasValue() ?
+                {
+                  "worklist": state.worklist,
+                  ?"assessment_next_url": state.?assessment_next_url
+                }
+              :
+                has(state.scope) ?
+                  (request("GET",
+                    state.?assessment_next_url.orValue(
+                      state.url.trim_right("/") + state.scope + "/providers/Microsoft.Security/assessments?" + {
+                        "api-version": [state.assessment_api_version],
+                      }.format_query()
+                    )
+                  ).do_request().as(resp, resp.StatusCode == 200 ?
+                    resp.Body.decode_json().as(body, {
+                      "worklist": body.value,
+                      ?"assessment_next_url": body.?nextLink
+                    })
+                  :
+                    {
+                      "events": {
+                        "error": {
+                          "code": string(resp.StatusCode),
+                          "id": string(resp.Status),
+                          "message": "GET " + state.url.trim_right("/") + state.scope + " :" + (
+                            size(resp.Body) != 0 ?
+                              string(resp.Body)
+                            :
+                              string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+                          ),
+                        },
+                      },
+                      "want_more": false,
+                      "assessment_api_version": state.assessment_api_version,
+                      "sub_assessment_api_version": state.sub_assessment_api_version,
+                      "scope": state.scope,
+                    }
+                  ))
+                :
+                  {
+                    "events": {
+                      "error": {
+                        "message": "A request scope is missing. Please provide a 'Subscription ID' or 'Management Group Name' while configuring",
+                      },
+                    },
+                    "want_more": false,
+                    "assessment_api_version": state.assessment_api_version,
+                    "sub_assessment_api_version": state.sub_assessment_api_version,
+                    "scope": state.scope,
+                  }
+            ).as(work,
+              has(work.events) ? state : // Exit early due to GET failure.
+                work.?worklist[0].hasValue() ?
+                  (work.?worklist[0].properties.additionalData.subAssessmentsLink.hasValue() ?
+                    request("GET",
+                      state.?subassessment_next_url.orValue(
+                        state.url.trim_right("/") + work.worklist[0].properties.additionalData.subAssessmentsLink + "?" + {
+                          "api-version": [state.sub_assessment_api_version],
+                        }.format_query()
+                      )
+                    ).do_request().as(resp, resp.StatusCode == 200 ?
+                      resp.Body.decode_json().as(body, {
+                        "events": body.?value[0].hasValue() ?
+                          body.value.map(e, {
+                            "message": work.worklist[0].with({
+                              "properties": {
+                                "additionalData": {
+                                  "subAssessment": e
+                                }
+                              }
+                            }).encode_json(),
+                          })
+                        :
+                          (
+                            has(state.subassessment_next_url) ?
+                              [{"message": "retry"}]
+                            :
+                              [{
+                                "message": work.worklist[0].encode_json(),
+                              }]
+                          ),
+                        ?"worklist": has(body.nextLink) ?
+                          optional.of(work.worklist)
+                        :
+                          tail(work.worklist)[?0].hasValue() ? optional.of(tail(work.worklist)) : optional.none(),
+                        "want_more": (tail(work.worklist)[?0].hasValue()) || has(work.assessment_next_url) || has(body.nextLink),
+                        ?"assessment_next_url": work.?assessment_next_url,
+                        ?"subassessment_next_url": body.?nextLink,
+                        "assessment_api_version": state.assessment_api_version,
+                        "sub_assessment_api_version": state.sub_assessment_api_version,
+                        "scope": state.scope,
+                      })
+                    :
+                      {
+                        "events": {
+                          "error": {
+                            "code": string(resp.StatusCode),
+                            "id": string(resp.Status),
+                            "message": "GET " + state.url.trim_right("/") + work.worklist[0].properties.additionalData.subAssessmentsLink + " :" + (
+                              size(resp.Body) != 0 ?
+                                string(resp.Body)
+                              :
+                                string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+                            ),
+                          },
+                        },
+                        "want_more": false,
+                        "assessment_api_version": state.assessment_api_version,
+                        "sub_assessment_api_version": state.sub_assessment_api_version,
+                        "scope": state.scope,
+                      }
+                    )
+                  :
+                    {
+                      "events": [{
+                        "message": work.worklist[0].encode_json(),
+                      }],
+                      ?"worklist": (tail(work.worklist)[?0].hasValue()) ? optional.of(tail(work.worklist)) : optional.none(),
+                      "want_more": (tail(work.worklist)[?0].hasValue()) || has(work.assessment_next_url),
+                      ?"assessment_next_url": work.?assessment_next_url,
+                      "assessment_api_version": state.assessment_api_version,
+                      "sub_assessment_api_version": state.sub_assessment_api_version,
+                      "scope": state.scope,
+                    }
+                  )
+                :
+                  {
+                    "events": [],
+                    "want_more": false,
+                    "assessment_api_version": state.assessment_api_version,
+                    "sub_assessment_api_version": state.sub_assessment_api_version,
+                    "scope": state.scope,
+                  }
+            )
+          publisher_pipeline.disable_host: true
+          redact:
+            fields: null
+          resource.ssl: null
+          resource.timeout: 120s
+          resource.tracer:
+            enabled: false
+            filename: ../../logs/cel/http-request-trace-*.ndjson
+            maxbackups: 5
+          resource.url: https://management.azure.com
+          state:
+            assessment_api_version: "2021-06-01"
+            scope: /subscriptions/5abcdef6-1234-5678-8912-e1234abcdef1
+            sub_assessment_api_version: 2019-01-01-preview
+          tags:
+            - preserve_original_event
+            - preserve_duplicate_custom_fields
+            - forwarded
+            - microsoft_defender_cloud-assessment
+      type: cel
+      use_output: default
+output_permissions:
+    default:
+        _elastic_agent_checks:
+            cluster:
+                - monitor
+        _elastic_agent_monitoring:
+            indices: []
+        uuid-for-permissions-on-related-indices:
+            indices:
+                - names:
+                    - logs-microsoft_defender_cloud.assessment-ep
+                  privileges:
+                    - auto_configure
+                    - create_doc
+secret_references:
+    - {}
+    - {}

packages/microsoft_defender_cloud/data_stream/assessment/_dev/test/policy/test-oauth-params.yml

@@ -0,0 +1,14 @@
+vars:
+  client_id: <client_id>
+  client_secret: <client_secret>
+  login_url: https://login.microsoftonline.com
+  url: https://management.azure.com
+  tenant_id: <tenant_id>
+  oauth_endpoint_params: |
+    grant_type: refresh_token
+    refresh_token: 'refresh_token_123'
+data_stream:
+  vars:
+    subscription_id: 5abcdef6-1234-5678-8912-e1234abcdef1
+    preserve_original_event: true
+    preserve_duplicate_custom_fields: true

packages/ti_abusech/data_stream/sslblacklist/_dev/test/system/test-default-config.yml

@@ -6,6 +6,7 @@ data_stream:
     preserve_original_event: true
 vars:
   enable_request_tracer: true
+  auth_key: test_auth_key
 assert:
   hit_count: 10
 wait_for_data_timeout: 2m