[salesforce] Implement setupaudittrail data stream for Salesforce

packages/salesforce/changelog.yml

@@ -1,5 +1,10 @@
 # newer versions go on top
 
+- version: 0.3.0
+  changes:
+    - description: Salesforce integration package with "setupaudittrail" data stream.
+      link: https://github.com/elastic/integrations/pull/4356
+      type: enhancement
 - version: 0.2.0
   changes:
     - description: Salesforce integration package with "logout_rest" data stream.

packages/salesforce/data_stream/setupaudittrail/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,5 @@
+dynamic_fields:
+  event.ingested: ".*"
+fields:
+  tags:
+    - preserve_original_event

packages/salesforce/data_stream/setupaudittrail/_dev/test/pipeline/test-setupaudittrail.log

@@ -0,0 +1 @@
+{"Action":"insertConnectedApplication","CreatedByContext":"Einstein","CreatedById":"0055j000000utlPAAQ","CreatedByIssuer":null,"CreatedDate":"2022-08-16T09:26:38.000+0000","DelegateUser":"user1","Display":"For user user@elastic.co, the User Verified Email status changed to verified","Id":"0Ym5j000019nwonCAA","ResponsibleNamespacePrefix":"namespaceprefix","Section":"Connected Apps","attributes":{"type":"SetupAuditTrail","url":"/services/data/v54.0/sobjects/SetupAuditTrail/0Ym5j000019nwonCAA"}}

packages/salesforce/data_stream/setupaudittrail/_dev/test/pipeline/test-setupaudittrail.log-expected.json

@@ -0,0 +1,42 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2022-08-16T09:26:38.000Z",
+            "ecs": {
+                "version": "8.5.0"
+            },
+            "event": {
+                "action": "insertConnectedApplication",
+                "created": "2022-08-16T09:26:38.000Z",
+                "dataset": "salesforce.setupaudittrail",
+                "id": "0Ym5j000019nwonCAA",
+                "kind": "event",
+                "module": "salesforce",
+                "original": "{\"Action\":\"insertConnectedApplication\",\"CreatedByContext\":\"Einstein\",\"CreatedById\":\"0055j000000utlPAAQ\",\"CreatedByIssuer\":null,\"CreatedDate\":\"2022-08-16T09:26:38.000+0000\",\"DelegateUser\":\"user1\",\"Display\":\"For user user@elastic.co, the User Verified Email status changed to verified\",\"Id\":\"0Ym5j000019nwonCAA\",\"ResponsibleNamespacePrefix\":\"namespaceprefix\",\"Section\":\"Connected Apps\",\"attributes\":{\"type\":\"SetupAuditTrail\",\"url\":\"/services/data/v54.0/sobjects/SetupAuditTrail/0Ym5j000019nwonCAA\"}}",
+                "type": [
+                    "admin"
+                ],
+                "url": "/services/data/v54.0/sobjects/SetupAuditTrail/0Ym5j000019nwonCAA"
+            },
+            "salesforce": {
+                "setup_audit_trail": {
+                    "access_mode": "REST",
+                    "created_by_context": "Einstein",
+                    "created_by_id": "0055j000000utlPAAQ",
+                    "delegate_user": "user1",
+                    "display": "For user user@elastic.co, the User Verified Email status changed to verified",
+                    "event_type": "SetupAuditTrail",
+                    "responsible_namespace_prefix": "namespaceprefix",
+                    "section": "Connected Apps"
+                }
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "id": "0055j000000utlPAAQ",
+                "name": "user@elastic.co"
+            }
+        }
+    ]
+}

packages/salesforce/data_stream/setupaudittrail/_dev/test/system/test-default-config.yml

@@ -0,0 +1,12 @@
+input: httpjson
+service: salesforce
+vars:
+  instance_url: http://{{Hostname}}:{{Port}}
+  client_id: temp_client_id
+  client_secret: forty_characters_long_secret_key
+  username: temp_user
+  password: temp_password
+  token_url: http://{{Hostname}}:{{Port}}/services/oauth2/token
+data_stream:
+  vars:
+    preserve_original_event: true

packages/salesforce/data_stream/setupaudittrail/agent/stream/httpjson.yml.hbs

@@ -0,0 +1,44 @@
+config_version: 2
+interval: {{period}}
+request.method: GET
+auth.oauth2:
+  enabled: true
+  client.id: {{client_id}}
+  client.secret: {{client_secret}}
+  token_url: {{token_url}}
+  user: {{username}}
+  password: {{password}}
+request.url: {{instance_url}}/services/data/v54.0/query?q=setupaudittrail+rest
+request.transforms:
+  - set:
+      target: url.params.q
+      value: "SELECT Action,CreatedByContext,CreatedById,CreatedByIssuer,CreatedDate,DelegateUser,Display,Id,ResponsibleNamespacePrefix,Section FROM SetupAuditTrail WHERE CreatedDate > [[.cursor.last_published_setupaudittrail]] ORDER BY CreatedDate ASC NULLS FIRST"
+      default: "SELECT Action,CreatedByContext,CreatedById,CreatedByIssuer,CreatedDate,DelegateUser,Display,Id,ResponsibleNamespacePrefix,Section FROM SetupAuditTrail ORDER BY CreatedDate ASC NULLS FIRST"
+response.split:
+  target: body.records
+response.pagination:
+- set:
+    target: url.value
+    value: '[[if (ne .last_response.body.done true)]]{{instance_url}}[[.last_response.body.nextRecordsUrl]][[end]]'
+    fail_on_template_error: true
+cursor:
+  last_published_setupaudittrail:
+    value: '[[.last_event.CreatedDate]]'
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+- add_fields:
+    target: salesforce
+    fields:
+      instance_url: {{instance_url}}
+{{#if processors}}
+{{processors}}
+{{/if}}

packages/salesforce/data_stream/setupaudittrail/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,150 @@
+---
+description: Pipeline for parsing Salesforce SetupAuditTrail logs.
+processors:
+- json:
+    field: message
+    target_field: json
+    ignore_failure: true
+- rename:
+    field: message
+    target_field: event.original
+    ignore_missing: true
+    ignore_failure: true
+- set:
+    field: salesforce.setup_audit_trail.access_mode
+    value: "REST"
+    ignore_failure: true
+    ignore_empty_value: true
+- date:
+    field: json.CreatedDate
+    target_field: "@timestamp"
+    formats:
+    - ISO8601
+    ignore_failure: true
+- rename:
+    field: json.attributes.type
+    target_field: salesforce.setup_audit_trail.event_type
+    ignore_missing: true
+    ignore_failure: true
+- rename:
+    field: json.CreatedByContext
+    target_field: salesforce.setup_audit_trail.created_by_context
+    ignore_missing: true
+    ignore_failure: true
+- rename:
+    field: json.CreatedById
+    target_field: salesforce.setup_audit_trail.created_by_id
+    ignore_missing: true
+    ignore_failure: true
+- rename:
+    field: json.CreatedByIssuer
+    target_field: salesforce.setup_audit_trail.created_by_issuer
+    ignore_missing: true
+    ignore_failure: true
+- rename:
+    field: json.DelegateUser
+    target_field: salesforce.setup_audit_trail.delegate_user
+    ignore_missing: true
+    ignore_failure: true
+- rename:
+    field: json.Display
+    target_field: salesforce.setup_audit_trail.display
+    ignore_missing: true
+    ignore_failure: true
+- rename:
+    field: json.ResponsibleNamespacePrefix
+    target_field: salesforce.setup_audit_trail.responsible_namespace_prefix
+    ignore_missing: true
+    ignore_failure: true
+- rename:
+    field: json.Section
+    target_field: salesforce.setup_audit_trail.section
+    ignore_missing: true
+    ignore_failure: true
+- set:
+    field: ecs.version
+    value: "8.5.0"
+    ignore_failure: true
+    ignore_empty_value: true
+- rename:
+    field: json.Id
+    target_field: event.id
+    ignore_missing: true
+    ignore_failure: true
+- rename:
+    field: json.Action
+    target_field: event.action
+    ignore_missing: true
+    ignore_failure: true
+- rename:
+    field: json.attributes.url
+    target_field: event.url
+    ignore_missing: true
+    ignore_failure: true
+- date:
+    field: json.CreatedDate
+    target_field: event.created
+    formats: 
+    - ISO8601
+    ignore_failure: true
+- set:
+    field: event.type
+    value: ["admin"]
+    ignore_failure: true
+    ignore_empty_value: true
+- set:
+    field: event.kind
+    value: "event"
+    ignore_failure: true
+    ignore_empty_value: true
+- set:
+    field: event.dataset
+    value: "salesforce.setupaudittrail"
+    ignore_failure: true
+    ignore_empty_value: true
+- set:
+    field: event.module
+    value: "salesforce"
+    ignore_failure: true
+    ignore_empty_value: true
+- set:
+    field: user.id
+    copy_from: salesforce.setup_audit_trail.created_by_id
+    ignore_failure: true
+    ignore_empty_value: true
+- dissect:
+    field: salesforce.setup_audit_trail.display
+    pattern: "For user %{user.name}, %{?}"
+    ignore_failure: true
+- script:
+    description: Drops null/empty values recursively.
+    lang: painless
+    source: |
+        boolean dropEmptyFields(Object object) {
+            if (object == null || object == "") {
+                return true;
+            } else if (object instanceof Map) {
+                ((Map) object).values().removeIf(value -> dropEmptyFields(value));
+                return (((Map) object).size() == 0);
+            } else if (object instanceof List) {
+                ((List) object).removeIf(value -> dropEmptyFields(value));
+                return (((List) object).length == 0);
+            }
+            return false;
+        }
+        dropEmptyFields(ctx);
+- remove:
+    field:
+      - json
+      - message
+    ignore_missing: true
+    ignore_failure: true
+- remove:
+    field: event.original
+    if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+    ignore_failure: true
+    ignore_missing: true
+on_failure:
+- set:
+    field: error.message
+    value: '{{{_ingest.on_failure_message}}}'

packages/salesforce/data_stream/setupaudittrail/fields/base-fields.yml

@@ -0,0 +1,15 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: input.type
+  type: keyword
+  description: Input type.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

packages/salesforce/data_stream/setupaudittrail/fields/ecs.yml

@@ -0,0 +1,28 @@
+- external: ecs
+  name: ecs.version
+- external: ecs
+  name: error.message
+- external: ecs
+  name: event.action
+- external: ecs
+  name: event.created
+- external: ecs
+  name: event.dataset
+- external: ecs
+  name: event.id
+- external: ecs
+  name: event.ingested
+- external: ecs
+  name: event.kind
+- external: ecs
+  name: event.module
+- external: ecs
+  name: event.type
+- external: ecs
+  name: event.url
+- external: ecs
+  name: tags
+- external: ecs
+  name: user.id
+- external: ecs
+  name: user.name

packages/salesforce/data_stream/setupaudittrail/fields/fields.yml

@@ -0,0 +1,36 @@
+- name: salesforce
+  type: group
+  fields:
+    - name: instance_url
+      type: keyword
+      description: The Instance URL of the Salesforce instance.
+    - name: setup_audit_trail
+      type: group
+      fields:
+        - name: access_mode
+          type: keyword
+          description: Type of API from which the event is collected.
+        - name: created_by_context
+          type: keyword
+          description: The context under which the Setup change was made. For example, if Einstein uses cloud-to-cloud services to make a change in Setup, the value of this field is Einstein.
+        - name: created_by_id
+          type: keyword
+          description: Unknown.
+        - name: created_by_issuer
+          type: keyword
+          description: Reserved for future use.
+        - name: delegate_user
+          type: keyword
+          description: The Login-As user who executed the action in Setup. If a Login-As user didn’t perform the action, this field is blank. This field is available in API version 35.0 and later.
+        - name: display
+          type: keyword
+          description: "The full description of changes made in Setup. For example, if the Action field has a value of PermSetCreate, the Display field has a value like “Created permission set MAD: with user license Salesforce.”"
+        - name: event_type
+          type: keyword
+          description: Event type.
+        - name: responsible_namespace_prefix
+          type: keyword
+          description: Unknown.
+        - name: section
+          type: keyword
+          description: The section in the Setup menu where the action occurred. For example, Manage Users or Company Profile.

packages/salesforce/data_stream/setupaudittrail/manifest.yml

@@ -0,0 +1,40 @@
+type: logs
+title: Salesforce setupaudittrail logs
+streams:
+  - input: httpjson
+    vars:
+      - name: period
+        type: text
+        title: Period
+        description: Period of fetching logs, i.e. 1s/1m/1h.
+        multi: false
+        required: true
+        show_user: false
+        default: 1h
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        required: true
+        show_user: false
+        default:
+          - salesforce-setupaudittrail
+          - forwarded
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`.
+        type: bool
+        multi: false
+        default: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+    template_path: httpjson.yml.hbs
+    title: Salesforce SetupAuditTrail logs
+    description: Collect Salesforce SetupAuditTrail logs.