[salesforce] Add salesforce package foundation along with login_rest data stream

.github/CODEOWNERS

@@ -151,6 +151,7 @@
 /packages/radware @elastic/security-external-integrations
 /packages/redis @elastic/obs-service-integrations
 /packages/redisenterprise @elastic/obs-service-integrations
+/packages/salesforce @elastic/obs-service-integrations
 /packages/santa @elastic/security-external-integrations
 /packages/security_detection_engine @elastic/protections
 /packages/sentinel_one @elastic/security-external-integrations

packages/salesforce/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@v8.4.0

packages/salesforce/_dev/build/docs/README.md

@@ -0,0 +1,39 @@
+# Salesforce Integration
+
+## Overview
+
+The Salesforce integration allows you to monitor [Salesforce](https://www.salesforce.com/) instance. Salesforce provides customer relationship management service and also provides enterprise applications focused on customer service, marketing automation, analytics, and application development.
+
+Use the Salesforce integration to get visibility into the Salesforce Org operations and hold Salesforce accountable to the Service Level Agreements. Then visualize that data in Kibana, create alerts to notify you if something goes wrong, and reference logs when troubleshooting an issue.
+
+For example, if you want to check the number of successful and failed login attempts over time, you could check the same based on the ingested events or the visualization. Then you can create visualizations, alerts and troubleshoot by looking at the documents ingested in Elasticsearch.
+
+## Data streams
+
+The Salesforce integration collects log events using REST and Streaming API of Salesforce.
+
+**Logs** help you keep a record of events happening in Salesforce.
+Log data streams collected by the Salesforce integration include [Login](https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_eventlogfile_login.htm) (using REST and Streaming API), [Logout](https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_eventlogfile_logout.htm) (using REST and Streaming API), [Apex](https://developer.salesforce.com/docs/atlas.en-us.238.0.object_reference.meta/object_reference/sforce_api_objects_apexclass.htm), and [SetupAuditTrail](https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_setupaudittrail.htm).
+
+This integration uses:
+- `httpjson` filebeat input to collect `login_rest`, `logout_rest`, `apex` and `setupaudittrail` events.
+- `cometd` filebeat input to collect `login_stream` and `logout_stream` events.
+
+## Compatibility
+
+This integration has been tested against Salesforce API version `v54.0`.
+
+## Requirements
+
+You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it.
+You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware.
+
+## Logs reference
+
+### Login Rest
+
+This is the `login_rest` data stream. It represents events containing details about your organization's user login history.
+
+{{event "login_rest"}}
+
+{{fields "login_rest"}}

packages/salesforce/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,15 @@
+version: '2.3'
+services:
+  salesforce:
+    image: docker.elastic.co/observability/stream:v0.8.0
+    hostname: salesforce
+    ports:
+      - 8010
+    volumes:
+      - ./files:/files:ro
+    environment:
+      PORT: 8010
+    command:
+      - http-server
+      - --addr=:8010
+      - --config=/files/config.yml

packages/salesforce/_dev/deploy/docker/files/config.yml

@@ -0,0 +1,71 @@
+rules:
+  - path: /services/oauth2/token
+    methods: ["POST"]
+    responses:
+      - status_code: 200
+        headers:
+          content-type: ['application/json']
+        body: '{"access_token":"access_token","instance_url":"https://temporary-intance-url","id":"https://login.salesforce.com/id/temp_id/temp_token","token_type":"Bearer","issued_at":"1633689089545","signature":"signature"}'
+  - path: /services/data/v54.0/query
+    methods: ["GET"]
+    query_params:
+      q: ["SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' ORDER BY LogDate ASC NULLS FIRST"]
+    responses:
+      - status_code: 200
+        body: '{"totalSize":1,"done":true,"records":[{"attributes":{"type":"EventLogFile","url":"/services/data/v54.0/sobjects/EventLogFile/0AT5j00002GVrfnGAD"},"Id":"0AT5j00002GVrfnGAD","CreatedDate":"2022-09-14T21:43:41.000+0000","LogDate":"2022-09-13T00:00:00.000+0000","LogFile":"/services/data/v54.0/sobjects/EventLogFile/0AT5j00002GVrfnGAD/LogFile"}]}'
+        headers:
+          content-type: ["text/json"]
+  - path: /services/data/v54.0/query
+    methods: ["GET"]
+    query_params:
+      q: ["SELECT Action,CreatedByContext,CreatedById,CreatedByIssuer,CreatedDate,DelegateUser,Display,Id,ResponsibleNamespacePrefix,Section FROM SetupAuditTrail ORDER BY CreatedDate ASC NULLS FIRST"]
+    responses:
+      - status_code: 200
+        body: '{"done":true,"records":[{"Action":"insertConnectedApplication","CreatedByContext":"Einstein","CreatedById":"0055j000000utlPAAQ","CreatedByIssuer":null,"CreatedDate":"2022-08-16T09:26:38.000+0000","DelegateUser":"user1","Display":"For user user@elastic.co, the User Verified Email status changed to verified","Id":"0Ym5j000019nwonCAA","ResponsibleNamespacePrefix":"namespaceprefix","Section":"Connected Apps","attributes":{"type":"SetupAuditTrail","url":"/services/data/v54.0/sobjects/SetupAuditTrail/0Ym5j000019nwonCAA"}}],"totalSize":160}'
+        headers:
+          content-type: ["text/json"]
+  - path: /services/data/v54.0/sobjects/EventLogFile/0AT5j00002GVrfnGAD/LogFile
+    methods: ["GET"]
+    responses:
+      - status_code: 200
+        headers:
+          content-type: ["text/csv"]
+        body: |-
+          "EVENT_TYPE","TIMESTAMP","REQUEST_ID","ORGANIZATION_ID","USER_ID","RUN_TIME","CPU_TIME","URI","SESSION_KEY","LOGIN_KEY","USER_TYPE","REQUEST_STATUS","DB_TOTAL_TIME","BROWSER_TYPE","API_TYPE","API_VERSION","USER_NAME","TLS_PROTOCOL","CIPHER_SUITE","AUTHENTICATION_METHOD_REFERENCE","TIMESTAMP_DERIVED","USER_ID_DERIVED","CLIENT_IP","URI_ID_DERIVED","LOGIN_STATUS","SOURCE_IP"
+          "Login","20211006071307.550","4ehU_U-nbQyAPFl1cJILm-","00D5j000000VI3n","0055j000000utlP","83","30","/index.jsp","","QfNecrLXSII6fsBq","Standard","Success","52435102","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36","f","9998.0","user@elastic.co","TLSv1.2","ECDHE-RSA-AES256-GCM-SHA384","","2021-10-06T07:13:07.550Z","0055j000000utlPAAQ","43.200.10.11","s4heK3WbH-lcJIL3-n","LOGIN_NO_ERROR","43.200.10.11"
+  - path: /services/data/v54.0/query
+    methods: ["GET"]
+    query_params:
+      q: ["SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' ORDER BY LogDate ASC NULLS FIRST"]
+    responses:
+      - status_code: 200
+        body: '{"totalSize":1,"done":true,"records":[{"attributes":{"type":"EventLogFile","url":"/services/data/v54.0/sobjects/EventLogFile/0AT5j00002GWEsRGAX"},"Id":"0AT5j00002GWEsRGAX","CreatedDate":"2022-09-19T21:03:41.000+0000","LogDate":"2022-09-18T00:00:00.000+0000","LogFile":"/services/data/v54.0/sobjects/EventLogFile/0AT5j00002GWEsRGAX/LogFile"}]}'
+        headers:
+          content-type: ["text/json"]
+  - path: /services/data/v54.0/sobjects/EventLogFile/0AT5j00002GWEsRGAX/LogFile
+    methods: ["GET"]
+    responses:
+      - status_code: 200
+        headers:
+          content-type: ["text/csv"]
+        body: |-
+          "EVENT_TYPE","TIMESTAMP","REQUEST_ID","ORGANIZATION_ID","USER_ID","USER_TYPE","SESSION_TYPE","SESSION_LEVEL","BROWSER_TYPE","PLATFORM_TYPE","RESOLUTION_TYPE","APP_TYPE","CLIENT_VERSION","API_TYPE","API_VERSION","USER_INITIATED_LOGOUT","SESSION_KEY","LOGIN_KEY","TIMESTAMP_DERIVED","USER_ID_DERIVED","CLIENT_IP"
+          "Logout","20220919073707.360","4exLFFQZ1234xFl1cJNwOV","00D5j001234VI3n","0055j000000utlP","S","O","1","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36","1015","9999","1000","9998","f","54.0","0","WvtsJ1235oW24EbH","Obv9123BzbaxqCo1","2022-09-19T07:37:07.360Z","0055j000000utlPAAQ","43.200.10.11"
+  - path: /services/data/v54.0/query
+    methods: ["GET"]
+    query_params:
+      q: ["SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST"]
+    responses:
+      - status_code: 200
+        body: '{"done":true,"records":[{"CreatedDate":"2022-10-01T23:22:27.000+0000","Id":"0AT5j00002GWEsRGAY","LogDate":"2022-09-30T00:00:00.000+0000","LogFile":"/services/data/v54.0/sobjects/EventLogFile/0AT5j00002GWEsRGAY/LogFile","attributes":{"type":"EventLogFile","url":"/services/data/v54.0/sobjects/EventLogFile/0AT5j00002GWEsRGAY"}}],"totalSize":1}'
+        headers:
+          content-type: ["text/json"]
+  - path: /services/data/v54.0/sobjects/EventLogFile/0AT5j00002GWEsRGAY/LogFile
+    methods: ["GET"]
+    responses:
+      - status_code: 200
+        headers:
+          content-type: ["text/csv"]
+        body: |-
+          "EVENT_TYPE","TIMESTAMP","REQUEST_ID","ORGANIZATION_ID","USER_ID","RUN_TIME","CPU_TIME","URI","SESSION_KEY","LOGIN_KEY","TYPE","METHOD","SUCCESS","TIME","REQUEST_SIZE","RESPONSE_SIZE","URL","TIMESTAMP_DERIVED","USER_ID_DERIVED","CLIENT_IP","URI_ID_DERIVED"
+          "ApexCallout","20211102091250.488","ABCDE","00D5j000000VABC","0055j000000ABCD","1305","10","CALLOUT-LOG","ABCDEF","ABCDEFGH","OData","GET","1","1293","10","256","https://temp.sh/odata/Accounts","2021-11-02T09:12:50.488Z","0055j012345utlPAAQ","127.0.0.1","0055j000000utlPABCD"

packages/salesforce/changelog.yml

@@ -0,0 +1,7 @@
+# newer versions go on top
+
+- version: 0.1.0
+  changes:
+    - description: Salesforce integration package with "login_rest" data stream.
+      link: https://github.com/elastic/integrations/pull/4261
+      type: enhancement

packages/salesforce/data_stream/login_rest/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,2 @@
+dynamic_fields:
+  event.ingested: ".*"

packages/salesforce/data_stream/login_rest/_dev/test/pipeline/test-login-rest.log

@@ -0,0 +1 @@
+{"API_TYPE":"f","API_VERSION":"9998.0","AUTHENTICATION_METHOD_REFERENCE":"","BROWSER_TYPE":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/IP_ADDRESS_REMOVED Safari/537.36","CIPHER_SUITE":"ECDHE-RSA-AES256-GCM-SHA384","CLIENT_IP":"175.16.199.0","CPU_TIME":"63","DB_TOTAL_TIME":"93531912","EVENT_TYPE":"Login","LOGIN_KEY":"NK2oPJrJze8tH+vC","LOGIN_STATUS":"LOGIN_NO_ERROR","ORGANIZATION_ID":"00D5j000000VI3n","REQUEST_ID":"4lGJ1R0ZSWVXwFl1cJIRH-","REQUEST_STATUS":"Success","RUN_TIME":"156","SESSION_KEY":"","SOURCE_IP":"175.16.199.0","TIMESTAMP":"20220913052243.429","TIMESTAMP_DERIVED":"2022-09-13T05:22:43.429Z","TLS_PROTOCOL":"TLSv1.2","URI":"/index.jsp","URI_ID_DERIVED":"s4heK3WbH-lcJIL3-n","USER_ID":"0055j000000utlP","USER_ID_DERIVED":"0055j000000utlPAAQ","USER_NAME":"user@elastic.co","USER_TYPE":"Standard"}

packages/salesforce/data_stream/login_rest/_dev/test/pipeline/test-login-rest.log-expected.json

@@ -0,0 +1,79 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2022-09-13T05:22:43.429Z",
+            "ecs": {
+                "version": "8.4.0"
+            },
+            "event": {
+                "action": "login-attempt",
+                "category": [
+                    "authentication"
+                ],
+                "dataset": "salesforce.login_rest",
+                "kind": "event",
+                "module": "salesforce",
+                "outcome": "success",
+                "type": [
+                    "info"
+                ],
+                "url": "/index.jsp"
+            },
+            "related": {
+                "ip": [
+                    "175.16.199.0"
+                ]
+            },
+            "salesforce": {
+                "login": {
+                    "access_mode": "rest",
+                    "api": {
+                        "type": "Feed",
+                        "version": "9998.0"
+                    },
+                    "client_ip": "175.16.199.0",
+                    "cpu_time": 63.0,
+                    "db_time": {
+                        "total": 9.3531912E7
+                    },
+                    "event_type": "Login",
+                    "key": "NK2oPJrJze8tH+vC",
+                    "organization_id": "00D5j000000VI3n",
+                    "request_id": "4lGJ1R0ZSWVXwFl1cJIRH-",
+                    "request_status": "Success",
+                    "run_time": 156.0,
+                    "uri_derived_id": "s4heK3WbH-lcJIL3-n",
+                    "user_id": "0055j000000utlP"
+                }
+            },
+            "source": {
+                "geo": {
+                    "city_name": "Changchun",
+                    "continent_name": "Asia",
+                    "country_iso_code": "CN",
+                    "country_name": "China",
+                    "location": {
+                        "lat": 43.88,
+                        "lon": 125.3228
+                    },
+                    "region_iso_code": "CN-22",
+                    "region_name": "Jilin Sheng"
+                },
+                "ip": "175.16.199.0"
+            },
+            "tls": {
+                "cipher": "ECDHE-RSA-AES256-GCM-SHA384",
+                "version": "1.2",
+                "version_protocol": "TLS"
+            },
+            "user": {
+                "email": "user@elastic.co",
+                "id": "0055j000000utlPAAQ",
+                "roles": "Standard"
+            },
+            "user_agent": {
+                "name": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/IP_ADDRESS_REMOVED Safari/537.36"
+            }
+        }
+    ]
+}

packages/salesforce/data_stream/login_rest/_dev/test/system/test-default-config.yml

@@ -0,0 +1,12 @@
+input: httpjson
+service: salesforce
+vars:
+  instance_url: http://{{Hostname}}:{{Port}}
+  client_id: temp_client_id
+  client_secret: forty_characters_long_secret_key
+  username: temp_user
+  password: temp_password
+  token_url: http://{{Hostname}}:{{Port}}/services/oauth2/token
+data_stream:
+  vars:
+    preserve_original_event: true

packages/salesforce/data_stream/login_rest/agent/stream/httpjson.yml.hbs

@@ -0,0 +1,39 @@
+config_version: 2
+interval: {{period}}
+request.method: GET
+auth.oauth2:
+  enabled: true
+  client.id: {{client_id}}
+  client.secret: {{client_secret}}
+  token_url: {{token_url}}
+  user: {{username}}
+  password: {{password}}
+request.url: {{instance_url}}/services/data/v54.0/query?q=login+rest
+request.transforms:
+  - set:
+      target: url.params.q
+      value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' AND LogDate > [[.cursor.last_published_login]] ORDER BY LogDate ASC NULLS FIRST"
+      default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' ORDER BY LogDate ASC NULLS FIRST"
+response.split:
+  target: body.records
+chain:
+  - step:
+      request.url: {{instance_url}}/services/data/v54.0/sobjects/EventLogFile/$.records[:].Id/LogFile
+      request.method: GET
+      replace: $.records[:].Id
+cursor:
+  last_published_login:
+    value: '[[.last_event.LogDate]]'
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+{{processors}}
+{{/if}}