[staging] Promote packages from snapshot (gcp-1.4.1, o365-1.4.1, zscaler-0.5.1)

packages/gcp/1.4.1/changelog.yml

@@ -0,0 +1,99 @@
+# newer versions go on top
+- version: "1.4.1"
+  changes:
+    - description: Fix quoting of the credentials_json value in policy templates.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2712
+- version: "1.4.0"
+  changes:
+    - description: Add gcp.dns integration
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2624
+- version: "1.3.1"
+  changes:
+    - description: Add Ingest Pipeline script to map IANA Protocol Numbers
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2470
+- version: "1.3.0"
+  changes:
+    - description: Update to ECS 8.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2406
+- version: "1.2.2"
+  changes:
+    - description: Regenerate test files using the new GeoIP database
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2339
+- version: "1.2.1"
+  changes:
+    - description: Change test public IPs to the supported subset
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2327
+- version: "1.2.0"
+  changes:
+    - description: Add 8.0.0 version constraint
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2251
+- version: "1.1.2"
+  changes:
+    - description: Update Title and Description.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1965
+- version: "1.1.1"
+  changes:
+    - description: Fix logic that checks for the 'forwarded' tag
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1818
+- version: "1.1.0"
+  changes:
+    - description: Update to ECS 1.12.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1661
+- version: "1.0.0"
+  changes:
+    - description: Move from experimental to GA
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1568
+    - description: remove experimental from data_sets
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1717
+- version: "0.3.3"
+  changes:
+    - description: Convert to generated ECS fields
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1478
+- version: '0.3.2'
+  changes:
+    - description: update to ECS 1.11.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1385
+- version: "0.3.1"
+  changes:
+    - description: Escape special characters in docs
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1405
+- version: "0.3.0"
+  changes:
+    - description: Update integration description
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1364
+- version: "0.2.0"
+  changes:
+    - description: Set "event.module" and "event.dataset"
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1240
+- version: "0.1.0"
+  changes:
+    - description: update to ECS 1.10.0 and adding event.original options
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1045
+- version: "0.0.2"
+  changes:
+    - description: update to ECS 1.9.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/846
+- version: "0.0.1"
+  changes:
+    - description: initial release
+      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/integrations/pull/459

packages/gcp/1.4.1/data_stream/audit/agent/stream/gcp-pubsub.yml.hbs

@@ -0,0 +1,27 @@
+project_id: {{project_id}}
+topic: {{topic}}
+subscription.name: {{subscription_name}}
+{{#if credentials_file}}
+credentials_file: {{credentials_file}}
+{{/if}}
+{{#if credentials_json}}
+credentials_json: '{{credentials_json}}'
+{{/if}}
+{{#if alternative_host}}
+alternative_host: {{alternative_host}}
+{{/if}}
+subscription.create: {{subscription_create}}
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/gcp/1.4.1/data_stream/audit/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,261 @@
+---
+description: Pipeline for Google Cloud audit logs
+
+processors:
+  - set:
+      field: event.ingested
+      value: "{{_ingest.timestamp}}"
+  - set:
+      field: ecs.version
+      value: '8.0.0'
+  - rename:
+      field: message
+      target_field: event.original
+      ignore_missing: true
+  - json:
+      field: event.original
+      target_field: json
+  - set:
+      field: event.kind
+      value: event
+  - date:
+      field: json.timestamp
+      timezone: UTC
+      formats:
+        - ISO8601
+  - rename:
+      field: json.logName
+      target_field: log.logger
+      ignore_missing: true
+  - set: 
+      field: event.id
+      copy_from: json.insertId
+      ignore_empty_value: true
+      ignore_failure: true
+  - convert: 
+      field: json.resource.labels.project_id
+      target_field: cloud.project.id
+      type: string
+      ignore_missing: true
+      ignore_failure: true
+  - convert: 
+      field: json.resource.labels.instance_id
+      target_field: cloud.instance.id
+      type: string
+      ignore_missing: true
+      ignore_failure: true
+  - rename: 
+      field: "json.protoPayload.@type"
+      target_field: gcp.audit.type
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.authenticationInfo.principalEmail
+      target_field: gcp.audit.authentication_info.principal_email
+      ignore_missing: true
+  - set:
+      field: user.email
+      value: "{{gcp.audit.authentication_info.principal_email}}"
+      if: ctx?.gcp?.audit?.authentication_info?.principal_email != null
+  - rename:
+      field: json.protoPayload.authenticationInfo.authoritySelector
+      target_field: gcp.audit.authentication_info.authority_selector
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.authorizationInfo
+      target_field: gcp.audit.authorization_info
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.methodName
+      target_field: gcp.audit.method_name
+      ignore_missing: true
+  - set:
+      field: event.action
+      value: "{{gcp.audit.method_name}}"
+      if: ctx?.gcp?.audit?.method_name != null
+  - convert:
+      field: json.protoPayload.numResponseItems
+      target_field: gcp.audit.num_response_items
+      type: long
+      ignore_missing: true
+  - rename:
+      field: "json.protoPayload.request.@type"
+      target_field: gcp.audit.request.proto_name
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.request.filter
+      target_field: gcp.audit.request.filter
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.request.name
+      target_field: gcp.audit.request.name
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.request.resourceName
+      target_field: gcp.audit.request.resource_name
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.requestMetadata.callerIp
+      target_field: gcp.audit.request_metadata.caller_ip
+      ignore_missing: true
+  - set:
+      field: source.ip
+      value: "{{gcp.audit.request_metadata.caller_ip}}"
+      if: ctx?.gcp?.audit?.request_metadata?.caller_ip != null
+  - rename:
+      field: json.protoPayload.requestMetadata.callerSuppliedUserAgent
+      target_field: gcp.audit.request_metadata.caller_supplied_user_agent
+      ignore_missing: true
+  - set:
+      field: user_agent.original
+      value: "{{gcp.audit.request_metadata.caller_supplied_user_agent}}"
+      if: ctx?.gcp?.audit?.request_metadata?.caller_supplied_user_agent != null
+  - rename:
+      field: "json.protoPayload.response.@type"
+      target_field: gcp.audit.response.proto_name
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.response.status
+      target_field: gcp.audit.response.status
+      ignore_missing: true
+  - rename:
+      field: gcp.audit.response.status
+      target_field: gcp.audit.response.status.value
+      if: ctx?.gcp?.audit?.response?.status instanceof String
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.response.details.group
+      target_field: gcp.audit.response.details.group
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.response.details.kind
+      target_field: gcp.audit.response.details.kind
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.response.details.name
+      target_field: gcp.audit.response.details.name
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.response.details.uid
+      target_field: gcp.audit.response.details.uid
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.resourceName
+      target_field: gcp.audit.resource_name
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.resourceLocation.currentLocations
+      target_field: gcp.audit.resource_location.current_locations
+      ignore_missing: true
+  - rename:
+      field: json.protoPayload.serviceName
+      target_field: gcp.audit.service_name
+      ignore_missing: true
+  - set:
+      field: service.name
+      value: "{{gcp.audit.service_name}}"
+      if: ctx?.gcp?.audit?.service_name != null
+  - convert:
+      field: json.protoPayload.status.code
+      target_field: gcp.audit.status.code
+      type: long
+      ignore_missing: true
+  - foreach:
+      field: gcp.audit.authorization_info
+      ignore_missing: true
+      ignore_failure: true
+      processor:
+        rename:
+          field: _ingest._value.resourceAttributes
+          target_field: _ingest._value.resource_attributes
+      if: ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List
+  - set: 
+      field: event.outcome
+      value: success
+      if: ctx?.gcp?.audit?.status?.code != null && ctx?.gcp?.audit?.status?.code == 0
+  - set: 
+      field: event.outcome
+      value: failure
+      if: ctx?.gcp?.audit?.status?.code != null && ctx?.gcp?.audit?.status?.code != 0
+  - set:
+      field: event.outcome
+      value: success
+      if: ctx?.gcp?.audit?.status?.code == null && ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List && ctx?.gcp?.audit?.authorization_info.size() == 1 && ctx?.gcp?.audit?.authorization_info[0].granted
+  - set:
+      field: event.outcome
+      value: failure
+      if: ctx?.gcp?.audit?.status?.code == null && ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List && ctx?.gcp?.audit?.authorization_info.size() == 1 && !ctx?.gcp?.audit?.authorization_info[0].granted
+  - set:
+      field: event.outcome
+      value: unknown
+      if: ctx?.event?.outcome == null
+  - rename:
+      field: json.protoPayload.status.message
+      target_field: gcp.audit.status.message
+      ignore_missing: true    
+  - user_agent:
+      field: user_agent.original
+      ignore_missing: true
+  # Orchestrator fields
+  - set:
+      field: orchestrator.type
+      value: kubernetes
+      if: ctx.json?.resource?.type == 'k8s_cluster'
+  - rename:
+      field: json.resource.labels.cluster_name
+      target_field: orchestrator.cluster.name
+      ignore_missing: true
+      if: ctx.json?.resource?.type == 'k8s_cluster'
+  - rename:
+      field: json.protoPayload.resourceName
+      target_field: orchestrator.resource.type_temp
+      ignore_missing: true
+      if: ctx.json?.resource?.type == 'k8s_cluster'
+  - grok:
+      field: orchestrator.resource.type_temp
+      patterns:
+        - '%{DATA}/%{API_VERSION:orchestrator.api_version}/namespaces/%{DATA:orchestrator.namespace}/%{RESOURCE_TYPE:orchestrator.resource.type}(/%{HOSTNAME:orchestrator.resource.name})?'
+        - '%{DATA}/%{API_VERSION:orchestrator.api_version}/%{RESOURCE_TYPE:orchestrator.resource.type}'
+        - 'apis/%{RESOURCE_TYPE:orchestrator.resource.type}/%{API_VERSION:orchestrator.api_version}'
+        - 'api/%{API_VERSION:orchestrator.api_version}'
+        - '%{RESOURCE_TYPE:orchestrator.resource.type}'
+      pattern_definitions:
+        API_VERSION: (v\d+([a-z]+)?(\d+)?)
+        RESOURCE_TYPE: ([a-z]+((\.[a-z0-9]+)+)?)
+      ignore_missing: true
+  - remove:
+      field: orchestrator.resource.type_temp
+      ignore_missing: true
+  - remove:
+      field: json
+      ignore_missing: true
+  # IP Geolocation Lookup
+  - geoip:
+      field: source.ip
+      target_field: source.geo
+      ignore_missing: true
+  # IP Autonomous System (AS) Lookup
+  - geoip:
+      database_file: GeoLite2-ASN.mmdb
+      field: source.ip
+      target_field: source.as
+      properties:
+        - asn
+        - organization_name
+      ignore_missing: true
+  - rename:
+      field: source.as.asn
+      target_field: source.as.number
+      ignore_missing: true
+  - rename:
+      field: source.as.organization_name
+      target_field: source.as.organization.name
+      ignore_missing: true
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"