Merge branch 'master' into resolve-perfmon-deprecation

Author: narph

.ci/packaging.groovy

@@ -328,21 +328,24 @@ def tagAndPush(Map args = [:]) {
   }
 
   variants.each { variant ->
+    // cloud docker images are stored in the private docker namespace.
+    def sourceNamespace = variant.equals('-cloud') ? 'beats-ci' : 'beats'
     tags.each { tag ->
       // TODO:
       // For backward compatibility let's ensure we tag only for amd64, then E2E can benefit from until
       // they support the versioning with the architecture
       if ("${arch}" == "amd64") {
-        doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}")
+        doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}", sourceNamespace: sourceNamespace)
       }
-      doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}")
+      doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}", sourceNamespace: sourceNamespace)
     }
   }
 }
 
 /**
 * @param beatName name of the Beat
 * @param variant name of the variant used to build the docker image name
+* @param sourceNamespace namespace to be used as source for the docker tag command
 * @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
 * @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
 */
@@ -351,7 +354,8 @@ def doTagAndPush(Map args = [:]) {
   def variant = args.variant
   def sourceTag = args.sourceTag
   def targetTag = args.targetTag
-  def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
+  def sourceNamespace = args.sourceNamespace
+  def sourceName = "${DOCKER_REGISTRY}/${sourceNamespace}/${beatName}${variant}:${sourceTag}"
   def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"
   def iterations = 0
   retryWithSleep(retries: 3, seconds: 5, backoff: true) {

.go-version

@@ -1 +1 @@
-1.16.6
+1.17.1

CHANGELOG-developer.next.asciidoc

@@ -63,6 +63,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Fix export dashboard command when running against Elastic Cloud hosted Kibana. {pull}22746[22746]
 - Remove `event.dataset` (ECS) annotion from `libbeat.logp`. {issue}27404[27404]
 - Errors should be thrown as errors. Metricsets inside Metricbeat will now throw errors as the `error` log level. {pull}27804[27804]
+- Avoid panicking in `add_fields` processor when input event.Fields is a nil map. {pull}28219[28219]
 
 ==== Added
 
@@ -121,3 +122,8 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update to go-concert 0.2.0 {pull}27162[27162]
 - Update Go version to 1.16.5. {issue}26182[26182] {pull}26186[26186]
 - Introduce `libbeat/beat.Beat.OutputConfigReloader` {pull}28048[28048]
+- Update Go version to 1.17.1. {pull}27543[27543]
+
+==== Deprecated
+
+- Deprecated the `common.Float` type. {issue}28279[28279] {pull}28280[28280]

CHANGELOG.next.asciidoc

@@ -23,6 +23,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove deprecated/undocumented IncludeCreatorMetadata setting from kubernetes metadata config options {pull}28006[28006]
 - Remove deprecated fields from kubernetes module {pull}28046[28046]
 - Remove deprecated config option aws_partition. {pull}28120[28120]
+- Update kubernetes.namespace from keyword to group field and add name, labels, annotations, uuid as its fields {pull}27917[27917]
 
 *Auditbeat*
 
@@ -92,6 +93,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove deprecated fields in Kafka module. {pull}27938[27938]
 - Remove deprecated fields in coredns module. {pull}28196[28196]
 - Remove old `httpjson` config implementation. {pull}28054[28054]
+- Fix initialization of http client in Cloudfoundry input. {issue}28271[28271] {pull}28277[28277]
 
 *Heartbeat*
 
@@ -116,6 +118,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove deprecated fields in Kafka module. {pull}27938[27938]
 - Remove deprecated config option default_region from aws module. {pull}28120[28120]
 - Remove deprecated config option perfmon.counters from windows/perfmon metricset. {pull}28282[28282]
+- Remove deprecated fields in Redis module. {issue}11835[11835] {pull}28246[28246]
 
 *Packetbeat*
 
@@ -139,6 +142,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Functionbeat*
 
+- Support for Google Cloud Functions have been removed, as it has been in Beta for a long time and been broken
+for a few releases. Please use other tools provided by Elastic to fetch data from GCP (e.g. Filebeat).
+
 
 ==== Bugfixes
 
@@ -210,6 +216,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Periodic metrics in logs will now report `libbeat.output.events.active` and `beat.memstats.rss`
   as gauges (rather than counters). {pull}22877[22877]
 - Beats dashboards use custom index when `setup.dashboards.index` is set. {issue}21232[21232] {pull}27901[27901]
+- Fix handling of float data types within processors. {issue}28279[28279] {pull}28280[28280]
+- Allow `clone3` syscall in seccomp filters. {pull}28117[28117]
 
 *Auditbeat*
 
@@ -307,6 +315,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update indentation for azure filebeat configuration. {pull}26604[26604]
 - Update Sophos xg module pipeline to deal with missing `date` and `time` fields. {pull}27834[27834]
 - sophos/xg fileset: Add missing pipeline for System Health logs. {pull}27827[27827] {issue}27826[27826]
+- Resolve issue with @timestamp for defender_atp. {pull}28272[28272]
+- Tolerate faults when Windows Event Log session is interrupted {issue}27947[27947] {pull}28191[28191]
+- Add support for username in cisco asa security negotiation logs {pull}26975[26975]
+- Relax time parsing and capture group and session type in Cisco ASA module {issue}24710[24710] {pull}28325[28325]
 
 *Heartbeat*
 
@@ -415,6 +427,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Packetbeat*
 
+- Handle truncated DNS records more gracefully. {issue}21495[21495] {pull}28297[28297]
 
 *Winlogbeat*
 
@@ -423,6 +436,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add source.ip validation for event ID 4778 in the Security module. {issue}19627[19627]
 - Protect against accessing undefined variables in Sysmon module. {issue}22219[22219] {pull}22236[22236]
 - Protect against accessing an undefined variable in Security module. {pull}22937[22937]
+- Tolerate faults when Windows Event Log session is interrupted {issue}27947[27947] {pull}28191[28191]
 
 *Functionbeat*
 
@@ -741,6 +755,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Make aws-cloudwatch input GA. {pull}28161[28161]
 - Move processing to ingest node for AWS vpcflow fileset. {pull}28168[28168]
 - Release zoom module as GA. {pull}28106[28106]
+- Add support for secondary object attribute handling in ThreatIntel MISP module {pull}28124[28124]
 
 *Heartbeat*
 

Jenkinsfile

@@ -460,13 +460,15 @@ def tagAndPush(Map args = [:]) {
   def variants = ["", "-oss", "-ubi8"]
 
   if(beatName == 'elastic-agent'){
-      variants.add("-complete")
-      variants.add("-cloud")
+    variants.add("-complete")
+    variants.add("-cloud")
   }
 
   variants.each { variant ->
+    // cloud docker images are stored in the private docker namespace.
+    def sourceNamespace = variant.equals('-cloud') ? 'beats-ci' : 'beats'
     tags.each { tag ->
-      doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}")
+      doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}", sourceNamespace: sourceNamespace)
     }
   }
 }
@@ -482,7 +484,8 @@ def doTagAndPush(Map args = [:]) {
   def variant = args.variant
   def sourceTag = args.sourceTag
   def targetTag = args.targetTag
-  def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
+  def sourceNamespace = args.sourceNamespace
+  def sourceName = "${DOCKER_REGISTRY}/${sourceNamespace}/${beatName}${variant}:${sourceTag}"
   def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"
 
   def iterations = 0