automate the ironbank docker context generation (#679) (#751)

(cherry picked from commit b0e9623)

Co-authored-by: Victor Martinez <[email protected]>

.ci/Jenkinsfile

@@ -175,7 +175,7 @@ pipeline {
                 withMageEnv(){
                   dir("${BASE_DIR}"){
                     withPackageEnv("${PLATFORM}") {
-                      cmd(label: 'Go package', script: 'mage package')
+                      cmd(label: 'Go package', script: 'mage package ironbank')
                       uploadPackagesToGoogleBucket(
                         credentialsId: env.JOB_GCS_EXT_CREDENTIALS,
                         repo: env.REPO,

.gitignore

@@ -62,4 +62,3 @@ internal/pkg/agent/transpiler/tests/exec-1.0-darwin-x86_64/exec
 
 # VSCode
 /.vscode
-

dev-tools/packaging/files/ironbank/config/docker-entrypoint

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# For information on the possible environment variables that can be passed into the container. Run the following
+# command for information on the options that are available.
+#
+# `./elastic-agent container --help`
+#
+
+elastic-agent container "$@"

dev-tools/packaging/templates/ironbank/Dockerfile.tmpl

@@ -0,0 +1,90 @@
+################################################################################
+# Build stage 0
+# Extract Elastic Agent and make various file manipulations.
+################################################################################
+ARG BASE_REGISTRY=registry1.dsop.io
+ARG BASE_IMAGE=ironbank/redhat/ubi/ubi8
+ARG BASE_TAG=8.6
+
+FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} as prep_files
+
+ARG ELASTIC_STACK={{ beat_version }}
+ARG ELASTIC_PRODUCT=elastic-agent
+ARG OS_AND_ARCH=linux-x86_64
+
+RUN mkdir /usr/share/${ELASTIC_PRODUCT}
+WORKDIR /usr/share/${ELASTIC_PRODUCT}
+COPY --chown=1000:0 ${ELASTIC_PRODUCT}-${ELASTIC_STACK}-${OS_AND_ARCH}.tar.gz  .
+RUN tar --strip-components=1 -zxf ${ELASTIC_PRODUCT}-${ELASTIC_STACK}-${OS_AND_ARCH}.tar.gz \
+  && rm  ${ELASTIC_PRODUCT}-${ELASTIC_STACK}-${OS_AND_ARCH}.tar.gz
+
+# Support arbitrary user ids
+# Ensure that group permissions are the same as user permissions.
+# This will help when relying on GID-0 to run Kibana, rather than UID-1000.
+# OpenShift does this, for example.
+# REF: https://docs.okd.io/latest/openshift_images/create-images.html
+RUN chmod -R g=u /usr/share/${ELASTIC_PRODUCT}
+
+# Create auxiliary folders and assigning default permissions.
+RUN mkdir -p /usr/share/${ELASTIC_PRODUCT}/data /usr/share/${ELASTIC_PRODUCT}/logs && \
+    chown -R root:root /usr/share/${ELASTIC_PRODUCT} && \
+    find /usr/share/${ELASTIC_PRODUCT} -type d -exec chmod 0750 {} \; && \
+    find /usr/share/${ELASTIC_PRODUCT} -type f -exec chmod 0640 {} \; && \
+    chmod 0750 /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT} && \
+    chmod 0770 /usr/share/${ELASTIC_PRODUCT}/data /usr/share/${ELASTIC_PRODUCT}/logs
+
+################################################################################
+# Build stage 1
+# Copy prepared files from the previous stage and complete the image.
+################################################################################
+FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}
+
+ARG ELASTIC_PRODUCT=elastic-agent
+
+COPY LICENSE /licenses/elastic-${ELASTIC_PRODUCT}
+
+# Add a dumb init process
+COPY tinit /tinit
+RUN chmod +x /tinit
+
+# Bring in product from the initial stage.
+COPY --from=prep_files --chown=1000:0 /usr/share/${ELASTIC_PRODUCT} /usr/share/${ELASTIC_PRODUCT}
+WORKDIR /usr/share/${ELASTIC_PRODUCT}
+RUN ln -s /usr/share/${ELASTIC_PRODUCT} /opt/${ELASTIC_PRODUCT}
+
+ENV ELASTIC_CONTAINER="true"
+RUN ln -s /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT} /usr/bin/${ELASTIC_PRODUCT}
+
+# Support arbitrary user ids
+# Ensure gid 0 write permissions for OpenShift.
+RUN chmod -R g+w /usr/share/${ELASTIC_PRODUCT}
+
+# config file ("${ELASTIC_PRODUCT}.yml") can only be writable by the root and group root
+# it is needed on some configurations where the container needs to run as root
+RUN chown root:root /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT}.yml \
+  && chmod go-w /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT}.yml
+
+# Remove the suid bit everywhere to mitigate "Stack Clash"
+RUN find / -xdev -perm -4000 -exec chmod u-s {} +
+
+# Provide a non-root user to run the process.
+RUN groupadd --gid 1000 ${ELASTIC_PRODUCT} && useradd --uid 1000 --gid 1000 --groups 0 --home-dir /usr/share/${ELASTIC_PRODUCT} --no-create-home ${ELASTIC_PRODUCT}
+
+# Elastic Agent permissions
+RUN find /usr/share//elastic-agent/data -type d -exec chmod 0770 {} \; && \
+    find /usr/share//elastic-agent/data -type f -exec chmod 0660 {} \; && \
+    chmod +x /usr/share//elastic-agent/data/elastic-agent-*/elastic-agent
+
+COPY jq /usr/local/bin
+RUN chown root:root /usr/local/bin/jq && chmod 0755 /usr/local/bin/jq
+
+COPY config/docker-entrypoint /usr/local/bin/docker-entrypoint
+RUN chmod 755 /usr/local/bin/docker-entrypoint
+
+USER ${ELASTIC_PRODUCT}
+ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT}
+
+ENTRYPOINT ["/tinit", "--", "/usr/local/bin/docker-entrypoint"]
+CMD [""]
+
+HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD test -w '/tmp/elastic-agent/elastic-agent.sock'

dev-tools/packaging/templates/ironbank/README.md.tmpl

@@ -0,0 +1,43 @@
+# elastic-agent
+
+**elastic-agent** is a single, unified way to add monitoring for logs, metrics, and other types of data to each host. A single agent makes it easier and faster to deploy monitoring across your infrastructure. The agent’s single, unified configuration makes it easier to add integrations for new data sources.
+
+For more information about elastic-agent, please visit
+https://www.elastic.co/guide/en/ingest-management/7.17/index.html.
+
+---
+
+**NOTE**
+
+This functionality is in beta and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Beta features are not subject to the support SLA of official GA features.
+
+---
+
+### Installation instructions
+
+Please follow the documentation on [Quick start](https://www.elastic.co/guide/en/fleet/{{ .MajorMinor }}/fleet-elastic-agent-quick-start.html).
+
+### Where to file issues and PRs
+
+- [Issues](https://github.com/elastic/elastic-agent/issues)
+- [PRs](https://github.com/elastic/elastic-agent/pulls)
+
+### DoD Restrictions
+
+### Where to get help
+
+- [elastic-agent Discuss Forums](https://discuss.elastic.co/tags/c/elastic-stack/beats/28/elastic-agent)
+- [elastic-agent Documentation](https://www.elastic.co/guide/en/ingest-management/current/index.html)
+
+### Still need help?
+
+You can learn more about the Elastic Community and also understand how to get more help
+visiting [Elastic Community](https://www.elastic.co/community).
+
+This software is governed by the [Elastic
+License](https://github.com/elastic/beats/blob/{{ .MajorMinor }}/licenses/ELASTIC-LICENSE.txt),
+and includes the full set of [free
+features](https://www.elastic.co/subscriptions).
+
+View the detailed release notes
+[here](https://www.elastic.co/guide/en/beats/libbeat/current/release-notes-{{ beat_version }}.html).

dev-tools/packaging/templates/ironbank/hardening_manifest.yaml.tmpl

@@ -0,0 +1,68 @@
+---
+apiVersion: v1
+
+# The repository name in registry1, excluding /ironbank/
+name: "elastic/beats/elastic-agent"
+
+# List of tags to push for the repository in registry1
+# The most specific version should be the first tag and will be shown
+# on ironbank.dsop.io
+tags:
+- "{{ beat_version }}"
+- "latest"
+
+# Build args passed to Dockerfile ARGs
+args:
+  BASE_IMAGE: "redhat/ubi/ubi8"
+  BASE_TAG: "8.6"
+  ELASTIC_STACK: "{{ beat_version }}"
+  ELASTIC_PRODUCT: "elastic-agent"
+
+# Docker image labels
+labels:
+  org.opencontainers.image.title: "elastic-agent"
+  ## Human-readable description of the software packaged in the image
+  org.opencontainers.image.description: "elastic-agent is a single, unified way to add monitoring for logs, metrics, and other types of data to each host"
+  ## License(s) under which contained software is distributed
+  org.opencontainers.image.licenses: "Elastic License"
+  ## URL to find more information on the image
+  org.opencontainers.image.url: "https://www.elastic.co/products/beats/elastic-agent"
+  ## Name of the distributing entity, organization or individual
+  org.opencontainers.image.vendor: "Elastic"
+  org.opencontainers.image.version: "{{ beat_version }}"
+  ## Keywords to help with search (ex. "cicd,gitops,golang")
+  mil.dso.ironbank.image.keywords: "log,metrics,monitoring,observabilty,o11y,oblt,beats,elastic,elasticsearch,golang"
+  ## This value can be "opensource" or "commercial"
+  mil.dso.ironbank.image.type: "commercial"
+  ## Product the image belongs to for grouping multiple images
+  mil.dso.ironbank.product.name: "beats"
+
+# List of resources to make available to the offline build context
+resources:
+  - filename: "elastic-agent-{{ beat_version }}-linux-x86_64.tar.gz"
+    url: "<artifact_path>/elastic-agent-{{ beat_version }}-linux-x86_64.tar.gz"
+    validation:
+      type: "sha512"
+      value: "<insert SHA 512 here>"
+  - filename: tinit
+    url: https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64
+    validation:
+      type: sha256
+      value: 93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c
+  - filename: jq
+    url: https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+    validation:
+      type: sha256
+      value: af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44
+
+# List of project maintainers
+maintainers:
+  - email: "[email protected]"
+    name: "Nassim Kammah"
+    username: "nassim.kammah"
+  - email: "[email protected]"
+    name: "Ivan Fernandez Calvo"
+    username: "ivan.fernandez"
+  - email: "[email protected]"
+    name: "Victor Martinez"
+    username: "victor.martinez"

magefile.go

@@ -827,3 +827,111 @@ func injectBuildVars(m map[string]string) {
 		m[k] = v
 	}
 }
+
+// Package packages elastic-agent for the IronBank distribution, relying on the
+// binaries having already been built.
+//
+// Use SNAPSHOT=true to build snapshots.
+func Ironbank() error {
+	if runtime.GOARCH != "amd64" {
+		fmt.Printf(">> IronBank images are only supported for amd64 arch (%s is not supported)\n", runtime.GOARCH)
+		return nil
+	}
+	if err := prepareIronbankBuild(); err != nil {
+		return errors.Wrap(err, "failed to prepare the IronBank context")
+	}
+	if err := saveIronbank(); err != nil {
+		return errors.Wrap(err, "failed to save artifacts for IronBank")
+	}
+	return nil
+}
+
+func saveIronbank() error {
+	fmt.Println(">> saveIronbank: save the IronBank container context.")
+
+	ironbank := getIronbankContextName()
+	buildDir := filepath.Join("build", ironbank)
+	if _, err := os.Stat(buildDir); os.IsNotExist(err) {
+		return fmt.Errorf("cannot find the folder with the ironbank context: %+v", err)
+	}
+
+	distributionsDir := "build/distributions"
+	if _, err := os.Stat(distributionsDir); os.IsNotExist(err) {
+		err := os.MkdirAll(distributionsDir, 0750)
+		if err != nil {
+			return fmt.Errorf("cannot create folder for docker artifacts: %+v", err)
+		}
+	}
+
+	// change dir to the buildDir location where the ironbank folder exists
+	// this will generate a tar.gz without some nested folders.
+	wd, _ := os.Getwd()
+	os.Chdir(buildDir)
+	defer os.Chdir(wd)
+
+	// move the folder to the parent folder, there are two parent folder since
+	// buildDir contains a two folders dir.
+	tarGzFile := filepath.Join("..", "..", distributionsDir, ironbank+".tar.gz")
+
+	// Save the build context as tar.gz artifact
+	err := devtools.Tar("./", tarGzFile)
+	if err != nil {
+		return fmt.Errorf("cannot compress the tar.gz file: %+v", err)
+	}
+
+	return errors.Wrap(devtools.CreateSHA512File(tarGzFile), "failed to create .sha512 file")
+}
+
+func getIronbankContextName() string {
+	version, _ := devtools.BeatQualifiedVersion()
+	defaultBinaryName := "{{.Name}}-ironbank-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}"
+	outputDir, _ := devtools.Expand(defaultBinaryName+"-docker-build-context", map[string]interface{}{
+		"Name":    "elastic-agent",
+		"Version": version,
+	})
+	return outputDir
+}
+
+func prepareIronbankBuild() error {
+	fmt.Println(">> prepareIronbankBuild: prepare the IronBank container context.")
+	buildDir := filepath.Join("build", getIronbankContextName())
+	templatesDir := filepath.Join("dev-tools", "packaging", "templates", "ironbank")
+
+	data := map[string]interface{}{
+		"MajorMinor": majorMinor(),
+	}
+
+	err := filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {
+		if !info.IsDir() {
+			target := strings.TrimSuffix(
+				filepath.Join(buildDir, filepath.Base(path)),
+				".tmpl",
+			)
+
+			err := devtools.ExpandFile(path, target, data)
+			if err != nil {
+				return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target)
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("cannot create templates for the IronBank: %+v", err)
+	}
+
+	// copy files
+	sourcePath := filepath.Join("dev-tools", "packaging", "files", "ironbank")
+	if err := devtools.Copy(sourcePath, buildDir); err != nil {
+		return fmt.Errorf("cannot create files for the IronBank: %+v", err)
+	}
+	return nil
+}
+
+func majorMinor() string {
+	if v, _ := devtools.BeatQualifiedVersion(); v != "" {
+		parts := strings.SplitN(v, ".", 3)
+		return parts[0] + "." + parts[1]
+	}
+	return ""
+}