From b67d2f1f9a0c6ce51070b233119b5803cc5c2f0b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 6 Jul 2022 18:10:25 +0100 Subject: [PATCH 1/7] ironbank: create docker build context --- .../packaging/templates/ironbank/Dockerfile | 90 ++++++ .../packaging/templates/ironbank/LICENSE | 280 ++++++++++++++++++ .../packaging/templates/ironbank/README.md | 43 +++ .../templates/ironbank/docker-entrypoint | 11 + .../ironbank/hardening_manifest.yaml | 68 +++++ magefile.go | 184 ++++++++++++ 6 files changed, 676 insertions(+) create mode 100644 dev-tools/packaging/templates/ironbank/Dockerfile create mode 100644 dev-tools/packaging/templates/ironbank/LICENSE create mode 100644 dev-tools/packaging/templates/ironbank/README.md create mode 100644 dev-tools/packaging/templates/ironbank/docker-entrypoint create mode 100644 dev-tools/packaging/templates/ironbank/hardening_manifest.yaml diff --git a/dev-tools/packaging/templates/ironbank/Dockerfile b/dev-tools/packaging/templates/ironbank/Dockerfile new file mode 100644 index 00000000000..a092320a720 --- /dev/null +++ b/dev-tools/packaging/templates/ironbank/Dockerfile @@ -0,0 +1,90 @@ +################################################################################ +# Build stage 0 +# Extract Elastic Agent and make various file manipulations. +################################################################################ +ARG BASE_REGISTRY=registry1.dsop.io +ARG BASE_IMAGE=ironbank/redhat/ubi/ubi8 +ARG BASE_TAG=8.6 + +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} as prep_files + +ARG ELASTIC_STACK={{ beat_version }} +ARG ELASTIC_PRODUCT=elastic-agent +ARG OS_AND_ARCH=linux-x86_64 + +RUN mkdir /usr/share/${ELASTIC_PRODUCT} +WORKDIR /usr/share/${ELASTIC_PRODUCT} +COPY --chown=1000:0 ${ELASTIC_PRODUCT}-${ELASTIC_STACK}-${OS_AND_ARCH}.tar.gz . +RUN tar --strip-components=1 -zxf ${ELASTIC_PRODUCT}-${ELASTIC_STACK}-${OS_AND_ARCH}.tar.gz \ + && rm ${ELASTIC_PRODUCT}-${ELASTIC_STACK}-${OS_AND_ARCH}.tar.gz + +# Support arbitrary user ids +# Ensure that group permissions are the same as user permissions. +# This will help when relying on GID-0 to run Kibana, rather than UID-1000. +# OpenShift does this, for example. +# REF: https://docs.okd.io/latest/openshift_images/create-images.html +RUN chmod -R g=u /usr/share/${ELASTIC_PRODUCT} + +# Create auxiliar folders and assing default permissions. +RUN mkdir -p /usr/share/${ELASTIC_PRODUCT}/data /usr/share/${ELASTIC_PRODUCT}/logs && \ + chown -R root:root /usr/share/${ELASTIC_PRODUCT} && \ + find /usr/share/${ELASTIC_PRODUCT} -type d -exec chmod 0750 {} \; && \ + find /usr/share/${ELASTIC_PRODUCT} -type f -exec chmod 0640 {} \; && \ + chmod 0750 /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT} && \ + chmod 0770 /usr/share/${ELASTIC_PRODUCT}/data /usr/share/${ELASTIC_PRODUCT}/logs + +################################################################################ +# Build stage 1 +# Copy prepared files from the previous stage and complete the image. +################################################################################ +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} + +ARG ELASTIC_PRODUCT=elastic-agent + +COPY LICENSE /licenses/elastic-${ELASTIC_PRODUCT} + +# Add a dumb init process +COPY tinit /tinit +RUN chmod +x /tinit + +# Bring in product from the initial stage. +COPY --from=prep_files --chown=1000:0 /usr/share/${ELASTIC_PRODUCT} /usr/share/${ELASTIC_PRODUCT} +WORKDIR /usr/share/${ELASTIC_PRODUCT} +RUN ln -s /usr/share/${ELASTIC_PRODUCT} /opt/${ELASTIC_PRODUCT} + +ENV ELASTIC_CONTAINER="true" +RUN ln -s /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT} /usr/bin/${ELASTIC_PRODUCT} + +# Support arbitrary user ids +# Ensure gid 0 write permissions for OpenShift. +RUN chmod -R g+w /usr/share/${ELASTIC_PRODUCT} + +# config file ("${ELASTIC_PRODUCT}.yml") can only be writable by the root and group root +# it is needed on some configurations where the container needs to run as root +RUN chown root:root /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT}.yml \ + && chmod go-w /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT}.yml + +# Remove the suid bit everywhere to mitigate "Stack Clash" +RUN find / -xdev -perm -4000 -exec chmod u-s {} + + +# Provide a non-root user to run the process. +RUN groupadd --gid 1000 ${ELASTIC_PRODUCT} && useradd --uid 1000 --gid 1000 --groups 0 --home-dir /usr/share/${ELASTIC_PRODUCT} --no-create-home ${ELASTIC_PRODUCT} + +# Elastic Agent permissions +RUN find /usr/share//elastic-agent/data -type d -exec chmod 0770 {} \; && \ + find /usr/share//elastic-agent/data -type f -exec chmod 0660 {} \; && \ + chmod +x /usr/share//elastic-agent/data/elastic-agent-*/elastic-agent + +COPY jq /usr/local/bin +RUN chown root:root /usr/local/bin/jq && chmod 0755 /usr/local/bin/jq + +COPY docker-entrypoint /usr/local/bin/docker-entrypoint +RUN chmod 755 /usr/local/bin/docker-entrypoint + +USER ${ELASTIC_PRODUCT} +ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT} + +ENTRYPOINT ["/tinit", "--", "/usr/local/bin/docker-entrypoint"] +CMD [""] + +HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD test -w '/tmp/elastic-agent/elastic-agent.sock' diff --git a/dev-tools/packaging/templates/ironbank/LICENSE b/dev-tools/packaging/templates/ironbank/LICENSE new file mode 100644 index 00000000000..ef2739c152e --- /dev/null +++ b/dev-tools/packaging/templates/ironbank/LICENSE @@ -0,0 +1,280 @@ +ELASTIC LICENSE AGREEMENT + +PLEASE READ CAREFULLY THIS ELASTIC LICENSE AGREEMENT (THIS "AGREEMENT"), WHICH +CONSTITUTES A LEGALLY BINDING AGREEMENT AND GOVERNS ALL OF YOUR USE OF ALL OF +THE ELASTIC SOFTWARE WITH WHICH THIS AGREEMENT IS INCLUDED ("ELASTIC SOFTWARE") +THAT IS PROVIDED IN OBJECT CODE FORMAT, AND, IN ACCORDANCE WITH SECTION 2 BELOW, +CERTAIN OF THE ELASTIC SOFTWARE THAT IS PROVIDED IN SOURCE CODE FORMAT. BY +INSTALLING OR USING ANY OF THE ELASTIC SOFTWARE GOVERNED BY THIS AGREEMENT, YOU +ARE ASSENTING TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE +WITH SUCH TERMS AND CONDITIONS, YOU MAY NOT INSTALL OR USE THE ELASTIC SOFTWARE +GOVERNED BY THIS AGREEMENT. IF YOU ARE INSTALLING OR USING THE SOFTWARE ON +BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE ACTUAL +AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT ON BEHALF OF +SUCH ENTITY. + +Posted Date: April 20, 2018 + +This Agreement is entered into by and between Elasticsearch BV ("Elastic") and +You, or the legal entity on behalf of whom You are acting (as applicable, +"You"). + +1. OBJECT CODE END USER LICENSES, RESTRICTIONS AND THIRD PARTY OPEN SOURCE +SOFTWARE + + 1.1 Object Code End User License. Subject to the terms and conditions of + Section 1.2 of this Agreement, Elastic hereby grants to You, AT NO CHARGE and + for so long as you are not in breach of any provision of this Agreement, a + License to the Basic Features and Functions of the Elastic Software. + + 1.2 Reservation of Rights; Restrictions. As between Elastic and You, Elastic + and its licensors own all right, title and interest in and to the Elastic + Software, and except as expressly set forth in Sections 1.1, and 2.1 of this + Agreement, no other license to the Elastic Software is granted to You under + this Agreement, by implication, estoppel or otherwise. You agree not to: (i) + reverse engineer or decompile, decrypt, disassemble or otherwise reduce any + Elastic Software provided to You in Object Code, or any portion thereof, to + Source Code, except and only to the extent any such restriction is prohibited + by applicable law, (ii) except as expressly permitted in this Agreement, + prepare derivative works from, modify, copy or use the Elastic Software Object + Code or the Commercial Software Source Code in any manner; (iii) except as + expressly permitted in Section 1.1 above, transfer, sell, rent, lease, + distribute, sublicense, loan or otherwise transfer, Elastic Software Object + Code, in whole or in part, to any third party; (iv) use Elastic Software + Object Code for providing time-sharing services, any software-as-a-service, + service bureau services or as part of an application services provider or + other service offering (collectively, "SaaS Offering") where obtaining access + to the Elastic Software or the features and functions of the Elastic Software + is a primary reason or substantial motivation for users of the SaaS Offering + to access and/or use the SaaS Offering ("Prohibited SaaS Offering"); (v) + circumvent the limitations on use of Elastic Software provided to You in + Object Code format that are imposed or preserved by any License Key, or (vi) + alter or remove any Marks and Notices in the Elastic Software. If You have any + question as to whether a specific SaaS Offering constitutes a Prohibited SaaS + Offering, or are interested in obtaining Elastic's permission to engage in + commercial or non-commercial distribution of the Elastic Software, please + contact elastic_license@elastic.co. + + 1.3 Third Party Open Source Software. The Commercial Software may contain or + be provided with third party open source libraries, components, utilities and + other open source software (collectively, "Open Source Software"), which Open + Source Software may have applicable license terms as identified on a website + designated by Elastic. Notwithstanding anything to the contrary herein, use of + the Open Source Software shall be subject to the license terms and conditions + applicable to such Open Source Software, to the extent required by the + applicable licensor (which terms shall not restrict the license rights granted + to You hereunder, but may contain additional rights). To the extent any + condition of this Agreement conflicts with any license to the Open Source + Software, the Open Source Software license will govern with respect to such + Open Source Software only. Elastic may also separately provide you with + certain open source software that is licensed by Elastic. Your use of such + Elastic open source software will not be governed by this Agreement, but by + the applicable open source license terms. + +2. COMMERCIAL SOFTWARE SOURCE CODE + + 2.1 Limited License. Subject to the terms and conditions of Section 2.2 of + this Agreement, Elastic hereby grants to You, AT NO CHARGE and for so long as + you are not in breach of any provision of this Agreement, a limited, + non-exclusive, non-transferable, fully paid up royalty free right and license + to the Commercial Software in Source Code format, without the right to grant + or authorize sublicenses, to prepare Derivative Works of the Commercial + Software, provided You (i) do not hack the licensing mechanism, or otherwise + circumvent the intended limitations on the use of Elastic Software to enable + features other than Basic Features and Functions or those features You are + entitled to as part of a Subscription, and (ii) use the resulting object code + only for reasonable testing purposes. + + 2.2 Restrictions. Nothing in Section 2.1 grants You the right to (i) use the + Commercial Software Source Code other than in accordance with Section 2.1 + above, (ii) use a Derivative Work of the Commercial Software outside of a + Non-production Environment, in any production capacity, on a temporary or + permanent basis, or (iii) transfer, sell, rent, lease, distribute, sublicense, + loan or otherwise make available the Commercial Software Source Code, in whole + or in part, to any third party. Notwithstanding the foregoing, You may + maintain a copy of the repository in which the Source Code of the Commercial + Software resides and that copy may be publicly accessible, provided that you + include this Agreement with Your copy of the repository. + +3. TERMINATION + + 3.1 Termination. This Agreement will automatically terminate, whether or not + You receive notice of such Termination from Elastic, if You breach any of its + provisions. + + 3.2 Post Termination. Upon any termination of this Agreement, for any reason, + You shall promptly cease the use of the Elastic Software in Object Code format + and cease use of the Commercial Software in Source Code format. For the + avoidance of doubt, termination of this Agreement will not affect Your right + to use Elastic Software, in either Object Code or Source Code formats, made + available under the Apache License Version 2.0. + + 3.3 Survival. Sections 1.2, 2.2. 3.3, 4 and 5 shall survive any termination or + expiration of this Agreement. + +4. DISCLAIMER OF WARRANTIES AND LIMITATION OF LIABILITY + + 4.1 Disclaimer of Warranties. TO THE MAXIMUM EXTENT PERMITTED UNDER APPLICABLE + LAW, THE ELASTIC SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, + AND ELASTIC AND ITS LICENSORS MAKE NO WARRANTIES WHETHER EXPRESSED, IMPLIED OR + STATUTORY REGARDING OR RELATING TO THE ELASTIC SOFTWARE. TO THE MAXIMUM EXTENT + PERMITTED UNDER APPLICABLE LAW, ELASTIC AND ITS LICENSORS SPECIFICALLY + DISCLAIM ALL IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE AND NON-INFRINGEMENT WITH RESPECT TO THE ELASTIC SOFTWARE, AND WITH + RESPECT TO THE USE OF THE FOREGOING. FURTHER, ELASTIC DOES NOT WARRANT RESULTS + OF USE OR THAT THE ELASTIC SOFTWARE WILL BE ERROR FREE OR THAT THE USE OF THE + ELASTIC SOFTWARE WILL BE UNINTERRUPTED. + + 4.2 Limitation of Liability. IN NO EVENT SHALL ELASTIC OR ITS LICENSORS BE + LIABLE TO YOU OR ANY THIRD PARTY FOR ANY DIRECT OR INDIRECT DAMAGES, + INCLUDING, WITHOUT LIMITATION, FOR ANY LOSS OF PROFITS, LOSS OF USE, BUSINESS + INTERRUPTION, LOSS OF DATA, COST OF SUBSTITUTE GOODS OR SERVICES, OR FOR ANY + SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES OF ANY KIND, IN CONNECTION WITH + OR ARISING OUT OF THE USE OR INABILITY TO USE THE ELASTIC SOFTWARE, OR THE + PERFORMANCE OF OR FAILURE TO PERFORM THIS AGREEMENT, WHETHER ALLEGED AS A + BREACH OF CONTRACT OR TORTIOUS CONDUCT, INCLUDING NEGLIGENCE, EVEN IF ELASTIC + HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +5. MISCELLANEOUS + + This Agreement completely and exclusively states the entire agreement of the + parties regarding the subject matter herein, and it supersedes, and its terms + govern, all prior proposals, agreements, or other communications between the + parties, oral or written, regarding such subject matter. This Agreement may be + modified by Elastic from time to time, and any such modifications will be + effective upon the "Posted Date" set forth at the top of the modified + Agreement. If any provision hereof is held unenforceable, this Agreement will + continue without said provision and be interpreted to reflect the original + intent of the parties. This Agreement and any non-contractual obligation + arising out of or in connection with it, is governed exclusively by Dutch law. + This Agreement shall not be governed by the 1980 UN Convention on Contracts + for the International Sale of Goods. All disputes arising out of or in + connection with this Agreement, including its existence and validity, shall be + resolved by the courts with jurisdiction in Amsterdam, The Netherlands, except + where mandatory law provides for the courts at another location in The + Netherlands to have jurisdiction. The parties hereby irrevocably waive any and + all claims and defenses either might otherwise have in any such action or + proceeding in any of such courts based upon any alleged lack of personal + jurisdiction, improper venue, forum non conveniens or any similar claim or + defense. A breach or threatened breach, by You of Section 2 may cause + irreparable harm for which damages at law may not provide adequate relief, and + therefore Elastic shall be entitled to seek injunctive relief without being + required to post a bond. You may not assign this Agreement (including by + operation of law in connection with a merger or acquisition), in whole or in + part to any third party without the prior written consent of Elastic, which + may be withheld or granted by Elastic in its sole and absolute discretion. + Any assignment in violation of the preceding sentence is void. Notices to + Elastic may also be sent to legal@elastic.co. + +6. DEFINITIONS + + The following terms have the meanings ascribed: + + 6.1 "Affiliate" means, with respect to a party, any entity that controls, is + controlled by, or which is under common control with, such party, where + "control" means ownership of at least fifty percent (50%) of the outstanding + voting shares of the entity, or the contractual right to establish policy for, + and manage the operations of, the entity. + + 6.2 "Basic Features and Functions" means those features and functions of the + Elastic Software that are eligible for use under a Basic license, as set forth + at https://www.elastic.co/subscriptions, as may be modified by Elastic from + time to time. + + 6.3 "Commercial Software" means the Elastic Software Source Code in any file + containing a header stating the contents are subject to the Elastic License or + which is contained in the repository folder labeled "x-pack", unless a LICENSE + file present in the directory subtree declares a different license. + + 6.4 "Derivative Work of the Commercial Software" means, for purposes of this + Agreement, any modification(s) or enhancement(s) to the Commercial Software, + which represent, as a whole, an original work of authorship. + + 6.5 "License" means a limited, non-exclusive, non-transferable, fully paid up, + royalty free, right and license, without the right to grant or authorize + sublicenses, solely for Your internal business operations to (i) install and + use the applicable Features and Functions of the Elastic Software in Object + Code, and (ii) permit Contractors and Your Affiliates to use the Elastic + software as set forth in (i) above, provided that such use by Contractors must + be solely for Your benefit and/or the benefit of Your Affiliates, and You + shall be responsible for all acts and omissions of such Contractors and + Affiliates in connection with their use of the Elastic software that are + contrary to the terms and conditions of this Agreement. + + 6.6 "License Key" means a sequence of bytes, including but not limited to a + JSON blob, that is used to enable certain features and functions of the + Elastic Software. + + 6.7 "Marks and Notices" means all Elastic trademarks, trade names, logos and + notices present on the Documentation as originally provided by Elastic. + + 6.8 "Non-production Environment" means an environment for development, testing + or quality assurance, where software is not used for production purposes. + + 6.9 "Object Code" means any form resulting from mechanical transformation or + translation of Source Code form, including but not limited to compiled object + code, generated documentation, and conversions to other media types. + + 6.10 "Source Code" means the preferred form of computer software for making + modifications, including but not limited to software source code, + documentation source, and configuration files. + + 6.11 "Subscription" means the right to receive Support Services and a License + to the Commercial Software. + + +GOVERNMENT END USER ADDENDUM TO THE ELASTIC LICENSE AGREEMENT + + This ADDENDUM TO THE ELASTIC LICENSE AGREEMENT (this "Addendum") applies +only to U.S. Federal Government, State Government, and Local Government +entities ("Government End Users") of the Elastic Software. This Addendum is +subject to, and hereby incorporated into, the Elastic License Agreement, +which is being entered into as of even date herewith, by Elastic and You (the +"Agreement"). This Addendum sets forth additional terms and conditions +related to Your use of the Elastic Software. Capitalized terms not defined in +this Addendum have the meaning set forth in the Agreement. + + 1. LIMITED LICENSE TO DISTRIBUTE (DSOP ONLY). Subject to the terms and +conditions of the Agreement (including this Addendum), Elastic grants the +Department of Defense Enterprise DevSecOps Initiative (DSOP) a royalty-free, +non-exclusive, non-transferable, limited license to reproduce and distribute +the Elastic Software solely through a software distribution repository +controlled and managed by DSOP, provided that DSOP: (i) distributes the +Elastic Software complete and unmodified, inclusive of the Agreement +(including this Addendum) and (ii) does not remove or alter any proprietary +legends or notices contained in the Elastic Software. + + 2. CHOICE OF LAW. The choice of law and venue provisions set forth shall +prevail over those set forth in Section 5 of the Agreement. + + "For U.S. Federal Government Entity End Users. This Agreement and any + non-contractual obligation arising out of or in connection with it, is + governed exclusively by U.S. Federal law. To the extent permitted by + federal law, the laws of the State of Delaware (excluding Delaware choice + of law rules) will apply in the absence of applicable federal law. + + For State and Local Government Entity End Users. This Agreement and any + non-contractual obligation arising out of or in connection with it, is + governed exclusively by the laws of the state in which you are located + without reference to conflict of laws. Furthermore, the Parties agree that + the Uniform Computer Information Transactions Act or any version thereof, + adopted by any state in any form ('UCITA'), shall not apply to this + Agreement and, to the extent that UCITA is applicable, the Parties agree to + opt out of the applicability of UCITA pursuant to the opt-out provision(s) + contained therein." + + 3. ELASTIC LICENSE MODIFICATION. Section 5 of the Agreement is hereby +amended to replace + + "This Agreement may be modified by Elastic from time to time, and any + such modifications will be effective upon the "Posted Date" set forth at + the top of the modified Agreement." + + with: + + "This Agreement may be modified by Elastic from time to time; provided, + however, that any such modifications shall apply only to Elastic Software + that is installed after the "Posted Date" set forth at the top of the + modified Agreement." + +V100820.0 diff --git a/dev-tools/packaging/templates/ironbank/README.md b/dev-tools/packaging/templates/ironbank/README.md new file mode 100644 index 00000000000..9b357727c8a --- /dev/null +++ b/dev-tools/packaging/templates/ironbank/README.md @@ -0,0 +1,43 @@ +# elastic-agent + +**elastic-agent** is a single, unified way to add monitoring for logs, metrics, and other types of data to each host. A single agent makes it easier and faster to deploy monitoring across your infrastructure. The agent’s single, unified configuration makes it easier to add integrations for new data sources. + +For more information about elastic-agent, please visit +https://www.elastic.co/guide/en/ingest-management/7.17/index.html. + +--- + +**NOTE** + +This functionality is in beta and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Beta features are not subject to the support SLA of official GA features. + +--- + +### Installation instructions + +Please follow the documentation on [Quick start](https://www.elastic.co/guide/en/ingest-management/7.17/ingest-management-getting-started.html). + +### Where to file issues and PRs + +- [Issues](https://github.com/elastic/beats/issues) +- [PRs](https://github.com/elastic/beats/pulls) + +### DoD Restrictions + +### Where to get help + +- [elastic-agent Discuss Forums](https://discuss.elastic.co/tags/c/elastic-stack/beats/28/elastic-agent) +- [elastic-agent Documentation](https://www.elastic.co/guide/en/ingest-management/current/index.html) + +### Still need help? + +You can learn more about the Elastic Community and also understand how to get more help +visiting [Elastic Community](https://www.elastic.co/community). + +This software is governed by the [Elastic +License](https://github.com/elastic/beats/blob/{{ .MajorMinor }}/licenses/ELASTIC-LICENSE.txt), +and includes the full set of [free +features](https://www.elastic.co/subscriptions). + +View the detailed release notes +[here](https://www.elastic.co/guide/en/beats/libbeat/current/release-notes-{{ beat_version }}.html). diff --git a/dev-tools/packaging/templates/ironbank/docker-entrypoint b/dev-tools/packaging/templates/ironbank/docker-entrypoint new file mode 100644 index 00000000000..7ebe21745f4 --- /dev/null +++ b/dev-tools/packaging/templates/ironbank/docker-entrypoint @@ -0,0 +1,11 @@ +#!/bin/bash + +set -eo pipefail + +# For information on the possible environment variables that can be passed into the container. Run the following +# command for information on the options that are available. +# +# `./elastic-agent container --help` +# + +elastic-agent container "$@" diff --git a/dev-tools/packaging/templates/ironbank/hardening_manifest.yaml b/dev-tools/packaging/templates/ironbank/hardening_manifest.yaml new file mode 100644 index 00000000000..3c753caa0fb --- /dev/null +++ b/dev-tools/packaging/templates/ironbank/hardening_manifest.yaml @@ -0,0 +1,68 @@ +--- +apiVersion: v1 + +# The repository name in registry1, excluding /ironbank/ +name: "elastic/beats/elastic-agent" + +# List of tags to push for the repository in registry1 +# The most specific version should be the first tag and will be shown +# on ironbank.dsop.io +tags: +- "{{ beat_version }}" +- "latest" + +# Build args passed to Dockerfile ARGs +args: + BASE_IMAGE: "redhat/ubi/ubi8" + BASE_TAG: "8.6" + ELASTIC_STACK: "{{ beat_version }}" + ELASTIC_PRODUCT: "elastic-agent" + +# Docker image labels +labels: + org.opencontainers.image.title: "elastic-agent" + ## Human-readable description of the software packaged in the image + org.opencontainers.image.description: "elastic-agent is a single, unified way to add monitoring for logs, metrics, and other types of data to each host" + ## License(s) under which contained software is distributed + org.opencontainers.image.licenses: "Elastic License" + ## URL to find more information on the image + org.opencontainers.image.url: "https://www.elastic.co/products/beats/elastic-agent" + ## Name of the distributing entity, organization or individual + org.opencontainers.image.vendor: "Elastic" + org.opencontainers.image.version: "{{ beat_version }}" + ## Keywords to help with search (ex. "cicd,gitops,golang") + mil.dso.ironbank.image.keywords: "log,metrics,monitoring,observabilty,o11y,oblt,beats,elastic,elasticsearch,golang" + ## This value can be "opensource" or "commercial" + mil.dso.ironbank.image.type: "commercial" + ## Product the image belongs to for grouping multiple images + mil.dso.ironbank.product.name: "beats" + +# List of resources to make available to the offline build context +resources: + - filename: "elastic-agent-{{ beat_version }}-linux-x86_64.tar.gz" + url: "/elastic-agent-{{ beat_version }}-linux-x86_64.tar.gz" + validation: + type: "sha512" + value: "" + - filename: tinit + url: https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 + validation: + type: sha256 + value: 93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c + - filename: jq + url: https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 + validation: + type: sha256 + value: af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 + +# List of project maintainers +maintainers: + - email: "nassim.kammah@elastic.co" + name: "Nassim Kammah" + username: "nassim.kammah" + - email: "ivan.fernandez@elastic.co" + name: "Ivan Fernandez Calvo" + username: "ivan.fernandez" + - email: "victor.martinez@elastic.co" + name: "Victor Martinez" + username: "victor.martinez" diff --git a/magefile.go b/magefile.go index aba39385159..8b3dac3b8b9 100644 --- a/magefile.go +++ b/magefile.go @@ -8,6 +8,8 @@ package main import ( + "archive/tar" + "compress/gzip" "context" "fmt" "io" @@ -817,3 +819,185 @@ func injectBuildVars(m map[string]string) { m[k] = v } } + +// Package packages elastic-agent for the IronBank distribution, relying on the +// binaries having already been built. +// +// Use SNAPSHOT=true to build snapshots. +func Ironbank() error { + if runtime.GOARCH != "amd64" { + fmt.Printf(">> IronBank images are only supported for amd64 arch (%s is not supported)\n", runtime.GOARCH) + return nil + } + if err := prepareIronbankBuild(); err != nil { + return errors.Wrap(err, "failed to prepare the IronBank context") + } + if err := saveIronbank(); err != nil { + return errors.Wrap(err, "failed to save artifacts for IronBank") + } + return nil +} + +func saveIronbank() error { + fmt.Println(">> saveIronbank: save the IronBank container context.") + + ironbank := getIronbankContextName() + buildDir := filepath.Join("build", "ironbank", ironbank) + if _, err := os.Stat(buildDir); os.IsNotExist(err) { + return fmt.Errorf("cannot find the folder with the ironbank context") + } + + distributionsDir := "build/distributions" + if _, err := os.Stat(distributionsDir); os.IsNotExist(err) { + err := os.MkdirAll(distributionsDir, 0750) + if err != nil { + return fmt.Errorf("cannot create folder for docker artifacts: %+v", err) + } + } + + // Save the build context as tar.gz artifact + tarGzSource, err := TarGz(buildDir, distributionsDir) + if err != nil { + return fmt.Errorf("cannot compress the tar.gz file") + } + + return errors.Wrap(devtools.CreateSHA512File(tarGzSource), "failed to create .sha512 file") +} + +func Tar(source, target string) error { + filename := filepath.Base(source) + target = filepath.Join(target, fmt.Sprintf("%s.tar", filename)) + tarfile, err := os.Create(target) + if err != nil { + return err + } + defer tarfile.Close() + + tarball := tar.NewWriter(tarfile) + defer tarball.Close() + + info, err := os.Stat(source) + if err != nil { + return nil + } + + var baseDir string + if info.IsDir() { + baseDir = filepath.Base(source) + } + + return filepath.Walk(source, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + header, err := tar.FileInfoHeader(info, info.Name()) + if err != nil { + return err + } + + if baseDir != "" { + header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source)) + } + + if err := tarball.WriteHeader(header); err != nil { + return err + } + + if info.IsDir() { + return nil + } + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(tarball, file) + return err + }) +} + +func Gzip(source, target string) error { + reader, err := os.Open(source) + if err != nil { + return err + } + + filename := filepath.Base(source) + target = filepath.Join(target, fmt.Sprintf("%s.gz", filename)) + writer, err := os.Create(target) + if err != nil { + return err + } + defer writer.Close() + + archiver := gzip.NewWriter(writer) + archiver.Name = filename + defer archiver.Close() + + _, err = io.Copy(archiver, reader) + return err +} + +func TarGz(source, target string) (string, error) { + filename := filepath.Base(source) + Tar(source, target) + tarSource := filepath.Join(target, filename+".tar") + + Gzip(tarSource, target) + tarGzSource := tarSource + ".gz" + + if err := os.RemoveAll(tarSource); err != nil { + return "", errors.Wrapf(err, "failed to clean existing build directory %s", tarSource) + } + return tarGzSource, nil +} + +func getIronbankContextName() string { + version, _ := devtools.BeatQualifiedVersion() + defaultBinaryName := "{{.Name}}-ironbank-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}" + outputDir, _ := devtools.Expand(defaultBinaryName+"-docker-build-context", map[string]interface{}{ + "Name": "elastic-agent", + "Version": version, + }) + return outputDir +} + +func prepareIronbankBuild() error { + fmt.Println(">> prepareIronbankBuild: prepare the IronBank container context.") + ironbank := getIronbankContextName() + templatesDir := filepath.Join("dev-tools", "packaging", "templates", "ironbank") + + data := map[string]interface{}{ + "MajorMinor": majorMinor(), + } + + err := filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error { + if !info.IsDir() { + target := strings.TrimSuffix( + filepath.Join("build", "ironbank", ironbank, filepath.Base(path)), + ".tmpl", + ) + + err := devtools.ExpandFile(path, target, data) + if err != nil { + return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target) + } + } + return nil + }) + + if err != nil { + return err + } + return nil +} + +func majorMinor() string { + if v, _ := devtools.BeatQualifiedVersion(); v != "" { + parts := strings.SplitN(v, ".", 3) + return parts[0] + "." + parts[1] + } + return "" +} From b696b14d1c48033d48fdfc5d00299259b209a55f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 6 Jul 2022 18:24:30 +0100 Subject: [PATCH 2/7] mage: use the Tar function within the top-level folder --- .gitignore | 2 ++ magefile.go | 101 +++------------------------------------------------- 2 files changed, 7 insertions(+), 96 deletions(-) diff --git a/.gitignore b/.gitignore index 89eaa67db73..a42ec504279 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,5 @@ internal/pkg/agent/transpiler/tests/exec-1.0-darwin-x86_64/exec # VSCode /.vscode +# Ironbank +elastic-agent-ironbank* diff --git a/magefile.go b/magefile.go index 8b3dac3b8b9..1372bb7996b 100644 --- a/magefile.go +++ b/magefile.go @@ -8,8 +8,6 @@ package main import ( - "archive/tar" - "compress/gzip" "context" "fmt" "io" @@ -842,7 +840,7 @@ func saveIronbank() error { fmt.Println(">> saveIronbank: save the IronBank container context.") ironbank := getIronbankContextName() - buildDir := filepath.Join("build", "ironbank", ironbank) + buildDir := filepath.Join(ironbank) if _, err := os.Stat(buildDir); os.IsNotExist(err) { return fmt.Errorf("cannot find the folder with the ironbank context") } @@ -854,104 +852,15 @@ func saveIronbank() error { return fmt.Errorf("cannot create folder for docker artifacts: %+v", err) } } + tarGzFile := filepath.Join(distributionsDir, ironbank+".tar.gz") // Save the build context as tar.gz artifact - tarGzSource, err := TarGz(buildDir, distributionsDir) + err := devtools.Tar(buildDir, tarGzFile) if err != nil { return fmt.Errorf("cannot compress the tar.gz file") } - return errors.Wrap(devtools.CreateSHA512File(tarGzSource), "failed to create .sha512 file") -} - -func Tar(source, target string) error { - filename := filepath.Base(source) - target = filepath.Join(target, fmt.Sprintf("%s.tar", filename)) - tarfile, err := os.Create(target) - if err != nil { - return err - } - defer tarfile.Close() - - tarball := tar.NewWriter(tarfile) - defer tarball.Close() - - info, err := os.Stat(source) - if err != nil { - return nil - } - - var baseDir string - if info.IsDir() { - baseDir = filepath.Base(source) - } - - return filepath.Walk(source, - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - header, err := tar.FileInfoHeader(info, info.Name()) - if err != nil { - return err - } - - if baseDir != "" { - header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source)) - } - - if err := tarball.WriteHeader(header); err != nil { - return err - } - - if info.IsDir() { - return nil - } - - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - _, err = io.Copy(tarball, file) - return err - }) -} - -func Gzip(source, target string) error { - reader, err := os.Open(source) - if err != nil { - return err - } - - filename := filepath.Base(source) - target = filepath.Join(target, fmt.Sprintf("%s.gz", filename)) - writer, err := os.Create(target) - if err != nil { - return err - } - defer writer.Close() - - archiver := gzip.NewWriter(writer) - archiver.Name = filename - defer archiver.Close() - - _, err = io.Copy(archiver, reader) - return err -} - -func TarGz(source, target string) (string, error) { - filename := filepath.Base(source) - Tar(source, target) - tarSource := filepath.Join(target, filename+".tar") - - Gzip(tarSource, target) - tarGzSource := tarSource + ".gz" - - if err := os.RemoveAll(tarSource); err != nil { - return "", errors.Wrapf(err, "failed to clean existing build directory %s", tarSource) - } - return tarGzSource, nil + return errors.Wrap(devtools.CreateSHA512File(tarGzFile), "failed to create .sha512 file") } func getIronbankContextName() string { @@ -976,7 +885,7 @@ func prepareIronbankBuild() error { err := filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error { if !info.IsDir() { target := strings.TrimSuffix( - filepath.Join("build", "ironbank", ironbank, filepath.Base(path)), + filepath.Join(ironbank, filepath.Base(path)), ".tmpl", ) From f3acaf2d98321ed2ccefecf97018c9bf5599dc63 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 6 Jul 2022 18:26:36 +0100 Subject: [PATCH 3/7] ci: run the ironbank goal --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 9454f8d2cc6..efeffbb3fb5 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -175,7 +175,7 @@ pipeline { withMageEnv(){ dir("${BASE_DIR}"){ withPackageEnv("${PLATFORM}") { - cmd(label: 'Go package', script: 'mage package') + cmd(label: 'Go package', script: 'mage package ironbank') uploadPackagesToGoogleBucket( credentialsId: env.JOB_GCS_EXT_CREDENTIALS, repo: env.REPO, From cc93c475a9dfcb9b282bd5c67af5dbb0cb038d63 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 6 Jul 2022 20:41:28 +0100 Subject: [PATCH 4/7] honour the ironbank folder layout --- .../ironbank => files/ironbank/config}/docker-entrypoint | 0 dev-tools/packaging/templates/ironbank/Dockerfile | 2 +- magefile.go | 8 +++++++- 3 files changed, 8 insertions(+), 2 deletions(-) rename dev-tools/packaging/{templates/ironbank => files/ironbank/config}/docker-entrypoint (100%) diff --git a/dev-tools/packaging/templates/ironbank/docker-entrypoint b/dev-tools/packaging/files/ironbank/config/docker-entrypoint similarity index 100% rename from dev-tools/packaging/templates/ironbank/docker-entrypoint rename to dev-tools/packaging/files/ironbank/config/docker-entrypoint diff --git a/dev-tools/packaging/templates/ironbank/Dockerfile b/dev-tools/packaging/templates/ironbank/Dockerfile index a092320a720..779bc680e6e 100644 --- a/dev-tools/packaging/templates/ironbank/Dockerfile +++ b/dev-tools/packaging/templates/ironbank/Dockerfile @@ -78,7 +78,7 @@ RUN find /usr/share//elastic-agent/data -type d -exec chmod 0770 {} \; && \ COPY jq /usr/local/bin RUN chown root:root /usr/local/bin/jq && chmod 0755 /usr/local/bin/jq -COPY docker-entrypoint /usr/local/bin/docker-entrypoint +COPY config/docker-entrypoint /usr/local/bin/docker-entrypoint RUN chmod 755 /usr/local/bin/docker-entrypoint USER ${ELASTIC_PRODUCT} diff --git a/magefile.go b/magefile.go index 1372bb7996b..57f17580cff 100644 --- a/magefile.go +++ b/magefile.go @@ -898,7 +898,13 @@ func prepareIronbankBuild() error { }) if err != nil { - return err + return fmt.Errorf("cannot create templates for the IronBank: %+v", err) + } + + // copy files + sourcePath := filepath.Join("dev-tools", "packaging", "files", "ironbank") + if err := devtools.Copy(sourcePath, ironbank); err != nil { + return fmt.Errorf("cannot create files for the IronBank: %+v", err) } return nil } From c0ceb62b44f2068292a6d7af8e940064fb4d92e0 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 6 Jul 2022 21:14:02 +0100 Subject: [PATCH 5/7] template: add extension and move LICENSE to files --- dev-tools/packaging/{templates => files}/ironbank/LICENSE | 0 .../packaging/templates/ironbank/{Dockerfile => Dockerfile.tmpl} | 0 .../packaging/templates/ironbank/{README.md => README.md.tmpl} | 0 .../{hardening_manifest.yaml => hardening_manifest.yaml.tmpl} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename dev-tools/packaging/{templates => files}/ironbank/LICENSE (100%) rename dev-tools/packaging/templates/ironbank/{Dockerfile => Dockerfile.tmpl} (100%) rename dev-tools/packaging/templates/ironbank/{README.md => README.md.tmpl} (100%) rename dev-tools/packaging/templates/ironbank/{hardening_manifest.yaml => hardening_manifest.yaml.tmpl} (100%) diff --git a/dev-tools/packaging/templates/ironbank/LICENSE b/dev-tools/packaging/files/ironbank/LICENSE similarity index 100% rename from dev-tools/packaging/templates/ironbank/LICENSE rename to dev-tools/packaging/files/ironbank/LICENSE diff --git a/dev-tools/packaging/templates/ironbank/Dockerfile b/dev-tools/packaging/templates/ironbank/Dockerfile.tmpl similarity index 100% rename from dev-tools/packaging/templates/ironbank/Dockerfile rename to dev-tools/packaging/templates/ironbank/Dockerfile.tmpl diff --git a/dev-tools/packaging/templates/ironbank/README.md b/dev-tools/packaging/templates/ironbank/README.md.tmpl similarity index 100% rename from dev-tools/packaging/templates/ironbank/README.md rename to dev-tools/packaging/templates/ironbank/README.md.tmpl diff --git a/dev-tools/packaging/templates/ironbank/hardening_manifest.yaml b/dev-tools/packaging/templates/ironbank/hardening_manifest.yaml.tmpl similarity index 100% rename from dev-tools/packaging/templates/ironbank/hardening_manifest.yaml rename to dev-tools/packaging/templates/ironbank/hardening_manifest.yaml.tmpl From c372d35e8fced44db0e47490e1a0b2b265bd19ce Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 19 Jul 2022 12:04:23 +0100 Subject: [PATCH 6/7] mage: support Tar without nested folders --- .gitignore | 3 --- magefile.go | 25 +++++++++++++++++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index a42ec504279..939765242f5 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,3 @@ internal/pkg/agent/transpiler/tests/exec-1.0-darwin-x86_64/exec # VSCode /.vscode - -# Ironbank -elastic-agent-ironbank* diff --git a/magefile.go b/magefile.go index 57f17580cff..a215f1639d7 100644 --- a/magefile.go +++ b/magefile.go @@ -840,9 +840,9 @@ func saveIronbank() error { fmt.Println(">> saveIronbank: save the IronBank container context.") ironbank := getIronbankContextName() - buildDir := filepath.Join(ironbank) + buildDir := filepath.Join("build", ironbank) if _, err := os.Stat(buildDir); os.IsNotExist(err) { - return fmt.Errorf("cannot find the folder with the ironbank context") + return fmt.Errorf("cannot find the folder with the ironbank context: %+v", err) } distributionsDir := "build/distributions" @@ -852,12 +852,21 @@ func saveIronbank() error { return fmt.Errorf("cannot create folder for docker artifacts: %+v", err) } } - tarGzFile := filepath.Join(distributionsDir, ironbank+".tar.gz") + + // change dir to the buildDir location where the ironbank folder exists + // this will generate a tar.gz without some nested folders. + wd, _ := os.Getwd() + os.Chdir(buildDir) + defer os.Chdir(wd) + + // move the folder to the parent folder, there are two parent folder since + // buildDir contains a two folders dir. + tarGzFile := filepath.Join("..", "..", distributionsDir, ironbank+".tar.gz") // Save the build context as tar.gz artifact - err := devtools.Tar(buildDir, tarGzFile) + err := devtools.Tar("./", tarGzFile) if err != nil { - return fmt.Errorf("cannot compress the tar.gz file") + return fmt.Errorf("cannot compress the tar.gz file: %+v", err) } return errors.Wrap(devtools.CreateSHA512File(tarGzFile), "failed to create .sha512 file") @@ -875,7 +884,7 @@ func getIronbankContextName() string { func prepareIronbankBuild() error { fmt.Println(">> prepareIronbankBuild: prepare the IronBank container context.") - ironbank := getIronbankContextName() + buildDir := filepath.Join("build", getIronbankContextName()) templatesDir := filepath.Join("dev-tools", "packaging", "templates", "ironbank") data := map[string]interface{}{ @@ -885,7 +894,7 @@ func prepareIronbankBuild() error { err := filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error { if !info.IsDir() { target := strings.TrimSuffix( - filepath.Join(ironbank, filepath.Base(path)), + filepath.Join(buildDir, filepath.Base(path)), ".tmpl", ) @@ -903,7 +912,7 @@ func prepareIronbankBuild() error { // copy files sourcePath := filepath.Join("dev-tools", "packaging", "files", "ironbank") - if err := devtools.Copy(sourcePath, ironbank); err != nil { + if err := devtools.Copy(sourcePath, buildDir); err != nil { return fmt.Errorf("cannot create files for the IronBank: %+v", err) } return nil From 73b7e11a4d32b41b0a91367d0b1ae166a2d5296d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 19 Jul 2022 15:17:24 +0100 Subject: [PATCH 7/7] Apply suggestions from code review --- dev-tools/packaging/templates/ironbank/Dockerfile.tmpl | 2 +- dev-tools/packaging/templates/ironbank/README.md.tmpl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev-tools/packaging/templates/ironbank/Dockerfile.tmpl b/dev-tools/packaging/templates/ironbank/Dockerfile.tmpl index 779bc680e6e..04c4dfde930 100644 --- a/dev-tools/packaging/templates/ironbank/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/ironbank/Dockerfile.tmpl @@ -25,7 +25,7 @@ RUN tar --strip-components=1 -zxf ${ELASTIC_PRODUCT}-${ELASTIC_STACK}-${OS_AND_A # REF: https://docs.okd.io/latest/openshift_images/create-images.html RUN chmod -R g=u /usr/share/${ELASTIC_PRODUCT} -# Create auxiliar folders and assing default permissions. +# Create auxiliary folders and assigning default permissions. RUN mkdir -p /usr/share/${ELASTIC_PRODUCT}/data /usr/share/${ELASTIC_PRODUCT}/logs && \ chown -R root:root /usr/share/${ELASTIC_PRODUCT} && \ find /usr/share/${ELASTIC_PRODUCT} -type d -exec chmod 0750 {} \; && \ diff --git a/dev-tools/packaging/templates/ironbank/README.md.tmpl b/dev-tools/packaging/templates/ironbank/README.md.tmpl index 9b357727c8a..271fdb8c0d7 100644 --- a/dev-tools/packaging/templates/ironbank/README.md.tmpl +++ b/dev-tools/packaging/templates/ironbank/README.md.tmpl @@ -15,12 +15,12 @@ This functionality is in beta and is subject to change. The design and code is l ### Installation instructions -Please follow the documentation on [Quick start](https://www.elastic.co/guide/en/ingest-management/7.17/ingest-management-getting-started.html). +Please follow the documentation on [Quick start](https://www.elastic.co/guide/en/fleet/{{ .MajorMinor }}/fleet-elastic-agent-quick-start.html). ### Where to file issues and PRs -- [Issues](https://github.com/elastic/beats/issues) -- [PRs](https://github.com/elastic/beats/pulls) +- [Issues](https://github.com/elastic/elastic-agent/issues) +- [PRs](https://github.com/elastic/elastic-agent/pulls) ### DoD Restrictions