diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile new file mode 100644 index 0000000000..483a31160e --- /dev/null +++ b/.cci.jenkinsfile @@ -0,0 +1,54 @@ +stage("Init") { + node { + checkout scm + pod = readFile(file: "ci/pod.yaml") + } +} + +def label = "pod-${UUID.randomUUID().toString()}" +podTemplate(cloud: 'openshift', yaml: pod, label: label, defaultContainer: 'jnlp') { + node(label) { container('fedora') { + checkout scm + + stage("Build") { + shwrap(""" + dnf install -y git + git submodule update --init + ./build.sh + """) + } + + stage("Test") { + parallel check: { + shwrap(""" + make check + make unittest + """) + }, + fcos: { + shwrap("chown builder: /srv") + // just split into separate invocations to make it easier to see where it fails + cosa_cmd("init https://github.com/coreos/fedora-coreos-config") + cosa_cmd("fetch") + cosa_cmd("build") + cosa_cmd("kola run") + cosa_cmd("buildextend-metal") + cosa_cmd("buildextend-installer") + cosa_cmd("buildextend-openstack") + cosa_cmd("buildextend-vmware") + cosa_cmd("compress") + } + } +}}} + +// XXX: move to shared lib +def shwrap(cmds) { + sh """ + set -xeuo pipefail + ${cmds} + """ +} + +def cosa_cmd(args) { + shwrap("cd /srv && sudo -u builder cosa ${args}") +} diff --git a/Dockerfile b/Dockerfile index 2b0327a0d7..9edf486780 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,12 @@ FROM registry.fedoraproject.org/fedora:30 WORKDIR /root/containerbuild +# We split into multiple steps here so that local dev workflows which involve +# rebuilding the container can leverage cached layers better. + # Only need a few of our scripts for the first few steps -COPY ./src/cmdlib.sh ./build.sh ./deps*.txt ./vmdeps.txt ./build-deps.txt /root/containerbuild/ +COPY ./src/cmdlib.sh /root/containerbuild/src/ +COPY ./build.sh ./deps*.txt ./vmdeps.txt ./build-deps.txt /root/containerbuild/ RUN ./build.sh configure_yum_repos RUN ./build.sh install_rpms @@ -13,6 +17,8 @@ RUN ./build.sh install_anaconda RUN ./build.sh make_and_makeinstall RUN ./build.sh configure_user +RUN make check +RUN make unittest RUN make clean # clean up scripts (it will get cached in layers, but oh well) diff --git a/Dockerfile.dev b/Dockerfile.dev index 210ddf4cb8..83192f1cf1 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -12,7 +12,8 @@ WORKDIR /root/containerbuild USER root RUN rm -rfv /lib/coreos-assembler /usr/bin/coreos-assembler -COPY ./src/cmdlib.sh ./build.sh ./deps*.txt ./vmdeps.txt ./build-deps.txt /root/containerbuild/ +COPY ./src/cmdlib.sh /root/containerbuild/src/ +COPY ./build.sh ./deps*.txt ./vmdeps.txt ./build-deps.txt /root/containerbuild/ RUN ./build.sh install_rpms COPY ./ /root/containerbuild/ diff --git a/Dockerfile.rhel b/Dockerfile.rhel deleted file mode 100644 index 5bbef60cfd..0000000000 --- a/Dockerfile.rhel +++ /dev/null @@ -1,31 +0,0 @@ -FROM registry.redhat.io/rhel7:latest -WORKDIR /root/containerbuild - -# Only need a few of our scripts for the first few steps -COPY ./src/cmdlib.sh ./build.sh ./deps*.txt ./vmdeps.txt ./build-deps.txt /root/containerbuild/ -COPY ./maipo/maipo.repo /etc/yum.repos.d/ -RUN ./build.sh configure_yum_repos -# ostree-packages are on another line because all repos get configured to exclude -# ostree/rpm-ostree in the configure step above -COPY ./maipo/ostree-packages.repo /etc/yum.repos.d/ -RUN ./build.sh install_rpms - -# Ok copy in the rest of them for the next few steps -COPY ./ /root/containerbuild/ -RUN ./build.sh write_archive_info -RUN ./build.sh make_and_makeinstall -RUN ./build.sh configure_user - -RUN make clean - -# clean up scripts (it will get cached in layers, but oh well) -WORKDIR /srv/ -RUN rm -rf /root/containerbuild - -# allow writing to /etc/passwd from arbitrary UID -# https://docs.openshift.com/container-platform/3.10/creating_images/guidelines.html -RUN chmod g=u /etc/passwd - -# run as `builder` user -USER builder -ENTRYPOINT ["/usr/bin/dumb-init", "scl", "enable", "rh-python36", "--", "/usr/bin/coreos-assembler"] diff --git a/build.sh b/build.sh index df787e2835..e6102c49bd 100755 --- a/build.sh +++ b/build.sh @@ -2,21 +2,15 @@ set -euo pipefail # Detect what platform we are on -if grep -q '^Fedora' /etc/redhat-release; then - ISFEDORA=1 - ISEL='' -elif grep -q '^Red Hat' /etc/redhat-release; then - ISFEDORA='' - ISEL=1 -else - echo 1>&2 "should be on either RHEL or Fedora" +if ! grep -q '^Fedora' /etc/redhat-release; then + echo 1>&2 "should be on either Fedora" exit 1 fi arch=$(uname -m) -if [ $# -eq 0 ]; then - echo Usage: "build.sh CMD" +if [ $# -gt 1 ]; then + echo Usage: "build.sh [CMD]" echo "Supported commands:" echo " configure_user" echo " configure_yum_repos" @@ -31,13 +25,11 @@ srcdir=$(pwd) release="30" configure_yum_repos() { - if [ -n "${ISFEDORA}" ]; then - # Add continuous tag for latest build tools and mark as required so we - # can depend on those latest tools being available in all container - # builds. - echo -e "[f$release-coreos-continuous]\nenabled=1\nmetadata_expire=1m\nbaseurl=https://kojipkgs.fedoraproject.org/repos-dist/f$release-coreos-continuous/latest/\$basearch/\ngpgcheck=0\nskip_if_unavailable=False\n" > /etc/yum.repos.d/coreos.repo + # Add continuous tag for latest build tools and mark as required so we + # can depend on those latest tools being available in all container + # builds. + echo -e "[f$release-coreos-continuous]\nenabled=1\nmetadata_expire=1m\nbaseurl=https://kojipkgs.fedoraproject.org/repos-dist/f$release-coreos-continuous/latest/\$basearch/\ngpgcheck=0\nskip_if_unavailable=False\n" > /etc/yum.repos.d/coreos.repo - fi } install_rpms() { @@ -49,29 +41,22 @@ install_rpms() { # xargs is part of findutils, which may not be installed yum -y install /usr/bin/xargs - # define the filter we want to use to filter out deps that don't - # apply to the platform we are on - [ -n "${ISFEDORA}" ] && filter='^#FEDORA ' - [ -n "${ISEL}" ] && filter='^#EL7 ' - # These are only used to build things in here. Today # we ship these in the container too to make it easier # to use the container as a development environment for itself. # Down the line we may strip these out, or have a separate # development version. - builddeps=$(sed "s/${filter}//" "${srcdir}"/build-deps.txt | grep -v '^#') + builddeps=$(grep -v '^#' "${srcdir}"/build-deps.txt) # Process our base dependencies + build dependencies and install - deps=$(sed "s/${filter}//" "${srcdir}"/deps.txt | grep -v '^#') - archdeps=$(sed "s/${filter}//" "${srcdir}/deps-$(arch)".txt | grep -v '^#') + deps=$(grep -v '^#' "${srcdir}"/deps.txt) + archdeps=$(grep -v '^#' "${srcdir}/deps-$(arch)".txt) echo "${builddeps}" "${deps}" "${archdeps}" | xargs yum -y install # Commented out for now, see above - #dnf remove -y $builddeps} + #dnf remove -y ${builddeps} # can't remove grubby on el7 because libguestfs-tools depends on it - if [ -n "${ISFEDORA}" ]; then - rpm -q grubby && yum remove -y grubby - fi + rpm -q grubby && yum remove -y grubby # Allow Kerberos Auth to work from a keytab. The keyring is not # available in a Container. @@ -86,7 +71,7 @@ install_rpms() { yum clean all # shellcheck source=src/cmdlib.sh - . "${srcdir}/cmdlib.sh" + . "${srcdir}/src/cmdlib.sh" depcheck "${deps} ${archdeps}" } @@ -100,12 +85,6 @@ _prep_make_and_make_install() { echo -e "\033[1merror: submodules not initialized. Run: git submodule update --init\033[0m" 1>&2 exit 1 fi - - # Can only (easily) get gobject-introspection in Python2 on EL7 - if [ -n "${ISEL}" ]; then - sed -i 's|^#!/usr/bin/python3|#!/usr/bin/python2|' src/commitmeta_to_json - sed -i 's|^#!/usr/bin/env python3|#!/usr/bin/python2|' src/cmd-oscontainer - fi } # For now keep using the f29 anaconda. There's no golden f30 image yet and it @@ -164,12 +143,7 @@ install_anaconda() { make_and_makeinstall() { _prep_make_and_make_install - # And the main scripts - if [ -n "${ISEL}" ]; then - echo "make && make check && make install" | scl enable rh-python36 bash - else - make && make check && make install - fi + make && make install } configure_user(){ @@ -197,8 +171,18 @@ write_archive_info() { . "${srcdir}/src/cmdlib.sh" mkdir -p /cosa /lib/coreos-assembler touch -f /lib/coreos-assembler/.clean - prepare_git_artifacts /root/containerbuild /cosa/coreos-assembler-git.tar.gz /cosa/coreos-assembler-git.json + prepare_git_artifacts "${srcdir}" /cosa/coreos-assembler-git.tar.gz /cosa/coreos-assembler-git.json } -# Run the function specified by the calling script -${1} +if [ $# -ne 0 ]; then + # Run the function specified by the calling script + ${1} +else + # Otherwise, just run all the steps + configure_yum_repos + install_rpms + write_archive_info + install_anaconda + make_and_makeinstall + configure_user +fi diff --git a/ci/pod.yaml b/ci/pod.yaml new file mode 100644 index 0000000000..9f85a966ce --- /dev/null +++ b/ci/pod.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +spec: + # XXX: tmp hack to get anyuid SCC; need to ask to get jenkins SA added + serviceAccountName: papr + containers: + - name: jnlp + image: jenkins-slave-base-centos7:latest + args: ['$(JENKINS_SECRET)', '$(JENKINS_NAME)'] + - name: fedora + image: registry.fedoraproject.org/fedora:30 + imagePullPolicy: Always + command: ['/usr/bin/sleep', 'infinity'] + securityContext: + runAsUser: 0 + nodeSelector: + oci_kvm_hook: allowed diff --git a/src/deps.txt b/src/deps.txt index 7df02802a3..d554f060ce 100644 --- a/src/deps.txt +++ b/src/deps.txt @@ -16,7 +16,7 @@ dnf-utils genisoimage # We expect people to use these explicitly in their repo configurations. -#FEDORA distribution-gpg-keys +distribution-gpg-keys # We need these for rojig selinux-policy-targeted rpm-build diff --git a/src/vmdeps.txt b/src/vmdeps.txt index 69e6045831..61f5d01cd3 100644 --- a/src/vmdeps.txt +++ b/src/vmdeps.txt @@ -16,6 +16,6 @@ dhcp-client bind-export-libs iproute selinux-policy selinux-policy-targeted policycoreutils # coreos-assembler -#FEDORA python3 python3-gobject-base buildah podman skopeo iptables iptables-libs +python3 python3-gobject-base buildah podman skopeo iptables iptables-libs gdisk xfsprogs e2fsprogs grub2 dosfstools shim-x64 grub2-efi-x64