Merge pull request #5444 from freedomofpress/builder_focal

Adds initial builder-focal molecule scenario

Makefile

@@ -329,6 +329,13 @@ build-debs-notest: ## Build SecureDrop Debian packages without running tests.
 	@$(SDROOT)/devops/scripts/build-debs.sh notest
 	@echo
 
+.PHONY: build-debs-focal
+build-debs-focal: ## Build and test SecureDrop Debian packages.
+	@echo "Building SecureDrop Debian packages..."
+	@$(SDROOT)/devops/scripts/build-debs.sh notest focal
+	@echo
+
+
 
 ########################
 #

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/defaults/main.yml

@@ -45,6 +45,4 @@ securedrop_app_rsync_opts:
 
 securedrop_local_build: "../../build"
 
-# Always build packages targeting Xenial.
-securedrop_build_xenial_support: True
 securedrop_package_dist: xenial

molecule/builder-focal/Dockerfile

@@ -0,0 +1,50 @@
+# ubuntu:focal-20200720
+FROM ubuntu@sha256:60f560e52264ed1cb7829a0d59b1ee7740d7580e0eb293aca2d722136edb1e24
+
+
+# additional meta-data makes it easier to clean up, find
+LABEL org="Freedom of the Press"
+LABEL image_name="focal-sd-builder-app"
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \
+        apache2-dev \
+        aptitude \
+        coreutils \
+        debhelper \
+        devscripts \
+        dh-python \
+        dh-systemd \
+        gdb \
+        git \
+        gnupg2 \
+        haveged \
+        inotify-tools \
+        libffi-dev \
+        libssl-dev \
+        make \
+        ntp \
+        paxctl \
+        python3-all \
+        python3-pip \
+        python3-setuptools \
+        rsync \
+        ruby \
+        sqlite \
+        sudo \
+        tzdata \
+        libevent-dev \
+        unzip
+
+
+# TEMPORARY: install dh-virtualenv from debian unstable, pending focal package:
+# https://github.com/spotify/dh-virtualenv/issues/298
+RUN echo "deb https://deb.debian.org/debian unstable main contrib" > /etc/apt/sources.list.d/debian-unstable.list
+COPY aptpreferences.conf /etc/apt/preferences.d/debian-unstable
+RUN apt-get install -y debian-archive-keyring
+RUN ln -s /usr/share/keyrings/debian-archive-keyring.gpg /etc/apt/trusted.gpg.d/
+
+RUN apt-get update && apt-get install -y dh-virtualenv
+
+RUN paxctl -cm /usr/bin/python3.8 && mkdir -p /tmp/build
+RUN apt-get clean \
+        && rm -rf /var/lib/apt/lists/*

molecule/builder-focal/Makefile

@@ -0,0 +1,20 @@
+DATE_STR := $(shell date +"%Y_%m_%d")
+BUILDER_IMAGE ?= "quay.io/freedomofpress/sd-docker-builder-focal:$(DATE_STR)"
+
+.PHONY: build-container
+build-container: ## Build Docker image for Debian package creation
+	@echo "███Building Docker image $(BUILDER_IMAGE) for Debian package creation..."
+	@docker build --no-cache -t $(BUILDER_IMAGE) .
+
+.PHONY: push-container
+push-container: ## Push the Docker image for Debian package creation to quay.io
+	@echo "███Pushing Docker image for Debian package creation to quay.io..."
+	@./push.sh
+
+.PHONY: help
+help: ## Print this message and exit.
+	@printf "Molecule scenario for building a Docker container for Debian package creation.\n"
+	@printf "Subcommands:\n\n"
+	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \
+		| sort \
+		| column -s ':' -t

molecule/builder-focal/ansible-override-vars.yml

@@ -0,0 +1,3 @@
+---
+securedrop_build_focal_support: True
+securedrop_staging_install_target_distro: "focal"

molecule/builder-focal/aptpreferences.conf

@@ -0,0 +1,7 @@
+Package: *
+Pin: release a=focal
+Pin-Priority: 700
+
+Package: *
+Pin: release a=unstable
+Pin-Priority: 1

molecule/builder-focal/create.yml

@@ -0,0 +1,28 @@
+---
+- name: Create
+  hosts: localhost
+  connection: local
+  gather_facts: False
+  vars:
+    molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
+    molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}"
+    molecule_scenario_directory: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | from_yaml }}"
+    image_hash: "{{ lookup('pipe', 'egrep -v ^# image_hash') }}"
+    default_image: "quay.io/freedomofpress/sd-docker-builder-focal@sha256:{{image_hash}}"
+    image: "{{ lookup('env', 'BUILDER_IMAGE') | default(default_image, true) }}"
+  tasks:
+    - debug:
+        msg: "Building with Docker image {{ image }}"
+
+    - name: Create builders
+      docker_container:
+        name: "{{ item.name }}"
+        hostname: "{{ item.name }}"
+        image: "{{ image }}"
+        state: started
+        command: "tail -f /dev/null"
+        privileged: "{{ item.privileged | default(omit) }}"
+        volumes: "{{ item.volumes | default(omit) }}"
+        capabilities: "{{ item.capabilities | default(omit) }}"
+      with_items: "{{ molecule_yml.platforms }}"

molecule/builder-focal/destroy.yml

@@ -0,0 +1,15 @@
+---
+- name: Destroy
+  hosts: localhost
+  connection: local
+  gather_facts: False
+  vars:
+    molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | from_yaml }}"
+  tasks:
+    - name: Destroy molecule instance(s)
+      docker_container:
+        name: "{{ item.name }}"
+        state: absent
+        force_kill: "{{ item.force_kill | default(True) }}"
+      with_items: "{{ molecule_yml.platforms }}"

molecule/builder-focal/image_hash

@@ -0,0 +1,2 @@
+# sha256 digest quay.io/freedomofpress/sd-docker-builder-focal:2020_08_13
+47f7e1af8cc54bf9cb0afdef599b76d063c614943e54443ea660d6b53615814c

molecule/builder-focal/molecule.yml

@@ -0,0 +1,79 @@
+---
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: focal-sd-app
+    groups:
+      - builders
+  - name: focal-sd-generic-ossec-agent
+    groups:
+      - builders
+  - name: focal-sd-generic-ossec-server
+    groups:
+      - builders
+  - name: focal-sd-generic-ossec-agent2
+    groups:
+      - builders
+  - name: focal-sd-generic-ossec-server2
+    groups:
+      - builders
+  - name: focal-sd-grsec
+    groups:
+      - builders
+  - name: focal-sd-config
+    groups:
+      - builders
+  - name: focal-sd-keyring
+    groups:
+      - builders
+  - name: focal-sd-sec-update
+    groups:
+      - builders
+  - name: focal-sd-dpkg-verification
+    groups:
+      - testers
+provisioner:
+  name: ansible
+  inventory:
+    links:
+      group_vars: ../../install_files/ansible-base/group_vars
+  options:
+    e: "@ansible-override-vars.yml"
+  config_options:
+    defaults:
+      interpreter_python: auto
+  env:
+    ANSIBLE_ROLES_PATH: ../../install_files/ansible-base/roles
+    ANSIBLE_ACTION_PLUGINS: ../../install_files/ansible-base/action_plugins
+    ANSIBLE_CALLBACK_WHITELIST: skippy
+    ANSIBLE_STDOUT_CALLBACK: skippy
+    ANSIBLE_GATHER_TIMEOUT: "120"
+  lint:
+    name: ansible-lint
+  playbooks:
+    converge: playbook.yml
+scenario:
+  name: builder-focal
+  converge_sequence:
+    - destroy
+    - create
+    - converge
+    - destroy
+  test_sequence:
+    - destroy
+    - create
+    - converge
+    - verify
+    - destroy
+verifier:
+  name: testinfra
+  options:
+    # provided by pytest-xdist
+    n: auto
+  env:
+    SECUREDROP_TARGET_PLATFORM: focal
+  directory: tests/
+  lint:
+    name: flake8

molecule/builder-focal/playbook.yml

@@ -0,0 +1,80 @@
+---
+- name: Build SecureDrop application Debian package from local repository.
+  hosts: builders
+  # Build as fast as possible with each host going individually
+  strategy: free
+  become: yes
+  tasks:
+    - name: Update apt-cache for our security checker
+      apt:
+        update_cache: yes
+      when: ansible_host.endswith("-sd-sec-update")
+  roles:
+    - role: build-securedrop-app-code-deb-pkg
+      tags: app-deb
+      when: ansible_host.endswith("-sd-app")
+
+    - role: build-ossec-deb-pkg
+      tags: ossec-server
+      purpose: server
+      when: ansible_host.endswith("-sd-generic-ossec-server")
+
+    - role: build-ossec-deb-pkg
+      tags: ossec-agent
+      purpose: agent
+      when: ansible_host.endswith("-sd-generic-ossec-agent")
+
+    - role: build-generic-pkg
+      tags: securedrop-ossec-server
+      package_name: securedrop-ossec-server
+      when: ansible_host.endswith("-sd-generic-ossec-server2") or ansible_host == "localhost"
+
+    - role: build-generic-pkg
+      tags: securedrop-ossec-agent
+      package_name: securedrop-ossec-agent
+      when: ansible_host.endswith("-sd-generic-ossec-agent2") or ansible_host == "localhost"
+
+    - role: build-generic-pkg
+      tags: securedrop-keyring
+      package_name: securedrop-keyring
+      when: ansible_host.endswith("-sd-keyring") or ansible_host == "localhost"
+
+    - role: build-generic-pkg
+      tags: securedrop-grsec
+      package_name: securedrop-grsec
+      when: ansible_host.endswith("-sd-grsec") or ansible_host == "localhost"
+
+    - role: build-generic-pkg
+      tags: securedrop-config
+      package_name: securedrop-config
+      when: ansible_host.endswith("-sd-config") or ansible_host == "localhost"
+  tags: rebuild
+
+# Typically we'd perform volume mounting here but to work around docker
+# remote calls (in CircleCI) we have to copy the files instead
+- name: Give dpkg verify container access to debs
+  hosts: testers
+  gather_facts: false
+  vars:
+    sd_build_root: "{{ playbook_dir + '/../../build' }}"
+    sd_target_distro: focal
+    sd_build_dest: "{{ sd_build_root + '/' + sd_target_distro }}"
+
+  tasks:
+    - name: Discover local debian build files
+      find:
+        paths: "{{ sd_build_dest }}"
+        patterns: '*.deb'
+      delegate_to: localhost
+      register: debian_files
+
+    - name: Create build dir
+      file:
+        state: directory
+        path: /tmp/build
+
+    - name: Drop debian files into container
+      copy:
+        src: "{{ item.path }}"
+        dest: /tmp/build/
+      with_items: "{{ debian_files.files }}"

molecule/builder-focal/push.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+DATE_STR=$(date +"%Y_%m_%d")
+QUAY_REPO=quay.io/freedomofpress/sd-docker-builder-focal
+
+set -e
+set -x
+
+docker push "${QUAY_REPO}:${DATE_STR}"
+
+echo "# sha256 digest ${QUAY_REPO}:${DATE_STR}" > image_hash
+docker inspect --format='{{index .RepoDigests 0}}' "${QUAY_REPO}:${DATE_STR}" \
+    | sed 's/.*://g' >> image_hash

molecule/builder-focal/tests/conftest.py

@@ -0,0 +1,22 @@
+"""
+Import variables from vars.yml and inject into pytest namespace
+"""
+
+import os
+import io
+import yaml
+
+
+def pytest_namespace():
+    """ Return dict of vars imported as 'securedrop_test_vars' into pytest
+        global namespace
+    """
+    filepath = os.path.join(os.path.dirname(__file__), "vars.yml")
+    with io.open(filepath, 'r') as f:
+        securedrop_test_vars = yaml.safe_load(f)
+
+    # Tack on target OS for use in tests
+    securedrop_target_platform = os.environ.get("SECUREDROP_TARGET_PLATFORM")
+    securedrop_test_vars["securedrop_target_platform"] = securedrop_target_platform
+    # Wrapping the return value to accommodate for pytest namespacing
+    return dict(securedrop_test_vars=securedrop_test_vars)

molecule/builder-focal/tests/test_build_dependencies.py

@@ -0,0 +1,40 @@
+import pytest
+import os
+
+
+SECUREDROP_TARGET_PLATFORM = os.environ.get("SECUREDROP_TARGET_PLATFORM")
+testinfra_hosts = [
+        "docker://{}-sd-app".format(SECUREDROP_TARGET_PLATFORM)
+]
+
+
+def test_sass_gem_installed(host):
+    """
+    Ensure the `sass` Ruby gem is installed, for compiling SASS to CSS.
+    """
+    c = host.run("gem list")
+    assert "sass (3.4.23)" in c.stdout
+    assert c.rc == 0
+
+
+def test_pip_dependencies_installed(host):
+    """
+    Ensure the development pip dependencies are installed
+    """
+    c = host.run("pip3 list installed")
+    assert "Flask-Babel" in c.stdout
+    assert c.rc == 0
+
+
+@pytest.mark.xfail(reason="This check conflicts with the concept of pegging"
+                          "dependencies")
+def test_build_all_packages_updated(host):
+    """
+    Ensure a dist-upgrade has already been run, by checking that no
+    packages are eligible for upgrade currently. This will ensure that
+    all upgrades, security and otherwise, have been applied to the VM
+    used to build packages.
+    """
+    c = host.run('aptitude --simulate -y dist-upgrade')
+    assert c.rc == 0
+    assert "No packages will be installed, upgraded, or removed." in c.stdout