support universal base image (#95)

* support building the operator with redhat UBI as the base image
* choose other UBI-based coordinates when running an operator that was built with UBI
* converge both github workflows into a single one
* publish docker images for both regular operator and ubi version from gh workflow

.github/workflows/operatorPRBuildAndDeploy.yml → .github/workflows/operatorBuildAndDeploy.yml

@@ -1,5 +1,9 @@
 name: Cass Operator Build & Deploy
-on: pull_request
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
 jobs:
   build_operator_docker:
     name: Build Cass Operator Docker Image
@@ -9,8 +13,11 @@ jobs:
       GOROOT: /usr/local/go1.13
     steps:
       - uses: actions/checkout@v2
+        if: github.event_name == 'pull_request'
         with:
           ref: ${{ github.event.pull_request.head.sha }}
+      - uses: actions/checkout@v2
+        if: github.event_name != 'pull_request'
       - name: Set up Go 1.13
         uses: actions/setup-go@v1
         with:
@@ -31,14 +38,20 @@ jobs:
         run: |
           export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
           mage operator:testGenerateClient
-      - name: Build docker
+      - name: Build docker - standard and ubi images
         env:
-          MO_BRANCH: ${{ github.event.pull_request.head.ref }}
+          PR_REF: ${{ github.event.pull_request.head.ref }}
+          MO_BASE_OS: 'registry.access.redhat.com/ubi7/ubi-minimal:7.8'
         run: |
+          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+            export MO_BRANCH=${PR_REF}
+          else
+            export MO_BRANCH="master"
+          fi;
           export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
           mage operator:testAndBuild
       - name: Deploy to ECR
-        if: github.event.pull_request.head.repo.full_name == 'datastax/cass-operator'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'datastax/cass-operator'
         env:
           MO_ECR_ID: ${{ secrets.ECR_ID }}
           MO_ECR_SECRET: ${{ secrets.ECR_SECRET }}
@@ -48,7 +61,7 @@ jobs:
           export MO_TAGS=$(cat ./build/tagsToPush.txt)
           mage operator:deployToECR
       - name: Deploy to GH Packages
-        if: github.event.pull_request.head.repo.full_name == 'datastax/cass-operator'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'datastax/cass-operator'
         env:
           MO_GH_USR: 'datastax/cass-operator'
           MO_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

mage/k8s/lib.go

@@ -25,6 +25,7 @@ import (
 
 const (
 	OperatorImage    = "datastax/cass-operator:latest"
+	OperatorImageUBI = "datastax/cass-operator:latest-ubi"
 	envLoadDevImages = "M_LOAD_DEV_IMAGES"
 	envK8sFlavor     = "M_K8S_FLAVOR"
 )
@@ -38,9 +39,21 @@ var supportedFlavors = map[string]ClusterActions{
 	"gke":  gcp.ClusterActions,
 }
 
+func getOperatorImage() string {
+	var img string
+	if baseOs := os.Getenv(operator.EnvBaseOs); baseOs != "" {
+		img = "datastax/cass-operator:latest-ubi"
+	} else {
+		img = "datastax/cass-operator:latest"
+	}
+	return img
+}
+
 func loadImagesFromBuildSettings(cfg ClusterActions, settings BuildSettings) {
 	for _, image := range settings.Dev.Images {
-		shutil.RunVPanic("docker", "pull", image)
+		// we likely don't always care if we fail to pull
+		// because we could be testing local images
+		_ = shutil.RunV("docker", "pull", image)
 		cfg.LoadImage(image)
 	}
 }
@@ -96,7 +109,8 @@ func SetupEmptyCluster() {
 	clusterActions.ApplyDefaultStorage()
 	//TODO make this part optional
 	operator.BuildDocker()
-	clusterActions.LoadImage(OperatorImage)
+	operatorImg := getOperatorImage()
+	clusterActions.LoadImage(operatorImg)
 }
 
 // Bootstrap a cluster, then run Ginkgo integration tests.
@@ -131,8 +145,8 @@ func SetupExampleCluster() {
 	mg.Deps(SetupEmptyCluster)
 	kubectl.CreateSecretLiteral("cassandra-superuser-secret", "devuser", "devpass").ExecVPanic()
 
+	overrides := map[string]string{"image": getOperatorImage()}
 	var namespace = "default"
-	var overrides = map[string]string{"image": "datastax/cass-operator:latest"}
 	err := helm_util.Install("./charts/cass-operator-chart", "cass-operator", namespace, overrides)
 	mageutil.PanicOnError(err)
 

mage/operator/deploy.go

@@ -58,9 +58,10 @@ func retagAndPush(tags []string, remoteUrl string) {
 func retagAndPushForGH(tags []string) {
 	pkgRepo := mageutil.RequireEnv(envGHPackageRepo)
 	reg := regexp.MustCompile(`.*\:`)
-	for _, tag := range tags {
+	for _, t := range tags {
+		tag := strings.TrimSpace(t)
 		updatedTag := reg.ReplaceAllString(tag, fmt.Sprintf("%s:", pkgRepo))
-		fullGHTag := fmt.Sprintf("%s/%s", ghPackagesRegistry, updatedTag)
+		fullGHTag := fmt.Sprintf("%s/%s", ghPackagesRegistry, strings.TrimSpace(updatedTag))
 		dockerTag(tag, fullGHTag)
 		fmt.Printf("- Pushing image %s\n", fullGHTag)
 		dockerutil.Push(fullGHTag).WithCfg(rootBuildDir).ExecVPanic()

mage/operator/lib.go

@@ -23,6 +23,8 @@ import (
 )
 
 const (
+	dockerBase                 = "./operator/docker/base/Dockerfile"
+	dockerUbi                  = "./operator/docker/ubi/Dockerfile"
 	rootBuildDir               = "./build"
 	sdkBuildDir                = "operator/build"
 	diagramsDir                = "./docs/developer/diagrams"
@@ -36,6 +38,7 @@ const (
 	envGitBranch               = "MO_BRANCH"
 	envVersionString           = "MO_VERSION"
 	envGitHash                 = "MO_HASH"
+	EnvBaseOs                  = "MO_BASE_OS"
 
 	errorUnstagedPreGenerate = `
   Unstaged changes detected.
@@ -73,7 +76,7 @@ func checkForUnstagedChanges(message string) {
 func writeBuildFile(fileName string, contents string) {
 	mageutil.EnsureDir(rootBuildDir)
 	outputPath := filepath.Join(rootBuildDir, fileName)
-	err := ioutil.WriteFile(outputPath, []byte(contents+"\n"), 0666)
+	err := ioutil.WriteFile(outputPath, []byte(contents), 0666)
 	if err != nil {
 		fmt.Printf("Failed to write file at %s\n", outputPath)
 		panic(err)
@@ -349,17 +352,34 @@ func calcFullVersion(settings cfgutil.BuildSettings, git GitData) FullVersion {
 	}
 }
 
-func runDockerBuild(version FullVersion) []string {
+func calcVersionAndTags(version FullVersion, ubiBase bool) (string, []string) {
 	repoPath := "datastax/cass-operator"
-	versionedTag := fmt.Sprintf("%s:%v", repoPath, version)
-	tagsToPush := []string{
-		versionedTag,
-		fmt.Sprintf("%s:%s", repoPath, version.Hash),
+	var versionedTag string
+	var tagsToPush []string
+
+	if ubiBase {
+		versionedTag = fmt.Sprintf("%s:%v-ubi", repoPath, version)
+		tagsToPush = []string{
+			versionedTag,
+			fmt.Sprintf("%s:%s-ubi", repoPath, version.Hash),
+			fmt.Sprintf("%s:latest-ubi", repoPath),
+		}
+	} else {
+		versionedTag = fmt.Sprintf("%s:%v", repoPath, version)
+		tagsToPush = []string{
+			versionedTag,
+			fmt.Sprintf("%s:%s", repoPath, version.Hash),
+			fmt.Sprintf("%s:latest", repoPath),
+		}
 	}
-	tags := append(tagsToPush, fmt.Sprintf("%s:latest", repoPath))
+
+	return versionedTag, tagsToPush
+}
+
+func runDockerBuild(versionedTag string, dockerTags []string, extraBuildArgs []string, dockerfile string) {
 	buildArgs := []string{fmt.Sprintf("VERSION_STAMP=%s", versionedTag)}
-	dockerutil.Build(".", "", "./operator/Dockerfile", tags, buildArgs).ExecVPanic()
-	return tagsToPush
+	buildArgs = append(buildArgs, extraBuildArgs...)
+	dockerutil.Build(".", "", dockerfile, dockerTags, buildArgs).ExecVPanic()
 }
 
 func runGoBuild(version string) {
@@ -423,12 +443,24 @@ func BuildDocker() {
 	settings := cfgutil.ReadBuildSettings()
 	git := getGitData()
 	version := calcFullVersion(settings, git)
-	operatorTags := runDockerBuild(version)
+
+	//build regular docker image
+	versionedTag, dockerTags := calcVersionAndTags(version, false)
+	runDockerBuild(versionedTag, dockerTags, nil, dockerBase)
+
+	if baseOs := os.Getenv(EnvBaseOs); baseOs != "" {
+		//build ubi docker image
+		args := []string{fmt.Sprintf("BASE_OS=%s", baseOs)}
+		ubiVersionedTag, ubiDockerTags := calcVersionAndTags(version, true)
+		runDockerBuild(ubiVersionedTag, ubiDockerTags, args, dockerUbi)
+		dockerTags = append(dockerTags, ubiDockerTags...)
+	}
+
 	// Write the versioned image tags to a file in our build
 	// directory so that other targets in the build process can identify
 	// what was built. This is particularly important to know
 	// for targets that retag and deploy to external docker repositories
-	outputText := strings.Join(operatorTags, "|")
+	outputText := strings.Join(dockerTags, "|")
 	writeBuildFile("tagsToPush.txt", outputText)
 }
 

operator/cmd/manager/main.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -120,6 +121,7 @@ func main() {
 	ctx := context.Background()
 	// Become the leader before proceeding
 	err = leader.Become(ctx, "cass-operator-lock")
+
 	if err != nil {
 		log.Error(err, "could not become leader")
 		os.Exit(1)
@@ -132,6 +134,10 @@ func main() {
 		log.Error(err, "Failed to ensure webhook CA configuration")
 	}
 
+	if err = readBaseOsIntoEnv(); err != nil {
+		log.Error(err, "Failed to read base OS into env")
+	}
+
 	// Set default manager options
 	options := manager.Options{
 		Namespace:          namespace,
@@ -200,6 +206,36 @@ func main() {
 	}
 }
 
+func readBaseOsIntoEnv() error {
+	baseOsArgFilePath := "/var/lib/cass-operator/base_os"
+
+	info, err := os.Stat(baseOsArgFilePath)
+	if os.IsNotExist(err) {
+		msg := fmt.Sprintf("Could not locate base OS arg file at %s", baseOsArgFilePath)
+		err = fmt.Errorf("%s. %v", msg, err)
+		return err
+	}
+
+	if info.IsDir() {
+		msg := fmt.Sprintf("Base OS arg path is a directory not a file: %s", baseOsArgFilePath)
+		err = fmt.Errorf("%s. %v", msg, err)
+		return err
+	}
+
+	rawVal, err := ioutil.ReadFile(baseOsArgFilePath)
+	if err != nil {
+		msg := fmt.Sprintf("Failed to read base OS arg file at %s", baseOsArgFilePath)
+		err = fmt.Errorf("%s. %v", msg, err)
+		return err
+	}
+
+	baseOs := strings.TrimSpace(string(rawVal))
+	os.Setenv(api.EnvBaseImageOs, baseOs)
+	log.Info(fmt.Sprintf("%s set to '%s'", api.EnvBaseImageOs, baseOs))
+
+	return nil
+}
+
 // addMetrics will create the Services and Service Monitors to allow the operator export the metrics by using
 // the Prometheus operator
 func addMetrics(ctx context.Context, cfg *rest.Config) {

operator/Dockerfile → operator/docker/base/Dockerfile

@@ -48,6 +48,9 @@ RUN MO_VERSION=${VERSION_STAMP} mage operator:buildGo
 FROM alpine:3.9
 
 ENV GOPATH=/go
+
+RUN mkdir -p /var/lib/cass-operator/
+RUN touch /var/lib/cass-operator/base_os
 WORKDIR /go
 
 # All we need from the builder image is operator executable

operator/docker/ubi/Dockerfile

@@ -0,0 +1,49 @@
+ARG BASE_OS
+FROM datastax/cass-operator:latest AS base
+
+#############################################################
+
+FROM ${BASE_OS} AS builder
+
+# Update the builder layer and create user
+RUN microdnf update && rm -rf /var/cache/yum && \
+    microdnf install shadow-utils && microdnf clean all && \
+    useradd -r -s /bin/false -U -G root cassandra
+
+#############################################################
+FROM ${BASE_OS}
+
+ARG BASE_OS
+ARG VERSION_STAMP=DEV
+
+LABEL maintainer="DataStax, Inc <[email protected]>"
+LABEL name="cass-operator"
+LABEL vendor="DataStax, Inc"
+LABEL release="${VERSION_STAMP}"
+LABEL summary="DataStax Kubernetes Operator for Apache Cassandra "
+LABEL description="The DataStax Kubernetes Operator for Apache Cassandra®. This operator handles the provisioning and day to day management of Apache Cassandra based clusters. Features include configuration deployment, node remediation, and automatic upgrades."
+
+# Update the builder layer and create user
+RUN microdnf update && rm -rf /var/cache/yum && \
+    microdnf install procps-ng && microdnf clean all
+
+# Copy user accounts information
+COPY --from=builder /etc/passwd /etc/passwd
+COPY --from=builder /etc/shadow /etc/shadow
+COPY --from=builder /etc/group /etc/group
+COPY --from=builder /etc/gshadow /etc/gshadow
+
+# Copy operator binary
+COPY --from=base /go/bin/operator /operator
+COPY ./operator/docker/ubi/LICENSE /licenses/
+
+RUN mkdir -p /var/lib/cass-operator/
+RUN echo ${BASE_OS} > /var/lib/cass-operator/base_os
+
+RUN chown cassandra:root /operator && \
+    chmod 0555 /operator
+
+USER cassandra:root
+
+
+ENTRYPOINT ["/operator"]