From bea9b6fe36e5410f76d142fde78013ecc385cecd Mon Sep 17 00:00:00 2001
From: Josh Dolitsky <josh@dolit.ski>
Date: Tue, 4 Apr 2023 18:06:18 -0500
Subject: [PATCH] Automate conformance testing on PR using zot

Signed-off-by: Josh Dolitsky <josh@dolit.ski>
---
 .github/workflows/build-pr.yml | 24 ++++++++++++++++++++++++
 .github/workflows/build.yml    | 24 ++++++++++++++++++++++++
 .gitignore                     |  2 ++
 Makefile                       | 18 ++++++++++++++++++
 4 files changed, 68 insertions(+)

diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml
index 96d65d82..4757ab96 100644
--- a/.github/workflows/build-pr.yml
+++ b/.github/workflows/build-pr.yml
@@ -23,8 +23,32 @@ jobs:
           go-version: ${{ matrix.go }}
 
       - name: run tests
+        id: tests
         run: |
           export PATH="$(go env GOPATH)/bin:${PATH}"
           make install.tools
           make .gitvalidation
           make docs conformance
+
+          set +e
+          make conformance-ci
+          CONFORMANCE_RC="$?"
+          set -e
+          if [[ -f report.html ]]; then
+            echo "Found report.html."
+            echo "has-report=true" >> $GITHUB_OUTPUT
+          fi
+
+          # TODO: Fail with CONFORMANCE_RC if non-zero
+          # (conformance not yet passing for this registry)
+          echo "Conformance return code: ${CONFORMANCE_RC}"
+          exit 0
+
+      - name: Upload OCI conformance results as build artifact
+        if: always() && steps.tests.outputs.has-report == 'true'
+        uses: actions/upload-artifact@v3
+        with:
+          name: oci-conformance-results-${{ matrix.go }}
+          path: |
+            ./report.html
+            ./junit.xml
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 41e33a72..5f095de7 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -23,8 +23,32 @@ jobs:
           go-version: ${{ matrix.go }}
 
       - name: run tests
+        id: tests
         run: |
           export PATH="$(go env GOPATH)/bin:${PATH}"
           make install.tools
           make .gitvalidation
           make docs conformance
+
+          set +e
+          make conformance-ci
+          CONFORMANCE_RC="$?"
+          set -e
+          if [[ -f report.html ]]; then
+            echo "Found report.html."
+            echo "has-report=true" >> $GITHUB_OUTPUT
+          fi
+
+          # TODO: Fail with CONFORMANCE_RC if non-zero
+          # (conformance not yet passing for this registry)
+          echo "Conformance return code: ${CONFORMANCE_RC}"
+          exit 0
+
+      - name: Upload OCI conformance results as build artifact
+        if: always() && steps.tests.outputs.has-report == 'true'
+        uses: actions/upload-artifact@v3
+        with:
+          name: oci-conformance-results-${{ matrix.go }}
+          path: |
+            ./report.html
+            ./junit.xml
diff --git a/.gitignore b/.gitignore
index 630f94c7..3d0ec451 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,5 @@ header.html
 tags
 go.mod
 go.sum
+junit.xml
+report.html
diff --git a/Makefile b/Makefile
index 3c6d6217..b932f13c 100644
--- a/Makefile
+++ b/Makefile
@@ -91,6 +91,24 @@ conformance-test:
 
 conformance-binary: $(OUTPUT_DIRNAME)/conformance.test
 
+TEST_REGISTRY_CONTAINER ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.0.0-rc3
+conformance-ci:
+	docker rm -f oci-conformance && \
+		echo '{"distSpecVersion":"1.1.0-dev","storage":{"rootDirectory":"/tmp/zot","gc":false,"dedupe":false},"http":{"address":"0.0.0.0","port":"5000"},"log":{"level":"debug"}}' > $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json
+		docker run -d \
+			-v $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json:/etc/zot/config.json \
+			--name=oci-conformance \
+			-p 5000:5000 \
+			$(TEST_REGISTRY_CONTAINER) && \
+		export OCI_ROOT_URL="http://localhost:5000" && \
+		export OCI_NAMESPACE="myorg/myrepo" && \
+		export OCI_TEST_PULL=1 && \
+		export OCI_TEST_PUSH=1 && \
+		export OCI_TEST_CONTENT_DISCOVERY=1 && \
+		export OCI_TEST_CONTENT_MANAGEMENT=1 && \
+		sleep 5 && \
+		$(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test
+
 $(OUTPUT_DIRNAME)/conformance.test:
 	cd conformance && \
 		CGO_ENABLED=0 go test -c -o $(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test \