Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Automate conformance testing on PR using zot #401

Merged
merged 9 commits into from
Apr 19, 2023
Merged

Automate conformance testing on PR using zot #401

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
bea9b6f
Automate conformance testing on PR using zot
jdolitsky Apr 4, 2023
8a73873
Merge branch 'main' of github.com:opencontainers/distribution-spec in…
jdolitsky Apr 12, 2023
f0122f6
move location check, allow missing bytes= prefix
jdolitsky Apr 12, 2023
a5582e3
expect units in range header
jdolitsky Apr 12, 2023
18fdcbd
fail CI if conformance fails
jdolitsky Apr 12, 2023
6520ed9
remove bytes= prefix on range header
jdolitsky Apr 13, 2023
b077648
Allow missing bytes= prefix
jdolitsky Apr 13, 2023
b0fa15c
Merge branch 'opencontainers:main' into automate-conformance
jdolitsky Apr 19, 2023
1880a43
modify range check to not need bytes prefix
jdolitsky Apr 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions .github/workflows/build-pr.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,29 @@ jobs:
go-version: ${{ matrix.go }}

- name: run tests
id: tests
run: |
export PATH="$(go env GOPATH)/bin:${PATH}"
make install.tools
make .gitvalidation
make docs conformance

set +e
make conformance-ci
CONFORMANCE_RC="$?"
set -e
if [[ -f report.html ]]; then
echo "Found report.html."
echo "has-report=true" >> $GITHUB_OUTPUT
fi
echo "Conformance return code: ${CONFORMANCE_RC}"
exit ${CONFORMANCE_RC}

- name: Upload OCI conformance results as build artifact
if: always() && steps.tests.outputs.has-report == 'true'
uses: actions/upload-artifact@v3
with:
name: oci-conformance-results-${{ matrix.go }}
path: |
./report.html
./junit.xml
21 changes: 21 additions & 0 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,29 @@ jobs:
go-version: ${{ matrix.go }}

- name: run tests
id: tests
run: |
export PATH="$(go env GOPATH)/bin:${PATH}"
make install.tools
make .gitvalidation
make docs conformance

set +e
make conformance-ci
CONFORMANCE_RC="$?"
set -e
if [[ -f report.html ]]; then
echo "Found report.html."
echo "has-report=true" >> $GITHUB_OUTPUT
fi
echo "Conformance return code: ${CONFORMANCE_RC}"
exit ${CONFORMANCE_RC}

- name: Upload OCI conformance results as build artifact
if: always() && steps.tests.outputs.has-report == 'true'
uses: actions/upload-artifact@v3
with:
name: oci-conformance-results-${{ matrix.go }}
path: |
./report.html
./junit.xml
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,5 @@ header.html
tags
go.mod
go.sum
junit.xml
report.html
18 changes: 18 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,24 @@ conformance-test:

conformance-binary: $(OUTPUT_DIRNAME)/conformance.test

TEST_REGISTRY_CONTAINER ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.0.0-rc3
conformance-ci:
docker rm -f oci-conformance && \
echo '{"distSpecVersion":"1.1.0-dev","storage":{"rootDirectory":"/tmp/zot","gc":false,"dedupe":false},"http":{"address":"0.0.0.0","port":"5000"},"log":{"level":"debug"}}' > $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json
docker run -d \
-v $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json:/etc/zot/config.json \
--name=oci-conformance \
-p 5000:5000 \
$(TEST_REGISTRY_CONTAINER) && \
export OCI_ROOT_URL="http://localhost:5000" && \
export OCI_NAMESPACE="myorg/myrepo" && \
export OCI_TEST_PULL=1 && \
export OCI_TEST_PUSH=1 && \
export OCI_TEST_CONTENT_DISCOVERY=1 && \
export OCI_TEST_CONTENT_MANAGEMENT=1 && \
sleep 5 && \
$(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test

$(OUTPUT_DIRNAME)/conformance.test:
cd conformance && \
CGO_ENABLED=0 go test -c -o $(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test \
Expand Down
14 changes: 5 additions & 9 deletions conformance/02_push_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,6 @@ var test02Push = func() {
location := resp.Header().Get("Location")
Expect(location).ToNot(BeEmpty())
prevResponse = resp

req = client.NewRequest(reggie.PATCH, resp.GetRelativeLocation()).
SetHeader("Content-Type", "application/octet-stream").
SetHeader("Content-Length", testBlobBChunk1Length).
Expand Down Expand Up @@ -220,7 +219,10 @@ var test02Push = func() {
Expect(err).To(BeNil())
Expect(resp.StatusCode()).To(Equal(http.StatusNoContent))
Expect(resp.Header().Get("Location")).ToNot(BeEmpty())
Expect(resp.Header().Get("Range")).To(Equal(fmt.Sprintf("bytes=%s", testBlobBChunk1Range)))
Expect(resp.Header().Get("Range")).To(SatisfyAny(
Equal(testBlobBChunk1Range), // Allow missing "bytes=" prefix
Equal(fmt.Sprintf("bytes=%s", testBlobBChunk1Range)),
))
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added this change to allow for missing "bytes=" prefix

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

chatting with @rchincha - looks like this is required: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range

working on zot version with this fix

Copy link
Contributor

@rchincha rchincha Apr 12, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range

<unit> doesn't look optional, so this appears to be a zot bug and addressed in project-zot/zot#1341

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That said, #203 (comment)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To follow the robustness principle, I believe we need to be strict in what registries output here, only allowing the output without bytes=. Registries and clients can be liberal in what they accept from each other, but for conformance, their output should follow the distribution-spec.

lastResponse = resp
})

Expand Down Expand Up @@ -277,15 +279,13 @@ var test02Push = func() {
Equal(http.StatusCreated),
Equal(http.StatusAccepted),
))
Expect(resp.GetRelativeLocation()).To(Equal(fmt.Sprintf("/v2/%s/blobs/%s", crossmountNamespace, testBlobADigest)))

lastResponse = resp
})

g.Specify("GET request to test digest within cross-mount namespace should return 200", func() {
SkipIfDisabled(push)
RunOnlyIf(lastResponse.StatusCode() == http.StatusCreated)

Expect(lastResponse.GetRelativeLocation()).To(Equal(fmt.Sprintf("/v2/%s/blobs/%s", crossmountNamespace, testBlobADigest)))
req := client.NewRequest(reggie.GET, lastResponse.GetRelativeLocation())
resp, err := client.Do(req)
Expect(err).To(BeNil())
Expand All @@ -295,7 +295,6 @@ var test02Push = func() {
g.Specify("Cross-mounting of nonexistent blob should yield session id", func() {
SkipIfDisabled(push)
RunOnlyIf(lastResponse.StatusCode() == http.StatusAccepted)

Expect(lastResponse.GetRelativeLocation()).To(HavePrefix(fmt.Sprintf("/v2/%s/blobs/uploads/", crossmountNamespace)))
})

Expand All @@ -304,7 +303,6 @@ var test02Push = func() {
RunOnlyIf(runAutomaticCrossmountTest)
RunOnlyIf(lastResponse.StatusCode() == http.StatusCreated)
RunOnlyIf(automaticCrossmountEnabled)

req := client.NewRequest(reggie.POST, "/v2/<name>/blobs/uploads/",
reggie.WithName(crossmountNamespace)).
SetQueryParam("mount", testBlobADigest)
Expand All @@ -318,7 +316,6 @@ var test02Push = func() {
RunOnlyIf(runAutomaticCrossmountTest)
RunOnlyIf(lastResponse.StatusCode() == http.StatusCreated)
RunOnlyIfNot(automaticCrossmountEnabled)

req := client.NewRequest(reggie.POST, "/v2/<name>/blobs/uploads/",
reggie.WithName(crossmountNamespace)).
SetQueryParam("mount", testBlobADigest)
Expand Down Expand Up @@ -442,7 +439,6 @@ var test02Push = func() {
))
})
}

})
})
}