Skip to content

Commit 8513c35

Browse files
developStormdevelopStorm
authored
Build: Multi-platform Docker Builds and Integration Test (zmap#428)
* build: publish docker image to ghcr * build: test cache * build: build standard zgrab image as runner * build: Integration Test use more reliable way to determine project root * build: zgrab should not be built in host * build: attach a tty * build: fix "the input device is not a TTY" * test: remove auto env setup * build: setup python2 * chore: comments * style: format python files with black * fix(test): mongodb schema * fix: pop3 banner checker according to rfc1939, merely +OK is a valid response in the AUTHORIZATION state * build: remove the naive build CI * docs: fix path and instructions in integrated test documents * build: docker image only on main branch push or tagging * build: add linter workflow for Python and Go * patch: template py file validity * build: only run linter on active PRs or main branch * patch: condense case
1 parent 51c2ff1 commit 8513c35

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+5186
-4537
lines changed

‎.dockerignore

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,5 @@
11
Dockerfile
2-
2+
.github
3+
integration_tests
4+
zgrab2_schemas
5+
*.md

‎.github/dependabot.yml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
version: 2
2+
updates:
3+
- package-ecosystem: gomod
4+
directory: /
5+
schedule:
6+
interval: weekly
7+
- package-ecosystem: github-actions
8+
directory: "/"
9+
schedule:
10+
interval: weekly
11+
- package-ecosystem: docker
12+
directory: "/"
13+
schedule:
14+
interval: weekly
15+
- package-ecosystem: docker
16+
directory: "/docker-runner"
17+
schedule:
18+
interval: weekly

‎.github/workflows/build.yml

-19
This file was deleted.

‎.github/workflows/docker-publish.yml

+153
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
# See also:
2+
# https://docs.docker.com/build/ci/github-actions/multi-platform/
3+
# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
4+
5+
name: Publish Docker image
6+
7+
on:
8+
push:
9+
branches:
10+
- master
11+
tags:
12+
13+
env:
14+
REGISTRY: ghcr.io
15+
IMAGE_NAME: ${{ github.repository }}
16+
17+
concurrency:
18+
group: ${{ github.workflow }}-${{ github.ref }}
19+
cancel-in-progress: true
20+
21+
jobs:
22+
build:
23+
runs-on: ubuntu-latest
24+
permissions:
25+
contents: read
26+
packages: write
27+
id-token: write
28+
attestations: write
29+
strategy:
30+
fail-fast: true
31+
matrix:
32+
platform:
33+
- linux/amd64
34+
- linux/arm/v6
35+
- linux/arm/v7
36+
- linux/arm64
37+
steps:
38+
- name: Prepare
39+
run: |
40+
platform=${{ matrix.platform }}
41+
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
42+
43+
- name: Checkout repository
44+
uses: actions/checkout@v4
45+
46+
- name: Extract metadata (tags, labels) for Docker
47+
id: meta
48+
uses: docker/metadata-action@v5
49+
with:
50+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
51+
52+
- name: Log in to the Container registry
53+
uses: docker/login-action@v3
54+
with:
55+
registry: ${{ env.REGISTRY }}
56+
username: ${{ github.actor }}
57+
password: ${{ secrets.GITHUB_TOKEN }}
58+
59+
- uses: docker/setup-qemu-action@v3
60+
- uses: docker/setup-buildx-action@v3
61+
- name: Build and push by digest
62+
id: build
63+
uses: docker/build-push-action@v5
64+
with:
65+
context: .
66+
cache-from: type=gha
67+
cache-to: type=gha,mode=max
68+
platforms: ${{ matrix.platform }}
69+
labels: ${{ steps.meta.outputs.labels }}
70+
outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
71+
72+
- name: Generate artifact attestation
73+
uses: actions/attest-build-provenance@v1
74+
with:
75+
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
76+
subject-digest: ${{ steps.build.outputs.digest }}
77+
push-to-registry: true
78+
79+
- name: Export digest
80+
run: |
81+
mkdir -p /tmp/digests
82+
digest="${{ steps.build.outputs.digest }}"
83+
touch "/tmp/digests/${digest#sha256:}"
84+
85+
- name: Upload digest
86+
uses: actions/upload-artifact@v4
87+
with:
88+
name: digests-${{ env.PLATFORM_PAIR }}
89+
path: /tmp/digests/*
90+
if-no-files-found: error
91+
retention-days: 1
92+
93+
publish:
94+
needs:
95+
- build
96+
runs-on: ubuntu-latest
97+
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
98+
permissions:
99+
contents: read
100+
packages: write
101+
id-token: write
102+
attestations: write
103+
steps:
104+
- name: Download digests
105+
uses: actions/download-artifact@v4
106+
with:
107+
path: /tmp/digests
108+
pattern: digests-*
109+
merge-multiple: true
110+
111+
- name: Set up Docker Buildx
112+
uses: docker/setup-buildx-action@v3
113+
114+
- name: Extract metadata (tags, labels) for Docker
115+
id: meta
116+
uses: docker/metadata-action@v5
117+
with:
118+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
119+
tags: |
120+
type=raw,value=latest,enable={{is_default_branch}}
121+
type=semver,pattern={{version}}
122+
type=semver,pattern={{major}}.{{minor}}
123+
type=semver,pattern={{major}}
124+
type=ref,event=tag
125+
type=ref,event=branch
126+
127+
- name: Log in to the Container registry
128+
uses: docker/login-action@v3
129+
with:
130+
registry: ${{ env.REGISTRY }}
131+
username: ${{ github.actor }}
132+
password: ${{ secrets.GITHUB_TOKEN }}
133+
134+
- name: Create manifest list and push
135+
id: push
136+
working-directory: /tmp/digests
137+
run: |
138+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
139+
$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
140+
141+
- name: Inspect image
142+
id: inspect
143+
run: |
144+
docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
145+
IMAGE_DIGEST=$(docker buildx imagetools inspect --format "{{json .Manifest}}" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} | jq -r '.digest')
146+
echo "IMAGE_DIGEST=$IMAGE_DIGEST" >> $GITHUB_OUTPUT
147+
148+
- name: Generate artifact attestation
149+
uses: actions/attest-build-provenance@v1
150+
with:
151+
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
152+
subject-digest: ${{ steps.inspect.outputs.IMAGE_DIGEST }}
153+
push-to-registry: true

‎.github/workflows/integration-test.yml

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: Integration Test
2+
on:
3+
push:
4+
branches:
5+
- master
6+
pull_request:
7+
8+
jobs:
9+
test:
10+
name: Test
11+
runs-on: ubuntu-latest
12+
steps:
13+
- name: Set up Go
14+
uses: actions/setup-go@v2
15+
with:
16+
go-version: 1.18
17+
18+
- name: Check out source
19+
uses: actions/checkout@v4
20+
21+
- name: Install dependencies
22+
run: |
23+
set -e
24+
sudo wget https://github.com/jmespath/jp/releases/download/0.2.1/jp-linux-amd64 -O /usr/local/bin/jp
25+
sudo chmod +x /usr/local/bin/jp
26+
# Install Python 2.7
27+
sudo apt update
28+
sudo apt install -y python2
29+
curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
30+
sudo python2 get-pip.py
31+
# Install Python dependencies
32+
pip2 install --user zschema
33+
pip2 install --user -r requirements.txt
34+
35+
- name: Run tests
36+
run: |
37+
make integration-test

‎.github/workflows/linter.yml

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: Lint
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
pull_request:
8+
9+
10+
jobs:
11+
py-black:
12+
name: Python Formatter
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v4
16+
- uses: psf/black@stable
17+
go-fmt:
18+
name: Go Formatter
19+
runs-on: ubuntu-latest
20+
if: false # Pending activation
21+
steps:
22+
- uses: actions/checkout@v4
23+
- name: Set up Go
24+
uses: actions/setup-go@v2
25+
with:
26+
go-version: 1.18
27+
- name: Run go fmt
28+
run: go fmt ./...

‎Makefile

+2-8
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ TEST_MODULES ?=
99

1010
all: zgrab2
1111

12-
.PHONY: all clean integration-test integration-test-clean docker-runner container-clean gofmt test
12+
.PHONY: all clean integration-test integration-test-clean docker-runner gofmt test
1313

1414
# Test currently only runs on the modules folder because some of the
1515
# third-party libraries in lib (e.g. http) are failing.
@@ -25,7 +25,7 @@ zgrab2: $(GO_FILES)
2525
rm -f zgrab2
2626
ln -s cmd/zgrab2/zgrab2$(EXECUTABLE_EXTENSION) zgrab2
2727

28-
docker-runner: zgrab2
28+
docker-runner: clean
2929
make -C docker-runner
3030

3131
integration-test: docker-runner
@@ -37,12 +37,6 @@ integration-test-clean:
3737
./integration_tests/cleanup.sh
3838
make -C docker-runner clean
3939

40-
# This is the target for re-building from source in the container
41-
container-clean:
42-
rm -f zgrab2
43-
cd cmd/zgrab2 && go build -v -a . && cd ../..
44-
ln -s cmd/zgrab2/zgrab2$(EXECUTABLE_EXTENSION) zgrab2
45-
4640
clean:
4741
cd cmd/zgrab2 && go clean
4842
rm -f zgrab2

‎README.md

+7-3
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ func init() {
146146

147147
### Output schema
148148

149-
To add a schema for the new module, add a module under schemas, and update [`schemas/__init__.py`](schemas/__init__.py) to ensure that it is loaded.
149+
To add a schema for the new module, add a module under schemas, and update [`zgrab2_schemas/zgrab2/__init__.py`](zgrab2_schemas/zgrab2/__init__.py) to ensure that it is loaded.
150150

151151
See [zgrab2_schemas/README.md](zgrab2_schemas/README.md) for details.
152152

@@ -158,11 +158,13 @@ The only hard requirement is that the `test.sh` script drops its output in `$ZGR
158158

159159
#### How to Run Integration Tests
160160

161-
To run integration tests, you must have [Docker](https://www.docker.com/) installed. Then, you can follow the following steps to run integration tests:
161+
To run integration tests, you must have [Docker](https://www.docker.com/) and **Python 2** on host installed. Then, you can follow the following steps to run integration tests:
162162

163163
```shell
164164
go get github.com/jmespath/jp && go build github.com/jmespath/jp
165-
pip install --user zschema
165+
# or, sudo wget https://github.com/jmespath/jp/releases/download/0.2.1/jp-linux-amd64 -O /usr/local/bin/jp && sudo chmod +x /usr/local/bin/jp
166+
pip2 install --user zschema
167+
pip2 install --user -r requirements.txt
166168
make integration-test
167169
```
168170

@@ -173,5 +175,7 @@ echo $?
173175
0
174176
```
175177

178+
Refer to our [Github Actions workflow](.github/workflows/integration-test.yml) for an example of how to prepare environment for integration tests.
179+
176180
## License
177181
ZGrab2.0 is licensed under Apache 2.0 and ISC. For more information, see the LICENSE file.

‎docker-runner/Dockerfile

-25
This file was deleted.

‎docker-runner/Makefile

+5-8
Original file line numberDiff line numberDiff line change
@@ -4,26 +4,23 @@ else
44
EXECUTABLE_EXTENSION :=
55
endif
66

7-
all: docker-runner.id
7+
PROJ_ROOT := $(shell git rev-parse --show-toplevel)
8+
9+
all: docker-runner.id service-base-image.id
810

911
.PHONY: clean clean-all base-apt-update
1012

1113
service-base-image.id: service-base.Dockerfile
1214
docker build -t zgrab2_service_base:latest -f service-base.Dockerfile -q . > service-base-image.id || (rm -f service-base-image.id && exit 1)
1315

14-
runner-base-image.id: runner-base.Dockerfile
15-
docker build -t zgrab2_runner_base:latest -f runner-base.Dockerfile -q . > runner-base-image.id || (rm -f runner-base-image.id && exit 1)
16-
1716
base-apt-update:
1817
IMAGE_ID=zgrab2_service_base:latest ./base-apt-update.sh
19-
IMAGE_ID=zgrab2_runner_base:latest ./base-apt-update.sh
2018

21-
docker-runner.id: Dockerfile ../cmd/zgrab2/zgrab2$(EXECUTABLE_EXTENSION) runner-base-image.id service-base-image.id
22-
docker build -t zgrab2_runner:latest -f Dockerfile -q .. > docker-runner.id || (rm -f docker-runner.id && exit 1)
19+
docker-runner.id: $(PROJ_ROOT)/Dockerfile
20+
docker build -t zgrab2_runner:latest -f $(PROJ_ROOT)/Dockerfile -q $(PROJ_ROOT) > docker-runner.id || (rm -f docker-runner.id && exit 1)
2321

2422
clean:
2523
if [ -f docker-runner.id ]; then docker rmi -f $$(cat docker-runner.id) && rm -f docker-runner.id; fi
2624

2725
clean-all: clean
2826
if [ -f service-base-image.id ]; then docker rmi -f $$(cat service-base-image.id) && rm -f service-base-image.id; fi
29-
if [ -f runner-base-image.id ]; then docker rmi -f $$(cat runner-base-image.id) && rm -f runner-base-image.id; fi

‎docker-runner/docker-run.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@
77
: "${CONTAINER_NAME:?}"
88

99
set -e
10-
docker run --rm --link $CONTAINER_NAME:target -e ZGRAB_TARGET=target zgrab2_runner $@
10+
echo 'target' | docker run --rm -i --link $CONTAINER_NAME:target zgrab2_runner $@

0 commit comments

Comments
 (0)