Skip to content

Commit 28bb746

Browse files
felipe-opfelipe-op
authored
feat: op-conductor-mon (#1)
* feat: op-conductor-mon * Update CODEOWNERS
1 parent 286dc71 commit 28bb746

29 files changed

+2213
-9
lines changed

Diff for: ‎.circleci/config.yml

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
version: 2.1
2+
3+
# this allows you to use CircleCI's dynamic configuration feature
4+
setup: true
5+
6+
orbs:
7+
path-filtering: circleci/[email protected]
8+
9+
workflows:
10+
check-updated-files:
11+
jobs:
12+
- path-filtering/filter:
13+
name: check-updated-files
14+
# 3-column, whitespace-delimited mapping. One mapping per
15+
# line:
16+
# <regex path-to-test> <parameter-to-set> <value-of-pipeline-parameter>
17+
mapping: |
18+
op-conductor-mon/.* run-build-op-conductor-mon true
19+
.circleci/.* run-all true
20+
.github/.* run-all true
21+
22+
base-revision: main
23+
# this is the path of the configuration we should trigger once
24+
# path filtering and pipeline parameter value updates are
25+
# complete. In this case, we are using the parent dynamic
26+
# configuration itself.
27+
config-path: .circleci/continue_config.yml

Diff for: ‎.circleci/continue_config.yml

+359
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,359 @@
1+
version: 2.1
2+
3+
orbs:
4+
go: circleci/[email protected]
5+
gcp-cli: circleci/[email protected]
6+
shellcheck: circleci/[email protected]
7+
path-filtering: circleci/[email protected]
8+
9+
parameters:
10+
run-build-op-conductor-mon:
11+
type: boolean
12+
default: false
13+
run-all:
14+
type: boolean
15+
default: false
16+
17+
commands:
18+
gcp-oidc-authenticate:
19+
description: "Authenticate with GCP using a CircleCI OIDC token."
20+
parameters:
21+
project_id:
22+
type: env_var_name
23+
default: GCP_PROJECT_ID
24+
workload_identity_pool_id:
25+
type: env_var_name
26+
default: GCP_WIP_ID
27+
workload_identity_pool_provider_id:
28+
type: env_var_name
29+
default: GCP_WIP_PROVIDER_ID
30+
service_account_email:
31+
type: env_var_name
32+
default: GCP_SERVICE_ACCOUNT_EMAIL
33+
gcp_cred_config_file_path:
34+
type: string
35+
default: /home/circleci/gcp_cred_config.json
36+
oidc_token_file_path:
37+
type: string
38+
default: /home/circleci/oidc_token.json
39+
steps:
40+
- run:
41+
name: "Create OIDC credential configuration"
42+
command: |
43+
# Store OIDC token in temp file
44+
echo $CIRCLE_OIDC_TOKEN > << parameters.oidc_token_file_path >>
45+
# Create a credential configuration for the generated OIDC ID Token
46+
gcloud iam workload-identity-pools create-cred-config \
47+
"projects/${<< parameters.project_id >>}/locations/global/workloadIdentityPools/${<< parameters.workload_identity_pool_id >>}/providers/${<< parameters.workload_identity_pool_provider_id >>}"\
48+
--output-file="<< parameters.gcp_cred_config_file_path >>" \
49+
--service-account="${<< parameters.service_account_email >>}" \
50+
--credential-source-file=<< parameters.oidc_token_file_path >>
51+
- run:
52+
name: "Authenticate with GCP using OIDC"
53+
command: |
54+
# Configure gcloud to leverage the generated credential configuration
55+
gcloud auth login --brief --cred-file "<< parameters.gcp_cred_config_file_path >>"
56+
# Configure ADC
57+
echo "export GOOGLE_APPLICATION_CREDENTIALS='<< parameters.gcp_cred_config_file_path >>'" | tee -a "$BASH_ENV"
58+
59+
60+
jobs:
61+
docker-build:
62+
environment:
63+
DOCKER_BUILDKIT: 1
64+
parameters:
65+
docker_name:
66+
description: Docker image name
67+
type: string
68+
docker_tags:
69+
description: Docker image tags as csv
70+
type: string
71+
docker_file:
72+
description: Path to Dockerfile
73+
type: string
74+
docker_context:
75+
description: Docker build context
76+
type: string
77+
registry:
78+
description: Docker registry
79+
type: string
80+
default: "us-docker.pkg.dev"
81+
repo:
82+
description: Docker repo
83+
type: string
84+
default: "oplabs-tools-artifacts/images"
85+
machine:
86+
image: default
87+
steps:
88+
- checkout
89+
- run:
90+
command: mkdir -p /tmp/docker_images
91+
- run:
92+
name: Build
93+
command: |
94+
# Check to see if DOCKER_HUB_READ_ONLY_TOKEN is set (i.e. we are in repo) before attempting to use secrets.
95+
# Building should work without this read only login, but may get rate limited.
96+
if [[ -v DOCKER_HUB_READ_ONLY_TOKEN ]]; then
97+
echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin
98+
fi
99+
IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
100+
DOCKER_TAGS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|-t ${IMAGE_BASE}:|")
101+
docker build \
102+
$(echo -ne $DOCKER_TAGS | tr '\n' ' ') \
103+
-f <<parameters.docker_file>> \
104+
<<parameters.docker_context>>
105+
- run:
106+
name: Save
107+
command: |
108+
IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
109+
DOCKER_LABELS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g")
110+
echo -ne $DOCKER_LABELS | tr ' ' '\n' | xargs -I {} docker save -o /tmp/docker_images/<<parameters.docker_name>>_{}.tar $IMAGE_BASE:{}
111+
- persist_to_workspace:
112+
root: /tmp/docker_images
113+
paths:
114+
- "."
115+
116+
docker-publish:
117+
parameters:
118+
docker_name:
119+
description: Docker image name
120+
type: string
121+
docker_tags:
122+
description: Docker image tags as csv
123+
type: string
124+
registry:
125+
description: Docker registry
126+
type: string
127+
default: "us-docker.pkg.dev"
128+
repo:
129+
description: Docker repo
130+
type: string
131+
default: "oplabs-tools-artifacts/images"
132+
machine:
133+
image: default
134+
steps:
135+
- attach_workspace:
136+
at: /tmp/docker_images
137+
- run:
138+
name: Docker load
139+
command: |
140+
DOCKER_LABELS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g")
141+
echo -ne $DOCKER_LABELS | tr ' ' '\n' | xargs -I {} docker load -i /tmp/docker_images/<<parameters.docker_name>>_{}.tar
142+
- gcp-oidc-authenticate
143+
# Below is CircleCI recommended way of specifying nameservers on an Ubuntu box:
144+
# https://support.circleci.com/hc/en-us/articles/7323511028251-How-to-set-custom-DNS-on-Ubuntu-based-images-using-netplan
145+
- run: sudo sed -i '13 i \ \ \ \ \ \ \ \ \ \ \ \ nameservers:' /etc/netplan/50-cloud-init.yaml
146+
- run: sudo sed -i '14 i \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ addresses:' /etc/netplan/50-cloud-init.yaml
147+
- run: sudo sed -i "s/addresses:/ addresses":" [8.8.8.8, 8.8.4.4] /g" /etc/netplan/50-cloud-init.yaml
148+
- run: cat /etc/netplan/50-cloud-init.yaml
149+
- run: sudo netplan apply
150+
- run:
151+
name: Publish
152+
command: |
153+
gcloud auth configure-docker <<parameters.registry>>
154+
IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
155+
DOCKER_TAGS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|${IMAGE_BASE}:|")
156+
echo -ne $DOCKER_TAGS | tr ' ' '\n' | xargs -L1 docker push
157+
158+
- when:
159+
condition:
160+
equal: ['main', <<pipeline.git.branch>>]
161+
steps:
162+
- gcp-oidc-authenticate:
163+
service_account_email: GCP_SERVICE_ATTESTOR_ACCOUNT_EMAIL
164+
- run:
165+
name: Sign
166+
command: |
167+
git clone --branch v1.0.3 --depth 1 https://github.com/ethereum-optimism/binary_signer
168+
cd binary_signer/signer
169+
170+
IMAGE_PATH="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>:<<pipeline.git.revision>>"
171+
echo $IMAGE_PATH
172+
pip3 install -r requirements.txt
173+
174+
python3 ./sign_image.py --command="sign"\
175+
--attestor-project-name="$ATTESTOR_PROJECT_NAME"\
176+
--attestor-name="$ATTESTOR_NAME"\
177+
--image-path="$IMAGE_PATH"\
178+
--signer-logging-level="INFO"\
179+
--attestor-key-id="//cloudkms.googleapis.com/v1/projects/$ATTESTOR_PROJECT_NAME/locations/global/keyRings/$ATTESTOR_NAME-key-ring/cryptoKeys/$ATTESTOR_NAME-key/cryptoKeyVersions/1"
180+
181+
182+
docker-tag-op-stack-release:
183+
parameters:
184+
registry:
185+
description: Docker registry
186+
type: string
187+
default: "us-docker.pkg.dev"
188+
repo:
189+
description: Docker repo
190+
type: string
191+
default: "oplabs-tools-artifacts/images"
192+
docker:
193+
- image: cimg/python:3.7
194+
resource_class: small
195+
steps:
196+
- gcp-cli/install
197+
- gcp-oidc-authenticate
198+
- checkout
199+
- run:
200+
name: Tag
201+
command: |
202+
gcloud auth configure-docker <<parameters.registry>>
203+
./ops/scripts/ci-docker-tag-op-stack-release.sh <<parameters.registry>>/<<parameters.repo>> $CIRCLE_TAG $CIRCLE_SHA1
204+
205+
go-lint:
206+
parameters:
207+
module:
208+
description: Go Module Name
209+
type: string
210+
docker:
211+
- image: cimg/go:1.21
212+
steps:
213+
- checkout
214+
- run:
215+
name: run generate
216+
command: |
217+
make generate || go generate ./...
218+
working_directory: <<parameters.module>>
219+
- run:
220+
name: run tidy
221+
command: |
222+
go mod tidy && git diff --exit-code
223+
working_directory: <<parameters.module>>
224+
- run:
225+
name: run lint
226+
command: |
227+
golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "3m0s" ./...
228+
working_directory: <<parameters.module>>
229+
230+
go-test:
231+
parameters:
232+
module:
233+
description: Go Module Name
234+
type: string
235+
docker:
236+
- image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest # only used to enable codecov.
237+
- image: cimg/postgres:14.6
238+
environment:
239+
POSTGRES_USER: opc
240+
POSTGRES_HOST_AUTH_METHOD: trust
241+
resource_class: small
242+
steps:
243+
- checkout
244+
- run:
245+
name: go version
246+
command: go version
247+
- run:
248+
name: prep results dir
249+
command: mkdir -p /tmp/test-results
250+
- run:
251+
name: run generate
252+
command: |
253+
make generate || go generate ./...
254+
working_directory: <<parameters.module>>
255+
- run:
256+
name: run tests
257+
command: |
258+
gotestsum --format=standard-verbose --junitfile=/tmp/test-results/<<parameters.module>>.xml \
259+
-- -coverpkg=github.com/ethereum-optimism/infrastructure-services/... -coverprofile=coverage.out ./...
260+
working_directory: <<parameters.module>>
261+
- run:
262+
name: upload coverage
263+
command: codecov --verbose --clean --flags <<parameters.module>>
264+
- store_test_results:
265+
path: /tmp/test-results
266+
267+
py-presubmit:
268+
parameters:
269+
poetry_root:
270+
description: Root of the Poetry project directory.
271+
type: string
272+
docker:
273+
- image: cimg/python:3.11
274+
resource_class: small
275+
steps:
276+
- checkout
277+
- run:
278+
name: prep results dir
279+
command: mkdir -p /tmp/test-results
280+
- run:
281+
name: run presubmit
282+
command: |
283+
poetry install
284+
poetry run presubmit
285+
working_directory: <<parameters.poetry_root>>
286+
287+
build-release:
288+
parameters:
289+
package_name:
290+
description: Package to build
291+
type: string
292+
artifact_path:
293+
description: Path to build artifact
294+
type: string
295+
default: ./bin
296+
release_env:
297+
description: Release environment
298+
type: string
299+
default: prod
300+
docker:
301+
- image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
302+
steps:
303+
- checkout
304+
- run:
305+
name: Build
306+
command: |
307+
VERSION=$(echo "$CIRCLE_TAG" | grep -Eow 'v.*' || true)
308+
make build-release VERSION=$VERSION RELEASE_ENV=<<parameters.release_env>>
309+
working_directory: <<parameters.package_name>>
310+
- persist_to_workspace:
311+
root: <<parameters.package_name>>/<<parameters.artifact_path>>
312+
paths:
313+
- "."
314+
315+
publish-release:
316+
parameters:
317+
package_name:
318+
description: Package to publish
319+
type: string
320+
artifact_path:
321+
description: Path to build artifact
322+
type: string
323+
default: ./bin
324+
docker:
325+
- image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
326+
steps:
327+
- attach_workspace:
328+
at: <<parameters.package_name>>/<<parameters.artifact_path>>
329+
- run:
330+
name: "Publish Release on GitHub"
331+
command: |
332+
go install github.com/tcnksm/[email protected]
333+
ghr -t "$GITHUB_TOKEN" -u "$CIRCLE_PROJECT_USERNAME" -r "$CIRCLE_PROJECT_REPONAME" -c "$CIRCLE_SHA1" -delete "$CIRCLE_TAG" <<parameters.package_name>>/<<parameters.artifact_path>>
334+
335+
workflows:
336+
op-conductor-mon:
337+
when:
338+
or: [<< pipeline.parameters.run-build-op-conductor-mon >>, << pipeline.parameters.run-all >>]
339+
jobs:
340+
- go-lint:
341+
name: op-conductor-mon-lint
342+
module: op-conductor-mon
343+
- go-test:
344+
name: op-conductor-mon-tests
345+
module: op-conductor-mon
346+
- docker-build:
347+
name: op-conductor-mon-docker-build
348+
docker_file: op-conductor-mon/Dockerfile
349+
docker_name: op-conductor-mon
350+
docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
351+
docker_context: .
352+
- docker-publish:
353+
name: op-conductor-mon-docker-publish
354+
docker_name: op-conductor-mon
355+
docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
356+
context:
357+
- oplabs-gcr
358+
requires:
359+
- op-conductor-mon-docker-build

0 commit comments

Comments
 (0)