Skip to content

Commit e191c2a

Browse files
mslippermslipper
authored
cci-stats: Add project (#115)
* cci-stats: Add project * Add CI * fix lint in proxyd * bump golangci-lint timeout
1 parent 8e5f685 commit e191c2a

File tree

15 files changed

+869
-7
lines changed

15 files changed

+869
-7
lines changed

Diff for: ‎.circleci/config.yml

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ workflows:
2121
peer-mgmt-service/.* run-build-pms true
2222
op-ufm/.* run-build-op-ufm true
2323
proxyd/.* run-build-proxyd true
24+
cci-stats/.* run-build-cci-stats true
2425
.circleci/.* run-all true
2526
.github/.* run-all true
2627
filters:

Diff for: ‎.circleci/continue_config.yml

+28-5
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ parameters:
2222
run-build-proxyd:
2323
type: boolean
2424
default: false
25+
run-build-cci-stats:
26+
type: boolean
27+
default: false
2528
run-all:
2629
type: boolean
2730
default: false
@@ -288,7 +291,7 @@ jobs:
288291
description: Go Module Name
289292
type: string
290293
docker:
291-
- image: cimg/go:1.21
294+
- image: cimg/go:1.23
292295
steps:
293296
- checkout
294297
- run:
@@ -305,9 +308,9 @@ jobs:
305308
name: run lint
306309
command: |
307310
if [ -f .golangci.yml ]; then
308-
golangci-lint run -c .golangci.yml -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "3m0s" ./...
311+
golangci-lint run -c .golangci.yml -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "5m0s" ./...
309312
else
310-
golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "3m0s" ./...
313+
golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "5m0s" ./...
311314
fi
312315
working_directory: <<parameters.module>>
313316

@@ -490,19 +493,35 @@ workflows:
490493
docker_name: proxyd
491494
docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
492495
docker_context: .
496+
cci-stats:
497+
when:
498+
or: [ << pipeline.parameters.run-build-cci-stats >>, << pipeline.parameters.run-all >> ]
499+
jobs:
500+
- go-lint:
501+
name: cci-stats-lint
502+
module: cci-stats
503+
- go-test:
504+
name: cci-stats-tests
505+
module: cci-stats
506+
- docker-build:
507+
name: cci-stats-docker-build
508+
docker_file: cci-stats/Dockerfile
509+
docker_name: cci-stats
510+
docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
511+
docker_context: .
493512
release:
494513
jobs:
495514
- log-config-results:
496515
filters:
497516
tags:
498-
only: /^(peer-mgmt-service|proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
517+
only: /^(cci-stats|peer-mgmt-service|proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
499518
branches:
500519
ignore: /.*/
501520
- hold:
502521
type: approval
503522
filters:
504523
tags:
505-
only: /^(peer-mgmt-service|proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
524+
only: /^(cci-stats|peer-mgmt-service|proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
506525
branches:
507526
ignore: /.*/
508527
- docker-build:
@@ -515,6 +534,7 @@ workflows:
515534
- proxyd
516535
- op-conductor-mon
517536
- peer-mgmt-service
537+
- cci-stats
518538
name: <<matrix.docker_name>>-docker-build
519539
filters:
520540
tags:
@@ -536,6 +556,7 @@ workflows:
536556
- proxyd
537557
- op-conductor-mon
538558
- peer-mgmt-service
559+
- cci-stats
539560
name: <<matrix.docker_name>>-docker-publish
540561
filters:
541562
tags:
@@ -555,6 +576,7 @@ workflows:
555576
- proxyd
556577
- op-conductor-mon
557578
- peer-mgmt-service
579+
- cci-stats
558580
name: <<matrix.docker_name>>-docker-tag
559581
filters:
560582
tags:
@@ -575,6 +597,7 @@ workflows:
575597
- proxyd
576598
- op-conductor-mon
577599
- peer-mgmt-service
600+
- cci-stats
578601
name: <<matrix.module>>-go-release
579602
filters:
580603
tags:

Diff for: ‎cci-stats/.goreleaser.yaml

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# yaml-language-server: $schema=https://goreleaser.com/static/schema-pro.json
2+
# vim: set ts=2 sw=2 tw=0 fo=cnqoj
3+
4+
version: 2
5+
6+
project_name: cci-stats
7+
8+
before:
9+
hooks:
10+
# You may remove this if you don't use go modules.
11+
- go mod tidy
12+
13+
builds:
14+
- id: main
15+
main: ./cmd/runner
16+
binary: cci-stats
17+
goos:
18+
- linux
19+
- windows
20+
- darwin
21+
goarch:
22+
- amd64
23+
- arm64
24+
ignore:
25+
- goos: windows
26+
goarch: arm64
27+
- goos: linux
28+
goarch: arm64
29+
mod_timestamp: "{{ .CommitTimestamp }}"
30+
ldflags:
31+
- -X main.GitCommit={{ .FullCommit }}
32+
- -X main.GitDate={{ .CommitDate }}
33+
- -X main.Version={{ .Version }}
34+
35+
archives:
36+
- format: tar.gz
37+
# this name template makes the OS and Arch compatible with the results of `uname`.
38+
name_template: "{{ .ProjectName }}-{{.Version}}-{{ tolower .Os }}-{{ .Arch }}"
39+
# use zip for windows archives
40+
wrap_in_directory: true
41+
format_overrides:
42+
- goos: windows
43+
format: zip
44+
45+
changelog:
46+
sort: asc
47+
48+
release:
49+
github:
50+
owner: ethereum-optimism
51+
name: infra
52+
make_latest: false
53+
54+
monorepo:
55+
tag_prefix: cci-stats/
56+
dir: cci-stats

Diff for: ‎cci-stats/Dockerfile

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
FROM golang:1.23.4-alpine3.20 as builder
2+
3+
ARG GITCOMMIT=docker
4+
ARG GITDATE=docker
5+
ARG GITVERSION=docker
6+
7+
COPY ./cci-stats /app
8+
9+
WORKDIR /app
10+
RUN apk add --no-cache make jq bash git alpine-sdk
11+
RUN make build
12+
13+
FROM alpine:3.20
14+
RUN addgroup -S app && adduser -S app -G app
15+
USER app
16+
WORKDIR /app
17+
18+
COPY --from=builder /app/bin/cci-stats /app
19+
20+
ENTRYPOINT ["/app/cci-stats"]

Diff for: ‎cci-stats/Makefile

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
LDFLAGSSTRING +=-X main.GitCommit=$(GITCOMMIT)
2+
LDFLAGSSTRING +=-X main.GitDate=$(GITDATE)
3+
LDFLAGSSTRING +=-X main.GitVersion=$(GITVERSION)
4+
LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
5+
6+
all: build
7+
8+
docker:
9+
docker build ../ -f Dockerfile -t cci-stats:latest
10+
11+
build:
12+
env GO111MODULE=on go build -v $(LDFLAGS) -o ./bin/cci-stats ./cmd/runner
13+
14+
clean:
15+
rm ./bin/cci-stats
16+
17+
test:
18+
go test -v ./...
19+
20+
lint:
21+
golangci-lint run ./...
22+
23+
.PHONY: \
24+
build \
25+
clean \
26+
test \
27+
lint \
28+
docker

Diff for: ‎cci-stats/README.md

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# cci-stats
2+
3+
`cci-stats` is a program that reads stats from the CircleCI API, and writes them to a Postgres database. It is used
4+
internally at OP Labs to keep track of CI pass rate, merge throughput, and other engineering health metrics.
5+
6+
To run the program, specify the following env vars:
7+
8+
- `CCI_KEY`: CircleCI API Key
9+
- `DATABASE_URI`: Postgres database URI
10+
- `PROJECT_SLUG`: Slug of the CCI project you want to grab stats for
11+
- `BRANCH_PATTERN`: Regex pattern to filter branches by
12+
- `WORKFLOW_PATTERN`: Regex pattern to filter workflows by
13+
- `FETCH_LIMIT_DAYS`: Maximum number of days to look into the past for new build data
14+
- `MAX_CONCURRENT_FETCH_JOBS`: How many concurrent requests to CCI to make at once. Used to tune rate limits
15+
- `SLOW_TEST_THRESHOLD_SECONDS`: Tests slower than this threshold are written to the database as "slow tests" for
16+
further debugging
17+
18+
Then run `go run cmd/runner/main.go`.

Diff for: ‎cci-stats/cmd/runner/main.go

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"os/signal"
8+
"regexp"
9+
"strconv"
10+
11+
"github.com/axelKingsley/go-circleci"
12+
"github.com/ethereum-optimism/infra/cci-stats/pkg/config"
13+
"github.com/ethereum-optimism/infra/cci-stats/pkg/db"
14+
"github.com/ethereum-optimism/infra/cci-stats/pkg/service"
15+
)
16+
17+
func main() {
18+
if err := run(); err != nil {
19+
fmt.Fprintf(os.Stderr, "error: %v\n", err)
20+
os.Exit(1)
21+
}
22+
}
23+
24+
func run() error {
25+
cciKey := requiredStrEnv("CCI_KEY")
26+
dbURI := requiredStrEnv("DATABASE_URI")
27+
projectSlug := requiredStrEnv("PROJECT_SLUG")
28+
branchPattern := requiredStrEnv("BRANCH_PATTERN")
29+
workflowPattern := requiredStrEnv("WORKFLOW_PATTERN")
30+
fetchLimitDays := requiredIntEnv("FETCH_LIMIT_DAYS")
31+
maxConcurrentFetchJobs := requiredIntEnv("MAX_CONCURRENT_FETCH_JOBS")
32+
slowTestThresholdSeconds := requiredIntEnv("SLOW_TEST_THRESHOLD_SECONDS")
33+
34+
ctx, cancel := context.WithCancel(context.Background())
35+
defer cancel()
36+
37+
dbConn, err := db.New(ctx, dbURI)
38+
if err != nil {
39+
return fmt.Errorf("failed to connect to db: %w", err)
40+
}
41+
defer dbConn.Close()
42+
43+
cfg := config.Config{
44+
ProjectSlug: projectSlug,
45+
BranchPatternRegex: regexp.MustCompile(branchPattern),
46+
WorkflowPatternRegex: regexp.MustCompile(workflowPattern),
47+
FetchLimitDays: fetchLimitDays,
48+
MaxConcurrentFetchJobs: maxConcurrentFetchJobs,
49+
SlowTestThresholdSeconds: float64(slowTestThresholdSeconds),
50+
}
51+
52+
clientCfg := circleci.DefaultConfig()
53+
clientCfg.Token = cciKey
54+
client, err := circleci.NewClient(clientCfg)
55+
if err != nil {
56+
return fmt.Errorf("failed to create circleci client: %w", err)
57+
}
58+
59+
errC := make(chan error)
60+
go func() {
61+
err := service.GenerateReport(ctx, cfg, client, dbConn)
62+
errC <- err
63+
}()
64+
65+
sigs := make(chan os.Signal, 1)
66+
signal.Notify(sigs, os.Interrupt)
67+
68+
for {
69+
select {
70+
case <-sigs:
71+
cancel()
72+
case err := <-errC:
73+
return err
74+
case <-ctx.Done():
75+
return nil
76+
}
77+
}
78+
}
79+
80+
func requiredStrEnv(envVar string) string {
81+
val := os.Getenv(envVar)
82+
if val == "" {
83+
panic(fmt.Errorf("%s must be set", envVar))
84+
}
85+
return val
86+
}
87+
88+
func requiredIntEnv(envVar string) int {
89+
val := os.Getenv(envVar)
90+
if val == "" {
91+
panic(fmt.Errorf("%s must be set", envVar))
92+
}
93+
out, err := strconv.Atoi(val)
94+
if err != nil {
95+
panic(fmt.Errorf("%s must be an integer", envVar))
96+
}
97+
return out
98+
}

Diff for: ‎cci-stats/go.mod

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
module github.com/ethereum-optimism/infra/cci-stats
2+
3+
go 1.23.1
4+
5+
require (
6+
github.com/axelKingsley/go-circleci v0.9.1
7+
github.com/jackc/pgx/v5 v5.7.1
8+
github.com/sourcegraph/conc v0.3.0
9+
)
10+
11+
require (
12+
github.com/google/go-querystring v1.1.0 // indirect
13+
github.com/jackc/pgpassfile v1.0.0 // indirect
14+
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
15+
github.com/jackc/puddle/v2 v2.2.2 // indirect
16+
go.uber.org/atomic v1.7.0 // indirect
17+
go.uber.org/multierr v1.9.0 // indirect
18+
golang.org/x/crypto v0.28.0 // indirect
19+
golang.org/x/sync v0.8.0 // indirect
20+
golang.org/x/text v0.19.0 // indirect
21+
)

Diff for: ‎cci-stats/go.sum

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
github.com/axelKingsley/go-circleci v0.9.1 h1:TsGXu2QeTaUO47BKVw9fBTKCH9tYiLltztFxTxE1GYA=
2+
github.com/axelKingsley/go-circleci v0.9.1/go.mod h1:fk2iqm3nLWa+Xj9wWUpnfDnuUdLizC6Rj8xEUxj6E18=
3+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
4+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
5+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6+
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
7+
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
8+
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
9+
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
10+
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
11+
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
12+
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
13+
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
14+
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
15+
github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs=
16+
github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA=
17+
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
18+
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
19+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
20+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
21+
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
22+
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
23+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
24+
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
25+
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
26+
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
27+
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
28+
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
29+
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
30+
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
31+
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
32+
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
33+
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
34+
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
35+
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
36+
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
37+
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
38+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
39+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
40+
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
41+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
42+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

0 commit comments

Comments
 (0)