Skip to content

Commit 4abc226

Browse files
phlaxphlax
authored
ci: Use upstream envoy.code.check (envoyproxy#19737)
This PR updates CI to use an upstream tool that integrates 4 of the current format CI checks: - flake8 - glint - shellcheck - yapf The tool can be run with: $ bazel run //tools/code:check # -- -c glint shellcheck python_yapf python_flake8 The -- --fix flag works for yapf only Also worth mentioning is that the following will only check changes against eg main (commits/branches/etc should work) $ bazel run //tools/code:check -- -s main This should allow us to use the tool in git hooks once enough of the checks have been integrated We can add more of the format checks to the tool after There are also various other planned improvements See https://github.com/envoyproxy/pytooling/milestones for general roadmap I would also like to rationalize the format check part of CI as we go (the pytest/tooling ci jobs can be removed fairly imminently) As the new tool produces quite a bit of logging ive had to make it run first - so as it doesnt bury the logs from other checkers For that reason, i added a message at the bottom of the log output to advise searching above A failing check run can be seen here: https://dev.azure.com/cncf/envoy/_build/results?buildId=103005&view=logs&j=c5dd2866-6ab3-5f3c-3a44-4cef0ec909b5&t=a9eb66d6-8944-5769-b3f7-476949dadcb8&l=7560 https://dev.azure.com/cncf/envoy/_build/results?buildId=103005&view=logs&j=c5dd2866-6ab3-5f3c-3a44-4cef0ec909b5&t=a9eb66d6-8944-5769-b3f7-476949dadcb8&l=7433 https://dev.azure.com/cncf/envoy/_build/results?buildId=103005&view=logs&j=c5dd2866-6ab3-5f3c-3a44-4cef0ec909b5&t=a9eb66d6-8944-5769-b3f7-476949dadcb8&l=7292 https://dev.azure.com/cncf/envoy/_build/results?buildId=103005&view=logs&j=c5dd2866-6ab3-5f3c-3a44-4cef0ec909b5&t=a9eb66d6-8944-5769-b3f7-476949dadcb8&l=124 https://dev.azure.com/cncf/envoy/_build/results?buildId=103005&view=logs&j=c5dd2866-6ab3-5f3c-3a44-4cef0ec909b5&t=a9eb66d6-8944-5769-b3f7-476949dadcb8&l=251 https://dev.azure.com/cncf/envoy/_build/results?buildId=103005&view=logs&j=c5dd2866-6ab3-5f3c-3a44-4cef0ec909b5&t=a9eb66d6-8944-5769-b3f7-476949dadcb8&l=7050 In addition to adding the envoy.code.check tool this PR also brings in newer versions of core libs that have been fixed/optimized/cleaned up significantly in this release release info/milestone for this PR https://github.com/envoyproxy/pytooling/milestone/1 https://github.com/envoyproxy/pytooling/releases/tag/2022-03-04.0 Next milestone is the next iteration of the dep.checker - mostly cleanups/fixes i think https://github.com/envoyproxy/pytooling/milestone/2 Signed-off-by: Ryan Northey <[email protected]>
1 parent 1e20ccd commit 4abc226

File tree

9 files changed

+62
-251
lines changed

9 files changed

+62
-251
lines changed

‎.flake8

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@
55
ignore = W503,W504,E121,E126,E241,E125,E127,E129,E251,E265,E303,E306,E402,E501,E502,E711,E713,E722,E741,F523,F541,F841,N803,N806,N817,W605
66

77
# TODO(phlax): exclude less
8-
exclude = build_docs,.git,generated,test,examples,venv
8+
exclude = build_docs,.git,generated,test,examples,venv,tools/dev

‎ci/format_pre.sh

+7-14
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,12 @@ trap_errors () {
2626
FAILED+=(" > ${sub}@ ${file} :${line}")
2727
else
2828
FAILED+=("${sub}@ ${file} :${line}${command}")
29-
if [[ "$CURRENT" == "glint" ]]; then
29+
if [[ "$CURRENT" == "check" ]]; then
30+
# shellcheck disable=SC2016
3031
FAILED+=(
31-
" Please fix your editor to ensure:"
32-
" - no trailing whitespace"
33-
" - no mixed tabs/spaces"
34-
" - all files end with a newline")
32+
""
33+
' *Code formatting check failed*: please search above logs for `CodeChecker ERROR`'
34+
"")
3535
fi
3636
fi
3737
((frame++))
@@ -42,19 +42,12 @@ trap_errors () {
4242
trap trap_errors ERR
4343
trap exit 1 INT
4444

45-
# TODO: move these to bazel
46-
CURRENT=glint
47-
"${ENVOY_SRCDIR}"/tools/code_format/glint.sh
48-
49-
CURRENT=shellcheck
50-
"${ENVOY_SRCDIR}"/tools/code_format/check_shellcheck_format.sh check
45+
CURRENT=check
46+
time bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/code:check
5147

5248
CURRENT=configs
5349
bazel run "${BAZEL_BUILD_OPTIONS[@]}" //configs:example_configs_validation
5450

55-
CURRENT=python
56-
bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/code_format:python_check -- --diff-file="$DIFF_OUTPUT" -werror --fix
57-
5851
CURRENT=extensions
5952
bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/extensions:extensions_check
6053

‎tools/base/requirements.in

+7-7
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,22 @@ abstracts>=0.0.12
22
aio.api.bazel
33
aio.api.github>=0.0.5
44
aio.api.nist
5-
aio.core>=0.5.8
6-
aio.run.checker>=0.3.5
7-
aio.run.runner>=0.2.6
5+
aio.core>=0.8.1
6+
aio.run.checker>=0.5.1
7+
aio.run.runner>=0.3.2
88
colorama
99
coloredlogs
1010
coverage
1111
envoy.base.utils>=0.1.0
12-
envoy.code_format.python_check>=0.0.8
12+
envoy.code.check
1313
envoy.dependency.check>=0.0.5
1414
envoy.dependency.pip_check>=0.1.2
1515
envoy.distribution.release>=0.0.7
1616
envoy.distribution.repo>=0.0.5
1717
envoy.distribution.verify>=0.0.8
18-
envoy.docs.sphinx-runner>=0.0.8
19-
envoy.gpg.identity>=0.0.6
20-
envoy.gpg.sign>=0.0.11
18+
envoy.docs.sphinx-runner>=0.1.0
19+
envoy.gpg.identity>=0.1.0
20+
envoy.gpg.sign>=0.1.0
2121
flake8
2222
frozendict
2323
gitpython

‎tools/base/requirements.txt

+34-29
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,9 @@ abstracts==0.0.12 \
1212
# aio-api-github
1313
# aio-api-nist
1414
# aio-core
15-
# aio-run-checker
1615
# aio-run-runner
1716
# envoy-base-utils
18-
# envoy-code-format-python-check
17+
# envoy-code-check
1918
# envoy-dependency-check
2019
# envoy-dependency-pip-check
2120
# envoy-distribution-release
@@ -38,34 +37,35 @@ aio-api-nist==0.0.1 \
3837
# via
3938
# -r requirements.in
4039
# envoy-dependency-check
41-
aio-core==0.5.8 \
42-
--hash=sha256:3e027710040ce43548f5704a8cd45fbb9c2a0a609a3d65735db0da3f79fa8d3f \
43-
--hash=sha256:a5f09e74b726accba91abf3bd32d2072d9add833bd5d092a6d9545e5efe26e7c
40+
aio-core==0.8.1 \
41+
--hash=sha256:085fb6573b7b8f07876f6f45d10692f20445cfe686ed43c46ed05d33379556b3 \
42+
--hash=sha256:136544fcc2f04a48fc9dd8d4413381bb1a6dd226c9eab733477167762a677854
4443
# via
4544
# -r requirements.in
4645
# aio-api-github
4746
# aio-api-nist
4847
# aio-run-runner
4948
# envoy-base-utils
50-
# envoy-code-format-python-check
49+
# envoy-code-check
5150
# envoy-dependency-check
5251
# envoy-distribution-release
5352
# envoy-distribution-repo
5453
# envoy-github-abstract
5554
# envoy-github-release
56-
aio-run-checker==0.3.5 \
57-
--hash=sha256:3ac33c5a8d5168d457c1fa2e874f76032d27615f57937c7a939123dec20cc8bf \
58-
--hash=sha256:3c8e09b4514dd33afdce21cae98b7fae90a9983973714d9935ba38726b084f07
55+
# envoy-gpg-identity
56+
aio-run-checker==0.5.1 \
57+
--hash=sha256:07e0c39031afa5370740e98a7e1628123e56646610fd8b9cf983ad85205d3adf \
58+
--hash=sha256:c9dc570aa1bf16355a35aec2417d897ecb77d03b82bd5a4f56986966437ab947
5959
# via
6060
# -r requirements.in
61-
# envoy-code-format-python-check
61+
# envoy-code-check
6262
# envoy-dependency-check
6363
# envoy-dependency-pip-check
6464
# envoy-distribution-distrotest
6565
# envoy-distribution-verify
66-
aio-run-runner==0.2.6 \
67-
--hash=sha256:5512fbef70a1eae793455caa29653d95da850cabb8063a74241d94b110de32fe \
68-
--hash=sha256:fb350bc70a998ca07f9185c277d4d114cd6cc8de3eb4d95906407b49abaa26b1
66+
aio-run-runner==0.3.2 \
67+
--hash=sha256:75ad35f9090d7dade92846d6700b6afbfe5751afdd97c7cf5fd8e2cf4b45d8f9 \
68+
--hash=sha256:a19aa3607fb0b585f695c264df72d043658c5cf7d19b4f34f29cdf4af8b39fa1
6969
# via
7070
# -r requirements.in
7171
# aio-run-checker
@@ -299,6 +299,7 @@ envoy-base-utils==0.1.0 \
299299
--hash=sha256:f5ca5e147e60af5c2f755cc61dcfc507b9253d4657869f923e56dfd7ac03b835
300300
# via
301301
# -r requirements.in
302+
# envoy-code-check
302303
# envoy-dependency-check
303304
# envoy-dependency-pip-check
304305
# envoy-distribution-distrotest
@@ -308,9 +309,9 @@ envoy-base-utils==0.1.0 \
308309
# envoy-docs-sphinx-runner
309310
# envoy-github-release
310311
# envoy-gpg-sign
311-
envoy-code-format-python-check==0.0.8 \
312-
--hash=sha256:2257f718567e469993bacb6aaa08d1af8d5195fd1f6936dde621b4449da45c49 \
313-
--hash=sha256:acfd00e3bbd6615eaf50b3d6bad0510b967afd687911fa01da5322a3911005d0
312+
envoy-code-check==0.0.1 \
313+
--hash=sha256:cc8094e2451ba88a24af2bde05765253391b10125d13947f1bd9ca7afdc07f20 \
314+
--hash=sha256:e8da3d11a9a17beae55ba82c9b7107142d92742d64cbe772df860d1259cd9eb9
314315
# via -r requirements.in
315316
envoy-dependency-check==0.0.5 \
316317
--hash=sha256:39a8b25ae88e0305eaf38a90c3c7956db3b3c53e967464368319e41f3309df36 \
@@ -339,9 +340,9 @@ envoy-distribution-verify==0.0.8 \
339340
envoy-docker-utils==0.0.2 \
340341
--hash=sha256:a12cb57f0b6e204d646cbf94f927b3a8f5a27ed15f60d0576176584ec16a4b76
341342
# via envoy-distribution-distrotest
342-
envoy-docs-sphinx-runner==0.0.8 \
343-
--hash=sha256:b9d051aa7ee995b647d00ea5a86fe3be30f399e344837185edd53ff3b6fc6988 \
344-
--hash=sha256:fb2dc69067d9593adebda90794e912d84bb9a1c0e9c5d23ac6da608b79a8fc98
343+
envoy-docs-sphinx-runner==0.1.0 \
344+
--hash=sha256:8be22a71038dac272f20c43589155aa0adca64223d0c85ef8864d8b1ca272b4e \
345+
--hash=sha256:c0031fc261621d00cc03ed1de47d947a720a4649d6be34d3dcd34986137bdc6a
345346
# via -r requirements.in
346347
envoy-github-abstract==0.0.21 \
347348
--hash=sha256:243dae9457243fb42e4643b1f45006c3b0d3151c808884217447d29a081b26a1 \
@@ -353,22 +354,22 @@ envoy-github-release==0.0.11 \
353354
--hash=sha256:1bed7a829bd77391c33107a58057b89de34a3045895f276816e07ee0119d844f \
354355
--hash=sha256:a3c117892b822b9f1d29bbc92f9c82c733d972316fbe2611c4895cf1e4b1113d
355356
# via envoy-distribution-release
356-
envoy-gpg-identity==0.0.6 \
357-
--hash=sha256:2195e09aaacbaa8d6588378ad96d206708ffaf863aead38445eb3e54e4223c34 \
358-
--hash=sha256:93032dd82c9c422c2145dbf426fd93c3673c53ba03cca2eab13f5fb8cf24f47c
357+
envoy-gpg-identity==0.1.0 \
358+
--hash=sha256:1df6989e1a6a3a9f5809ac056829b840f57326d41ae1181db415a722216bff48 \
359+
--hash=sha256:9e11625e25ad6a031f1c85c57067673f50c0427b0e71ef0a7e926ce77ebb75a9
359360
# via
360361
# -r requirements.in
361362
# envoy-gpg-sign
362-
envoy-gpg-sign==0.0.11 \
363-
--hash=sha256:6e7d1defb15eea4a64b0042271e1206a1e3892000e53990c5988c2b6e726734b \
364-
--hash=sha256:95c86ddd7031abf5249b292986b5de1279ca9ccc0dd1a5a5d977cd90a1a48122
363+
envoy-gpg-sign==0.1.0 \
364+
--hash=sha256:14b8efee80916fa78857198fb98357c9ddfaa9dec34eb8a453bda1a364f4b973 \
365+
--hash=sha256:36bb56534a6c5947c18bd10c43185345f092664de86b49318e2afc6cac7d9158
365366
# via -r requirements.in
366367
flake8==4.0.1 \
367368
--hash=sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d \
368369
--hash=sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d
369370
# via
370371
# -r requirements.in
371-
# envoy-code-format-python-check
372+
# envoy-code-check
372373
# flake8-polyfill
373374
# pep8-naming
374375
flake8-polyfill==1.0.2 \
@@ -641,7 +642,9 @@ psutil==5.9.0 \
641642
--hash=sha256:ea42d747c5f71b5ccaa6897b216a7dadb9f52c72a0fe2b872ef7d3e1eacf3ba3 \
642643
--hash=sha256:ef216cc9feb60634bda2f341a9559ac594e2eeaadd0ba187a4c2eb5b5d40b91c \
643644
--hash=sha256:ff0d41f8b3e9ebb6b6110057e40019a432e96aae2008951121ba4e56040b84f3
644-
# via envoy-dependency-check
645+
# via
646+
# aio-core
647+
# envoy-dependency-check
645648
py==1.10.0 \
646649
--hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3 \
647650
--hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a
@@ -896,7 +899,9 @@ types-orjson==3.6.2 \
896899
types-psutil==5.8.20 \
897900
--hash=sha256:35e16c5d58c21cd638b7a74a8b451b3c099e0f8140121f1d084174c60625a5c8 \
898901
--hash=sha256:b9c5d6c1f8bd6154f845a79dfbe46e7628f8d58978c6cadde0afa4abe6400294
899-
# via envoy-dependency-check
902+
# via
903+
# aio-core
904+
# envoy-dependency-check
900905
typing-extensions==3.10.0.2 \
901906
--hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \
902907
--hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \
@@ -950,7 +955,7 @@ yapf==0.32.0 \
950955
--hash=sha256:a3f5085d37ef7e3e004c4ba9f9b3e40c54ff1901cd111f05145ae313a7c67d1b
951956
# via
952957
# -r requirements.in
953-
# envoy-code-format-python-check
958+
# envoy-code-check
954959
yarl==1.6.3 \
955960
--hash=sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e \
956961
--hash=sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434 \

‎tools/code/BUILD

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
load("//bazel:envoy_build_system.bzl", "envoy_package")
2+
load("//tools/base:envoy_python.bzl", "envoy_entry_point")
3+
load("@envoy_repo//:path.bzl", "PATH")
4+
5+
licenses(["notice"]) # Apache 2
6+
7+
envoy_package()
8+
9+
envoy_entry_point(
10+
name = "check",
11+
args = ["--path=%s" % PATH],
12+
pkg = "envoy.code.check",
13+
)

‎tools/code_format/BUILD

-11
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
load("@base_pip3//:requirements.bzl", "requirement")
2-
load("@rules_python//python:defs.bzl", "py_binary")
31
load("//bazel:envoy_build_system.bzl", "envoy_package")
42

53
licenses(["notice"]) # Apache 2
@@ -11,12 +9,3 @@ exports_files([
119
"header_order.py",
1210
"envoy_build_fixer.py",
1311
])
14-
15-
py_binary(
16-
name = "python_check",
17-
srcs = ["python_check.py"],
18-
deps = [
19-
"@envoy_repo",
20-
requirement("envoy.code_format.python_check"),
21-
],
22-
)

‎tools/code_format/check_shellcheck_format.sh

-59
This file was deleted.

‎tools/code_format/glint.sh

-75
This file was deleted.

0 commit comments

Comments
 (0)