-
Notifications
You must be signed in to change notification settings - Fork 8
331 lines (282 loc) · 9.71 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
name: CI
on:
push:
branches:
- main
pull_request:
branches:
- main
concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true
jobs:
commitlint:
if: ${{ github.event_name == 'pull_request' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
with:
fetch-depth: 0
- uses: cachix/install-nix-action@6a9a9e84a173d90b3ffb42c5ddaf9ea033fad011 # v23
with:
nix_path: nixpkgs=channel:nixos-unstable-small
- uses: cachix/cachix-action@v12
with:
name: protoletariat
extraPullNames: nix-community,poetry2nix
- name: commitlint
run: nix shell -L -f '<nixpkgs>' commitlint -c commitlint --from=${{ github.event.pull_request.base.sha }} --to=${{ github.sha }} --verbose
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
- uses: cachix/install-nix-action@6a9a9e84a173d90b3ffb42c5ddaf9ea033fad011 # v23
with:
nix_path: nixpkgs=channel:nixos-unstable-small
- uses: cachix/cachix-action@v12
with:
name: protoletariat
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
extraPullNames: nix-community,poetry2nix
- run: nix develop -c pre-commit run --all-files
nix:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
python-version:
- "38"
- "39"
- "310"
- "311"
steps:
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
- uses: cachix/install-nix-action@6a9a9e84a173d90b3ffb42c5ddaf9ea033fad011 # v23
with:
nix_path: nixpkgs=channel:nixos-unstable-small
- uses: cachix/cachix-action@v12
with:
name: protoletariat
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
extraPullNames: nix-community,poetry2nix
- name: build package and run tests
run: nix build --print-build-logs --no-link --keep-going '.#protoletariat${{ matrix.python-version }}'
- name: show closure info
run: nix path-info --closure-size --human-readable --recursive '.#protoletariat${{ matrix.python-version }}'
docker-image:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
- uses: cachix/install-nix-action@6a9a9e84a173d90b3ffb42c5ddaf9ea033fad011 # v23
with:
nix_path: nixpkgs=channel:nixos-unstable-small
- uses: cachix/cachix-action@v12
with:
name: protoletariat
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
extraPullNames: nix-community,poetry2nix
- name: build and load docker image
run: nix run --print-build-logs --keep-going '.#protoletariat-image.copyToDockerDaemon'
- run: docker images protoletariat
- name: smoke test docker image
run: |
set -euo pipefail
imagetag="$(nix eval --raw '.#protoletariat-image.imageTag')"
docker run --rm "protoletariat:${imagetag}"
conda-ubuntu-macos:
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- macos-latest
python-version:
- "3.8"
- "3.9"
- "3.10"
- "3.11"
buf:
- buf
include:
- os: ubuntu-latest
python-version: "3.9"
buf: no-buf
- os: macos-latest
python-version: "3.9"
buf: no-buf
runs-on: ${{ matrix.os }}
defaults:
run:
shell: bash -l {0}
steps:
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
- uses: bufbuild/[email protected]
if: ${{ matrix.buf == 'buf' }}
with:
github_token: ${{ github.token }}
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- run: pip3 install poetry2conda poetry
- name: Generate conda environment file
run: |
set -euo pipefail
poetry2conda --dev pyproject.toml - | tee environment.yml
- uses: conda-incubator/setup-miniconda@v2
with:
mamba-version: "*"
miniforge-variant: Mambaforge
miniforge-version: latest
channel-priority: strict
activate-environment: protoletariat
python-version: ${{ matrix.python-version }}
environment-file: environment.yml
- run: mamba install grpc-cpp grpcio-tools
- run: pip install .
- run: pytest -ra
conda-windows:
strategy:
fail-fast: false
matrix:
python-version:
- "3.8"
- "3.9"
- "3.10"
- "3.11"
buf:
- buf
include:
- python-version: "3.9"
buf: no-buf
runs-on: windows-latest
steps:
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
- uses: bufbuild/[email protected]
if: ${{ matrix.buf == 'buf' }}
with:
github_token: ${{ github.token }}
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- run: pip3 install poetry2conda poetry
- name: Generate conda environment file
run: poetry2conda --dev pyproject.toml - > environment.yml
- uses: conda-incubator/setup-miniconda@v2
with:
mamba-version: "*"
miniforge-variant: Mambaforge
miniforge-version: latest
channel-priority: strict
activate-environment: protoletariat
python-version: ${{ matrix.python-version }}
environment-file: environment.yml
- run: mamba install grpc-cpp grpcio-tools pip
- run: pip install .
- run: pytest -ra
tensorflow:
runs-on: ubuntu-latest
env:
PYTHON_OUT: /tmp/tensorflow-gen
TENSORFLOW_HOME: ${{ github.workspace }}/tensorflow
steps:
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
with:
path: protoletariat
- uses: cachix/install-nix-action@6a9a9e84a173d90b3ffb42c5ddaf9ea033fad011 # v23
with:
nix_path: nixpkgs=channel:nixos-unstable-small
- uses: cachix/cachix-action@v12
with:
name: protoletariat
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
extraPullNames: nix-community,poetry2nix
- name: Clone Tensorflow
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
with:
repository: tensorflow/tensorflow
path: tensorflow
- run: mkdir -p "$PYTHON_OUT"
- name: Generate tensorflow Python proto code
working-directory: protoletariat
run: |
set -euo pipefail
mapfile -t files < <(find "$TENSORFLOW_HOME" -name '*.proto')
nix develop -c \
protoc \
--python_out "$PYTHON_OUT" \
--proto_path "$TENSORFLOW_HOME" "${files[@]}"
- name: Test tensorflow rewriting
working-directory: protoletariat
run: |
set -euo pipefail
mapfile -t files < <(find "$TENSORFLOW_HOME" -name '*.proto')
nix run '.#protoletariat' \
-- \
--in-place \
--create-package \
--python-out "$PYTHON_OUT" \
protoc \
--proto-path "$TENSORFLOW_HOME" "${files[@]}"
- name: Check for absolute imports
run: grep -vq -R '^import tensorflow' "$PYTHON_OUT"
dry-release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
with:
fetch-depth: 0
- uses: cachix/install-nix-action@6a9a9e84a173d90b3ffb42c5ddaf9ea033fad011 # v23
with:
nix_path: nixpkgs=channel:nixos-unstable-small
- uses: cachix/cachix-action@v12
with:
name: protoletariat
extraPullNames: nix-community,poetry2nix
- name: Configure git info
run: |
set -euo pipefail
# not incredibly important what user we use here
#
# we're making a commit in a temporary worktree that is thrown away
# if the process exits successfully
#
# git requires user information to make commits
git config user.name 'phillip-ground[bot]'
git config user.email 'phillip-ground[bot]@users.noreply.github.com'
- name: run semantic-release
run: ./ci/release/dry_run.sh
release:
runs-on: ubuntu-latest
if: ${{ github.event_name != 'pull_request' }}
needs:
- nix
- conda-ubuntu-macos
- conda-windows
- docker-image
- pre-commit
- tensorflow
- dry-release
steps:
- uses: actions/[email protected]
id: generate_token
with:
app-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
with:
fetch-depth: 0
token: ${{ steps.generate_token.outputs.token }}
- uses: cachix/install-nix-action@6a9a9e84a173d90b3ffb42c5ddaf9ea033fad011 # v23
with:
nix_path: nixpkgs=channel:nixos-unstable-small
- uses: cachix/cachix-action@v12
with:
name: protoletariat
extraPullNames: nix-community,poetry2nix
- name: run semantic-release
run: ./ci/release/run.sh
env:
POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_TOKEN }}
GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}