Skip to content

Commit 9891ea8

Browse files
b-longb-long
authored
Chore/update docs and release process (#72)
* Cleanup docs * Refine workflows for release management and testing - Implement `release-please` workflow for automated releases. - Create `publish-test` and `publish` workflows to handle package builds and releases. - Introduce `test-suite` workflow to run tests before publishing. - Update configuration files for release management. * Add 'ruff' as dev dependency * Configure ruff to ignore generated files * Fail fast if linting fails * Document release process * Bump version to 0.3.0a7 * Publish new alpha * Allow replacing artifacts with the same name * Remove the duplicate integration-test job * Attempt alpha release * chore: improve pre-commit configuration * chore: revert 'rm CONNECT_RPC_MIGRATION.md' * chore: disable TestPyPIBuild unless workflow_dispatch * chore: bump version 0.3.0a7 -> 0.3.0a8 * chore: bump version 0.3.0a8 -> 0.3.0a9 * chore: target this branch * chore: target develop branch
1 parent ace14e7 commit 9891ea8

17 files changed

+657
-68
lines changed

‎.github/workflows/build-python.yaml‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,15 +39,5 @@ jobs:
3939
with:
4040
name: python-wheel
4141
path: dist/*.whl
42+
overwrite: true
4243

43-
integration-test:
44-
strategy:
45-
fail-fast: true
46-
matrix:
47-
python3_version: ["3.10", "3.11", "3.12", "3.13"]
48-
# python3_version: ["3.13"]
49-
needs: build
50-
uses: ./.github/workflows/platform-integration-test.yaml
51-
with:
52-
wheel: ${{ needs.build.outputs.wheel }}
53-
python_version: ${{ matrix.python3_version }}

‎.github/workflows/publish-test.yaml‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
name: TestPyPIBuild
22

33
on:
4-
push:
5-
branches:
6-
- develop
7-
- chore/rewrite-in-python
4+
# push:
5+
# branches:
6+
# - develop
7+
# - chore/update-docs-and-release-process
8+
# repository_dispatch:
9+
# types: [alpha-release-created, stable-release-created]
810
workflow_dispatch:
911

1012
jobs:
@@ -41,8 +43,8 @@ jobs:
4143
id-token: write
4244
needs: [build]
4345
runs-on: ubuntu-22.04
44-
# If branch is 'develop'
45-
if: github.ref == 'refs/heads/develop'
46+
# If branch is 'develop' or triggered by repository dispatch
47+
if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/chore/update-docs-and-release-process' || github.event_name == 'repository_dispatch'
4648

4749
steps:
4850
- uses: actions/checkout@v4

‎.github/workflows/publish.yaml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
name: PyPIBuild
22

33
on:
4-
push:
5-
branches:
6-
- main
4+
release:
5+
types: [published]
6+
repository_dispatch:
7+
types: [alpha-release-created, stable-release-created]
78
workflow_dispatch:
89

910
jobs:
@@ -106,8 +107,8 @@ jobs:
106107
id-token: write
107108
needs: [build_macos, build_linux_x86_64, build_linux_arm]
108109
runs-on: ubuntu-22.04
109-
# If branch is 'main'
110-
if: github.ref == 'refs/heads/main'
110+
# If branch is 'main' or triggered by repository dispatch
111+
if: github.ref == 'refs/heads/main' || github.event_name == 'repository_dispatch' || github.event_name == 'release'
111112

112113
steps:
113114
- uses: actions/checkout@v4

‎.github/workflows/release-please.yaml‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
name: Release Please
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- develop
8+
- chore/update-docs-and-release-process
9+
workflow_dispatch:
10+
11+
permissions:
12+
contents: write
13+
pull-requests: write
14+
15+
jobs:
16+
# Run full test suite before any release operations
17+
test-suite:
18+
uses: ./.github/workflows/test-suite.yaml
19+
20+
release-please:
21+
runs-on: ubuntu-latest
22+
needs: test-suite
23+
if: needs.test-suite.outputs.tests_passed == 'true'
24+
outputs:
25+
releases_created: ${{ steps.release.outputs.releases_created }}
26+
paths_released: ${{ steps.release.outputs.paths_released }}
27+
steps:
28+
- uses: googleapis/release-please-action@v4
29+
id: release
30+
with:
31+
config-file: .release-please-config.json
32+
manifest-file: .release-please-manifest.json
33+
target-branch: develop # FIXME: Change to 'main' after initial setup
34+
token: ${{ secrets.GITHUB_TOKEN }}
35+
36+
# Trigger appropriate publish workflows based on release type
37+
trigger-publish:
38+
needs: release-please
39+
if: ${{ needs.release-please.outputs.releases_created }}
40+
runs-on: ubuntu-latest
41+
steps:
42+
- uses: actions/checkout@v4
43+
44+
- name: Check if alpha release
45+
id: check_alpha
46+
run: |
47+
# Get the version from the main package
48+
VERSION=$(cat .release-please-manifest.json | jq -r '."."')
49+
echo "version=$VERSION" >> $GITHUB_OUTPUT
50+
if [[ "$VERSION" =~ [0-9]+\.[0-9]+\.[0-9]+a[0-9]+ ]]; then
51+
echo "is_alpha=true" >> $GITHUB_OUTPUT
52+
echo "Alpha version detected: $VERSION"
53+
else
54+
echo "is_alpha=false" >> $GITHUB_OUTPUT
55+
echo "Stable version detected: $VERSION"
56+
fi
57+
58+
# For develop branch: trigger TestPyPI build (both alpha and stable releases go to TestPyPI from develop)
59+
- name: Trigger TestPyPI Build (Develop)
60+
if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/chore/update-docs-and-release-process'
61+
uses: peter-evans/repository-dispatch@v3
62+
with:
63+
token: ${{ secrets.GITHUB_TOKEN }}
64+
event-type: ${{ steps.check_alpha.outputs.is_alpha == 'true' && 'alpha-release-created' || 'stable-release-created' }}
65+
client-payload: '{"version": "${{ steps.check_alpha.outputs.version }}", "releases": "${{ needs.release-please.outputs.paths_released }}"}'
66+
67+
# For main branch: trigger PyPI build (both alpha and stable releases go to PyPI from main)
68+
- name: Trigger PyPI Build (Main)
69+
if: github.ref == 'refs/heads/main'
70+
uses: peter-evans/repository-dispatch@v3
71+
with:
72+
token: ${{ secrets.GITHUB_TOKEN }}
73+
event-type: ${{ steps.check_alpha.outputs.is_alpha == 'true' && 'alpha-release-created' || 'stable-release-created' }}
74+
client-payload: '{"version": "${{ steps.check_alpha.outputs.version }}", "releases": "${{ needs.release-please.outputs.paths_released }}"}'

‎.github/workflows/test-suite.yaml‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
name: Test Suite
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- develop
8+
pull_request:
9+
workflow_call:
10+
outputs:
11+
tests_passed:
12+
description: "Whether all tests passed"
13+
value: ${{ jobs.report.outputs.success }}
14+
workflow_dispatch:
15+
16+
jobs:
17+
# Step 1: Fast lint and format checks (fail fast on code style)
18+
lint-check:
19+
runs-on: ubuntu-22.04
20+
steps:
21+
- uses: actions/checkout@v4
22+
23+
- name: Set up uv
24+
uses: astral-sh/setup-uv@v6
25+
with:
26+
enable-cache: true
27+
cache-dependency-glob: "uv.lock"
28+
29+
- name: Run linting (fail fast)
30+
run: |
31+
uv sync --frozen
32+
uv run ruff check
33+
uv run ruff format --check
34+
35+
# Step 2: Build (only after linting passes)
36+
build:
37+
runs-on: ubuntu-22.04
38+
needs: lint-check
39+
outputs:
40+
wheel: ${{ steps.find_wheel.outputs.wheel_path }}
41+
steps:
42+
- name: Checkout this repo
43+
uses: actions/checkout@v4
44+
45+
- name: Set up uv
46+
uses: astral-sh/setup-uv@v6
47+
with:
48+
enable-cache: true
49+
cache-dependency-glob: "uv.lock"
50+
51+
- name: Build otdf-python wheel using uv
52+
run: |
53+
uv sync --frozen
54+
uv build
55+
shell: bash
56+
57+
- name: Find built wheel
58+
id: find_wheel
59+
run: |
60+
wheel_path=$(ls dist/*.whl | head -n1)
61+
echo "wheel_path=$wheel_path" >> $GITHUB_OUTPUT
62+
shell: bash
63+
64+
- name: Upload wheel as artifact
65+
uses: actions/upload-artifact@v4
66+
with:
67+
name: python-wheel
68+
path: dist/*.whl
69+
70+
# Step 3: Unit tests (only after build succeeds)
71+
unit-tests:
72+
runs-on: ubuntu-22.04
73+
needs: build
74+
steps:
75+
- uses: actions/checkout@v4
76+
77+
- name: Set up uv
78+
uses: astral-sh/setup-uv@v6
79+
with:
80+
enable-cache: true
81+
cache-dependency-glob: "uv.lock"
82+
83+
- name: Run unit tests
84+
run: |
85+
uv sync --frozen
86+
uv run pytest -m "not integration" --tb=short -v tests/
87+
88+
# Step 4: Integration tests (only after unit tests pass)
89+
integration-tests:
90+
strategy:
91+
fail-fast: true
92+
matrix:
93+
python3_version: ["3.10", "3.11", "3.12", "3.13"]
94+
needs: [build, unit-tests]
95+
uses: ./.github/workflows/platform-integration-test.yaml
96+
with:
97+
wheel: ${{ needs.build.outputs.wheel }}
98+
python_version: ${{ matrix.python3_version }}
99+
100+
report:
101+
runs-on: ubuntu-22.04
102+
needs: [lint-check, build, unit-tests, integration-tests]
103+
if: always()
104+
outputs:
105+
success: ${{ steps.check.outputs.success }}
106+
steps:
107+
- name: Check all jobs succeeded
108+
id: check
109+
run: |
110+
if [[ "${{ needs.lint-check.result }}" == "success" && "${{ needs.build.result }}" == "success" && "${{ needs.unit-tests.result }}" == "success" && "${{ needs.integration-tests.result }}" == "success" ]]; then
111+
echo "success=true" >> $GITHUB_OUTPUT
112+
echo "✅ All tests passed!"
113+
else
114+
echo "success=false" >> $GITHUB_OUTPUT
115+
echo "❌ Some tests failed:"
116+
echo " Lint Check: ${{ needs.lint-check.result }}"
117+
echo " Build: ${{ needs.build.result }}"
118+
echo " Unit Tests: ${{ needs.unit-tests.result }}"
119+
echo " Integration Tests: ${{ needs.integration-tests.result }}"
120+
exit 1
121+
fi

‎.pre-commit-config.yaml‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
---
2+
default_install_hook_types:
3+
- pre-commit
4+
- commit-msg
5+
- post-rewrite
26
exclude: |
37
(?x)^(
48
otdf_python/.*
@@ -17,26 +21,6 @@ repos:
1721
- id: check-yaml
1822
- id: end-of-file-fixer
1923
- id: trailing-whitespace
20-
21-
# - repo: https://github.com/tekwizely/pre-commit-golang
22-
# rev: master
23-
# hooks:
24-
# - id: go-lint
25-
26-
- repo: https://github.com/dnephin/pre-commit-golang
27-
rev: v0.5.1
28-
hooks:
29-
- id: go-fmt
30-
31-
- id: go-vet
32-
# - id: golangci-lint
33-
# timeout is needed for CI
34-
# args: [-E, gosec, -E, goconst, -E, govet, --timeout, 300s]
35-
# - id: go-imports
36-
# - id: go-cyclo
37-
# args: [-over=15]
38-
# - id: validate-toml
39-
- id: no-go-testing
4024
- repo: https://github.com/codespell-project/codespell
4125
rev: v2.4.1
4226
hooks:
@@ -45,9 +29,15 @@ repos:
4529

4630
- repo: https://github.com/astral-sh/ruff-pre-commit
4731
# Ruff version.
48-
rev: v0.12.10
32+
rev: v0.12.12
4933
hooks:
5034
# Run the linter.
5135
- id: ruff
5236
# Run the formatter.
5337
- id: ruff-format
38+
- repo: https://github.com/compilerla/conventional-pre-commit
39+
rev: v4.2.0
40+
hooks:
41+
- id: conventional-pre-commit
42+
stages: [commit-msg,post-rewrite]
43+
args: [--verbose,--scopes=feat,fix,docs,style,test,chore,ci]

‎.release-please-config.json‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{
2+
"release-type": "python",
3+
"include-v-in-tag": true,
4+
"packages": {
5+
".": {
6+
"release-type": "python",
7+
"package-name": "otdf-python",
8+
"extra-files": [
9+
"src/otdf_python/cli.py",
10+
"tests/test_cli.py"
11+
]
12+
},
13+
"otdf-python-proto": {
14+
"release-type": "python",
15+
"package-name": "otdf-python-proto"
16+
}
17+
},
18+
"changelog-sections": [
19+
{ "type": "feat", "section": "Features", "hidden": false },
20+
{ "type": "fix", "section": "Bug Fixes", "hidden": false },
21+
{ "type": "perf", "section": "Performance Improvements", "hidden": false },
22+
{ "type": "revert", "section": "Reverts", "hidden": false },
23+
{ "type": "docs", "section": "Documentation", "hidden": false },
24+
{ "type": "style", "section": "Styles", "hidden": true },
25+
{ "type": "chore", "section": "Miscellaneous Chores", "hidden": true },
26+
{ "type": "refactor", "section": "Code Refactoring", "hidden": false },
27+
{ "type": "test", "section": "Tests", "hidden": true },
28+
{ "type": "build", "section": "Build System", "hidden": false },
29+
{ "type": "ci", "section": "Continuous Integration", "hidden": true }
30+
]
31+
}

‎.release-please-manifest.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
".": "0.3.0a9",
3+
"otdf-python-proto": "0.3.0a9"
4+
}

‎PROTOBUF_SETUP.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ curl -o proto-gen/proto-files/kas.proto https://raw.githubusercontent.com/opentd
8686

8787
## Dependencies
8888

89-
The proto-gen sub-module includes these dependencies:
89+
The otdf-python-proto sub-module includes these dependencies:
9090
- `grpcio>=1.74.0` - gRPC runtime
9191
- `grpcio-tools>=1.74.0` - Protocol buffer compiler
9292
- `protobuf>=6.31.1` - Protocol buffer runtime
@@ -128,7 +128,7 @@ The generated Python files include:
128128
- **`kas_pb2.py`** - Protocol buffer message classes
129129
- **`kas_pb2_grpc.py`** - gRPC service client and server classes
130130

131-
These files are automatically synced to `src/otdf_python/proto/` in the main project.
131+
These files are automatically synced to `otdf-python-proto/generated/` and used by the main project in `src/otdf_python/`.
132132

133133
## Fallback Strategy
134134

0 commit comments

Comments
 (0)