diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 40d382792..d30ae3ceb 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -54,6 +54,11 @@ on: - cron: "30 6 * * *" # 0630 UTC - cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday) - cron: "0 18 * * 0" # 1800 UTC (Sunday) + +concurrency: + group: ${{ github.workflow }}-pr-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: resolve-versions: timeout-minutes: 10 @@ -69,7 +74,7 @@ jobs: java: ${{ steps.version-info.outputs.java-version-info }} js: ${{ steps.version-info.outputs.js-version-info }} env: - PLATFORM_REF: "${{ inputs.platform-ref || 'main lts' }}" + PLATFORM_REF: "${{ inputs.platform-ref }}" JS_REF: "${{ inputs.js-ref }}" OTDFCTL_REF: "${{ inputs.otdfctl-ref }}" JAVA_REF: "${{ inputs.java-ref }}" @@ -91,13 +96,13 @@ jobs: echo "DEFAULT_TAGS=main latest" >> "$GITHUB_ENV" elif [[ $CRON_MONDAY_WEDNESDAY == 'true' ]]; then echo "Running Monday/Wednesday tests" - echo "DEFAULT_TAGS=main lts" >> "$GITHUB_ENV" + echo "DEFAULT_TAGS=main" >> "$GITHUB_ENV" elif [[ $CRON_WEEKLY == 'true' ]]; then echo "Running weekly tests" - echo "DEFAULT_TAGS=main latest lts" >> "$GITHUB_ENV" + echo "DEFAULT_TAGS=main latest" >> "$GITHUB_ENV" else echo "Running PR, Workflow Dispatch, or manual trigger" - echo "DEFAULT_TAGS=main" >> "$GITHUB_ENV" + echo "DEFAULT_TAGS=main latest" >> "$GITHUB_ENV" fi env: CRON_NIGHTLY: ${{ github.event.schedule == '30 6 * * *' }} @@ -245,9 +250,11 @@ jobs: uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b with: python-version: "3.14" - - uses: bufbuild/buf-setup-action@2211e06e8cf26d628cda2eea15c95f8c42b080b3 + - uses: bufbuild/buf-action@8f4a1456a0ab6a1eb80ba68e53832e6fcfacc16c # v1.3.0 with: - github_token: ${{ secrets.GITHUB_TOKEN }} + setup_only: true + token: ${{ secrets.BUF_TOKEN }} + version: "1.56.0" - name: Set up JDK uses: actions/setup-java@5896cecc08fd8a1fbdfaf517e29b571164b031f7 @@ -269,6 +276,14 @@ jobs: sdk: js version-info: "${{ needs.resolve-versions.outputs.js }}" + - name: Cache npm + uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 + with: + path: ~/.npm + key: npm-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/js/src/**/package-lock.json') }} + restore-keys: | + npm-${{ runner.os }}- + ######## SETUP THE JS CLI ############# - name: build and setup the web-sdk cli id: build-web-sdk @@ -285,6 +300,16 @@ jobs: sdk: go version-info: "${{ needs.resolve-versions.outputs.go }}" + - name: Cache Go modules + uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: go-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/go/src/*/go.sum') }} + restore-keys: | + go-${{ runner.os }}- + - name: Resolve otdfctl heads id: resolve-otdfctl-heads run: |- @@ -326,6 +351,14 @@ jobs: sdk: java version-info: "${{ needs.resolve-versions.outputs.java }}" + - name: Cache Maven repository + uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 + with: + path: ~/.m2/repository + key: maven-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/java/src/**/pom.xml') }} + restore-keys: | + maven-${{ runner.os }}- + - name: pre-release protocol buffers for java-sdk if: | (env.FOCUS_SDK == 'go' || env.FOCUS_SDK == 'java') && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) @@ -352,9 +385,6 @@ jobs: - name: Prepare java cli run: | make - env: - BUF_INPUT_HTTPS_USERNAME: opentdf-bot - BUF_INPUT_HTTPS_PASSWORD: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }} working-directory: otdftests/xtest/sdk/java ######## Configure test environment ############# diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..9d7d07ef7 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,39 @@ +# Repository Guidelines + +## Project Structure & Module Organization + +- `xtest/`: Cross-client compatibility test harness (Python + `pytest`), with fixtures in `xtest/fixtures/` and golden data in `xtest/golden/`. +- `xtest/sdk/`: Helper scripts and Makefiles for checking out/building SDKs under test (e.g., `xtest/sdk/scripts/checkout-all.sh`, `cd xtest/sdk && make`). +- `vulnerability/`: Playwright-based security regression tests (`vulnerability/tests/`). +- `.github/workflows/`: CI workflows (lint/type-check, xtest matrix runs, vulnerability runs). + +## Build, Test, and Development Commands + +- Enter the dev environment: `devbox shell` (installs Python/JDK/Node per `devbox.json`). +- Install xtest deps: `cd xtest && uv sync` (or `uv sync --extra dev` for dev tools) +- Run xtest: `cd xtest && uv run pytest` + - Focus a subset: `uv run pytest --sdks "go js" --focus go` (see `xtest/conftest.py` options) + - HTML report: `uv run pytest --html tmp/test-report.html --self-contained-html` +- Build SDK CLIs (after checkout): `cd xtest/sdk && make` +- Run vulnerability tests: `cd vulnerability && npm ci && npm test` (requires a running platform; see `README.md` and `vulnerability/README.md`). + +## Coding Style & Naming Conventions + +- Python code in `xtest/` uses 4-space indentation, `snake_case`, and `pytest`-style fixtures. +- CI enforces these checks in `xtest/`: + - `ruff check` and `ruff format --check` + - `pyright` + - Local equivalent: `cd xtest && uv sync --extra dev && uv run ruff check . && uv run ruff format --check . && uv run pyright` + +## Testing Guidelines + +- `pytest` tests live in `xtest/test_*.py`; add new fixtures under `xtest/fixtures/`. +- Tests assume a platform backend is reachable (Docker + Keycloak). Use `xtest/test.env` as a template: + - `cd xtest && set -a && source test.env && set +a` + +## Commit & Pull Request Guidelines + +- Use semantic commit/PR titles (enforced by CI): `feat(xtest): ...`, `fix(vulnerability): ...`, `docs: ...` (types: `fix|feat|chore|docs`; scopes include `xtest`, `vulnerability`, `go`, `java`, `web`, `ci`). +- DCO sign-off is required: `git commit -s -m "feat(xtest): ..."` (see `CONTRIBUTING.md`). +- PRs should include a clear description, linked issue (if any), and relevant logs/screenshots for test failures; reviewers are defined in `CODEOWNERS`. + diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 000000000..47dc3e3d8 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/xtest/sdk/scripts/post-checkout-java.sh b/xtest/sdk/scripts/post-checkout-java.sh index 434f42a21..0336602e9 100755 --- a/xtest/sdk/scripts/post-checkout-java.sh +++ b/xtest/sdk/scripts/post-checkout-java.sh @@ -15,6 +15,24 @@ else SED_CMD="sed -i" fi +# Map Java SDK version to compatible platform protocol branch +# Must match the mappings in resolve-version.py +get_platform_branch() { + local version="$1" + case "$version" in + 0.7.8|0.7.7) echo "protocol/go/v0.2.29" ;; + 0.7.6) echo "protocol/go/v0.2.25" ;; + 0.7.5|0.7.4) echo "protocol/go/v0.2.18" ;; + 0.7.3|0.7.2) echo "protocol/go/v0.2.17" ;; + 0.6.1|0.6.0) echo "protocol/go/v0.2.14" ;; + 0.5.0) echo "protocol/go/v0.2.13" ;; + 0.4.0|0.3.0|0.2.0) echo "protocol/go/v0.2.10" ;; + 0.1.0) echo "protocol/go/v0.2.3" ;; + *) echo "main" ;; # Default to main for unknown/newer versions + esac + return 0 +} + # Loop through all subdirectories in the base directory find "$BASE_DIR" -mindepth 1 -maxdepth 1 -type d -not -name "*.git" | while read -r SRC_DIR; do POM_FILE="$SRC_DIR/sdk/pom.xml" @@ -25,21 +43,51 @@ find "$BASE_DIR" -mindepth 1 -maxdepth 1 -type d -not -name "*.git" | while read continue fi - # Check if the platform.branch property is already defined - if grep -q "" "$POM_FILE"; then - echo "platform.branch already defined in $POM_FILE, skipping." + # Extract version from directory name (e.g., "v0.7.5" -> "0.7.5", "main" -> "main") + DIR_NAME=$(basename "$SRC_DIR") + VERSION="${DIR_NAME#v}" # Remove leading 'v' if present + PLATFORM_BRANCH=$(get_platform_branch "$VERSION") + + # Check if the correct platform.branch is already set + if grep -q "$PLATFORM_BRANCH" "$POM_FILE"; then + echo "platform.branch already set to $PLATFORM_BRANCH in $POM_FILE, skipping." continue fi - echo "Updating $POM_FILE..." + # If we don't have a specific mapping for this version (defaults to "main"), + # check if the pom.xml already has a valid protocol/go branch set - don't overwrite it + if [[ "$PLATFORM_BRANCH" == "main" ]]; then + if grep -q "protocol/go/" "$POM_FILE"; then + EXISTING_BRANCH=$(grep -o "[^<]*" "$POM_FILE" | sed 's/<[^>]*>//g') + echo "platform.branch already set to $EXISTING_BRANCH in $POM_FILE (no mapping for version $VERSION), skipping." + continue + fi + fi - # Add the platform.branch property to the section - $SED_CMD '//a \ - main' "$POM_FILE" + echo "Updating $POM_FILE (version=$VERSION, platform.branch=$PLATFORM_BRANCH)..." - # Replace hardcoded branch=main with branch=${platform.branch} in the maven-antrun-plugin configuration - # shellcheck disable=SC2016 # Literal $; it is for a variable expansion in the maven file - $SED_CMD 's/branch=main/branch=${platform.branch}/g' "$POM_FILE" + # Check if platform.branch property exists (possibly with wrong value) + if grep -q "" "$POM_FILE"; then + # Replace existing platform.branch value with the correct one + $SED_CMD "s|[^<]*|$PLATFORM_BRANCH|g" "$POM_FILE" + echo "Updated existing platform.branch to $PLATFORM_BRANCH in $POM_FILE" + else + # Add the platform.branch property to the section + $SED_CMD "//a \\ + $PLATFORM_BRANCH" "$POM_FILE" + + # Only replace branch=main if the property now exists (sed above may have failed silently if no section) + if grep -q "" "$POM_FILE"; then + # Replace hardcoded branch=main with branch=${platform.branch} in the maven-antrun-plugin configuration + # shellcheck disable=SC2016 # Literal $; it is for a variable expansion in the maven file + $SED_CMD 's/branch=main/branch=${platform.branch}/g' "$POM_FILE" + echo "Added platform.branch=$PLATFORM_BRANCH and updated branch references in $POM_FILE" + else + # No section exists, directly replace branch=main with the actual branch value + $SED_CMD "s|branch=main|branch=$PLATFORM_BRANCH|g" "$POM_FILE" + echo "No section, directly replaced branch=main with branch=$PLATFORM_BRANCH in $POM_FILE" + fi + fi done echo "Update complete." diff --git a/xtest/sdk/scripts/resolve-version.py b/xtest/sdk/scripts/resolve-version.py index 34eefafc5..ca6655c13 100755 --- a/xtest/sdk/scripts/resolve-version.py +++ b/xtest/sdk/scripts/resolve-version.py @@ -107,10 +107,10 @@ def is_resolve_success(val: ResolveResult) -> TypeGuard[ResolveSuccess]: } lts_versions = { - "go": "0.15.0", - "java": "0.7.5", - "js": "0.2.0", - "platform": "0.4.34", + "go": "0.24.0", + "java": "0.9.0", + "js": "0.4.0", + "platform": "0.9.0", }