From 4ffbeffac2896c55473e5ac34daeca3c1fd30ab9 Mon Sep 17 00:00:00 2001 From: jakedoublev Date: Fri, 6 Sep 2024 14:30:01 -0700 Subject: [PATCH 01/13] test --- .github/workflows/xtest.yml | 50 +++---------------------------------- 1 file changed, 4 insertions(+), 46 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 782ae71d..0701e39e 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -57,53 +57,11 @@ jobs: distribution: "adopt" server-id: github - ######## CHECKOUT THE PLATFORM ############# - - name: Check out platform - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - with: - repository: opentdf/platform - path: platform - ref: ${{ env.PLATFORM_REF }} - - name: Set up go - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 - with: - go-version: "1.22.3" - check-latest: false - cache-dependency-path: | - platform/service/go.sum - platform/examples/go.sum - platform/protocol/go/go.sum - platform/sdk/go.sum - - ######## SPIN UP BACKEND ############# - - run: | - .github/scripts/init-temp-keys.sh - cp opentdf-dev.yaml opentdf.yaml - working-directory: platform - - name: Added Trusted Certs - run: | - sudo chmod -R 777 ./keys - sudo apt-get install -y ca-certificates - sudo cp ./keys/localhost.crt /usr/local/share/ca-certificates - sudo update-ca-certificates - working-directory: platform - - run: docker compose up -d --wait --wait-timeout 240 - working-directory: platform - - run: go run ./service provision keycloak - working-directory: platform - - run: go run ./service provision fixtures - working-directory: platform - - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 - name: start server in background + ######## SPIN UP PLATFORM BACKEND ############# + - name: Check out and start up platform with deps/containers + uses: opentdf/platform/test/start-up-with-containers@feat/reusable-startup with: - run: > - go build -o opentdf -v service/main.go - && .github/scripts/watch.sh opentdf.yaml ./opentdf start - wait-on: | - tcp:localhost:8080 - log-output-if: true - wait-for: 90s - working-directory: platform + platform-ref: ${{ env.PLATFORM_REF }} ######## CHECKOUT CLIENT-WEB ############# - name: Check out client-web From 08898e0b6b88189cf17aa229311049fb5b60bbed Mon Sep 17 00:00:00 2001 From: jakedoublev Date: Fri, 6 Sep 2024 14:34:00 -0700 Subject: [PATCH 02/13] test --- .github/workflows/xtest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 0701e39e..f73aa9b7 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -100,8 +100,8 @@ jobs: ######## SETUP THE GO CLI ############# - name: Prepare go cli run: |- - go mod edit -replace github.com/opentdf/platform/protocol/go=../platform/protocol/go - go mod edit -replace github.com/opentdf/platform/sdk=../platform/sdk + go mod edit -replace github.com/opentdf/platform/protocol/go=../otdf-test-platform/protocol/go + go mod edit -replace github.com/opentdf/platform/sdk=../otdf-test-platform/sdk go mod tidy go build . cp ./otdfctl ../otdftests/xtest/sdk/go/otdfctl From 7b76a846c0370bf2490a963c9b9973c65958fa57 Mon Sep 17 00:00:00 2001 From: jakedoublev Date: Fri, 6 Sep 2024 14:51:54 -0700 Subject: [PATCH 03/13] additional KAS --- .github/workflows/xtest.yml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index f73aa9b7..41ae0a9d 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -147,21 +147,11 @@ jobs: working-directory: otdftests/xtest ######## ATTRIBUTE BASED CONFIGURATION ############# - - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 - name: start another KAS server in background + - name: Check out and start up platform with deps/containers + uses: opentdf/platform/test/start-additional-kas@feat/reusable-startup with: - run: > - opentdf-beta.yaml yq e ' - (.server.port = 8282) - | (.mode = ["kas"]) - | (.sdk_config = {"endpoint":"http://localhost:8080","plaintext":true,"client_id":"opentdf","client_secret":"secret"}) - ' - && .github/scripts/watch.sh opentdf-beta.yaml ./opentdf --config-file ./opentdf-beta.yaml start - wait-on: | - tcp:localhost:8282 - log-output-if: true - wait-for: 90s - working-directory: platform + kas-port: 8282 + kas-name: beta - name: Run attribute based configuration tests run: |- From a1207c768c792287c590fc6191ad64921246d6a4 Mon Sep 17 00:00:00 2001 From: jakedoublev Date: Mon, 9 Sep 2024 08:25:53 -0700 Subject: [PATCH 04/13] test with fix --- .github/workflows/xtest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 6c3f870f..ab081da8 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -59,7 +59,7 @@ jobs: ######## SPIN UP PLATFORM BACKEND ############# - name: Check out and start up platform with deps/containers - uses: opentdf/platform/test/start-up-with-containers@feat/reusable-startup + uses: opentdf/platform/test/start-up-with-containers@fix/reusable-ci with: platform-ref: ${{ env.PLATFORM_REF }} @@ -148,7 +148,7 @@ jobs: ######## ATTRIBUTE BASED CONFIGURATION ############# - name: Check out and start up platform with deps/containers - uses: opentdf/platform/test/start-additional-kas@feat/reusable-startup + uses: opentdf/platform/test/start-additional-kas@fix/reusable-ci with: kas-port: 8282 kas-name: beta From 852297cd0e31c65e9b7ce20a4f4224ef163c634a Mon Sep 17 00:00:00 2001 From: jakedoublev Date: Mon, 9 Sep 2024 08:36:56 -0700 Subject: [PATCH 05/13] test --- xtest/test_abac.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xtest/test_abac.py b/xtest/test_abac.py index 6ac8136b..cbecad78 100644 --- a/xtest/test_abac.py +++ b/xtest/test_abac.py @@ -11,7 +11,7 @@ def load_cached_kas_keys() -> abac.PublicKey: keyset: list[abac.KasPublicKey] = [] - with open("../../platform/kas-cert.pem", "r") as rsaFile: + with open("../../otdf-test-platform/kas-cert.pem", "r") as rsaFile: keyset.append( abac.KasPublicKey( alg=abac.KAS_PUBLIC_KEY_ALG_ENUM_RSA_2048, @@ -19,7 +19,7 @@ def load_cached_kas_keys() -> abac.PublicKey: pem=rsaFile.read(), ) ) - with open("../../platform/kas-ec-cert.pem", "r") as ecFile: + with open("../../otdf-test-platform/kas-ec-cert.pem", "r") as ecFile: keyset.append( abac.KasPublicKey( alg=abac.KAS_PUBLIC_KEY_ALG_ENUM_EC_SECP256R1, From 660c571209a0522c55882158eff543e0bdcab80d Mon Sep 17 00:00:00 2001 From: jakedoublev Date: Mon, 9 Sep 2024 08:53:42 -0700 Subject: [PATCH 06/13] read key store from output --- .github/workflows/xtest.yml | 3 +++ xtest/test_abac.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index ab081da8..b6440cc2 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -59,6 +59,7 @@ jobs: ######## SPIN UP PLATFORM BACKEND ############# - name: Check out and start up platform with deps/containers + id: start-platform uses: opentdf/platform/test/start-up-with-containers@fix/reusable-ci with: platform-ref: ${{ env.PLATFORM_REF }} @@ -145,6 +146,8 @@ jobs: run: |- pytest test_tdfs.py working-directory: otdftests/xtest + env: + KEY_STORE_DIR: '../../${{ steps.start-platform.outputs.key-store-dir }}' ######## ATTRIBUTE BASED CONFIGURATION ############# - name: Check out and start up platform with deps/containers diff --git a/xtest/test_abac.py b/xtest/test_abac.py index cbecad78..96a7a4ee 100644 --- a/xtest/test_abac.py +++ b/xtest/test_abac.py @@ -1,6 +1,7 @@ import filecmp import random import string +import os import abac import tdfs @@ -8,10 +9,13 @@ otdfctl = abac.OpentdfCommandLineTool() +# reusable action stores keys in 'otdf-test-platform' [https://github.com/opentdf/platform/blob/main/test/start-up-with-containers/action.yaml] def load_cached_kas_keys() -> abac.PublicKey: keyset: list[abac.KasPublicKey] = [] - with open("../../otdf-test-platform/kas-cert.pem", "r") as rsaFile: + # Read in KEY_STORE_DIR from environment variable with default value + key_store_dir = os.getenv("KEY_STORE_DIR", "../../otdf-test-platform") + with open(f"{key_store_dir}/kas-cert.pem", "r") as rsaFile: keyset.append( abac.KasPublicKey( alg=abac.KAS_PUBLIC_KEY_ALG_ENUM_RSA_2048, @@ -19,7 +23,7 @@ def load_cached_kas_keys() -> abac.PublicKey: pem=rsaFile.read(), ) ) - with open("../../otdf-test-platform/kas-ec-cert.pem", "r") as ecFile: + with open(f"{key_store_dir}/kas-ec-cert.pem", "r") as ecFile: keyset.append( abac.KasPublicKey( alg=abac.KAS_PUBLIC_KEY_ALG_ENUM_EC_SECP256R1, From 1b2e0637bc2b2d7eae147ee7af101939e00ea218 Mon Sep 17 00:00:00 2001 From: jakedoublev Date: Mon, 9 Sep 2024 09:00:25 -0700 Subject: [PATCH 07/13] universalize hardcoded dir with step output --- .github/workflows/xtest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index b6440cc2..2c34d0c1 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -101,8 +101,8 @@ jobs: ######## SETUP THE GO CLI ############# - name: Prepare go cli run: |- - go mod edit -replace github.com/opentdf/platform/protocol/go=../otdf-test-platform/protocol/go - go mod edit -replace github.com/opentdf/platform/sdk=../otdf-test-platform/sdk + go mod edit -replace github.com/opentdf/platform/protocol/go=../${{ steps.start-platform.outputs.platform-working-dir }}/protocol/go + go mod edit -replace github.com/opentdf/platform/sdk=../${{ steps.start-platform.outputs.platform-working-dir }}/sdk go mod tidy go build . cp ./otdfctl ../otdftests/xtest/sdk/go/otdfctl @@ -147,7 +147,7 @@ jobs: pytest test_tdfs.py working-directory: otdftests/xtest env: - KEY_STORE_DIR: '../../${{ steps.start-platform.outputs.key-store-dir }}' + KEY_STORE_DIR: '../../${{ steps.start-platform.outputs.platform-working-dir }}' ######## ATTRIBUTE BASED CONFIGURATION ############# - name: Check out and start up platform with deps/containers From d8d55d489868c99e2e50441490980a06ef56b944 Mon Sep 17 00:00:00 2001 From: jakedoublev Date: Mon, 9 Sep 2024 09:19:51 -0700 Subject: [PATCH 08/13] lint --- xtest/test_abac.py | 49 ++++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/xtest/test_abac.py b/xtest/test_abac.py index 96a7a4ee..0448d1a0 100644 --- a/xtest/test_abac.py +++ b/xtest/test_abac.py @@ -1,7 +1,7 @@ +import os import filecmp import random import string -import os import abac import tdfs @@ -9,33 +9,30 @@ otdfctl = abac.OpentdfCommandLineTool() -# reusable action stores keys in 'otdf-test-platform' [https://github.com/opentdf/platform/blob/main/test/start-up-with-containers/action.yaml] + +def load_kas_key(filepath: str, alg: str, kid: str) -> abac.KasPublicKey: + """Helper function to load a KAS public key from a file.""" + try: + with open(filepath, "r") as file: + pem_data = file.read() + return abac.KasPublicKey(alg=alg, kid=kid, pem=pem_data) + except FileNotFoundError: + raise FileNotFoundError(f"File not found: {filepath}") + except Exception as e: + raise RuntimeError(f"Error loading KAS key from {filepath}: {e}") + def load_cached_kas_keys() -> abac.PublicKey: - keyset: list[abac.KasPublicKey] = [] - # Read in KEY_STORE_DIR from environment variable with default value - key_store_dir = os.getenv("KEY_STORE_DIR", "../../otdf-test-platform") - with open(f"{key_store_dir}/kas-cert.pem", "r") as rsaFile: - keyset.append( - abac.KasPublicKey( - alg=abac.KAS_PUBLIC_KEY_ALG_ENUM_RSA_2048, - kid="r1", - pem=rsaFile.read(), - ) - ) - with open(f"{key_store_dir}/kas-ec-cert.pem", "r") as ecFile: - keyset.append( - abac.KasPublicKey( - alg=abac.KAS_PUBLIC_KEY_ALG_ENUM_EC_SECP256R1, - kid="e1", - pem=ecFile.read(), - ) - ) - return abac.PublicKey( - cached=abac.KasPublicKeySet( - keys=keyset, - ) - ) + key_store_dir = os.getenv("KEY_STORE_DIR", "../../platform") + kas_rsa_path = os.path.join(key_store_dir, "kas-cert.pem") + kas_ec_path = os.path.join(key_store_dir, "kas-ec-cert.pem") + + keyset = [ + load_kas_key(kas_rsa_path, abac.KAS_PUBLIC_KEY_ALG_ENUM_RSA_2048, "r1"), + load_kas_key(kas_ec_path, abac.KAS_PUBLIC_KEY_ALG_ENUM_EC_SECP256R1, "e1") + ] + + return abac.PublicKey(cached=abac.KasPublicKeySet(keys=keyset)) def test_autoconfigure_one_attribute(tmp_dir, pt_file): From 1aaf334328975242ae5af163e31c60d12f834931 Mon Sep 17 00:00:00 2001 From: jakedoublev Date: Mon, 9 Sep 2024 09:24:16 -0700 Subject: [PATCH 09/13] lint --- xtest/test_abac.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xtest/test_abac.py b/xtest/test_abac.py index 7984e4d6..53f2534b 100644 --- a/xtest/test_abac.py +++ b/xtest/test_abac.py @@ -1,4 +1,3 @@ -import os import filecmp import pytest From 1a0d4a41fd3265dfa314429c5289197056ae29c1 Mon Sep 17 00:00:00 2001 From: jakedoublev Date: Mon, 9 Sep 2024 09:30:38 -0700 Subject: [PATCH 10/13] properly set env var --- .github/workflows/xtest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index c55d2cc0..8329260b 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -160,6 +160,8 @@ jobs: run: |- pytest test_abac.py working-directory: otdftests/xtest + env: + PLATFORM_DIR: '../../${{ steps.start-platform.outputs.platform-working-dir }}' ###### TODO: move these unbound tests to v2 platform # unbound-test-js: # timeout-minutes: 60 From c62c130786eba8a9ffc2de42742564f2b58e8b19 Mon Sep 17 00:00:00 2001 From: jakedoublev Date: Mon, 9 Sep 2024 10:14:01 -0700 Subject: [PATCH 11/13] lint --- .github/workflows/xtest.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 8329260b..7e5bb79b 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -20,7 +20,7 @@ on: type: string default: main schedule: - - cron: '30 6 * * *' + - cron: "30 6 * * *" jobs: cross-client-test: timeout-minutes: 60 @@ -41,20 +41,20 @@ jobs: - name: Set up Node 20 uses: actions/setup-node@v3 with: - node-version: '20.x' + node-version: "20.x" registry-url: https://npm.pkg.github.com - name: Set up Python 3.10 uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: "3.10" - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa with: github_token: ${{ secrets.GITHUB_TOKEN }} - name: Set up JDK uses: actions/setup-java@5896cecc08fd8a1fbdfaf517e29b571164b031f7 with: - java-version: '11' - distribution: 'adopt' + java-version: "11" + distribution: "adopt" server-id: github ######## SPIN UP PLATFORM BACKEND ############# From ba9e55da0bfb7105e2a1156d1cef3a9602117479 Mon Sep 17 00:00:00 2001 From: jakedoublev Date: Mon, 9 Sep 2024 10:54:26 -0700 Subject: [PATCH 12/13] run against main --- .github/workflows/xtest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 7e5bb79b..d0ce8572 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -60,7 +60,7 @@ jobs: ######## SPIN UP PLATFORM BACKEND ############# - name: Check out and start up platform with deps/containers id: start-platform - uses: opentdf/platform/test/start-up-with-containers@fix/reusable-ci + uses: opentdf/platform/test/start-up-with-containers@main with: platform-ref: ${{ env.PLATFORM_REF }} @@ -151,7 +151,7 @@ jobs: ######## ATTRIBUTE BASED CONFIGURATION ############# - name: Check out and start up platform with deps/containers - uses: opentdf/platform/test/start-additional-kas@fix/reusable-ci + uses: opentdf/platform/test/start-additional-kas@main with: kas-port: 8282 kas-name: beta From 41ae9ce41c2c76e2f67ac1479462209c09619f53 Mon Sep 17 00:00:00 2001 From: jakedoublev Date: Mon, 9 Sep 2024 11:02:32 -0700 Subject: [PATCH 13/13] tweak --- .github/workflows/xtest.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index d0ce8572..2831770b 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -59,7 +59,7 @@ jobs: ######## SPIN UP PLATFORM BACKEND ############# - name: Check out and start up platform with deps/containers - id: start-platform + id: run-platform uses: opentdf/platform/test/start-up-with-containers@main with: platform-ref: ${{ env.PLATFORM_REF }} @@ -101,8 +101,8 @@ jobs: ######## SETUP THE GO CLI ############# - name: Prepare go cli run: |- - go mod edit -replace github.com/opentdf/platform/protocol/go=../${{ steps.start-platform.outputs.platform-working-dir }}/protocol/go - go mod edit -replace github.com/opentdf/platform/sdk=../${{ steps.start-platform.outputs.platform-working-dir }}/sdk + go mod edit -replace github.com/opentdf/platform/protocol/go=../${{ steps.run-platform.outputs.platform-working-dir }}/protocol/go + go mod edit -replace github.com/opentdf/platform/sdk=../${{ steps.run-platform.outputs.platform-working-dir }}/sdk go mod tidy go build . cp ./otdfctl ../otdftests/xtest/sdk/go/otdfctl @@ -147,7 +147,7 @@ jobs: pytest test_tdfs.py working-directory: otdftests/xtest env: - PLATFORM_DIR: '../../${{ steps.start-platform.outputs.platform-working-dir }}' + PLATFORM_DIR: '../../${{ steps.run-platform.outputs.platform-working-dir }}' ######## ATTRIBUTE BASED CONFIGURATION ############# - name: Check out and start up platform with deps/containers @@ -161,7 +161,7 @@ jobs: pytest test_abac.py working-directory: otdftests/xtest env: - PLATFORM_DIR: '../../${{ steps.start-platform.outputs.platform-working-dir }}' + PLATFORM_DIR: '../../${{ steps.run-platform.outputs.platform-working-dir }}' ###### TODO: move these unbound tests to v2 platform # unbound-test-js: # timeout-minutes: 60