From d6531cd7a1f1a0a70aac0d29e927cc6c14494a9d Mon Sep 17 00:00:00 2001 From: Yong Date: Wed, 2 Jul 2025 16:05:57 -0500 Subject: [PATCH 01/10] Update Makefile for python client with auto setup --- client/python/Makefile | 50 ++++++++++++++++++++++++++++++++++++++--- client/python/README.md | 12 ++++------ 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/client/python/Makefile b/client/python/Makefile index 7d65922abc..cb131a5c6c 100644 --- a/client/python/Makefile +++ b/client/python/Makefile @@ -15,10 +15,38 @@ # specific language governing permissions and limitations # under the License. +VENV_DIR := .venv + +.PHONY: regenerate-client test-integration lint setup-env install-poetry-deps + +# Target to create the virtual environment directory +$(VENV_DIR): + @echo "Setting up Python virtual environment at $(VENV_DIR)..." + python3 -m venv $(VENV_DIR) + @echo "Virtual environment created." + +# Target to ensure the virtual environment and poetry/dependencies are set up +setup-env: $(VENV_DIR) install-poetry-deps + +# Target to install poetry and project dependencies +install-poetry-deps: # This target doesn't *create* $(VENV_DIR), but *uses* it. + @echo "Installing Poetry and project dependencies into $(VENV_DIR)..." + # Ensure pip is up-to-date within the venv + $(VENV_DIR)/bin/pip install --upgrade pip + # Install poetry if not already present + @if [ ! -f "$(VENV_DIR)/bin/poetry" ]; then \ + $(VENV_DIR)/bin/pip install poetry; \ + fi + # Install needed dependencies using poetry + $(VENV_DIR)/bin/poetry install --all-extras + @echo "Poetry and dependencies installed." + +# Target to regenerate the client code regenerate-client: ../templates/regenerate.sh -test-integration: +# Target to run integration tests +test-integration: setup-env docker compose -f docker-compose.yml kill docker compose -f docker-compose.yml rm -f docker compose -f docker-compose.yml up -d @@ -30,6 +58,22 @@ test-integration: @echo "Polaris is healthy. Starting integration tests..." poetry run pytest integration_tests/ ${PYTEST_ARGS} +# Target to run linting checks +lint: setup-env + $(VENV_DIR)/bin/poetry run pre-commit run --files integration_tests/* cli/* + +# Clean targets +clean: clean-venv + @echo "Cleaning up Python cache files..." + find . -type f -name "*.pyc" -delete + find . -type d -name "__pycache__" -delete -lint: - poetry run pre-commit run --files integration_tests/* +clean-venv: + @echo "Attempting to remove virtual environment directory: $(VENV_DIR)..." + # SAFETY CHECK: Ensure VENV_DIR is not empty and exists before attempting to remove + @if [ -n "$(VENV_DIR)" ] && [ -d "$(VENV_DIR)" ]; then \ + rm -rf "$(VENV_DIR)"; \ + echo "Virtual environment removed."; \ + else \ + echo "Virtual environment directory '$(VENV_DIR)' not found or VENV_DIR is empty. No action taken."; \ + fi diff --git a/client/python/README.md b/client/python/README.md index cec9526816..db89d63d72 100644 --- a/client/python/README.md +++ b/client/python/README.md @@ -6,9 +6,9 @@ to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -27,14 +27,10 @@ The Apache Polaris Python package provides a client for interacting with the Apa - poetry >= 2.0 ### Installation -First we need to generate the OpenAPI client code from the OpenAPI specification. +First we need to generate the OpenAPI client code from the OpenAPI specification. ``` make regenerate-client ``` -Install the project with test dependencies: -``` -poetry install --all-extras -``` ### Auto-formatting and Linting ``` @@ -44,4 +40,4 @@ make lint ### Running Integration Tests ``` make test-integration -``` \ No newline at end of file +``` From 8525c705c0165d9ce15dfc1f305fb251dbea99d3 Mon Sep 17 00:00:00 2001 From: Yong Date: Wed, 2 Jul 2025 16:46:58 -0500 Subject: [PATCH 02/10] Update Makefile for python client with auto setup --- client/python/.pre-commit-config.yaml | 8 +++++--- client/python/Makefile | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/client/python/.pre-commit-config.yaml b/client/python/.pre-commit-config.yaml index 84b1ab95eb..7918bb4eb8 100644 --- a/client/python/.pre-commit-config.yaml +++ b/client/python/.pre-commit-config.yaml @@ -23,10 +23,12 @@ repos: - id: end-of-file-fixer - id: debug-statements - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.13 + rev: v0.12.1 hooks: - - id: ruff - args: [ --fix, --exit-non-zero-on-fix ] + # Run the linter. + - id: ruff-check + args: [ --fix ] + # Run the formatter. - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.16.0 diff --git a/client/python/Makefile b/client/python/Makefile index cb131a5c6c..91159e84c5 100644 --- a/client/python/Makefile +++ b/client/python/Makefile @@ -17,7 +17,7 @@ VENV_DIR := .venv -.PHONY: regenerate-client test-integration lint setup-env install-poetry-deps +.PHONY: regenerate-client test-integration lint setup-env install-poetry-deps clean clean-venv # Target to create the virtual environment directory $(VENV_DIR): From c90e74b9854e6d4d3ada8706a3df8c0a8e355c87 Mon Sep 17 00:00:00 2001 From: Yong Date: Wed, 2 Jul 2025 16:52:06 -0500 Subject: [PATCH 03/10] Update Makefile for python client with auto setup --- client/python/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/python/Makefile b/client/python/Makefile index 91159e84c5..52da4bf04a 100644 --- a/client/python/Makefile +++ b/client/python/Makefile @@ -60,7 +60,7 @@ test-integration: setup-env # Target to run linting checks lint: setup-env - $(VENV_DIR)/bin/poetry run pre-commit run --files integration_tests/* cli/* + $(VENV_DIR)/bin/poetry run pre-commit run --files integration_tests/* # Clean targets clean: clean-venv From 3a7b6dd5ff938b09c317c0f454f849f897d475fd Mon Sep 17 00:00:00 2001 From: Yong Date: Wed, 2 Jul 2025 16:57:34 -0500 Subject: [PATCH 04/10] Update Makefile for python client with auto setup --- client/python/.pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/python/.pre-commit-config.yaml b/client/python/.pre-commit-config.yaml index 7918bb4eb8..008ba4736e 100644 --- a/client/python/.pre-commit-config.yaml +++ b/client/python/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: hooks: # Run the linter. - id: ruff-check - args: [ --fix ] + args: [ --fix, --exit-non-zero-on-fix ] # Run the formatter. - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy From f2829a4431a50256f80c1a96462fb81966cfd125 Mon Sep 17 00:00:00 2001 From: Yong Date: Wed, 2 Jul 2025 17:20:31 -0500 Subject: [PATCH 05/10] Update Makefile for python client with auto setup --- client/python/.pre-commit-config.yaml | 1 + client/python/Makefile | 2 +- client/python/cli/constants.py | 205 +++++++++++++++----------- client/python/cli/polaris_cli.py | 8 +- 4 files changed, 122 insertions(+), 94 deletions(-) diff --git a/client/python/.pre-commit-config.yaml b/client/python/.pre-commit-config.yaml index 008ba4736e..5ffb2bfc6c 100644 --- a/client/python/.pre-commit-config.yaml +++ b/client/python/.pre-commit-config.yaml @@ -36,3 +36,4 @@ repos: - id: mypy args: [--disallow-untyped-defs, --ignore-missing-imports, --install-types, --non-interactive] + files: 'integration_tests/.*\.py' diff --git a/client/python/Makefile b/client/python/Makefile index 52da4bf04a..91159e84c5 100644 --- a/client/python/Makefile +++ b/client/python/Makefile @@ -60,7 +60,7 @@ test-integration: setup-env # Target to run linting checks lint: setup-env - $(VENV_DIR)/bin/poetry run pre-commit run --files integration_tests/* + $(VENV_DIR)/bin/poetry run pre-commit run --files integration_tests/* cli/* # Clean targets clean: clean-venv diff --git a/client/python/cli/constants.py b/client/python/cli/constants.py index f82ff2caaf..93b36d998c 100644 --- a/client/python/cli/constants.py +++ b/client/python/cli/constants.py @@ -53,8 +53,8 @@ class CatalogConnectionType(Enum): Represents a ConnectionType for an EXTERNAL catalog -- see ConnectionConfigInfo in the spec """ - HADOOP = 'hadoop' - ICEBERG = 'iceberg-rest' + HADOOP = "hadoop" + ICEBERG = "iceberg-rest" class AuthenticationType(Enum): @@ -62,9 +62,9 @@ class AuthenticationType(Enum): Represents a AuthenticationType for an EXTERNAL catalog -- see AuthenticationParameters in the spec """ - OAUTH = 'oauth' - BEARER = 'bearer' - SIGV4 = 'sigv4' + OAUTH = "oauth" + BEARER = "bearer" + SIGV4 = "sigv4" class ServiceIdentityType(Enum): @@ -72,7 +72,7 @@ class ServiceIdentityType(Enum): Represents a Service Identity Type for an EXTERNAL catalog -- see ServiceIdentityInfo in the spec """ - AWS_IAM = 'aws_iam' + AWS_IAM = "aws_iam" class Commands: @@ -129,57 +129,57 @@ class Arguments: These values should be snake_case, but they will get mapped to kebab-case in `Parser.parse` """ - TYPE = 'type' - DEFAULT_BASE_LOCATION = 'default_base_location' - STORAGE_TYPE = 'storage_type' - ALLOWED_LOCATION = 'allowed_location' - ROLE_ARN = 'role_arn' - EXTERNAL_ID = 'external_id' - USER_ARN = 'user_arn' - TENANT_ID = 'tenant_id' - MULTI_TENANT_APP_NAME = 'multi_tenant_app_name' - CONSENT_URL = 'consent_url' - SERVICE_ACCOUNT = 'service_account' - CATALOG_ROLE = 'catalog_role' - CATALOG = 'catalog' - PRINCIPAL = 'principal' - CLIENT_ID = 'client_id' - PRINCIPAL_ROLE = 'principal_role' - PROPERTY = 'property' - SET_PROPERTY = 'set_property' - REMOVE_PROPERTY = 'remove_property' - PRIVILEGE = 'privilege' - NAMESPACE = 'namespace' - TABLE = 'table' - VIEW = 'view' - CASCADE = 'cascade' - CLIENT_SECRET = 'client_secret' - ACCESS_TOKEN = 'access_token' - HOST = 'host' - PORT = 'port' - BASE_URL = 'base_url' - PARENT = 'parent' - LOCATION = 'location' - REGION = 'region' - PROFILE = 'profile' - PROXY = 'proxy' - HADOOP_WAREHOUSE = 'hadoop_warehouse' - ICEBERG_REMOTE_CATALOG_NAME = 'iceberg_remote_catalog_name' - CATALOG_CONNECTION_TYPE = 'catalog_connection_type' - CATALOG_AUTHENTICATION_TYPE = 'catalog_authentication_type' - CATALOG_SERVICE_IDENTITY_TYPE = 'catalog_service_identity_type' - CATALOG_SERVICE_IDENTITY_IAM_ARN = 'catalog_service_identity_iam_arn' - CATALOG_URI = 'catalog_uri' - CATALOG_TOKEN_URI = 'catalog_token_uri' - CATALOG_CLIENT_ID = 'catalog_client_id' - CATALOG_CLIENT_SECRET = 'catalog_client_secret' - CATALOG_CLIENT_SCOPE = 'catalog_client_scope' - CATALOG_BEARER_TOKEN = 'catalog_bearer_token' - CATALOG_ROLE_ARN = 'catalog_role_arn' - CATALOG_ROLE_SESSION_NAME = 'catalog_role_session_name' - CATALOG_EXTERNAL_ID = 'catalog_external_id' - CATALOG_SIGNING_REGION = 'catalog_signing_region' - CATALOG_SIGNING_NAME = 'catalog_signing_name' + TYPE = "type" + DEFAULT_BASE_LOCATION = "default_base_location" + STORAGE_TYPE = "storage_type" + ALLOWED_LOCATION = "allowed_location" + ROLE_ARN = "role_arn" + EXTERNAL_ID = "external_id" + USER_ARN = "user_arn" + TENANT_ID = "tenant_id" + MULTI_TENANT_APP_NAME = "multi_tenant_app_name" + CONSENT_URL = "consent_url" + SERVICE_ACCOUNT = "service_account" + CATALOG_ROLE = "catalog_role" + CATALOG = "catalog" + PRINCIPAL = "principal" + CLIENT_ID = "client_id" + PRINCIPAL_ROLE = "principal_role" + PROPERTY = "property" + SET_PROPERTY = "set_property" + REMOVE_PROPERTY = "remove_property" + PRIVILEGE = "privilege" + NAMESPACE = "namespace" + TABLE = "table" + VIEW = "view" + CASCADE = "cascade" + CLIENT_SECRET = "client_secret" + ACCESS_TOKEN = "access_token" + HOST = "host" + PORT = "port" + BASE_URL = "base_url" + PARENT = "parent" + LOCATION = "location" + REGION = "region" + PROFILE = "profile" + PROXY = "proxy" + HADOOP_WAREHOUSE = "hadoop_warehouse" + ICEBERG_REMOTE_CATALOG_NAME = "iceberg_remote_catalog_name" + CATALOG_CONNECTION_TYPE = "catalog_connection_type" + CATALOG_AUTHENTICATION_TYPE = "catalog_authentication_type" + CATALOG_SERVICE_IDENTITY_TYPE = "catalog_service_identity_type" + CATALOG_SERVICE_IDENTITY_IAM_ARN = "catalog_service_identity_iam_arn" + CATALOG_URI = "catalog_uri" + CATALOG_TOKEN_URI = "catalog_token_uri" + CATALOG_CLIENT_ID = "catalog_client_id" + CATALOG_CLIENT_SECRET = "catalog_client_secret" + CATALOG_CLIENT_SCOPE = "catalog_client_scope" + CATALOG_BEARER_TOKEN = "catalog_bearer_token" + CATALOG_ROLE_ARN = "catalog_role_arn" + CATALOG_ROLE_SESSION_NAME = "catalog_role_session_name" + CATALOG_EXTERNAL_ID = "catalog_external_id" + CATALOG_SIGNING_REGION = "catalog_signing_region" + CATALOG_SIGNING_NAME = "catalog_signing_name" class Hints: @@ -237,39 +237,64 @@ class Update: DEFAULT_BASE_LOCATION = "A new default base location for the catalog" class External: - CATALOG_CONNECTION_TYPE = 'The type of external catalog in [ICEBERG, HADOOP].' - CATALOG_AUTHENTICATION_TYPE = 'The type of authentication in [OAUTH, BEARER, SIGV4]' - CATALOG_SERVICE_IDENTITY_TYPE = 'The type of service identity in [AWS_IAM]' - - CATALOG_SERVICE_IDENTITY_IAM_ARN = ('When using the AWS_IAM service identity type, this is the ARN ' - 'of the IAM user or IAM role Polaris uses to assume roles and ' - 'then access external resources.') - - CATALOG_URI = 'The URI of the external catalog' - HADOOP_WAREHOUSE = 'The warehouse to use when federating to a HADOOP catalog' - ICEBERG_REMOTE_CATALOG_NAME = 'The remote catalog name when federating to an Iceberg REST catalog' - - - CATALOG_TOKEN_URI = '(For authentication type OAUTH) Token server URI' - CATALOG_CLIENT_ID = '(For authentication type OAUTH) oauth client id' - CATALOG_CLIENT_SECRET = '(For authentication type OAUTH) oauth client secret (input-only)' - CATALOG_CLIENT_SCOPE = ('(For authentication type OAUTH) oauth scopes to specify when exchanging ' - 'for a short-lived access token. Multiple can be provided by specifying' - ' this option more than once') - - CATALOG_BEARER_TOKEN = '(For authentication type BEARER) Bearer token (input-only)' - - CATALOG_ROLE_ARN = ('(For authentication type SIGV4) The aws IAM role arn assumed by polaris ' - 'userArn when signing requests') - CATALOG_ROLE_SESSION_NAME = ('(For authentication type SIGV4) The role session name to be used ' - 'by the SigV4 protocol for signing requests') - CATALOG_EXTERNAL_ID = ('(For authentication type SIGV4) An optional external id used to establish ' - 'a trust relationship with AWS in the trust policy') - CATALOG_SIGNING_REGION = ('(For authentication type SIGV4) Region to be used by the SigV4 protocol ' - 'for signing requests') - CATALOG_SIGNING_NAME = ('(For authentication type SIGV4) The service name to be used by the SigV4 ' - 'protocol for signing requests, the default signing name is "execute-api" ' - 'is if not provided') + CATALOG_CONNECTION_TYPE = ( + "The type of external catalog in [ICEBERG, HADOOP]." + ) + CATALOG_AUTHENTICATION_TYPE = ( + "The type of authentication in [OAUTH, BEARER, SIGV4]" + ) + CATALOG_SERVICE_IDENTITY_TYPE = "The type of service identity in [AWS_IAM]" + + CATALOG_SERVICE_IDENTITY_IAM_ARN = ( + "When using the AWS_IAM service identity type, this is the ARN " + "of the IAM user or IAM role Polaris uses to assume roles and " + "then access external resources." + ) + + CATALOG_URI = "The URI of the external catalog" + HADOOP_WAREHOUSE = ( + "The warehouse to use when federating to a HADOOP catalog" + ) + ICEBERG_REMOTE_CATALOG_NAME = ( + "The remote catalog name when federating to an Iceberg REST catalog" + ) + + CATALOG_TOKEN_URI = "(For authentication type OAUTH) Token server URI" + CATALOG_CLIENT_ID = "(For authentication type OAUTH) oauth client id" + CATALOG_CLIENT_SECRET = ( + "(For authentication type OAUTH) oauth client secret (input-only)" + ) + CATALOG_CLIENT_SCOPE = ( + "(For authentication type OAUTH) oauth scopes to specify when exchanging " + "for a short-lived access token. Multiple can be provided by specifying" + " this option more than once" + ) + + CATALOG_BEARER_TOKEN = ( + "(For authentication type BEARER) Bearer token (input-only)" + ) + + CATALOG_ROLE_ARN = ( + "(For authentication type SIGV4) The aws IAM role arn assumed by polaris " + "userArn when signing requests" + ) + CATALOG_ROLE_SESSION_NAME = ( + "(For authentication type SIGV4) The role session name to be used " + "by the SigV4 protocol for signing requests" + ) + CATALOG_EXTERNAL_ID = ( + "(For authentication type SIGV4) An optional external id used to establish " + "a trust relationship with AWS in the trust policy" + ) + CATALOG_SIGNING_REGION = ( + "(For authentication type SIGV4) Region to be used by the SigV4 protocol " + "for signing requests" + ) + CATALOG_SIGNING_NAME = ( + "(For authentication type SIGV4) The service name to be used by the SigV4 " + 'protocol for signing requests, the default signing name is "execute-api" ' + "is if not provided" + ) class Principals: class Create: diff --git a/client/python/cli/polaris_cli.py b/client/python/cli/polaris_cli.py index 47e8038650..83341ada41 100644 --- a/client/python/cli/polaris_cli.py +++ b/client/python/cli/polaris_cli.py @@ -158,9 +158,11 @@ def _get_client_builder(options): # Authenticate accordingly if options.base_url: if options.host is not None or options.port is not None: - raise Exception(f'Please provide either {Argument.to_flag_name(Arguments.BASE_URL)} or' - f' {Argument.to_flag_name(Arguments.HOST)} &' - f' {Argument.to_flag_name(Arguments.PORT)}, but not both') + raise Exception( + f"Please provide either {Argument.to_flag_name(Arguments.BASE_URL)} or" + f" {Argument.to_flag_name(Arguments.HOST)} &" + f" {Argument.to_flag_name(Arguments.PORT)}, but not both" + ) polaris_management_url = f"{options.base_url}/api/management/v1" polaris_catalog_url = f"{options.base_url}/api/catalog/v1" From 7b5eaeaf876f93387cb8dad1973c777d2ebf778d Mon Sep 17 00:00:00 2001 From: Yong Date: Mon, 7 Jul 2025 22:55:00 -0500 Subject: [PATCH 06/10] Remove implicit poetry setup in workflow and add helper in Makefile --- .github/workflows/python-client.yml | 15 ++------ client/python/Makefile | 56 ++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 29 deletions(-) diff --git a/.github/workflows/python-client.yml b/.github/workflows/python-client.yml index 37ead61eda..2ed52b1608 100644 --- a/.github/workflows/python-client.yml +++ b/.github/workflows/python-client.yml @@ -58,15 +58,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install Poetry - run: | - pip install --user --upgrade -r regtests/requirements.txt - - # TODO: add cache for poetry dependencies once we have poetry.lock in the repo - - name: Install dependencies - working-directory: client/python - run: poetry install --all-extras - - name: Lint working-directory: client/python run: | @@ -81,9 +72,9 @@ jobs: - name: Image build run: | ./gradlew \ - :polaris-server:assemble \ - :polaris-server:quarkusAppPartsBuild --rerun \ - -Dquarkus.container-image.build=true + :polaris-server:assemble \ + :polaris-server:quarkusAppPartsBuild --rerun \ + -Dquarkus.container-image.build=true - name: Integration Tests working-directory: client/python diff --git a/client/python/Makefile b/client/python/Makefile index 91159e84c5..870bac8be6 100644 --- a/client/python/Makefile +++ b/client/python/Makefile @@ -15,9 +15,30 @@ # specific language governing permissions and limitations # under the License. +.SILENT: + +# Configures the shell for recipes to use bash, enabling bash commands and ensuring +# that recipes exit on any command failure (including within pipes). +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +# Version information +VERSION ?= $(shell cat pyproject.toml | grep version | sed 's/version *= *"\(.*\)"/\1/') +BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%S%:z") +GIT_COMMIT := $(shell git rev-parse HEAD) + +# Variables VENV_DIR := .venv -.PHONY: regenerate-client test-integration lint setup-env install-poetry-deps clean clean-venv +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-30s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +.PHONY: version +version: ## Print version information. + @echo "Apache Polaris version: ${VERSION}" + @echo "Build date: ${BUILD_DATE}" + @echo "Git commit: ${GIT_COMMIT}" # Target to create the virtual environment directory $(VENV_DIR): @@ -25,28 +46,28 @@ $(VENV_DIR): python3 -m venv $(VENV_DIR) @echo "Virtual environment created." -# Target to ensure the virtual environment and poetry/dependencies are set up +.PHONY: setup-env setup-env: $(VENV_DIR) install-poetry-deps -# Target to install poetry and project dependencies -install-poetry-deps: # This target doesn't *create* $(VENV_DIR), but *uses* it. +.PHONY: install-poetry-deps +install-poetry-deps: @echo "Installing Poetry and project dependencies into $(VENV_DIR)..." # Ensure pip is up-to-date within the venv $(VENV_DIR)/bin/pip install --upgrade pip # Install poetry if not already present @if [ ! -f "$(VENV_DIR)/bin/poetry" ]; then \ - $(VENV_DIR)/bin/pip install poetry; \ + $(VENV_DIR)/bin/pip install --upgrade -r ../../regtests/requirements.txt; \ fi # Install needed dependencies using poetry $(VENV_DIR)/bin/poetry install --all-extras @echo "Poetry and dependencies installed." -# Target to regenerate the client code -regenerate-client: +.PHONY: regenerate-client +regenerate-client: ## Regenerate the client code ../templates/regenerate.sh -# Target to run integration tests -test-integration: setup-env +.PHONY: test-integration +test-integration: setup-env ## Run integration tests docker compose -f docker-compose.yml kill docker compose -f docker-compose.yml rm -f docker compose -f docker-compose.yml up -d @@ -58,16 +79,11 @@ test-integration: setup-env @echo "Polaris is healthy. Starting integration tests..." poetry run pytest integration_tests/ ${PYTEST_ARGS} -# Target to run linting checks -lint: setup-env +.PHONY: lint +lint: setup-env ## Run linting checks $(VENV_DIR)/bin/poetry run pre-commit run --files integration_tests/* cli/* -# Clean targets -clean: clean-venv - @echo "Cleaning up Python cache files..." - find . -type f -name "*.pyc" -delete - find . -type d -name "__pycache__" -delete - +.PHONY: clean-venv clean-venv: @echo "Attempting to remove virtual environment directory: $(VENV_DIR)..." # SAFETY CHECK: Ensure VENV_DIR is not empty and exists before attempting to remove @@ -77,3 +93,9 @@ clean-venv: else \ echo "Virtual environment directory '$(VENV_DIR)' not found or VENV_DIR is empty. No action taken."; \ fi + +.PHONY: clean +clean: clean-venv ## Cleanup + @echo "Cleaning up Python cache files..." + find . -type f -name "*.pyc" -delete + find . -type d -name "__pycache__" -delete From 153cf3fd6828e388fe32c53669c4f06f5a6d74c2 Mon Sep 17 00:00:00 2001 From: Yong Date: Mon, 7 Jul 2025 23:15:53 -0500 Subject: [PATCH 07/10] Wrap client client into Makefile target --- .github/workflows/python-client.yml | 3 +-- client/python/Makefile | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-client.yml b/.github/workflows/python-client.yml index 2ed52b1608..7586e3b184 100644 --- a/.github/workflows/python-client.yml +++ b/.github/workflows/python-client.yml @@ -66,8 +66,7 @@ jobs: - name: Generated Client Tests working-directory: client/python run: | - export SCRIPT_DIR="non-existing-mock-directory" - poetry run pytest test/ + make test-client - name: Image build run: | diff --git a/client/python/Makefile b/client/python/Makefile index 870bac8be6..53d7a4e63c 100644 --- a/client/python/Makefile +++ b/client/python/Makefile @@ -66,6 +66,10 @@ install-poetry-deps: regenerate-client: ## Regenerate the client code ../templates/regenerate.sh +.PHONY: test-client +test-client: setup-env ## Run client tests + SCRIPT_DIR="non-existing-mock-directory" $(VENV_DIR)/bin/poetry run pytest test/ + .PHONY: test-integration test-integration: setup-env ## Run integration tests docker compose -f docker-compose.yml kill From e3169524b8d36bb08a7a8d43cfc7a37a5b1d5023 Mon Sep 17 00:00:00 2001 From: Yong Date: Mon, 7 Jul 2025 23:39:04 -0500 Subject: [PATCH 08/10] Get poetry version from pyproject.toml --- client/python/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/python/Makefile b/client/python/Makefile index 53d7a4e63c..ab5f224ba3 100644 --- a/client/python/Makefile +++ b/client/python/Makefile @@ -26,6 +26,7 @@ SHELL = /usr/bin/env bash -o pipefail VERSION ?= $(shell cat pyproject.toml | grep version | sed 's/version *= *"\(.*\)"/\1/') BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%S%:z") GIT_COMMIT := $(shell git rev-parse HEAD) +POETRY_VERSION := $(shell cat pyproject.toml | grep requires-poetry | sed 's/requires-poetry *= *"\(.*\)"/\1/') # Variables VENV_DIR := .venv @@ -39,6 +40,7 @@ version: ## Print version information. @echo "Apache Polaris version: ${VERSION}" @echo "Build date: ${BUILD_DATE}" @echo "Git commit: ${GIT_COMMIT}" + @echo "Poetry version: ${POETRY_VERSION}" # Target to create the virtual environment directory $(VENV_DIR): @@ -56,7 +58,7 @@ install-poetry-deps: $(VENV_DIR)/bin/pip install --upgrade pip # Install poetry if not already present @if [ ! -f "$(VENV_DIR)/bin/poetry" ]; then \ - $(VENV_DIR)/bin/pip install --upgrade -r ../../regtests/requirements.txt; \ + $(VENV_DIR)/bin/pip install --upgrade "poetry${POETRY_VERSION}"; \ fi # Install needed dependencies using poetry $(VENV_DIR)/bin/poetry install --all-extras From df730c44f5fbce03f93af611d8dffe7edef7ddca Mon Sep 17 00:00:00 2001 From: Yong Date: Mon, 7 Jul 2025 23:50:40 -0500 Subject: [PATCH 09/10] Fix poetry reference --- client/python/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/python/Makefile b/client/python/Makefile index ab5f224ba3..9d222fa39a 100644 --- a/client/python/Makefile +++ b/client/python/Makefile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -.SILENT: +# .SILENT: # Configures the shell for recipes to use bash, enabling bash commands and ensuring # that recipes exit on any command failure (including within pipes). @@ -83,7 +83,7 @@ test-integration: setup-env ## Run integration tests echo "Still waiting for HTTP 200 from /q/health..."; \ done @echo "Polaris is healthy. Starting integration tests..." - poetry run pytest integration_tests/ ${PYTEST_ARGS} + $(VENV_DIR)/bin/poetry run pytest integration_tests/ ${PYTEST_ARGS} .PHONY: lint lint: setup-env ## Run linting checks From 6acd297902e75976034161a38fcc02b306ade59e Mon Sep 17 00:00:00 2001 From: Yong Date: Wed, 9 Jul 2025 19:43:57 -0500 Subject: [PATCH 10/10] Fixed version for poetry --- client/python/README.md | 2 +- client/python/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/python/README.md b/client/python/README.md index db89d63d72..4489452a13 100644 --- a/client/python/README.md +++ b/client/python/README.md @@ -24,7 +24,7 @@ The Apache Polaris Python package provides a client for interacting with the Apa ### Prerequisites - Python 3.9 or later -- poetry >= 2.0 +- poetry >= 2.1 ### Installation First we need to generate the OpenAPI client code from the OpenAPI specification. diff --git a/client/python/pyproject.toml b/client/python/pyproject.toml index a5e20d97fe..9740437bfb 100644 --- a/client/python/pyproject.toml +++ b/client/python/pyproject.toml @@ -42,7 +42,7 @@ homepage = "https://polaris.apache.org/" repository = "https://github.com/apache/polaris/" [tool.poetry] -requires-poetry = ">=2.1" +requires-poetry = "==2.1.3" packages = [{ include = "polaris" }] [tool.poetry.group.test.dependencies]