From 2be84019691604f2ecd46d21258363f8992486d4 Mon Sep 17 00:00:00 2001 From: ChenXing Yang <20001020ycx@gmail.com> Date: Tue, 27 Jan 2026 11:35:55 -0800 Subject: [PATCH 01/10] add code for update clp.properties in the velox worker --- tools/deployment/presto-clp/scripts/init.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/deployment/presto-clp/scripts/init.py b/tools/deployment/presto-clp/scripts/init.py index ed7eeae207..11d1d2f948 100755 --- a/tools/deployment/presto-clp/scripts/init.py +++ b/tools/deployment/presto-clp/scripts/init.py @@ -286,7 +286,13 @@ def _add_clp_s3_env_vars( env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_AUTH_PROVIDER"] = "clp_package" env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_ACCESS_KEY_ID"] = s3_access_key_id - s3_end_point = f"https://{s3_bucket}.s3.{s3_region_code}.amazonaws.com/" + env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_BUCKET"] = s3_bucket + + # Use endpoint_url if provided, otherwise construct AWS-style URL + s3_end_point = _get_config_value(clp_config, f"{s3_config_key}.endpoint_url") + if s3_end_point is None: + s3_end_point = f"https://{s3_bucket}.s3.{s3_region_code}.amazonaws.com/" + env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_END_POINT"] = s3_end_point env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_SECRET_ACCESS_KEY"] = s3_secret_access_key @@ -359,6 +365,7 @@ def _generate_worker_clp_properties( "clp.storage-type": "PRESTO_WORKER_CLPPROPERTIES_STORAGE_TYPE", "clp.s3-auth-provider": "PRESTO_WORKER_CLPPROPERTIES_S3_AUTH_PROVIDER", "clp.s3-access-key-id": "PRESTO_WORKER_CLPPROPERTIES_S3_ACCESS_KEY_ID", + "clp.s3-bucket": "PRESTO_WORKER_CLPPROPERTIES_S3_BUCKET", "clp.s3-end-point": "PRESTO_WORKER_CLPPROPERTIES_S3_END_POINT", "clp.s3-secret-access-key": "PRESTO_WORKER_CLPPROPERTIES_S3_SECRET_ACCESS_KEY", } From 6298bc4f4e344d6bdb800b4f4d9c8b1454ab16a6 Mon Sep 17 00:00:00 2001 From: ChenXing Yang <20001020ycx@gmail.com> Date: Tue, 27 Jan 2026 11:46:28 -0800 Subject: [PATCH 02/10] fix error handling --- tools/deployment/presto-clp/scripts/init.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tools/deployment/presto-clp/scripts/init.py b/tools/deployment/presto-clp/scripts/init.py index 11d1d2f948..71b4c07ce6 100755 --- a/tools/deployment/presto-clp/scripts/init.py +++ b/tools/deployment/presto-clp/scripts/init.py @@ -281,19 +281,16 @@ def _add_clp_s3_env_vars( s3_secret_access_key = _get_required_config_value( clp_config, f"{s3_credentials_key}.secret_access_key", clp_config_file_path ) + s3_endpoint_url = _get_required_config_value( + clp_config, f"{s3_config_key}.endpoint_url", clp_config_file_path + ) except KeyError: return False env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_AUTH_PROVIDER"] = "clp_package" env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_ACCESS_KEY_ID"] = s3_access_key_id env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_BUCKET"] = s3_bucket - - # Use endpoint_url if provided, otherwise construct AWS-style URL - s3_end_point = _get_config_value(clp_config, f"{s3_config_key}.endpoint_url") - if s3_end_point is None: - s3_end_point = f"https://{s3_bucket}.s3.{s3_region_code}.amazonaws.com/" - - env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_END_POINT"] = s3_end_point + env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_END_POINT"] = s3_endpoint_url env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_SECRET_ACCESS_KEY"] = s3_secret_access_key return True From 931948828ce10ba1805d0bacb258d442bd10ae89 Mon Sep 17 00:00:00 2001 From: ChenXing Yang <20001020ycx@gmail.com> Date: Tue, 27 Jan 2026 11:58:03 -0800 Subject: [PATCH 03/10] delete unused var --- tools/deployment/presto-clp/scripts/init.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/deployment/presto-clp/scripts/init.py b/tools/deployment/presto-clp/scripts/init.py index 71b4c07ce6..f4f144e123 100755 --- a/tools/deployment/presto-clp/scripts/init.py +++ b/tools/deployment/presto-clp/scripts/init.py @@ -260,9 +260,6 @@ def _add_clp_s3_env_vars( s3_bucket = _get_required_config_value( clp_config, f"{s3_config_key}.bucket", clp_config_file_path ) - s3_region_code = _get_required_config_value( - clp_config, f"{s3_config_key}.region_code", clp_config_file_path - ) aws_auth_key = f"{s3_config_key}.aws_authentication" aws_auth_type_key = f"{aws_auth_key}.type" From 85116469ae1df678685d6e728e4a82c895d76ce3 Mon Sep 17 00:00:00 2001 From: ChenXing Yang <20001020ycx@gmail.com> Date: Tue, 27 Jan 2026 13:25:15 -0800 Subject: [PATCH 04/10] address comment --- tools/deployment/presto-clp/scripts/init.py | 31 ++++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/tools/deployment/presto-clp/scripts/init.py b/tools/deployment/presto-clp/scripts/init.py index f4f144e123..d533deaf94 100755 --- a/tools/deployment/presto-clp/scripts/init.py +++ b/tools/deployment/presto-clp/scripts/init.py @@ -21,6 +21,9 @@ PRESTO_QUERY_MEMORY_RATIO = 0.5 PRESTO_SYSTEM_MEMORY_RATIO = 0.9 +# S3 URL constant (matching s3_utils.py) +AWS_ENDPOINT = "amazonaws.com" + # Set up console logging logging_console_handler = logging.StreamHandler() logging_formatter = logging.Formatter( @@ -278,21 +281,41 @@ def _add_clp_s3_env_vars( s3_secret_access_key = _get_required_config_value( clp_config, f"{s3_credentials_key}.secret_access_key", clp_config_file_path ) - s3_endpoint_url = _get_required_config_value( - clp_config, f"{s3_config_key}.endpoint_url", clp_config_file_path - ) except KeyError: return False + s3_endpoint_url = _get_config_value(clp_config, f"{s3_config_key}.endpoint_url") + s3_region_code = _get_config_value(clp_config, f"{s3_config_key}.region_code") + s3_end_point = _handle_s3_endpoint_url(s3_endpoint_url, s3_region_code, s3_bucket) + env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_AUTH_PROVIDER"] = "clp_package" env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_ACCESS_KEY_ID"] = s3_access_key_id env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_BUCKET"] = s3_bucket - env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_END_POINT"] = s3_endpoint_url + env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_END_POINT"] = s3_end_point env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_SECRET_ACCESS_KEY"] = s3_secret_access_key return True +def _handle_s3_endpoint_url( + endpoint_url: str | None, region_code: str | None, bucket_name: str +) -> str: + """ + Returns the S3 endpoint URL, or constructs an AWS S3 URL if not provided. + + :param endpoint_url: + :param region_code: AWS region code, or None. + :param bucket_name: S3 bucket name (used to construct AWS S3 URL). + :return: The S3 endpoint URL. + """ + if endpoint_url is not None: + return endpoint_url + + if region_code is None: + return f"https://{bucket_name}.s3.{AWS_ENDPOINT}/" + return f"https://{bucket_name}.s3.{region_code}.{AWS_ENDPOINT}/" + + def _add_memory_env_vars(env_vars: dict[str, str]) -> bool: """ Adds memory-related environment variables based on Presto guidelines. From f8ee2e5b24f28a6fcdd045519d5056fc87c4336f Mon Sep 17 00:00:00 2001 From: ChenXing Yang <20001020ycx@gmail.com> Date: Tue, 27 Jan 2026 13:25:59 -0800 Subject: [PATCH 05/10] nit fix --- tools/deployment/presto-clp/scripts/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/presto-clp/scripts/init.py b/tools/deployment/presto-clp/scripts/init.py index d533deaf94..909e59b795 100755 --- a/tools/deployment/presto-clp/scripts/init.py +++ b/tools/deployment/presto-clp/scripts/init.py @@ -21,7 +21,7 @@ PRESTO_QUERY_MEMORY_RATIO = 0.5 PRESTO_SYSTEM_MEMORY_RATIO = 0.9 -# S3 URL constant (matching s3_utils.py) +# S3 URL constant AWS_ENDPOINT = "amazonaws.com" # Set up console logging From fc402c909899704fc112ab8fe511f9618b37312f Mon Sep 17 00:00:00 2001 From: ChenXing Yang <60459812+20001020ycx@users.noreply.github.com> Date: Tue, 27 Jan 2026 15:53:14 -0800 Subject: [PATCH 06/10] Update tools/deployment/presto-clp/scripts/init.py Co-authored-by: Junhao Liao --- tools/deployment/presto-clp/scripts/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/presto-clp/scripts/init.py b/tools/deployment/presto-clp/scripts/init.py index 909e59b795..cf02cac9b5 100755 --- a/tools/deployment/presto-clp/scripts/init.py +++ b/tools/deployment/presto-clp/scripts/init.py @@ -303,7 +303,7 @@ def _handle_s3_endpoint_url( """ Returns the S3 endpoint URL, or constructs an AWS S3 URL if not provided. - :param endpoint_url: + :param endpoint_url: Custom S3 endpoint URL. :param region_code: AWS region code, or None. :param bucket_name: S3 bucket name (used to construct AWS S3 URL). :return: The S3 endpoint URL. From 064feb865750ee75a8c8d791af3a875074756210 Mon Sep 17 00:00:00 2001 From: ChenXing Yang <60459812+20001020ycx@users.noreply.github.com> Date: Tue, 27 Jan 2026 15:53:22 -0800 Subject: [PATCH 07/10] Update tools/deployment/presto-clp/scripts/init.py Co-authored-by: Junhao Liao --- tools/deployment/presto-clp/scripts/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/presto-clp/scripts/init.py b/tools/deployment/presto-clp/scripts/init.py index cf02cac9b5..45b18ab50b 100755 --- a/tools/deployment/presto-clp/scripts/init.py +++ b/tools/deployment/presto-clp/scripts/init.py @@ -304,7 +304,7 @@ def _handle_s3_endpoint_url( Returns the S3 endpoint URL, or constructs an AWS S3 URL if not provided. :param endpoint_url: Custom S3 endpoint URL. - :param region_code: AWS region code, or None. + :param region_code: AWS region code. :param bucket_name: S3 bucket name (used to construct AWS S3 URL). :return: The S3 endpoint URL. """ From db8b8f711318eb347d958f5b6f584bc3c0b797ea Mon Sep 17 00:00:00 2001 From: ChenXing Yang <20001020ycx@gmail.com> Date: Tue, 27 Jan 2026 16:01:19 -0800 Subject: [PATCH 08/10] address review comment --- tools/deployment/presto-clp/scripts/init.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/deployment/presto-clp/scripts/init.py b/tools/deployment/presto-clp/scripts/init.py index 45b18ab50b..9624fee6d5 100755 --- a/tools/deployment/presto-clp/scripts/init.py +++ b/tools/deployment/presto-clp/scripts/init.py @@ -22,7 +22,7 @@ PRESTO_SYSTEM_MEMORY_RATIO = 0.9 # S3 URL constant -AWS_ENDPOINT = "amazonaws.com" +AWS_S3_DOMAIN = "amazonaws.com" # Set up console logging logging_console_handler = logging.StreamHandler() @@ -286,7 +286,7 @@ def _add_clp_s3_env_vars( s3_endpoint_url = _get_config_value(clp_config, f"{s3_config_key}.endpoint_url") s3_region_code = _get_config_value(clp_config, f"{s3_config_key}.region_code") - s3_end_point = _handle_s3_endpoint_url(s3_endpoint_url, s3_region_code, s3_bucket) + s3_end_point = _resolve_s3_endpoint_url(s3_endpoint_url, s3_region_code, s3_bucket) env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_AUTH_PROVIDER"] = "clp_package" env_vars["PRESTO_WORKER_CLPPROPERTIES_S3_ACCESS_KEY_ID"] = s3_access_key_id @@ -297,7 +297,7 @@ def _add_clp_s3_env_vars( return True -def _handle_s3_endpoint_url( +def _resolve_s3_endpoint_url( endpoint_url: str | None, region_code: str | None, bucket_name: str ) -> str: """ @@ -309,11 +309,11 @@ def _handle_s3_endpoint_url( :return: The S3 endpoint URL. """ if endpoint_url is not None: - return endpoint_url + return endpoint_url.rstrip("/") if region_code is None: - return f"https://{bucket_name}.s3.{AWS_ENDPOINT}/" - return f"https://{bucket_name}.s3.{region_code}.{AWS_ENDPOINT}/" + return f"https://{bucket_name}.s3.{AWS_S3_DOMAIN}" + return f"https://{bucket_name}.s3.{region_code}.{AWS_S3_DOMAIN}" def _add_memory_env_vars(env_vars: dict[str, str]) -> bool: From b1a0881da153550fdcd54dda8830e763a0294501 Mon Sep 17 00:00:00 2001 From: ChenXing Yang <20001020ycx@gmail.com> Date: Tue, 27 Jan 2026 16:03:57 -0800 Subject: [PATCH 09/10] nit --- tools/deployment/presto-clp/scripts/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/presto-clp/scripts/init.py b/tools/deployment/presto-clp/scripts/init.py index 9624fee6d5..cbbd81dc51 100755 --- a/tools/deployment/presto-clp/scripts/init.py +++ b/tools/deployment/presto-clp/scripts/init.py @@ -301,7 +301,7 @@ def _resolve_s3_endpoint_url( endpoint_url: str | None, region_code: str | None, bucket_name: str ) -> str: """ - Returns the S3 endpoint URL, or constructs an AWS S3 URL if not provided. + Returns the resolved S3 endpoint URL, or constructs an AWS S3 URL if not provided. :param endpoint_url: Custom S3 endpoint URL. :param region_code: AWS region code. From 469aa3b91960b1dbf00689c66d47aec24878a254 Mon Sep 17 00:00:00 2001 From: ChenXing Yang <60459812+20001020ycx@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:09:45 -0800 Subject: [PATCH 10/10] Update tools/deployment/presto-clp/scripts/init.py Co-authored-by: Junhao Liao --- tools/deployment/presto-clp/scripts/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/presto-clp/scripts/init.py b/tools/deployment/presto-clp/scripts/init.py index cbbd81dc51..ba9e754e1a 100755 --- a/tools/deployment/presto-clp/scripts/init.py +++ b/tools/deployment/presto-clp/scripts/init.py @@ -305,7 +305,7 @@ def _resolve_s3_endpoint_url( :param endpoint_url: Custom S3 endpoint URL. :param region_code: AWS region code. - :param bucket_name: S3 bucket name (used to construct AWS S3 URL). + :param bucket_name: :return: The S3 endpoint URL. """ if endpoint_url is not None: