From 10ab065464e724c9ad50b74b5b787ac3321f0a82 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Fri, 25 Jul 2025 17:24:57 -0400 Subject: [PATCH 1/5] refactor --- .../clp_package_utils/scripts/start_clp.py | 4 +++- .../clp-py-utils/clp_py_utils/clp_config.py | 10 +++++++++- .../job_orchestration/scheduler/constants.py | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index b988651fca..65be81245c 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -957,7 +957,6 @@ def start_webui( ] necessary_mounts = [ mounts.clp_home, - mounts.stream_output_dir, ] if StorageType.S3 == stream_storage.type: auth = stream_storage.s3_config.aws_authentication @@ -973,6 +972,9 @@ def start_webui( necessary_mounts.append(mounts.aws_config_dir) if aws_env_vars: necessary_env_vars.extend(aws_env_vars) + else: + necessary_mounts.append(mounts.stream_output_dir) + append_docker_options(container_cmd, necessary_mounts, necessary_env_vars) container_cmd.append(clp_config.execution_container) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 5776b9c364..8b77ccb378 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -366,6 +366,12 @@ def validate_authentication(cls, values): raise ValueError(f"profile and credentials must not be set when type is '{auth_enum}.'") return values + def dump_to_primitive_dict(self): + d = self.dict() + if self.credentials is not None: + d["credentials"] = self.credentials.dict() + return d + class S3Config(BaseModel): region_code: str @@ -391,7 +397,9 @@ class S3IngestionConfig(BaseModel): aws_authentication: AwsAuthentication def dump_to_primitive_dict(self): - return self.dict() + d = self.dict() + d["aws_authentication"] = self.aws_authentication.dump_to_primitive_dict() + return d class FsStorage(BaseModel): diff --git a/components/job-orchestration/job_orchestration/scheduler/constants.py b/components/job-orchestration/job_orchestration/scheduler/constants.py index cd85016a3e..dbb5f03621 100644 --- a/components/job-orchestration/job_orchestration/scheduler/constants.py +++ b/components/job-orchestration/job_orchestration/scheduler/constants.py @@ -17,11 +17,23 @@ class CompressionJobStatus(IntEnum): SUCCEEDED = auto() FAILED = auto() + def __str__(self) -> str: + return str(self.value) + + def to_str(self) -> str: + return str(self.name) + class CompressionJobCompletionStatus(IntEnum): SUCCEEDED = 0 FAILED = auto() + def __str__(self) -> str: + return str(self.value) + + def to_str(self) -> str: + return str(self.name) + class CompressionTaskStatus(IntEnum): PENDING = 0 @@ -29,6 +41,12 @@ class CompressionTaskStatus(IntEnum): SUCCEEDED = auto() FAILED = auto() + def __str__(self) -> str: + return str(self.value) + + def to_str(self) -> str: + return str(self.name) + # When adding new states always add them to the end of this enum # and make necessary changes in the UI, Query Scheduler, and Reducer From a7e68c26cef88c2b707b317bdd91e8da839823c0 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Fri, 25 Jul 2025 18:20:52 -0400 Subject: [PATCH 2/5] Remove unnecessary change --- components/clp-py-utils/clp_py_utils/clp_config.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 8b77ccb378..5776b9c364 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -366,12 +366,6 @@ def validate_authentication(cls, values): raise ValueError(f"profile and credentials must not be set when type is '{auth_enum}.'") return values - def dump_to_primitive_dict(self): - d = self.dict() - if self.credentials is not None: - d["credentials"] = self.credentials.dict() - return d - class S3Config(BaseModel): region_code: str @@ -397,9 +391,7 @@ class S3IngestionConfig(BaseModel): aws_authentication: AwsAuthentication def dump_to_primitive_dict(self): - d = self.dict() - d["aws_authentication"] = self.aws_authentication.dump_to_primitive_dict() - return d + return self.dict() class FsStorage(BaseModel): From d3b72a4fa9fd6f50202fec5ea736e3b606e781b6 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Mon, 28 Jul 2025 10:28:12 -0400 Subject: [PATCH 3/5] Revert one change --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 65be81245c..b988651fca 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -957,6 +957,7 @@ def start_webui( ] necessary_mounts = [ mounts.clp_home, + mounts.stream_output_dir, ] if StorageType.S3 == stream_storage.type: auth = stream_storage.s3_config.aws_authentication @@ -972,9 +973,6 @@ def start_webui( necessary_mounts.append(mounts.aws_config_dir) if aws_env_vars: necessary_env_vars.extend(aws_env_vars) - else: - necessary_mounts.append(mounts.stream_output_dir) - append_docker_options(container_cmd, necessary_mounts, necessary_env_vars) container_cmd.append(clp_config.execution_container) From adcbf75b4a614d01a960e2641afb388e463491c6 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Fri, 1 Aug 2025 19:16:11 -0400 Subject: [PATCH 4/5] fix --- .../job_orchestration/scheduler/constants.py | 55 ++++++------------- 1 file changed, 16 insertions(+), 39 deletions(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/constants.py b/components/job-orchestration/job_orchestration/scheduler/constants.py index dbb5f03621..ca9180c118 100644 --- a/components/job-orchestration/job_orchestration/scheduler/constants.py +++ b/components/job-orchestration/job_orchestration/scheduler/constants.py @@ -11,11 +11,11 @@ class QueueName: QUERY = "query" -class CompressionJobStatus(IntEnum): - PENDING = 0 - RUNNING = auto() - SUCCEEDED = auto() - FAILED = auto() +class StatusIntEnum(IntEnum): + """ + Delegates __str__ to int.__str__, matching the behavior of IntEnum in Python 3.11+. + TODO: Remove this when our minimum supported Python version is 3.11+. + """ def __str__(self) -> str: return str(self.value) @@ -24,33 +24,28 @@ def to_str(self) -> str: return str(self.name) -class CompressionJobCompletionStatus(IntEnum): - SUCCEEDED = 0 +class CompressionJobStatus(StatusIntEnum): + PENDING = 0 + RUNNING = auto() + SUCCEEDED = auto() FAILED = auto() - def __str__(self) -> str: - return str(self.value) - def to_str(self) -> str: - return str(self.name) +class CompressionJobCompletionStatus(StatusIntEnum): + SUCCEEDED = 0 + FAILED = auto() -class CompressionTaskStatus(IntEnum): +class CompressionTaskStatus(StatusIntEnum): PENDING = 0 RUNNING = auto() SUCCEEDED = auto() FAILED = auto() - def __str__(self) -> str: - return str(self.value) - - def to_str(self) -> str: - return str(self.name) - # When adding new states always add them to the end of this enum # and make necessary changes in the UI, Query Scheduler, and Reducer -class QueryJobStatus(IntEnum): +class QueryJobStatus(StatusIntEnum): PENDING = 0 RUNNING = auto() SUCCEEDED = auto() @@ -62,14 +57,8 @@ class QueryJobStatus(IntEnum): def from_str(label: str) -> QueryJobStatus: return QueryJobStatus[label.upper()] - def __str__(self) -> str: - return str(self.value) - - def to_str(self) -> str: - return str(self.name) - -class QueryTaskStatus(IntEnum): +class QueryTaskStatus(StatusIntEnum): PENDING = 0 RUNNING = auto() SUCCEEDED = auto() @@ -80,20 +69,8 @@ class QueryTaskStatus(IntEnum): def from_str(label: str) -> QueryTaskStatus: return QueryTaskStatus[label.upper()] - def __str__(self) -> str: - return str(self.value) - def to_str(self) -> str: - return str(self.name) - - -class QueryJobType(IntEnum): +class QueryJobType(StatusIntEnum): SEARCH_OR_AGGREGATION = 0 EXTRACT_IR = auto() EXTRACT_JSON = auto() - - def __str__(self) -> str: - return str(self.value) - - def to_str(self) -> str: - return str(self.name) From fed9010e68f95d5389eaf2839dfbd05a0bbca092 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Sat, 2 Aug 2025 21:50:17 -0400 Subject: [PATCH 5/5] review comments --- .../job-orchestration/job_orchestration/scheduler/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/scheduler/constants.py b/components/job-orchestration/job_orchestration/scheduler/constants.py index ca9180c118..74cc2bfdcc 100644 --- a/components/job-orchestration/job_orchestration/scheduler/constants.py +++ b/components/job-orchestration/job_orchestration/scheduler/constants.py @@ -21,7 +21,7 @@ def __str__(self) -> str: return str(self.value) def to_str(self) -> str: - return str(self.name) + return self.name class CompressionJobStatus(StatusIntEnum):