reversions - disable Glue in PR checks (#5476)

.env.example

@@ -42,8 +42,6 @@ AWS_RESOURCE_NAME=YOUR_COST_MANAGEMENT_AWS_ARN
 SCHEMA_SUFFIX="" # if DEVELOPMENT=True, this can be left empty and will default to $USER; otherwise, set this value to something unique
 TRINO_SCHEMA_PREFIX=""
 TRINO_S3A_OR_S3=s3
-HIVE_PROPERTIES_FILE=hive.properties
-GLUE_PROPERTIES_FILE=glue.properties
 
 AWS_CATALOG_ID=589173575009
 MINIO_ENDPOINT=http://koku-minio:9000

Jenkinsfile

@@ -42,7 +42,7 @@ pipeline {
 
         IQE_PLUGINS="cost_management"
         BUILD_URL="https://ci.ext.devshift.net/job/koku-pipeline-pr-check-main/${BUILD_NUMBER}/"
-        IQE_ENV_VARS="JOB_NAME=koku-ci-jenkins,BUILD_NUMBER=${BUILD_NUMBER},BUILD_URL=${BUILD_URL},SCHEMA_SUFFIX=_${IMAGE_TAG}"
+        IQE_ENV_VARS="JOB_NAME=koku-ci-jenkins,BUILD_NUMBER=${BUILD_NUMBER},BUILD_URL=${BUILD_URL}"
         IQE_PARALLEL_ENABLED="false"
 
         GITHUB_API_ROOT='https://api.github.com/repos/project-koku/koku'

ci/functions.sh

@@ -101,7 +101,7 @@ function run_smoke_tests_stage() {
     bonfire deploy \
         ${APP_NAME} \
         --ref-env insights-production \
-        --set-template-ref ${COMPONENT_NAME}=${ghprbActualCommit} \
+        --set-template-ref ${APP_NAME}/${COMPONENT_NAME}=${ghprbActualCommit} \
         --set-image-tag ${IMAGE}=${IMAGE_TAG} \
         --namespace ${NAMESPACE} \
         ${COMPONENTS_ARG} \
@@ -112,10 +112,6 @@ function run_smoke_tests_stage() {
         --set-parameter koku/DBM_IMAGE_TAG=${IMAGE_TAG} \
         --set-parameter koku/DBM_INVOCATION=${DBM_INVOCATION} \
         --set-parameter koku/IMAGE=${IMAGE} \
-        --set-parameter koku/SCHEMA_SUFFIX=_${IMAGE_TAG} \
-        --set-parameter koku/TRINO_S3A_OR_S3=s3 \
-        --set-parameter trino/HIVE_PROPERTIES_FILE=glue.properties \
-        --set-parameter trino/GLUE_PROPERTIES_FILE=hive.properties \
         --no-single-replicas \
         --source=appsre \
         --timeout 600

docker-compose.yml

@@ -681,8 +681,7 @@ services:
       - ./dev/containers/trino/etc/config.properties:/etc/trino/config.properties
       - ./dev/containers/trino/etc/jvm.config:/etc/trino/jvm.config
       - ./dev/containers/trino/etc/log.properties:/etc/trino/log.properties
-      - ./dev/containers/trino/etc/catalog/hive.properties:/etc/trino/catalog/${HIVE_PROPERTIES_FILE-hive.properties}
-      - ./dev/containers/trino/etc/catalog/glue.properties:/etc/trino/catalog/${GLUE_PROPERTIES_FILE-glue.properties}
+      - ./dev/containers/trino/etc/catalog/hive.properties:/etc/trino/catalog/hive.properties
       - ./dev/containers/trino/etc/catalog/postgres.properties:/etc/trino/catalog/postgres.properties
       - ./dev/containers/trino/data:/data/trino/data
       - ./dev/containers/trino/logs:/data/trino/logs

koku/koku/configurator.py

@@ -7,17 +7,12 @@
 """
 import pathlib
 
-import boto3
-
 from .env import ENVIRONMENT
 
+
 CLOWDER_ENABLED = ENVIRONMENT.bool("CLOWDER_ENABLED", default=False)
 if CLOWDER_ENABLED:
-    from app_common_python import DependencyEndpoints
-    from app_common_python import KafkaServers
-    from app_common_python import KafkaTopics
-    from app_common_python import LoadedConfig
-    from app_common_python import ObjectBuckets
+    from app_common_python import ObjectBuckets, LoadedConfig, KafkaTopics, KafkaServers, DependencyEndpoints
 
 
 class Configurator:
@@ -83,9 +78,6 @@ def get_cloudwatch_log_group():
         """Obtain cloudwatch log group."""
         pass
 
-    def get_object_store_endpoint_default(self):
-        pass
-
     @staticmethod
     def get_object_store_endpoint():
         """Obtain object store endpoint."""
@@ -106,19 +98,11 @@ def get_object_store_tls():
         """Obtain object store secret key."""
         pass
 
-    def get_object_store_access_key_default(self, requested_name: str = ""):
-        """Obtain object store access key."""
-        pass
-
     @staticmethod
     def get_object_store_access_key(requested_name: str = ""):
         """Obtain object store access key."""
         pass
 
-    def get_object_store_secret_key_default(self, requested_name: str = ""):
-        """Obtain object store secret key."""
-        pass
-
     @staticmethod
     def get_object_store_secret_key(requested_name: str = ""):
         """Obtain object store secret key."""
@@ -266,9 +250,6 @@ def get_cloudwatch_log_group():
         """Obtain cloudwatch log group."""
         return ENVIRONMENT.get_value("CW_LOG_GROUP", default="platform-dev")
 
-    def get_object_store_endpoint_default(self):
-        return self.get_object_store_endpoint()
-
     @staticmethod
     def get_object_store_endpoint():
         """Obtain object store endpoint."""
@@ -295,19 +276,11 @@ def get_object_store_tls():
         # return ENVIRONMENT.bool("S3_SECURE", default=False)
         pass
 
-    def get_object_store_access_key_default(self, requested_name: str = ""):
-        """Obtain object store access key."""
-        return self.get_object_store_access_key(requested_name)
-
     @staticmethod
     def get_object_store_access_key(requested_name: str = ""):
         """Obtain object store access key."""
         return ENVIRONMENT.get_value("S3_ACCESS_KEY", default=None)
 
-    def get_object_store_secret_key_default(self, requested_name: str = ""):
-        """Obtain object store secret key."""
-        return self.get_object_store_secret_key(requested_name)
-
     @staticmethod
     def get_object_store_secret_key(requested_name: str = ""):
         """Obtain object store secret key."""
@@ -386,12 +359,6 @@ def get_endpoint_port(app, name, default):
 class ClowderConfigurator(Configurator):
     """Obtain configuration based on using Clowder and app-common."""
 
-    def __init__(self):
-        session = boto3.Session()
-        credentials = session.get_credentials()
-        self.credentials = credentials.get_frozen_credentials()
-        self.is_minio = ENVIRONMENT.get_value("TRINO_S3A_OR_S3", default="s3a") == "s3a"
-
     @staticmethod
     def get_feature_flag_host():
         """Obtain feature flag (Unleash) host."""
@@ -464,11 +431,6 @@ def get_cloudwatch_log_group():
         """Obtain cloudwatch log group."""
         return LoadedConfig.logging.cloudwatch.logGroup
 
-    def get_object_store_endpoint_default(self):
-        if self.is_minio:
-            return self.get_object_store_endpoint()
-        return ENVIRONMENT.get_value("S3_ENDPOINT", default="https://s3.amazonaws.com")
-
     @staticmethod
     def get_object_store_endpoint():
         """Obtain object store endpoint."""
@@ -503,11 +465,6 @@ def get_object_store_tls():
         else:
             return False
 
-    def get_object_store_access_key_default(self, requested_name: str = ""):
-        if self.is_minio:
-            return self.get_object_store_access_key(requested_name)
-        return self.credentials.access_key
-
     @staticmethod
     def get_object_store_access_key(requested_name: str = ""):
         """Obtain object store access key."""
@@ -518,11 +475,6 @@ def get_object_store_access_key(requested_name: str = ""):
         if LoadedConfig.objectStore.accessKey:
             return LoadedConfig.objectStore.accessKey
 
-    def get_object_store_secret_key_default(self, requested_name: str = ""):
-        if self.is_minio:
-            return self.get_object_store_secret_key(requested_name)
-        return self.credentials.secret_key
-
     @staticmethod
     def get_object_store_secret_key(requested_name: str = ""):
         """Obtain object store secret key."""
@@ -658,7 +610,7 @@ class ConfigFactory:
     @staticmethod
     def get_configurator():
         """Returns configurator based on mode from env variable."""
-        return ClowderConfigurator() if CLOWDER_ENABLED else EnvConfigurator()
+        return ClowderConfigurator if CLOWDER_ENABLED else EnvConfigurator
 
 
 CONFIGURATOR = ConfigFactory.get_configurator()

koku/koku/settings.py

@@ -493,11 +493,11 @@
 REQUESTED_ROS_BUCKET = ENVIRONMENT.get_value("REQUESTED_ROS_BUCKET", default="ros-report")
 REQUESTED_SUBS_BUCKET = ENVIRONMENT.get_value("REQUESTED_SUBS_BUCKET", default="subs-report")
 S3_TIMEOUT = ENVIRONMENT.int("S3_CONNECTION_TIMEOUT", default=60)
-S3_ENDPOINT = CONFIGURATOR.get_object_store_endpoint_default()
+S3_ENDPOINT = CONFIGURATOR.get_object_store_endpoint()
 S3_REGION = ENVIRONMENT.get_value("S3_REGION", default="us-east-1")
 S3_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_BUCKET)
-S3_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key_default(REQUESTED_BUCKET)
-S3_SECRET = CONFIGURATOR.get_object_store_secret_key_default(REQUESTED_BUCKET)
+S3_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_BUCKET)
+S3_SECRET = CONFIGURATOR.get_object_store_secret_key(REQUESTED_BUCKET)
 # HCS
 S3_HCS_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_BUCKET)
 S3_HCS_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_BUCKET)

koku/masu/test/api/test_db_performance.py

@@ -13,7 +13,8 @@
 from koku.configurator import CONFIGURATOR
 from masu.api.db_performance.db_performance import DBPerformanceStats
 
-TEST_CONFIGURATOR = type("TEST_CONFIGURATOR", (CONFIGURATOR.__class__,), dict(CONFIGURATOR.__dict__))
+
+TEST_CONFIGURATOR = type("TEST_CONFIGURATOR", CONFIGURATOR.__bases__, dict(CONFIGURATOR.__dict__))
 
 
 def _get_database_name():

smoke_test.sh

@@ -10,7 +10,7 @@ IQE_MARKER_EXPRESSION="cost_smoke"
 IQE_FILTER_EXPRESSION=""
 IQE_CJI_TIMEOUT="5h"
 IQE_PARALLEL_ENABLED="false"
-IQE_ENV_VARS="JOB_NAME=${JOB_NAME},BUILD_NUMBER=${BUILD_NUMBER},SCHEMA_SUFFIX=_${BUILD_NUMBER}"
+IQE_ENV_VARS="JOB_NAME=${JOB_NAME},BUILD_NUMBER=${BUILD_NUMBER}"
 
 # Get bonfire helper scripts
 CICD_URL="https://raw.githubusercontent.com/RedHatInsights/cicd-tools/main"
@@ -42,10 +42,6 @@ bonfire deploy \
     --optional-deps-method hybrid \
     --no-single-replicas \
     --set-parameter rbac/MIN_REPLICAS=1 \
-    --set-parameter koku/SCHEMA_SUFFIX=_${BUILD_NUMBER} \
-    --set-parameter koku/TRINO_S3A_OR_S3=s3 \
-    --set-parameter trino/HIVE_PROPERTIES_FILE=glue.properties \
-    --set-parameter trino/GLUE_PROPERTIES_FILE=hive.properties \
     ${COMPONENTS_ARG} \
     ${COMPONENTS_RESOURCES_ARG} \
     ${EXTRA_DEPLOY_ARGS}