From b2c096c20c2be31d32f550dfbd2c3e4caa6bb6c3 Mon Sep 17 00:00:00 2001 From: Greg Dean Date: Mon, 15 Mar 2021 11:21:53 -0400 Subject: [PATCH 1/2] splitting cloud and cloud resource migrations into seperate pipelines since someone may be using the cloud realm but not have cloud resource specs --- .../Version900To950/DatabaseMigration.php | 40 +++++++++++++- .../etl.d/xdmod-migration-9_0_0-9_5_0.json | 52 ++++++++++++------- 2 files changed, 71 insertions(+), 21 deletions(-) diff --git a/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php b/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php index 6576c899f0..4b8ff08672 100644 --- a/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php +++ b/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php @@ -7,6 +7,7 @@ use CCR\DB; use ETL\Utilities; +use OpenXdmod\Setup\Console; /** * Migrate databases from version 9.0.0 to 9.5.0. @@ -17,9 +18,13 @@ public function execute() { parent::execute(); + $console = Console::factory(); + $dbh = DB::factory('datawarehouse'); $mysql_helper = \CCR\DB\MySQLHelper::factory($dbh); - if ($mysql_helper->tableExists('modw_cloud.cloud_resource_specs')) { + + if ($mysql_helper->tableExists('modw_cloud.event')) { + Utilities::runEtlPipeline( ['cloud-migration-9_0_0-9_5_0'], $this->logger, @@ -28,5 +33,38 @@ public function execute() ] ); } + + if ($mysql_helper->tableExists('modw_cloud.cloud_resource_specs')) { + + $staging_resource_sql = "SELECT + COUNT(*) + FROM + modw_cloud.staging_resource_specifications + GROUP BY + resource_id, hostname, fact_date + HAVING + COUNT(*) > 1"; + + $staging_result = $dbh->query($staging_resource_sql); + + if(count($staging_result) > 0) { + $console->displayMessage(<<<"EOT" +This version of Open XDMoD changes the schema on two tables related to cloud utilization metrics. It appears that +data in the table modw_cloud.staging_resource_specifications will violate these schema changes. The violation is that +there cannot be two rows with the same resource ID, hostname, and date. Before you next ingest your cloud resource +specification files you should either remove any extra rows or truncate this table and then shred and ingest all of +your cloud resource specification files. +EOT + ); + } + + Utilities::runEtlPipeline( + ['cloud-resource-specs-migration-9_0_0-9_5_0'], + $this->logger, + [ + 'last-modified-start-date' => '2017-01-01 00:00:00' + ] + ); + } } } diff --git a/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json b/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json index 49d8d3834d..fd9eea19a1 100644 --- a/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json +++ b/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json @@ -23,7 +23,26 @@ "schema": "modw_cloud" } } - } + }, + "cloud-resource-specs-migration-9_0_0-9_5_0": { + "namespace": "ETL\\Ingestor", + "options_class": "IngestorOptions", + "endpoints": { + "source": { + "type": "mysql", + "name": "Cloud DB", + "config": "datawarehouse", + "schema": "modw_cloud" + }, + "destination": { + "type": "mysql", + "name": "Cloud DB", + "config": "datawarehouse", + "schema": "modw_cloud", + "truncate_destination": true + } + } + } }, "migration-9_0_0-9_5_0": [ { @@ -165,21 +184,14 @@ "explode_column": { "session_id_list": "session_id" } - }, + } + ], + "cloud-resource-specs-migration-9_0_0-9_5_0": [ { "name": "CloudResourceSpecsReconstructor", "class": "CloudResourceSpecsStateTransformIngestor", "definition_file": "cloud_common/resource_specifications_transformer.json", - "description": "Sets a start and end time for memory and vcpu paring for a compute node on a cloud resource", - "endpoints": { - "destination": { - "type": "mysql", - "name": "Cloud DB", - "config": "datawarehouse", - "schema": "modw_cloud", - "truncate_destination": true - } - } + "description": "Sets a start and end time for memory and vcpu paring for a compute node on a cloud resource" }, { "#": "Asset data must be aggregated post ingestion", @@ -194,14 +206,14 @@ "day", "month", "quarter", "year" ], "endpoints": { - "destination": { - "type": "mysql", - "name": "Aggregate DB", - "config": "datawarehouse", - "schema": "modw_aggregates", - "create_schema_if_not_exists": true, - "truncate_destination": true - } + "destination": { + "type": "mysql", + "name": "Aggregate DB", + "config": "datawarehouse", + "schema": "modw_aggregates", + "create_schema_if_not_exists": true, + "truncate_destination": true + } } } ] From fc2df0e73c3165962b4ec2e5ecaec2c8d77a3645 Mon Sep 17 00:00:00 2001 From: Greg Dean Date: Tue, 16 Mar 2021 11:58:11 -0400 Subject: [PATCH 2/2] style updates --- .../Migration/Version900To950/DatabaseMigration.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php b/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php index 4b8ff08672..13704e4dc0 100644 --- a/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php +++ b/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php @@ -39,11 +39,11 @@ public function execute() $staging_resource_sql = "SELECT COUNT(*) FROM - modw_cloud.staging_resource_specifications + modw_cloud.staging_resource_specifications GROUP BY - resource_id, hostname, fact_date + resource_id, hostname, fact_date HAVING - COUNT(*) > 1"; + COUNT(*) > 1"; $staging_result = $dbh->query($staging_resource_sql);