diff --git a/configuration/etl/etl.d/cloud_ingest_resource_specs.json b/configuration/etl/etl.d/cloud_ingest_resource_specs.json index 4e29ab817f..0c4ea0e451 100644 --- a/configuration/etl/etl.d/cloud_ingest_resource_specs.json +++ b/configuration/etl/etl.d/cloud_ingest_resource_specs.json @@ -67,6 +67,11 @@ } } }, + { + "name": "CloudResourceSpecsStaging", + "class": "DatabaseIngestor", + "definition_file": "cloud_common/staging_resource_specifications.json" + }, { "name": "CloudResourceSpecsAddEndDay", "namespace": "ETL\\Maintenance", diff --git a/configuration/etl/etl.d/xdmod-migration-8_5_1-9_0_0.json b/configuration/etl/etl.d/xdmod-migration-8_5_1-9_0_0.json index 57d84012cd..8dbf8b57a9 100644 --- a/configuration/etl/etl.d/xdmod-migration-8_5_1-9_0_0.json +++ b/configuration/etl/etl.d/xdmod-migration-8_5_1-9_0_0.json @@ -81,6 +81,7 @@ "cloud_common/session_records.json", "cloud_common/event.json", "cloud_common/raw_resource_specs.json", + "cloud_common/staging_resource_specifications.json", "cloud_common/cloud_resource_specs.json", "cloud_common/account.json", "cloud_common/instance_type.json", diff --git a/configuration/etl/etl_action_defs.d/cloud_common/resource_specifications.json b/configuration/etl/etl_action_defs.d/cloud_common/resource_specifications.json index 436b1dd6a7..118c34f604 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/resource_specifications.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/resource_specifications.json @@ -8,18 +8,18 @@ "hostname": "r1.hostname", "vcpus": "IF(r1.memory_mb = -1 AND r1.vcpus = -1, r2.vcpus, r1.vcpus)", "memory_mb": "IF(r1.memory_mb = -1 AND r1.vcpus = -1, r2.memory_mb, r1.memory_mb)", - "start_date_ts": "UNIX_TIMESTAMP(IF(r1.memory_mb = -1 AND r1.vcpus = -1, r2.fact_date, r1.fact_date))", - "end_date_ts": "UNIX_TIMESTAMP(CASE WHEN MIN(r2.fact_date) IS NOT NULL AND (r1.memory_mb != -1 AND r1.vcpus != -1) THEN MIN(r2.fact_date) - INTERVAL 1 DAY WHEN r1.memory_mb = -1 AND r1.vcpus = -1 AND (SELECT MAX(fact_date) FROM modw_cloud.raw_resource_specs) != MAX(r2.fact_date) THEN MAX(r1.fact_date) - INTERVAL 1 DAY ELSE CURDATE() END)", - "start_day_id": "IF(r1.memory_mb = -1 AND r1.vcpus = -1, YEAR(r2.fact_date) * 100000 + DAYOFYEAR(r2.fact_date), YEAR(r1.fact_date) * 100000 + DAYOFYEAR(r1.fact_date))", - "end_day_id": "CASE WHEN MIN(r2.fact_date) IS NOT NULL AND (r1.memory_mb != -1 AND r1.vcpus != -1) THEN YEAR(MIN(r2.fact_date) - INTERVAL 1 DAY) * 100000 + DAYOFYEAR(MIN(r2.fact_date) - INTERVAL 1 DAY) WHEN r1.memory_mb = -1 AND r1.vcpus = -1 AND (SELECT MAX(fact_date) FROM modw_cloud.raw_resource_specs) != MAX(r2.fact_date) THEN YEAR(MAX(r1.fact_date) - INTERVAL 1 DAY) * 100000 + DAYOFYEAR(MAX(r1.fact_date) - INTERVAL 1 DAY) ELSE YEAR(CURDATE()) * 100000 + DAYOFYEAR(CURDATE()) END" + "start_date_ts": "UNIX_TIMESTAMP(IF(r1.memory_mb = -1 AND r1.vcpus = -1, MIN(r2.fact_date), MIN(r1.fact_date)))", + "end_date_ts": "UNIX_TIMESTAMP(CASE WHEN MIN(r2.fact_date) IS NOT NULL AND (r1.memory_mb != -1 AND r1.vcpus != -1) THEN MIN(r2.fact_date) - INTERVAL 1 DAY WHEN r1.memory_mb = -1 AND r1.vcpus = -1 AND (SELECT MAX(fact_date) FROM modw_cloud.staging_resource_specifications) != MAX(r2.fact_date) THEN MAX(r1.fact_date) - INTERVAL 1 DAY ELSE CURDATE() END)", + "start_day_id": "IF(r1.memory_mb = -1 AND r1.vcpus = -1, YEAR(MIN(r2.fact_date)) * 100000 + DAYOFYEAR(MIN(r2.fact_date)), YEAR(MIN(r1.fact_date)) * 100000 + DAYOFYEAR(MIN(r1.fact_date)))", + "end_day_id": "CASE WHEN MIN(r2.fact_date) IS NOT NULL AND (r1.memory_mb != -1 AND r1.vcpus != -1) THEN YEAR(MIN(r2.fact_date) - INTERVAL 1 DAY) * 100000 + DAYOFYEAR(MIN(r2.fact_date) - INTERVAL 1 DAY) WHEN r1.memory_mb = -1 AND r1.vcpus = -1 AND (SELECT MAX(fact_date) FROM modw_cloud.staging_resource_specifications) != MAX(r2.fact_date) THEN YEAR(MAX(r1.fact_date) - INTERVAL 1 DAY) * 100000 + DAYOFYEAR(MAX(r1.fact_date) - INTERVAL 1 DAY) ELSE YEAR(CURDATE()) * 100000 + DAYOFYEAR(CURDATE()) END" }, "joins": [{ - "name": "raw_resource_specs", + "name": "staging_resource_specifications", "schema": "${SOURCE_SCHEMA}", "alias": "r1" }, { - "name": "raw_resource_specs", + "name": "staging_resource_specifications", "schema": "${SOURCE_SCHEMA}", "alias": "r2", "type": "LEFT", diff --git a/configuration/etl/etl_action_defs.d/cloud_common/staging_resource_specifications.json b/configuration/etl/etl_action_defs.d/cloud_common/staging_resource_specifications.json new file mode 100644 index 0000000000..4793f4f1c6 --- /dev/null +++ b/configuration/etl/etl_action_defs.d/cloud_common/staging_resource_specifications.json @@ -0,0 +1,21 @@ +{ + "#": "@@session.time_zone is used instead of using the time zone listed in resources.json to keep consistency", + "#": "with how time zone conversion works for other places in the cloud realm", + "table_definition": { + "$ref": "${table_definition_dir}/cloud_common/staging_resource_specifications.json#/table_definition" + }, + "source_query": { + "records": { + "resource_id": "raw.resource_id", + "hostname": "raw.hostname", + "vcpus": "raw.vcpus", + "memory_mb": "raw.memory_mb", + "fact_date": "DATE(CONVERT_TZ(raw.fact_date, '+00:00', @@session.time_zone))" + }, + "joins": [{ + "name": "raw_resource_specs", + "schema": "${SOURCE_SCHEMA}", + "alias": "raw" + }] + } +} diff --git a/configuration/etl/etl_sql.d/cloud_common/resource_specification_end_day.sql b/configuration/etl/etl_sql.d/cloud_common/resource_specification_end_day.sql index 1f6f8ba300..7944ff6664 100644 --- a/configuration/etl/etl_sql.d/cloud_common/resource_specification_end_day.sql +++ b/configuration/etl/etl_sql.d/cloud_common/resource_specification_end_day.sql @@ -1,7 +1,7 @@ -- This sql statement inserts -1 values for the memory_mb and vcpus for a day that a compute node has been -- removed from the most recently ingested resource specifications file. The -1 helps when setting start and -- end times of a cpu and memory configuration for a compute node. -INSERT INTO modw_cloud.raw_resource_specs (hostname, resource_id, memory_mb, vcpus, fact_date) SELECT +INSERT INTO modw_cloud.staging_resource_specifications (hostname, resource_id, memory_mb, vcpus, fact_date) SELECT rs.hostname, rs.resource_id, rs.memory_mb, @@ -20,9 +20,9 @@ FROM rs.resource_id, rs2.hostname FROM - modw_cloud.raw_resource_specs AS rs + modw_cloud.staging_resource_specifications AS rs LEFT JOIN - (SELECT r.resource_id, r.hostname, r.fact_date FROM modw_cloud.raw_resource_specs AS r GROUP BY r.resource_id, r.hostname) AS rs2 ON rs.resource_id = rs2.resource_id + (SELECT r.resource_id, r.hostname, r.fact_date FROM modw_cloud.staging_resource_specifications AS r GROUP BY r.resource_id, r.hostname) AS rs2 ON rs.resource_id = rs2.resource_id GROUP BY rs.resource_id, rs.fact_date, @@ -30,7 +30,7 @@ FROM HAVING MIN(rs2.fact_date) <= rs.fact_date) AS rss1 LEFT JOIN - `modw_cloud`.`raw_resource_specs` AS rss2 + `modw_cloud`.`staging_resource_specifications` AS rss2 ON rss1.resource_id = rss2.resource_id AND rss1.fact_date = rss2.fact_date AND rss1.hostname = rss2.hostname WHERE @@ -38,11 +38,11 @@ FROM GROUP BY rss1.resource_id, rss1.hostname, rss1.fact_date) as rs LEFT JOIN - (SELECT MAX(r.fact_date) AS fact_date, r.hostname, r.resource_id FROM modw_cloud.raw_resource_specs AS r GROUP BY r.resource_id, r.hostname) AS r1 + (SELECT MAX(r.fact_date) AS fact_date, r.hostname, r.resource_id FROM modw_cloud.staging_resource_specifications AS r GROUP BY r.resource_id, r.hostname) AS r1 ON rs.resource_id = r1.resource_id and rs.hostname = r1.hostname LEFT JOIN - `modw_cloud`.`raw_resource_specs` AS rs2 + `modw_cloud`.`staging_resource_specifications` AS rs2 ON r1.resource_id = rs2.resource_id AND r1.hostname = rs2.hostname AND r1.fact_date = rs2.fact_date AND rs2.memory_mb != -1 AND rs2.vcpus != -1 WHERE diff --git a/configuration/etl/etl_tables.d/cloud_common/raw_resource_specs.json b/configuration/etl/etl_tables.d/cloud_common/raw_resource_specs.json index 1e11386d39..4c213c885f 100644 --- a/configuration/etl/etl_tables.d/cloud_common/raw_resource_specs.json +++ b/configuration/etl/etl_tables.d/cloud_common/raw_resource_specs.json @@ -32,7 +32,7 @@ }, { "name": "fact_date", - "type": "date", + "type": "datetime", "nullable": false } ], diff --git a/configuration/etl/etl_tables.d/cloud_common/staging_resource_specifications.json b/configuration/etl/etl_tables.d/cloud_common/staging_resource_specifications.json new file mode 100644 index 0000000000..b1fa119f01 --- /dev/null +++ b/configuration/etl/etl_tables.d/cloud_common/staging_resource_specifications.json @@ -0,0 +1,53 @@ +{ + "#": "Record types are global to all cloud resources", + + "table_definition": { + "name": "staging_resource_specifications", + "engine": "MyISAM", + "comment": "Record type: accounting, administrative, derived, etc.", + "columns": [ + { + "name": "hostname", + "type": "varchar(225)", + "nullable": false + }, + { + "name": "resource_id", + "type": "int(11)", + "nullable": false, + "comment": "Unknown = -1 for global dimensions" + }, + { + "name": "memory_mb", + "type": "int(11)", + "nullable": false, + "comment": "Amount of memory available on the associated node." + }, + { + "name": "vcpus", + "type": "int(5)", + "nullable": false, + "default": null, + "comment": "Number of vcpus available on the associated node." + }, + { + "name": "fact_date", + "type": "date", + "nullable": false + } + ], + "indexes": [ + { + "name": "PRIMARY", + "columns": [ + "resource_id", + "hostname", + "memory_mb", + "vcpus", + "fact_date" + ], + "is_unique": true + } + ] + } +} diff --git a/docs/configuration.md b/docs/configuration.md index e51d0bb392..86a37955b3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -422,6 +422,9 @@ The default is that resources are assumed to not allow node sharing. If the SUPReMM module is in use and a resource does allow node sharing then this should be set to `true`. +For cloud resources the timezone is not used and times are converted to +the local timezone that the server is in. + ```json [ { diff --git a/html/index.php b/html/index.php index 03bf1fd02a..d2f1df71e2 100644 --- a/html/index.php +++ b/html/index.php @@ -235,16 +235,29 @@ function isReferrer($referrer) - getName(); - return $carry; - }, array()); - ?> -