diff --git a/configuration/etl/etl.d/cloud_ingest_resource_specs.json b/configuration/etl/etl.d/cloud_ingest_resource_specs.json
index 4e29ab817f..0c4ea0e451 100644
--- a/configuration/etl/etl.d/cloud_ingest_resource_specs.json
+++ b/configuration/etl/etl.d/cloud_ingest_resource_specs.json
@@ -67,6 +67,11 @@
}
}
},
+ {
+ "name": "CloudResourceSpecsStaging",
+ "class": "DatabaseIngestor",
+ "definition_file": "cloud_common/staging_resource_specifications.json"
+ },
{
"name": "CloudResourceSpecsAddEndDay",
"namespace": "ETL\\Maintenance",
diff --git a/configuration/etl/etl.d/xdmod-migration-8_5_1-9_0_0.json b/configuration/etl/etl.d/xdmod-migration-8_5_1-9_0_0.json
index 57d84012cd..8dbf8b57a9 100644
--- a/configuration/etl/etl.d/xdmod-migration-8_5_1-9_0_0.json
+++ b/configuration/etl/etl.d/xdmod-migration-8_5_1-9_0_0.json
@@ -81,6 +81,7 @@
"cloud_common/session_records.json",
"cloud_common/event.json",
"cloud_common/raw_resource_specs.json",
+ "cloud_common/staging_resource_specifications.json",
"cloud_common/cloud_resource_specs.json",
"cloud_common/account.json",
"cloud_common/instance_type.json",
diff --git a/configuration/etl/etl_action_defs.d/cloud_common/resource_specifications.json b/configuration/etl/etl_action_defs.d/cloud_common/resource_specifications.json
index 436b1dd6a7..118c34f604 100644
--- a/configuration/etl/etl_action_defs.d/cloud_common/resource_specifications.json
+++ b/configuration/etl/etl_action_defs.d/cloud_common/resource_specifications.json
@@ -8,18 +8,18 @@
"hostname": "r1.hostname",
"vcpus": "IF(r1.memory_mb = -1 AND r1.vcpus = -1, r2.vcpus, r1.vcpus)",
"memory_mb": "IF(r1.memory_mb = -1 AND r1.vcpus = -1, r2.memory_mb, r1.memory_mb)",
- "start_date_ts": "UNIX_TIMESTAMP(IF(r1.memory_mb = -1 AND r1.vcpus = -1, r2.fact_date, r1.fact_date))",
- "end_date_ts": "UNIX_TIMESTAMP(CASE WHEN MIN(r2.fact_date) IS NOT NULL AND (r1.memory_mb != -1 AND r1.vcpus != -1) THEN MIN(r2.fact_date) - INTERVAL 1 DAY WHEN r1.memory_mb = -1 AND r1.vcpus = -1 AND (SELECT MAX(fact_date) FROM modw_cloud.raw_resource_specs) != MAX(r2.fact_date) THEN MAX(r1.fact_date) - INTERVAL 1 DAY ELSE CURDATE() END)",
- "start_day_id": "IF(r1.memory_mb = -1 AND r1.vcpus = -1, YEAR(r2.fact_date) * 100000 + DAYOFYEAR(r2.fact_date), YEAR(r1.fact_date) * 100000 + DAYOFYEAR(r1.fact_date))",
- "end_day_id": "CASE WHEN MIN(r2.fact_date) IS NOT NULL AND (r1.memory_mb != -1 AND r1.vcpus != -1) THEN YEAR(MIN(r2.fact_date) - INTERVAL 1 DAY) * 100000 + DAYOFYEAR(MIN(r2.fact_date) - INTERVAL 1 DAY) WHEN r1.memory_mb = -1 AND r1.vcpus = -1 AND (SELECT MAX(fact_date) FROM modw_cloud.raw_resource_specs) != MAX(r2.fact_date) THEN YEAR(MAX(r1.fact_date) - INTERVAL 1 DAY) * 100000 + DAYOFYEAR(MAX(r1.fact_date) - INTERVAL 1 DAY) ELSE YEAR(CURDATE()) * 100000 + DAYOFYEAR(CURDATE()) END"
+ "start_date_ts": "UNIX_TIMESTAMP(IF(r1.memory_mb = -1 AND r1.vcpus = -1, MIN(r2.fact_date), MIN(r1.fact_date)))",
+ "end_date_ts": "UNIX_TIMESTAMP(CASE WHEN MIN(r2.fact_date) IS NOT NULL AND (r1.memory_mb != -1 AND r1.vcpus != -1) THEN MIN(r2.fact_date) - INTERVAL 1 DAY WHEN r1.memory_mb = -1 AND r1.vcpus = -1 AND (SELECT MAX(fact_date) FROM modw_cloud.staging_resource_specifications) != MAX(r2.fact_date) THEN MAX(r1.fact_date) - INTERVAL 1 DAY ELSE CURDATE() END)",
+ "start_day_id": "IF(r1.memory_mb = -1 AND r1.vcpus = -1, YEAR(MIN(r2.fact_date)) * 100000 + DAYOFYEAR(MIN(r2.fact_date)), YEAR(MIN(r1.fact_date)) * 100000 + DAYOFYEAR(MIN(r1.fact_date)))",
+ "end_day_id": "CASE WHEN MIN(r2.fact_date) IS NOT NULL AND (r1.memory_mb != -1 AND r1.vcpus != -1) THEN YEAR(MIN(r2.fact_date) - INTERVAL 1 DAY) * 100000 + DAYOFYEAR(MIN(r2.fact_date) - INTERVAL 1 DAY) WHEN r1.memory_mb = -1 AND r1.vcpus = -1 AND (SELECT MAX(fact_date) FROM modw_cloud.staging_resource_specifications) != MAX(r2.fact_date) THEN YEAR(MAX(r1.fact_date) - INTERVAL 1 DAY) * 100000 + DAYOFYEAR(MAX(r1.fact_date) - INTERVAL 1 DAY) ELSE YEAR(CURDATE()) * 100000 + DAYOFYEAR(CURDATE()) END"
},
"joins": [{
- "name": "raw_resource_specs",
+ "name": "staging_resource_specifications",
"schema": "${SOURCE_SCHEMA}",
"alias": "r1"
},
{
- "name": "raw_resource_specs",
+ "name": "staging_resource_specifications",
"schema": "${SOURCE_SCHEMA}",
"alias": "r2",
"type": "LEFT",
diff --git a/configuration/etl/etl_action_defs.d/cloud_common/staging_resource_specifications.json b/configuration/etl/etl_action_defs.d/cloud_common/staging_resource_specifications.json
new file mode 100644
index 0000000000..4793f4f1c6
--- /dev/null
+++ b/configuration/etl/etl_action_defs.d/cloud_common/staging_resource_specifications.json
@@ -0,0 +1,21 @@
+{
+ "#": "@@session.time_zone is used instead of using the time zone listed in resources.json to keep consistency",
+ "#": "with how time zone conversion works for other places in the cloud realm",
+ "table_definition": {
+ "$ref": "${table_definition_dir}/cloud_common/staging_resource_specifications.json#/table_definition"
+ },
+ "source_query": {
+ "records": {
+ "resource_id": "raw.resource_id",
+ "hostname": "raw.hostname",
+ "vcpus": "raw.vcpus",
+ "memory_mb": "raw.memory_mb",
+ "fact_date": "DATE(CONVERT_TZ(raw.fact_date, '+00:00', @@session.time_zone))"
+ },
+ "joins": [{
+ "name": "raw_resource_specs",
+ "schema": "${SOURCE_SCHEMA}",
+ "alias": "raw"
+ }]
+ }
+}
diff --git a/configuration/etl/etl_sql.d/cloud_common/resource_specification_end_day.sql b/configuration/etl/etl_sql.d/cloud_common/resource_specification_end_day.sql
index 1f6f8ba300..7944ff6664 100644
--- a/configuration/etl/etl_sql.d/cloud_common/resource_specification_end_day.sql
+++ b/configuration/etl/etl_sql.d/cloud_common/resource_specification_end_day.sql
@@ -1,7 +1,7 @@
-- This sql statement inserts -1 values for the memory_mb and vcpus for a day that a compute node has been
-- removed from the most recently ingested resource specifications file. The -1 helps when setting start and
-- end times of a cpu and memory configuration for a compute node.
-INSERT INTO modw_cloud.raw_resource_specs (hostname, resource_id, memory_mb, vcpus, fact_date) SELECT
+INSERT INTO modw_cloud.staging_resource_specifications (hostname, resource_id, memory_mb, vcpus, fact_date) SELECT
rs.hostname,
rs.resource_id,
rs.memory_mb,
@@ -20,9 +20,9 @@ FROM
rs.resource_id,
rs2.hostname
FROM
- modw_cloud.raw_resource_specs AS rs
+ modw_cloud.staging_resource_specifications AS rs
LEFT JOIN
- (SELECT r.resource_id, r.hostname, r.fact_date FROM modw_cloud.raw_resource_specs AS r GROUP BY r.resource_id, r.hostname) AS rs2 ON rs.resource_id = rs2.resource_id
+ (SELECT r.resource_id, r.hostname, r.fact_date FROM modw_cloud.staging_resource_specifications AS r GROUP BY r.resource_id, r.hostname) AS rs2 ON rs.resource_id = rs2.resource_id
GROUP BY
rs.resource_id,
rs.fact_date,
@@ -30,7 +30,7 @@ FROM
HAVING
MIN(rs2.fact_date) <= rs.fact_date) AS rss1
LEFT JOIN
- `modw_cloud`.`raw_resource_specs` AS rss2
+ `modw_cloud`.`staging_resource_specifications` AS rss2
ON
rss1.resource_id = rss2.resource_id AND rss1.fact_date = rss2.fact_date AND rss1.hostname = rss2.hostname
WHERE
@@ -38,11 +38,11 @@ FROM
GROUP BY
rss1.resource_id, rss1.hostname, rss1.fact_date) as rs
LEFT JOIN
- (SELECT MAX(r.fact_date) AS fact_date, r.hostname, r.resource_id FROM modw_cloud.raw_resource_specs AS r GROUP BY r.resource_id, r.hostname) AS r1
+ (SELECT MAX(r.fact_date) AS fact_date, r.hostname, r.resource_id FROM modw_cloud.staging_resource_specifications AS r GROUP BY r.resource_id, r.hostname) AS r1
ON
rs.resource_id = r1.resource_id and rs.hostname = r1.hostname
LEFT JOIN
- `modw_cloud`.`raw_resource_specs` AS rs2
+ `modw_cloud`.`staging_resource_specifications` AS rs2
ON
r1.resource_id = rs2.resource_id AND r1.hostname = rs2.hostname AND r1.fact_date = rs2.fact_date AND rs2.memory_mb != -1 AND rs2.vcpus != -1
WHERE
diff --git a/configuration/etl/etl_tables.d/cloud_common/raw_resource_specs.json b/configuration/etl/etl_tables.d/cloud_common/raw_resource_specs.json
index 1e11386d39..4c213c885f 100644
--- a/configuration/etl/etl_tables.d/cloud_common/raw_resource_specs.json
+++ b/configuration/etl/etl_tables.d/cloud_common/raw_resource_specs.json
@@ -32,7 +32,7 @@
},
{
"name": "fact_date",
- "type": "date",
+ "type": "datetime",
"nullable": false
}
],
diff --git a/configuration/etl/etl_tables.d/cloud_common/staging_resource_specifications.json b/configuration/etl/etl_tables.d/cloud_common/staging_resource_specifications.json
new file mode 100644
index 0000000000..b1fa119f01
--- /dev/null
+++ b/configuration/etl/etl_tables.d/cloud_common/staging_resource_specifications.json
@@ -0,0 +1,53 @@
+{
+ "#": "Record types are global to all cloud resources",
+
+ "table_definition": {
+ "name": "staging_resource_specifications",
+ "engine": "MyISAM",
+ "comment": "Record type: accounting, administrative, derived, etc.",
+ "columns": [
+ {
+ "name": "hostname",
+ "type": "varchar(225)",
+ "nullable": false
+ },
+ {
+ "name": "resource_id",
+ "type": "int(11)",
+ "nullable": false,
+ "comment": "Unknown = -1 for global dimensions"
+ },
+ {
+ "name": "memory_mb",
+ "type": "int(11)",
+ "nullable": false,
+ "comment": "Amount of memory available on the associated node."
+ },
+ {
+ "name": "vcpus",
+ "type": "int(5)",
+ "nullable": false,
+ "default": null,
+ "comment": "Number of vcpus available on the associated node."
+ },
+ {
+ "name": "fact_date",
+ "type": "date",
+ "nullable": false
+ }
+ ],
+ "indexes": [
+ {
+ "name": "PRIMARY",
+ "columns": [
+ "resource_id",
+ "hostname",
+ "memory_mb",
+ "vcpus",
+ "fact_date"
+ ],
+ "is_unique": true
+ }
+ ]
+ }
+}
diff --git a/docs/configuration.md b/docs/configuration.md
index e51d0bb392..86a37955b3 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -422,6 +422,9 @@ The default is that resources are assumed to not allow node sharing. If
the SUPReMM module is in use and a resource does allow node sharing then
this should be set to `true`.
+For cloud resources the timezone is not used and times are converted to
+the local timezone that the server is in.
+
```json
[
{
diff --git a/html/index.php b/html/index.php
index 03bf1fd02a..d2f1df71e2 100644
--- a/html/index.php
+++ b/html/index.php
@@ -235,16 +235,29 @@ function isReferrer($referrer)
- getName();
- return $carry;
- }, array());
- ?>
-