Remove use of Slurm ReqGRES field (#1479)

classes/OpenXdmod/Shredder/Slurm.php

@@ -53,7 +53,6 @@ class Slurm extends Shredder
         'ncpus',
         'reqcpus',
         'reqmem',
-        'reqgres',
         'reqtres',
         'alloctres',
         'timelimit',
@@ -87,7 +86,6 @@ class Slurm extends Shredder
         'ncpus',
         'req_cpus',
         'req_mem',
-        'req_gres',
         'req_tres',
         'alloc_tres',
         'timelimit',

classes/Xdmod/SlurmResourceParser.php

@@ -1,72 +1,15 @@
 <?php
 /**
- * Slurm resource (GRES and TRES) parser.
+ * Slurm resource (TRES) parser.
  */
 
 namespace Xdmod;
 
 /**
- * Contains functions related to Slurm's GRES and TRES data.
+ * Contains functions related to Slurm's TRES data.
  */
 class SlurmResourceParser
 {
-
-    /**
-     * Parse generic resource (GRES) scheduling fields from sacct.
-     *
-     * Expected GRES format is a list of resources separated by commas with
-     * the each resource using the form name[:type:count]
-     *
-     * e.g. gpu:p100:2,bandwidth:1G
-     *
-     * @see https://slurm.schedmd.com/gres.html
-     *
-     * If the count (or anything else) contains pairs of parenthesis these will
-     * be removed along with everything contained between them.
-     *
-     * e.g. gpu:p100:2(IDX:0-1),hbm:0 is treated as gpu:p100:2,hbm:0
-     *
-     * @see https://github.com/PySlurm/pyslurm/issues/104
-     *
-     * @param string $gres ReqGRES field from sacct.
-     * @return array[] Parsed data. An array of arrays for each resource split
-     *     on ":".
-     */
-    public function parseGres($gres) {
-        if ($gres === '') {
-            return [];
-        }
-
-        // Remove anything contained in parenthesis along with the parenthesis.
-        $gres = preg_replace('/\(.*?\)/', '', $gres);
-
-        return array_map(
-            function ($resource) {
-                return explode(':', $resource);
-            },
-            explode(',', $gres)
-        );
-    }
-
-    /**
-     * Determine the GPU count from parsed GRES data.
-     *
-     * @see \Xdmod\SlurmResourceParser::parseGres
-     *
-     * @param array $gres Parsed GRES data.
-     * @return int The GPU count.
-     */
-    public function getGpuCountFromGres(array $gres)
-    {
-        foreach ($gres as $resource) {
-            if ($resource[0] === 'gpu' && count($resource) > 1) {
-                return (int)$resource[count($resource) - 1];
-            }
-        }
-
-        return 0;
-    }
-
     /**
      * Parse requested trackable resources (ReqTRES or AllocTres) field from sacct.
      *

configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json

@@ -62,6 +62,27 @@
                     "schema": "modw"
                 }
             }
+        },
+        {
+            "name": "update-mod_shredder-shredded_job_slurm-table",
+            "description": "Update mod_shredder.shredded_job_slurm table",
+            "namespace": "ETL\\Maintenance",
+            "class": "ExecuteSql",
+            "options_class": "MaintenanceOptions",
+            "sql_file_list": [
+                {
+                    "sql_file": "migrations/9.0.0-9.5.0/mod_shredder/alter-shredded-job-slurm-table.sql",
+                    "delimiter": ";"
+                }
+            ],
+            "endpoints": {
+                "destination": {
+                    "type": "mysql",
+                    "name": "XDMoD Data Warehouse",
+                    "config": "database",
+                    "schema": "mod_shredder"
+                }
+            }
         }
     ]
 }

configuration/etl/etl_sql.d/jobs/shredder/job-slurm.sql

@@ -25,7 +25,6 @@ CREATE TABLE ${DESTINATION_SCHEMA}.`shredded_job_slurm` (
   `ngpus` int(10) unsigned NOT NULL DEFAULT '0',
   `req_cpus` int(10) unsigned DEFAULT NULL,
   `req_mem` varchar(32) DEFAULT NULL,
-  `req_gres` text NOT NULL,
   `req_tres` text NOT NULL,
   `alloc_tres` text NOT NULL,
   `timelimit` int(10) unsigned DEFAULT NULL,

configuration/etl/etl_sql.d/migrations/9.0.0-9.5.0/mod_shredder/alter-shredded-job-slurm-table.sql

@@ -0,0 +1,23 @@
+-- Drop column from the `shredded_job_slurm` table.  This table is not managed
+-- by an ETL definition because it contains a key that is currently not
+-- supported by the ETL system.
+
+-- Wrap the ALTER TABLE statement in a prepared statement that conditionally
+-- drops the column so that this file can be executed multiple times without
+-- causing errors.
+SET @statement = (
+    SELECT IF(
+        (
+            SELECT COUNT(*)
+            FROM information_schema.columns
+            WHERE table_schema = '${DESTINATION_SCHEMA}'
+                AND table_name = 'shredded_job_slurm'
+                AND column_name = 'req_gres'
+        ) > 0,
+        'ALTER TABLE `${DESTINATION_SCHEMA}`.`shredded_job_slurm` DROP COLUMN `req_gres`',
+        'SELECT 1'
+    )
+);
+PREPARE dropColumnIfExists FROM @statement;
+EXECUTE dropColumnIfExists;
+DEALLOCATE PREPARE dropColumnIfExists;

docs/resource-manager-slurm.md

@@ -46,7 +46,7 @@ $ TZ=UTC sacct --clusters *cluster* --allusers \
     --parsable2 --noheader --allocations --duplicates \
     --format jobid,jobidraw,cluster,partition,account,group,gid,user,uid,\
 submit,eligible,start,end,elapsed,exitcode,state,nnodes,ncpus,reqcpus,reqmem,\
-reqgres,reqtres,alloctres,timelimit,nodelist,jobname \
+reqtres,alloctres,timelimit,nodelist,jobname \
     --starttime 2013-01-01T00:00:00 --endtime 2013-01-01T23:59:59 \
     >/tmp/slurm.log
 

docs/upgrade.md

@@ -115,6 +115,16 @@ You may upgrade directly from 9.0.0.
 
 The `xdmod-upgrade` script will migrate user editable configuration files to the new version and ask for the location of `chromium`.
 
+### Slurm Input File Format Changes
+
+The input file format for Slurm data has changed to remove the `ReqGRES` field.
+
+**If you are generating Slurm input for the `xdmod-shredder` command then you
+will need to make the appropriate changes.**  Refer to the [Slurm
+Notes](resource-manager-slurm.html#input-format) for the example `sacct`
+command.  If you are using the `xdmod-slurm-helper` command then no changes are
+necessary.
+
 ### Database Changes
 
 [github-latest-release]: https://github.com/ubccr/xdmod/releases/latest