Merge pull request #1411 from ubccr/gateway-drilldown

Populate joblist table for gateway jobs

classes/DataWarehouse/Query/Gateways/JobDataset.php

@@ -0,0 +1,155 @@
+<?php
+namespace DataWarehouse\Query\Gateways;
+
+use DataWarehouse\Data\RawStatisticsConfiguration;
+use DataWarehouse\Query\Model\FormulaField;
+use DataWarehouse\Query\Model\Schema;
+use DataWarehouse\Query\Model\Table;
+use DataWarehouse\Query\Model\TableField;
+use DataWarehouse\Query\Model\WhereCondition;
+use Exception;
+
+/**
+  * @see DataWarehouse::Query::RawQuery
+  */
+class JobDataset extends \DataWarehouse\Query\RawQuery
+{
+    private $documentation = array();
+
+    public function __construct(
+        array $parameters,
+        $stat = "all"
+    ) {
+        parent::__construct('Gateways', 'modw_gateways', 'gatewayfact_by_day', array());
+
+        // The same fact table row may correspond to multiple rows in the
+        // aggregate table (e.g. a job that runs over two days).
+        $this->setDistinct(true);
+
+        $config = RawStatisticsConfiguration::factory();
+
+        // The data table is always aliased to "agg".
+        $tables = ['agg' => $this->getDataTable()];
+
+        foreach ($config->getQueryTableDefinitions('Gateways') as $tableDef) {
+            $alias = $tableDef['alias'];
+            $table = new Table(
+                new Schema($tableDef['schema']),
+                $tableDef['name'],
+                $alias
+            );
+            $tables[$alias] = $table;
+            $this->addTable($table);
+
+            $join = $tableDef['join'];
+            $this->addWhereCondition(new WhereCondition(
+                new TableField($table, $join['primaryKey']),
+                '=',
+                new TableField($tables[$join['foreignTableAlias']], $join['foreignKey'])
+            ));
+        }
+
+        // This table is defined in the configuration file, but used in the section below.
+        $factTable = $tables['jt'];
+
+        if (isset($parameters['primary_key'])) {
+            $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'job_id'), "=", $parameters['primary_key']));
+        } elseif (isset($parameters['job_identifier'])) {
+            $matches = array();
+            if (preg_match('/^(\d+)(?:[\[_](\d+)\]?)?$/', $parameters['job_identifier'], $matches)) {
+                $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'resource_id'), '=', $parameters['resource_id']));
+                if (isset($matches[2])) {
+                    $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'local_jobid'), '=', $matches[1]));
+                    $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'local_job_array_index'), '=', $matches[2]));
+                } else {
+                    $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'local_job_id_raw'), '=', $matches[1]));
+                }
+            } else {
+                throw new Exception('invalid "job_identifier" query parameter');
+            }
+        } elseif (isset($parameters['start_date']) && isset($parameters['end_date'])) {
+            date_default_timezone_set('UTC');
+            $startDate = date_parse_from_format('Y-m-d', $parameters['start_date']);
+            $startDateTs = mktime(
+                0,
+                0,
+                0,
+                $startDate['month'],
+                $startDate['day'],
+                $startDate['year']
+            );
+            if ($startDateTs === false) {
+                throw new Exception('invalid "start_date" query parameter');
+            }
+
+            $endDate = date_parse_from_format('Y-m-d', $parameters['end_date']);
+            $endDateTs = mktime(
+                23,
+                59,
+                59,
+                $endDate['month'],
+                $endDate['day'],
+                $endDate['year']
+            );
+            if ($startDateTs === false) {
+                throw new Exception('invalid "end_date" query parameter');
+            }
+
+            $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), ">=", $startDateTs));
+            $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), "<=", $endDateTs));
+        } else {
+            throw new Exception('invalid query parameters');
+        }
+
+        if ($stat == "accounting" || $stat == 'batch') {
+            foreach ($config->getQueryFieldDefinitions('Gateways') as $field) {
+                $alias = $field['name'];
+                if (isset($field['tableAlias']) && isset($field['column'])) {
+                    $this->addField(new TableField(
+                        $tables[$field['tableAlias']],
+                        $field['column'],
+                        $alias
+                    ));
+                } elseif (isset($field['formula'])) {
+                    $this->addField(new FormulaField($field['formula'], $alias));
+                } else {
+                    throw new Exception(sprintf(
+                        'Missing tableAlias and column or formula for "%s", definition: %s',
+                        $alias,
+                        json_encode($field)
+                    ));
+                }
+                $this->documentation[$alias] = $field;
+            }
+        } else {
+            $this->addField(new TableField($factTable, "job_id", "jobid"));
+            $this->addField(new TableField($factTable, "local_jobid", "local_job_id"));
+
+            $rt = new Table(new Schema("modw"), "resourcefact", "rf");
+            $this->joinTo($rt, "task_resource_id", "code", "resource");
+
+            // Instead of Person we use Gateway table, for Gateways realm
+            $pt = new Table(new Schema('modw_gateways'), 'gateway', 'p');
+            // here we are joining to the gatewayfact_by_day table on person_id
+            $this->joinTo($pt, "person_id", "long_name", "name", "proxy_person_id");
+
+            $st = new Table(new Schema('modw'), 'systemaccount', 'sa');
+            $this->joinTo($st, "systemaccount_id", "username", "username");
+        }
+    }
+
+    /**
+     * helper function to join the data table to another table
+     */
+    private function joinTo($othertable, $joinkey, $otherkey, $colalias, $idcol = "id")
+    {
+        $this->addTable($othertable);
+        $this->addWhereCondition(new WhereCondition(new TableField($this->getDataTable(), $joinkey), '=', new TableField($othertable, $idcol)));
+        $this->addField(new TableField($othertable, $otherkey, $colalias));
+    }
+
+    public function getColumnDocumentation()
+    {
+        return $this->documentation;
+    }
+}

classes/DataWarehouse/Query/Gateways/JobMetadata.php

@@ -0,0 +1,93 @@
+<?php
+
+namespace DataWarehouse\Query\Gateways;
+
+use \XDUser;
+
+/**
+ * @see DataWarehouse::Query::iJobMetadata
+ */
+class JobMetadata implements \DataWarehouse\Query\iJobMetadata
+{
+    public function getJobMetadata(XDUser $user, $jobid)
+    {
+        $job = $this->lookupJob($user, $jobid);
+        if ($job == null) {
+            return array();
+        }
+
+        return array(
+            \DataWarehouse\Query\RawQueryTypes::ACCOUNTING => true
+        );
+    }
+
+    /**
+     * Note there is no job summary data available in the Gateways realm
+     */
+    public function getJobSummary(XDUser $user, $jobid)
+    {
+        return array();
+    }
+
+    /**
+     * Note there is no job executable data available in the Gateways realm
+     */
+    public function getJobExecutableInfo(XDUser $user, $jobid)
+    {
+        return array();
+    }
+
+    /**
+     * Note there is no job timeseries data available in the Gateways realm
+     */
+    public function getJobTimeseriesMetaData(XDUser $user, $jobid)
+    {
+        return array();
+    }
+
+    /**
+     * Note there is no job timeseries data available in the Gateways realm
+     */
+    public function getJobTimeseriesMetricMeta(XDUser $user, $jobid, $tsid)
+    {
+        return array();
+    }
+
+    /**
+     * Note there is no job timeseries data available in the Gateways realm
+     */
+    public function getJobTimeseriesMetricNodeMeta(XDUser $user, $jobid, $tsid, $nodeid)
+    {
+        return array();
+    }
+
+    /**
+     * Note there is no job timeseries data available in the Gateways realm
+     */
+    public function getJobTimeseriesData(XDUser $user, $jobid, $tsid, $nodeid, $cpuid)
+    {
+        return array();
+    }
+
+    /**
+     * Lookup the job in the datawarehouse to check that it exists and the
+     * user has permission to view it.
+     *
+     * @param XDUser $user The user to lookup the job for.
+     * @param $jobid the unique identifier for the job.
+     *
+     * @return array() the accounting data for the job or null if no job exists or permission denied
+     */
+    private function lookupJob(XDUser $user, $jobid)
+    {
+        $query = new \DataWarehouse\Query\Gateways\JobDataset(array('primary_key' => $jobid));
+        $query->setMultipleRoleParameters($user->getAllRoles(), $user);
+        $stmt = $query->getRawStatement(1, 0);
+
+        $job = $stmt->fetchAll(\PDO::FETCH_ASSOC);
+        if (count($job) != 1) {
+            return null;
+        }
+        return $job[0];
+    }
+}

classes/DataWarehouse/Query/Gateways/RawData.php

@@ -0,0 +1,130 @@
+<?php
+namespace DataWarehouse\Query\Gateways;
+
+use \DataWarehouse\Query\Model\Table;
+use \DataWarehouse\Query\Model\TableField;
+use \DataWarehouse\Query\Model\FormulaField;
+use \DataWarehouse\Query\Model\WhereCondition;
+use \DataWarehouse\Query\Model\Schema;
+use Psr\Log\LoggerInterface;
+
+/**
+ * The RawData class is reponsible for generating a query that returns
+ * the set of fact table rows given the where conditions on the aggregate
+ * table.
+ */
+class RawData extends \DataWarehouse\Query\Query implements \DataWarehouse\Query\iQuery
+{
+    public function __construct(
+        $realmId,
+        $aggregationUnitName,
+        $startDate,
+        $endDate,
+        $groupById = null,
+        $statisticId = null,
+        array $parameters = array(),
+        LoggerInterface $logger = null
+    ) {
+        $realmId = 'Gateways';
+        $schema = 'modw_gateways';
+        $dataTablePrefix = 'gatewayfact_by_';
+
+        parent::__construct(
+            $realmId,
+            $aggregationUnitName,
+            $startDate,
+            $endDate,
+            $groupById,
+            null,
+            $parameters
+        );
+
+        // The same fact table row may correspond to multiple rows in the
+        // aggregate table (e.g. a job that runs over two days).
+        $this->setDistinct(true);
+
+        // Override values set in Query::__construct() to use the fact table rather than the
+        // aggregation table prefix from the Realm configuration.
+
+        $this->setDataTable($schema, sprintf("%s%s", $dataTablePrefix, $aggregationUnitName));
+        $this->_aggregation_unit = \DataWarehouse\Query\TimeAggregationUnit::factory(
+            $aggregationUnitName,
+            $startDate,
+            $endDate,
+            sprintf("%s.%s", $schema, $dataTablePrefix)
+        );
+
+        $dataTable = $this->getDataTable();
+        $joblistTable = new Table($dataTable->getSchema(), $dataTable->getName() . "_joblist", "jl");
+        $factTable = new Table(new Schema('modw'), "job_tasks", "jt");
+
+        $resourcefactTable = new Table(new Schema('modw'), 'resourcefact', 'rf');
+        $this->addTable($resourcefactTable);
+
+        $this->addWhereCondition(new WhereCondition(
+            new TableField($dataTable, "task_resource_id"),
+            '=',
+            new TableField($resourcefactTable, "id")
+        ));
+
+        // For Gateways realm the analog (with person_id) is the Gateway table:
+        $personTable = new Table(new Schema('modw_gateways'), 'gateway', 'p');
+
+        $this->addTable($personTable);
+        $this->addWhereCondition(new WhereCondition(
+            new TableField($dataTable, "person_id"),
+            '=',
+            new TableField($personTable, "proxy_person_id")
+        ));
+
+        $this->addField(new TableField($resourcefactTable, "code", 'resource'));
+        $this->addField(new TableField($personTable, "long_name", "name"));
+
+        $this->addField(new TableField($factTable, "job_id", "jobid"));
+        $this->addField(new TableField($factTable, "local_jobid", "local_job_id"));
+        $this->addField(new TableField($factTable, 'start_time_ts'));
+        $this->addField(new TableField($factTable, 'end_time_ts'));
+        $this->addField(new FormulaField('-1', 'cpu_user'));
+
+        // This is used by Integrations and not currently shown on the XDMoD interface
+        $this->addField(new TableField($factTable, 'name', 'job_name'));
+
+        $this->addTable($joblistTable);
+        $this->addTable($factTable);
+
+        $this->addWhereCondition(new WhereCondition(
+            new TableField($joblistTable, "agg_id"),
+            "=",
+            new TableField($dataTable, "id")
+        ));
+        $this->addWhereCondition(new WhereCondition(
+            new TableField($joblistTable, "jobid"),
+            "=",
+            new TableField($factTable, "job_id")
+        ));
+
+        switch ($statisticId) {
+            case "job_count":
+                $this->addWhereCondition(new WhereCondition("jt.end_time_ts", "BETWEEN", "duration.day_start_ts AND duration.day_end_ts"));
+                break;
+            case "started_job_count":
+                $this->addWhereCondition(new WhereCondition("jt.start_time_ts", "BETWEEN", "duration.day_start_ts AND duration.day_end_ts"));
+                break;
+            default:
+                // All other metrics show running job count
+                break;
+        }
+
+        $this->prependOrder(
+            new \DataWarehouse\Query\Model\OrderBy(
+                new TableField($factTable, 'end_time_ts'),
+                'DESC',
+                'end_time_ts'
+            )
+        );
+    }
+
+    public function getQueryType(){
+        return 'timeseries';
+    }
+}

classes/DataWarehouse/Query/Gateways/Statistic.php

@@ -0,0 +1,10 @@
+<?php
+namespace DataWarehouse\Query\Gateways;
+
+class Statistic extends \DataWarehouse\Query\Statistic
+{
+    public function getWeightStatName()
+    {
+        return 'running_job_count';
+    }
+}