updates needed for xsede groupbys (#1288)

classes/DataWarehouse/Access/MetricExplorer.php

@@ -701,7 +701,7 @@ public static function getDimensionValues(
         // Check if the realms were specified, and if not, use all realms.
         $realmsSpecified = !empty($realms);
         if (!$realmsSpecified) {
-            $realms = self::getRealmsFromUser($user);
+            $realms = Realms::getRealmIdsForUser($user);
         }
 
         // Determine which aggregation unit to use for dimension values queries.
@@ -904,7 +904,7 @@ public static function getDimensionName(
         XDUser $user,
         $dimension_id
     ) {
-        $realms = self::getRealmsFromUser($user);
+        $realms = Realms::getRealmIdsForUser($user);
 
         foreach ($realms as $realm) {
             try {
@@ -935,7 +935,7 @@ public static function getDimensionRealms(
         XDUser $user,
         $dimension_id
     ) {
-        $realms = self::getRealmsFromUser($user);
+        $realms = Realms::getRealmIdsForUser($user);
 
         $dimensionRealms = array();
         foreach ($realms as $realm) {
@@ -977,7 +977,7 @@ public static function getDimensionValueName(
         $value_id,
         $getLongName = false
     ) {
-        $realms = self::getRealmsFromUser($user);
+        $realms = Realms::getRealmIdsForUser($user);
 
         $dimensionValueName = null;
         foreach ($realms as $realm) {
@@ -1002,17 +1002,6 @@ public static function getDimensionValueName(
         return $dimensionValueName;
     }
 
-    /**
-     * Get a list of realms available to a user.
-     *
-     * @param  XDUser $user       The user whose realms are being retrieved.
-     *
-     * @return array              The realms available to the user.
-     */
-    public static function getRealmsFromUser(XDUser $user) {
-        return Realms::getRealmsForUser($user);
-    }
-
     /**
      * Get a GroupBy object for a given realm and dimension.
      *

classes/DataWarehouse/Visualization/HighChart2.php

@@ -1094,7 +1094,7 @@ public function configure(
                 }
                 if($this->_multiCategory) {
                     $dataSeriesName = (
-                        DataWarehouse::getCategoryForRealm($data_description->realm)
+                        $data_description->category
                         . ': '
                         . $dataSeriesName
                     );

classes/DataWarehouse/Visualization/HighChartTimeseries2.php

@@ -536,7 +536,7 @@ public function configure(
                         if($multiCategory)
                         {
                             $dataSeriesName = (
-                                DataWarehouse::getCategoryForRealm($data_description->realm)
+                                $data_description->category
                                 . ': '
                                 . $dataSeriesName
                             );

classes/ETL/DbModel/Query.php

@@ -109,7 +109,7 @@ protected function filterAndVerifyValue($property, $value)
             case 'overseer_restrictions':
                 if ( ! is_object($value) ) {
                     $this->logAndThrowException(
-                        sprintf("%s name must be an object, '%s' given", $property, gettype($value))
+                        sprintf("%s must be an object, '%s' given", $property, gettype($value))
                     );
                 }
                 break;
@@ -120,11 +120,17 @@ protected function filterAndVerifyValue($property, $value)
             case 'macros':
             case 'joins':
                 // Note that we are only checking that the value is an array here and not
-                // the array elements. That must come later.
+                // the array elements. The elements can be mixed content that must be checked later.
 
                 if ( ! is_array($value) ) {
                     $this->logAndThrowException(
-                        sprintf("%s name must be an array, '%s' given", $property, gettype($value))
+                        sprintf("%s must be an array, '%s' given", $property, gettype($value))
+                    );
+                }
+                // If this is a join property it must contain elements
+                if ( $property === 'joins' && count($value) === 0 ) {
+                    $this->logAndThrowException(
+                        sprintf("%s must have at least one item in it", $property)
                     );
                 }
 
@@ -142,7 +148,7 @@ protected function filterAndVerifyValue($property, $value)
             case 'query_hint':
                 if ( ! is_string($value) ) {
                     $this->logAndThrowException(
-                        sprintf("%s name must be a string, '%s' given", $property, gettype($value))
+                        sprintf("%s must be a string, '%s' given", $property, gettype($value))
                     );
                 }
                 break;

classes/Models/Services/Acls.php

@@ -984,11 +984,11 @@ public static function getQueryDescripters(XDUser $user, $realmName = null, $gro
         }
 
         $query = <<<SQL
-        SELECT DISTINCT
-            r.display as realm,
-            gb.name as group_by,
-            !agb.enabled as not_enabled,
-            agb.visible
+SELECT
+    r.name as realm,
+    gb.name as group_by,
+    !MAX(agb.enabled) as not_enabled,
+    MAX(agb.visible) as visible
 FROM group_bys gb
   JOIN realms r ON gb.realm_id = r.realm_id
   JOIN acl_group_bys agb
@@ -1055,7 +1055,8 @@ public static function getQueryDescripters(XDUser $user, $realmName = null, $gro
             $query .= " AND s.name = :statistic_name\n";
             $params[':statistic_name'] = $statisticName;
         }
-
+        $query .= "\nGROUP BY 1,2";
+        $query .= "\nORDER BY 1,2 DESC";
         $results = array();
         $sorted = array();
 

classes/Models/Services/Realms.php

@@ -51,6 +51,54 @@ public static function getRealmsForUser(\XDUser $user)
         }, array());
     }
 
+    public static function getRealmIdsForUser(\XDUser $user)
+    {
+        $query = <<<SQL
+            SELECT DISTINCT
+              r.name AS realm
+            FROM acl_group_bys agb
+              JOIN user_acls ua ON agb.acl_id = ua.acl_id
+              JOIN realms r ON r.realm_id = agb.realm_id
+            WHERE ua.user_id = :user_id
+            ORDER BY r.realm_id
+SQL;
+        $params = array(
+            ':user_id'=> $user->getUserID()
+        );
+
+        $db = DB::factory('database');
+        $rows = $db->query($query, $params);
+
+        return array_reduce($rows, function ($carry, $item) {
+            $carry[] = $item['realm'];
+            return $carry;
+        }, array());
+    }
+
+
+    public static function getRealmObjectsForUser(\XDUser $user)
+    {
+        $query = <<<SQL
+            SELECT DISTINCT
+              r.*
+            FROM acl_group_bys agb
+              JOIN user_acls ua ON agb.acl_id = ua.acl_id
+              JOIN realms r ON r.realm_id = agb.realm_id
+            WHERE ua.user_id = :user_id
+            ORDER BY r.realm_id
+SQL;
+        $params = array(
+            ':user_id'=> $user->getUserID()
+        );
+
+        $db = DB::factory('database');
+        $rows = $db->query($query, $params);
+
+        return array_reduce($rows, function ($carry, $item) {
+            $carry[$item['name']] = new Realm($item);
+            return $carry;
+        }, array());
+    }
     /**
      * Retrieve the Realms that are currently considered "enabled" for the current installation.
      *

classes/Realm/GroupBy.php

@@ -260,7 +260,6 @@ public function __construct($shortName, \stdClass $config, Realm $realm, Logger
 
         $messages = array();
         $configTypes = array(
-            'attribute_table' => 'string',
             'attribute_to_aggregate_table_key_map' => 'array',
             'attribute_values_query' => 'object',
             'description_html' => 'string',
@@ -310,9 +309,6 @@ public function __construct($shortName, \stdClass $config, Realm $realm, Logger
                         $this->additionalJoinConstraints[] = $constraint;
                     }
                     break;
-                case 'attribute_table':
-                    $this->attributeTableName = trim($value);
-                    break;
                 case 'attribute_values_query':
                     $this->attributeValuesQuery = new DbQuery($value, '`', $logger);
                     break;
@@ -473,6 +469,7 @@ public function __construct($shortName, \stdClass $config, Realm $realm, Logger
         }
         $this->attributeValuesQuery->joins = $joins;
         $this->attributeValuesQueryAsStdClass = $this->attributeValuesQuery->toStdClass();
+        $this->attributeTableName = trim($this->attributeValuesQuery->joins[0]->name);
 
         if ( $this->attributeDescriptionSql && count($this->attributeToAggregateKeyMap) > 1 ){
             $this->logAndThrowException('The attribute_description_query does not work with more than one ');
@@ -924,7 +921,13 @@ public function applyTo(\DataWarehouse\Query\iQuery $query, $multi_group = false
         // added by Query::setDuration()
 
         if ( ! $this->isAggregationUnit ) {
-            $query->addTable($this->attributeTableObj);
+            foreach ($this->attributeValuesQuery->joins as $join) {
+                $query->addTable(new Table(
+                    new Schema($join->schema),
+                    $join->name,
+                    $join->name
+                ));
+            }
         }
 
         if ( $this->isAggregationUnit && 'none' == $this->id ) {
@@ -970,8 +973,16 @@ public function applyTo(\DataWarehouse\Query\iQuery $query, $multi_group = false
                 // The aggregation unit where condition is already added by Query::setDuration()
 
                 if ( ! $this->isAggregationUnit ) {
+                    $pieces = explode('.', $attributeKey);
+                    if ( count($pieces) === 2 ) {
+                        $alternateAttributeTableObj = new Table($this->attributeTableObj->getSchema(), $pieces[0], $pieces[0]);
+                        $attributeKey = $pieces[1];
+                    } else {
+                        $alternateAttributeTableObj = null;
+                    }
+
                     $where = new WhereCondition(
-                        new TableField($tableObj, $attributeKey),
+                        new TableField(!empty($alternateAttributeTableObj) ? $alternateAttributeTableObj : $tableObj, $attributeKey),
                         '=',
                         new TableField($query->getDataTable(), $aggregateKey)
                     );
@@ -986,9 +997,9 @@ public function applyTo(\DataWarehouse\Query\iQuery $query, $multi_group = false
         if ( null !== $this->additionalJoinConstraints ) {
             foreach ( $this->additionalJoinConstraints as $constraint ) {
                 $where = new WhereCondition(
-                    new TableField($this->attributeTableObj, $constraint->attribute_expr),
+                    new TableField(!empty($constraint->attribute_table) ? new Table($this->attributeTableObj->getSchema(), $constraint->attribute_table, $constraint->attribute_table) : $this->attributeTableObj, $constraint->attribute_expr),
                     $constraint->operation,
-                    new TableField($query->getDataTable(), $constraint->aggregate_expr)
+                    new TableField(!empty($constraint->aggregate_table) ? new Table($query->getDataTable()->getSchema(), $constraint->aggregate_table, $constraint->aggregate_table) : $query->getDataTable(), $constraint->aggregate_expr)
                 );
                 $this->logger->trace(sprintf("%s Add additional JOIN condition '%s'", $this, $where));
                 $query->addWhereCondition($where);
@@ -1166,21 +1177,6 @@ public function addWhereJoin(\DataWarehouse\Query\iQuery $query, $aggregateTable
             $query->addWhereCondition($where);
         }
 
-        // If additional join constraints were specified add those here.
-
-        if ( null !== $this->additionalJoinConstraints ) {
-            foreach ( $this->additionalJoinConstraints as $constraint ) {
-                $where = new WhereCondition(
-                    new TableField($this->attributeTableObj, $constraint->attribute_expr),
-                    $constraint->operation,
-                    new TableField($query->getDataTable(), $constraint->aggregate_expr)
-                );
-                $this->logger->trace(sprintf("%s Add additional JOIN condition '%s'", $this, $where));
-                $query->addWhereCondition($where);
-            }
-        }
-
-
         // Normalize the WHERE constraint. We may be able to set this as an array in the parameter
         // list.
 

classes/User/Elements/QueryDescripter.php

@@ -109,7 +109,7 @@ public function getShowMenu()
 
     public function getRealmName()
     {
-        return $this->realm->getId();
+        return $this->realm->getName();
     }
 
     public function getGroupByName()

configuration/datawarehouse.d/ref/Cloud-group-bys.json

@@ -3,7 +3,6 @@
         "name": "Instance Type",
         "description_html": "The instance type of a virtual machine.",
         "attribute_table_schema": "modw_cloud",
-        "attribute_table": "instance_type",
         "alternate_group_by_columns": [
             "display"
         ],
@@ -53,7 +52,6 @@
             }
         ],
         "attribute_table_schema": "modw_cloud",
-        "attribute_table": "domains",
         "attribute_to_aggregate_table_key_map": [
             {
                 "id": "domain_id"
@@ -87,7 +85,6 @@
         "$ref": "datawarehouse.d/ref/group-by-time-period.json#/month"
     },
     "person": {
-        "attribute_table": "person",
         "attribute_table_schema": "modw",
         "attribute_to_aggregate_table_key_map": [
             {
@@ -115,7 +112,6 @@
         "name": "User"
     },
     "project": {
-        "attribute_table": "account",
         "attribute_table_schema": "modw_cloud",
         "alternate_group_by_columns": [
             "display"
@@ -153,44 +149,17 @@
         "$ref": "datawarehouse.d/ref/group-by-time-period.json#/quarter"
     },
     "resource": {
-        "attribute_table": "resourcefact",
-        "attribute_table_schema": "modw",
-        "attribute_to_aggregate_table_key_map": [
-            {
-                "id": "host_resource_id"
-            }
-        ],
-        "attribute_values_query": {
-            "joins": [
+        "$overwrite": {
+            "attribute_to_aggregate_table_key_map": [
                 {
-                    "alias": "rf",
-                    "name": "resourcefact"
-                },
-                {
-                    "alias": "rs",
-                    "name": "resourcespecs",
-                    "on": "rf.id = rs.resource_id"
+                    "id": "host_resource_id"
                 }
             ],
-            "orderby": [
-                "rf.code",
-                "rf.name"
-            ],
-            "records": {
-                "id": "rf.id",
-                "name": "rf.code",
-                "order_id": "id",
-                "short_name": "rf.code"
-            },
-            "where": [
-                "rs.processors IS NOT NULL"
-            ]
+            "description_html": "A resource is defined as any infrastructure that hosts virtual machines."
         },
-        "description_html": "A resource is defined as any infrastructure that hosts virtual machines.",
-        "name": "Resource"
+        "$ref-with-overwrite": "datawarehouse.d/ref/group-by-common.json#/resource"
     },
     "provider": {
-        "attribute_table": "serviceprovider",
         "attribute_table_schema": "modw",
         "attribute_to_aggregate_table_key_map": [
             {
@@ -218,7 +187,6 @@
         "show_in_catalog": false
     },
     "submission_venue": {
-        "attribute_table": "submission_venue",
         "attribute_table_schema": "modw",
         "attribute_to_aggregate_table_key_map": [
             {
@@ -249,7 +217,6 @@
         "$ref": "datawarehouse.d/ref/group-by-common.json#/username"
     },
     "vm_size": {
-        "attribute_table": "processor_buckets",
         "attribute_table_schema": "modw_cloud",
         "attribute_to_aggregate_table_key_map": [
             {
@@ -284,7 +251,6 @@
         "name": "VM Size: Cores"
     },
     "vm_size_memory": {
-        "attribute_table": "memory_buckets",
         "attribute_table_schema": "modw_cloud",
         "attribute_to_aggregate_table_key_map": [
             {