Implements: #1353

micafer · micafer · commit 5817d52482fe · 2024-10-31T11:28:15.000+01:00
diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py
@@ -42,6 +42,7 @@
 from IM.openid.JWT import JWT
 from IM.openid.OpenIDClient import OpenIDClient
 from IM.vault import VaultCredentials
+from IM.Stats import Stats
 
 
 if Config.MAX_SIMULTANEOUS_LAUNCHES > 1:
@@ -2036,3 +2037,21 @@ def EstimateResouces(radl_data, auth):
                         cont += 1
 
         return res
+
+    @staticmethod
+    def GetStats(init_date, end_date, auth):
+        """
+        Get the statistics from the IM DB.
+        Args:
+        - init_date(str): Only will be returned infrastructure created afther this date.
+        - end_date(str): Only will be returned infrastructure created before this date.
+        - auth(Authentication): parsed authentication tokens.
+        Return: a list of dict with the stats.
+        """
+        # First check the auth data
+        auth = InfrastructureManager.check_auth_data(auth)
+        stats = Stats.get_stats(init_date, end_date, auth)
+        if not stats:
+            raise Exception("ERROR connecting with the database!.")
+        else:
+            return stats
diff --git a/IM/REST.py b/IM/REST.py
@@ -21,6 +21,7 @@
 import flask
 import os
 import yaml
+import datetime
 
 from cheroot.wsgi import Server as WSGIServer, PathInfoDispatcher
 from cheroot.ssl.builtin import BuiltinSSLAdapter
@@ -1077,6 +1078,49 @@ def RESTChangeInfrastructureAuth(infid=None):
         return return_error(400, "Error modifying infrastructure owner: %s" % get_ex_error(ex))
 
 
+@app.route('/stats', methods=['GET'])
+def RESTGetStats():
+    try:
+        auth = get_auth_header()
+    except Exception:
+        return return_error(401, "No authentication data provided")
+
+    try:
+        init_date = None
+        if "init_date" in flask.request.args.keys():
+            init_date = flask.request.args.get("init_date").lower()
+            init_date = init_date.replace("/", "-")
+            parts = init_date.split("-")
+            try:
+                year = int(parts[0])
+                month = int(parts[1])
+                day = int(parts[2])
+                datetime.date(year, month, day)
+            except Exception:
+                return return_error(400, "Incorrect format in init_date parameter: YYYY/MM/dd")
+        else:
+            init_date = "1970-01-01"
+
+        end_date = None
+        if "end_date" in flask.request.args.keys():
+            end_date = flask.request.args.get("end_date").lower()
+            end_date = end_date.replace("/", "-")
+            parts = end_date.split("-")
+            try:
+                year = int(parts[0])
+                month = int(parts[1])
+                day = int(parts[2])
+                datetime.date(year, month, day)
+            except Exception:
+                return return_error(400, "Incorrect format in end_date parameter: YYYY/MM/dd")
+
+        stats = InfrastructureManager.GetStats(init_date, end_date, auth)
+        return format_output(stats, default_type="application/json", field_name="stats")
+    except Exception as ex:
+        logger.exception("Error getting stats")
+        return return_error(400, "Error getting stats: %s" % get_ex_error(ex))
+
+
 @app.errorhandler(403)
 def error_mesage_403(error):
     return return_error(403, error.description)
diff --git a/IM/ServiceRequests.py b/IM/ServiceRequests.py
@@ -60,6 +60,7 @@ class IMBaseRequest(AsyncRequest):
     CHANGE_INFRASTRUCTURE_AUTH = "ChangeInfrastructureAuth"
     GET_INFRASTRUCTURE_OWNERS = "GetInfrastructureOwners"
     ESTIMATE_RESOURCES = "EstimateResouces"
+    GET_STATS = "GetStats"
 
     @staticmethod
     def create_request(function, arguments=()):
@@ -119,6 +120,8 @@ def create_request(function, arguments=()):
             return Request_GetInfrastructureOwners(arguments)
         elif function == IMBaseRequest.ESTIMATE_RESOURCES:
             return Request_EstimateResouces(arguments)
+        elif function == IMBaseRequest.GET_STATS:
+            return Request_GetStats(arguments)
         else:
             raise NotImplementedError("Function not Implemented")
 
@@ -473,3 +476,14 @@ def _call_function(self):
         self._error_mesage = "Error getting the resources estimation"
         (radl_data, auth_data) = self.arguments
         return IM.InfrastructureManager.InfrastructureManager.EstimateResouces(radl_data, Authentication(auth_data))
+
+
+class Request_GetStats(IMBaseRequest):
+    """
+    Request class for the GetStats function
+    """
+
+    def _call_function(self):
+        self._error_mesage = "Error getting stats"
+        (init_date, end_date, auth_data) = self.arguments
+        return IM.InfrastructureManager.InfrastructureManager.GetStats(init_date, end_date, Authentication(auth_data))
diff --git a/IM/Stats.py b/IM/Stats.py
@@ -0,0 +1,142 @@
+# IM - Infrastructure Manager
+# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os.path
+import datetime
+import json
+import yaml
+import logging
+
+from IM.db import DataBase
+from IM.auth import Authentication
+from IM.config import Config
+from IM.InfrastructureList import InfrastructureList
+from radl.radl_parse import parse_radl
+
+
+class Stats():
+
+    logger = logging.getLogger('InfrastructureManager')
+    """Logger object."""
+
+    @staticmethod
+    def _get_data(str_data, init_date=None, end_date=None):
+        dic = json.loads(str_data)
+        resp = {'creation_date': None}
+        if 'creation_date' in dic and dic['creation_date']:
+            creation_date = datetime.datetime.fromtimestamp(float(dic['creation_date']))
+            resp['creation_date'] = str(creation_date)
+            if init_date and creation_date < init_date:
+                return None
+            if end_date and creation_date > end_date:
+                return None
+
+        resp['tosca_name'] = None
+        if 'extra_info' in dic and dic['extra_info'] and "TOSCA" in dic['extra_info']:
+            try:
+                tosca = yaml.safe_load(dic['extra_info']['TOSCA'])
+                icon = tosca.get("metadata", {}).get("icon", "")
+                resp['tosca_name'] = os.path.basename(icon)[:-4]
+            except Exception:
+                Stats.logger.exception("Error loading TOSCA.")
+
+        resp['vm_count'] = 0
+        resp['cpu_count'] = 0
+        resp['memory_size'] = 0
+        resp['cloud_type'] = None
+        resp['cloud_host'] = None
+        resp['hybrid'] = False
+        for str_vm_data in dic['vm_list']:
+            vm_data = json.loads(str_vm_data)
+            cloud_data = json.loads(vm_data["cloud"])
+
+            # only get the cloud of the first VM
+            if not resp['cloud_type']:
+                resp['cloud_type'] = cloud_data["type"]
+            if not resp['cloud_host']:
+                resp['cloud_host'] = cloud_data["server"]
+            elif resp['cloud_host'] != cloud_data["server"]:
+                resp['hybrid'] = True
+
+            vm_sys = parse_radl(vm_data['info']).systems[0]
+            if vm_sys.getValue('cpu.count'):
+                resp['cpu_count'] += vm_sys.getValue('cpu.count')
+            if vm_sys.getValue('memory.size'):
+                resp['memory_size'] += vm_sys.getFeature('memory.size').getValue('M')
+            resp['vm_count'] += 1
+
+        inf_auth = Authentication.deserialize(dic['auth']).getAuthInfo('InfrastructureManager')[0]
+        resp['im_user'] = inf_auth.get('username')
+        return resp
+
+    @staticmethod
+    def get_stats(init_date="1970-01-01", end_date=None, auth=None):
+        """
+        Get the statistics from the IM DB.
+
+        Args:
+
+        - init_date(str): Only will be returned infrastructure created afther this date.
+        - end_date(str): Only will be returned infrastructure created afther this date.
+        - auth(Authentication): parsed authentication tokens.
+
+        Return: a list of dict with the stats with the following format:
+            {'creation_date': '2022-03-07 13:16:14',
+             'tosca_name': 'kubernetes',
+             'vm_count': 2,
+             'cpu_count': 4,
+             'memory_size': 1024,
+             'cloud_type': 'OSCAR',
+             'cloud_host': 'sharp-elbakyan5.im.grycap.net',
+             'hybrid': False,
+             'im_user': '__OPENID__mcaballer',
+             'inf_id': '1',
+             'last_date': '2022-03-23'}
+        """
+        stats = []
+        db = DataBase(Config.DATA_DB)
+        if db.connect():
+            if db.db_type == DataBase.MONGO:
+                filt = InfrastructureList._gen_filter_from_auth(auth)
+                res = db.find("inf_list", filt, {"id": True, "date": True, "date": True}, [('id', -1)])
+            else:
+                where = InfrastructureList._gen_where_from_auth(auth)
+                res = db.select("select data, date, id from inf_list %s order by rowid desc" % where)
+
+            for elem in res:
+                if db.db_type == DataBase.MONGO:
+                    data = elem["data"]
+                    date = elem["date"]
+                    inf_id = elem["id"]
+                else:
+                    data = elem[0]
+                    date = elem[1]
+                    inf_id = elem[2]
+                try:
+                    init = datetime.datetime.strptime(init_date, "%Y-%m-%d")
+                    end = datetime.datetime.strptime(end_date, "%Y-%m-%d") if end_date else None
+                    res = Stats._get_data(data, init, end)
+                    if res:
+                        res['inf_id'] = inf_id
+                        res['last_date'] = str(date)
+                        stats.append(res)
+                except Exception:
+                    Stats.logger.exception("ERROR reading infrastructure info from Inf ID: %s" % inf_id)
+            db.close()
+            return stats
+        else:
+            Stats.logger.error("ERROR connecting with the database!.")
+            return None
diff --git a/IM/im_service.py b/IM/im_service.py
@@ -229,6 +229,12 @@ def EstimateResources(radl_data, auth_data):
     return WaitRequest(request)
 
 
+def GetStats(init_date, end_date, auth_data):
+    request = IMBaseRequest.create_request(
+        IMBaseRequest.GET_STATS, (init_date, end_date, auth_data))
+    return WaitRequest(request)
+
+
 def launch_daemon():
     """
     Launch the IM daemon
@@ -290,6 +296,7 @@ def launch_daemon():
         server.register_function(ChangeInfrastructureAuth)
         server.register_function(GetInfrastructureOwners)
         server.register_function(EstimateResources)
+        server.register_function(GetStats)
 
         # Launch the API XMLRPC thread
         server.serve_forever_in_thread()
diff --git a/test/unit/REST.py b/test/unit/REST.py
@@ -838,6 +838,21 @@ def test_GetInfrastructureOwners(self, GetInfrastructureOwners):
         res = self.client.get('/infrastructures/1/authorization', headers=headers)
         self.assertEqual(res.json, {"authorization": ["user1", "user2"]})
 
+    @patch("IM.InfrastructureManager.InfrastructureManager.GetStats")
+    def test_GetStats(self, GetStats):
+        """Test REST GetStats."""
+        headers = {"AUTHORIZATION": "type = InfrastructureManager; username = user; password = pass"}
+        GetStats.return_value = [{"key": 1}]
+
+        res = self.client.get('/stats?init_date=2010-01-01&end_date=2022-01-01', headers=headers)
+
+        self.assertEqual(res.json, {"stats": [{"key": 1}]})
+        self.assertEqual(GetStats.call_args_list[0][0][0], '2010-01-01')
+        self.assertEqual(GetStats.call_args_list[0][0][1], '2022-01-01')
+        self.assertEqual(GetStats.call_args_list[0][0][2].auth_list, [{"type": "InfrastructureManager",
+                                                                       "username": "user",
+                                                                       "password": "pass"}])
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/test/unit/ServiceRequests.py b/test/unit/ServiceRequests.py
@@ -214,6 +214,13 @@ def test_estimate_resources(self, inflist):
             IM.ServiceRequests.IMBaseRequest.ESTIMATE_RESOURCES, ("", ""))
         req._call_function()
 
+    @patch('IM.InfrastructureManager.InfrastructureManager')
+    def test_get_stats(self, inflist):
+        import IM.ServiceRequests
+        req = IM.ServiceRequests.IMBaseRequest.create_request(
+            IM.ServiceRequests.IMBaseRequest.GET_STATS, ("", "", ""))
+        req._call_function()
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/unit/test_im_logic.py b/test/unit/test_im_logic.py
@@ -23,6 +23,7 @@
 import sys
 import json
 import base64
+import yaml
 
 from mock import Mock, patch, MagicMock
 
@@ -1560,6 +1561,47 @@ def test_estimate_resources(self):
                 'storage': [{'sizeInGigabytes': 100}]
             }})
 
+    @patch('IM.Stats.DataBase')
+    @patch('IM.InfrastructureManager.InfrastructureManager.check_auth_data')
+    def test_get_stats(self, check_auth_data, DataBase):
+        radl = """
+            system node (
+            memory.size = 512M and
+            cpu.count = 2
+            )"""
+
+        auth = Authentication([{'type': 'InfrastructureManager', 'token': 'atoken',
+                                'username': '__OPENID__mcaballer', 'password': 'pass'}])
+        check_auth_data.return_value = auth
+
+        db = MagicMock()
+        inf_data = {
+            "id": "1",
+            "auth": auth.serialize(),
+            "creation_date": 1646655374,
+            "extra_info": {"TOSCA": yaml.dump({"metadata": {"icon": "kubernetes.png"}})},
+            "vm_list": [
+                json.dumps({"cloud": '{"type": "OSCAR", "server": "sharp-elbakyan5.im.grycap.net"}', "info": radl}),
+                json.dumps({"cloud": '{"type": "OSCAR", "server": "sharp-elbakyan5.im.grycap.net"}', "info": radl})
+            ]
+        }
+        db.select.return_value = [(json.dumps(inf_data), '2022-03-23', '1')]
+        DataBase.return_value = db
+
+        stats = IM.GetStats('2001-01-01', '2122-01-01', auth)
+        expected_res = [{'creation_date': '2022-03-07 13:16:14',
+                         'tosca_name': 'kubernetes',
+                         'vm_count': 2,
+                         'cpu_count': 4,
+                         'memory_size': 1024,
+                         'cloud_type': 'OSCAR',
+                         'cloud_host': 'sharp-elbakyan5.im.grycap.net',
+                         'hybrid': False,
+                         'im_user': '__OPENID__mcaballer',
+                         'inf_id': '1',
+                         'last_date': '2022-03-23'}]
+        self.assertEqual(stats, expected_res)
+
 
 if __name__ == "__main__":
     unittest.main()
