Remove the resource_type and the agent column from resource

CHANGELOG.md

@@ -1,7 +1,7 @@
 v 2019.2
 Changes in this release:
 - Various bugfixes (#1046, #968, #1045)
-- Migration from mongodb to postgres (#1030)
+- Migration from mongodb to postgres (#1023, #1024, #1025, #1030)
 - added metering using pyformance
 - Added influxdb reporter for protocol endpoint metrics
 - Remove the configuration option agent-run-at-start (#1055)

src/inmanta/data.py

@@ -32,6 +32,7 @@
 import pkgutil
 import inmanta.db.versions
 
+from inmanta.resources import Id
 from inmanta import const
 import asyncpg
 
@@ -1467,7 +1468,6 @@ class Resource(BaseDocument):
     resource_id = Field(field_type=str, required=True)
     resource_version_id = Field(field_type=str, required=True, part_of_primary_key=True)
 
-    resource_type = Field(field_type=str, required=True)
     agent = Field(field_type=str, required=True)
 
     # Field based on content from the resource actions
@@ -1483,6 +1483,15 @@ class Resource(BaseDocument):
     # the list contains full rv id's
     provides = Field(field_type=list, default=[])  # List of resource versions
 
+    @property
+    def resource_type(self):
+        return self._resource_type
+
+    def __init__(self, from_postgres=False, **kwargs):
+        super(Resource, self).__init__(from_postgres, **kwargs)
+        parsed_id = Id.parse_id(self.resource_version_id)
+        self._resource_type = parsed_id.entity_type
+
     def make_hash(self):
         character = "|".join(sorted([str(k) + "||" + str(v)
                                      for k, v in self.attributes.items() if k not in ["requires", "provides", "version"]]))
@@ -1594,8 +1603,9 @@ async def get_resources_report(cls, environment):
                     else:
                         deployed = latest
 
+                    parsed_id = Id.parse_id(resource_id)
                     result.append({"resource_id": resource_id,
-                                   "resource_type": latest["resource_type"],
+                                   "resource_type": parsed_id.entity_type,
                                    "agent": latest["agent"],
                                    "latest_version": latest["model"],
                                    "deployed_version": deployed["model"] if "last_deploy" in deployed else None,
@@ -1608,21 +1618,13 @@ async def get_resources_for_version(cls,
                                         environment,
                                         version,
                                         agent=None,
-                                        include_attributes=True,
-                                        no_obj=False,
-                                        include_undefined=True):
-        projection = "*"
-        if not include_attributes:
-            projection = ','.join(["id", "environment", "model", "resource_id", "resource_version_id",
-                                   "resource_type", "agent", "last_deploy", "status", "provides"])
+                                        no_obj=False):
         if agent:
             (filter_statement, values) = cls._get_composed_filter(environment=environment, model=version, agent=agent)
         else:
             (filter_statement, values) = cls._get_composed_filter(environment=environment, model=version)
-        if not include_undefined:
-            filter_statement += " AND status NOT IN ($" + str(len(values) + 1) + ",$" + str(len(values) + 2) + ")"
-            values += [cls._get_value(const.ResourceState.undefined), cls._get_value(const.ResourceState.skipped_for_undefined)]
-        query = "SELECT " + projection + " FROM " + cls.table_name() + " WHERE " + filter_statement
+
+        query = f"SELECT * FROM {Resource.table_name()} WHERE {filter_statement}"
         resources = []
         async with cls._connection_pool.acquire() as con:
             async with con.transaction():
@@ -1631,6 +1633,8 @@ async def get_resources_for_version(cls,
                         record = dict(record)
                         record["attributes"] = json.loads(record["attributes"])
                         record["id"] = record["resource_version_id"]
+                        parsed_id = Id.parse_id(record["resource_version_id"])
+                        record["resource_type"] = parsed_id.entity_type
                         resources.append(record)
                     else:
                         resources.append(cls(from_postgres=True, **record))
@@ -1682,11 +1686,11 @@ async def get_with_state(cls, environment, version):
 
     @classmethod
     def new(cls, environment, resource_version_id, **kwargs):
-        from inmanta.resources import Id
         vid = Id.parse_id(resource_version_id)
 
         attr = dict(environment=environment, model=vid.version, resource_id=vid.resource_str(),
-                    resource_version_id=resource_version_id, resource_type=vid.entity_type, agent=vid.agent_name)
+                    resource_version_id=resource_version_id, agent=vid.agent_name)
+
         attr.update(kwargs)
 
         return cls(**attr)
@@ -1788,6 +1792,7 @@ def to_dict(self):
         self.make_hash()
         dct = super(Resource, self).to_dict()
         dct["id"] = dct["resource_version_id"]
+        dct["resource_type"] = self._resource_type
         return dct
 
 

src/inmanta/db/versions/v1.py

@@ -48,7 +48,6 @@ async def update(connection):
     model integer NOT NULL,
     resource_id varchar NOT NULL,
     resource_version_id varchar NOT NULL,
-    resource_type varchar NOT NULL,
     agent varchar NOT NULL,
     last_deploy timestamp,
     attributes JSONB,

src/inmanta/server/server.py

@@ -936,7 +936,7 @@ def get_version(self, env, version_id, include_logs=None, log_filter=None, limit
         if version is None:
             return 404, {"message": "The given configuration model does not exist yet."}
 
-        resources = yield data.Resource.get_resources_for_version(env.id, version_id, include_attributes=True, no_obj=True)
+        resources = yield data.Resource.get_resources_for_version(env.id, version_id, no_obj=True)
         if resources is None:
             return 404, {"message": "The given configuration model does not exist yet."}
 

tests/test_data.py

@@ -25,7 +25,6 @@
 import types
 import pkgutil
 
-
 from inmanta import data, const
 from inmanta.const import LogLevel
 from asyncpg import PostgresSyntaxError
@@ -1048,7 +1047,7 @@ async def test_get_resources(init_dataclasses_and_load_schema):
 
 
 @pytest.mark.asyncio
-async def test_get_resources_for_version(init_dataclasses_and_load_schema):
+async def test_model_get_resources_for_version(init_dataclasses_and_load_schema):
     project = data.Project(name="test")
     await project.insert()
 
@@ -1111,9 +1110,46 @@ async def make_with_status(i, status):
     assert len(resources) == 4
     assert sorted([x.resource_version_id for x in resources]) == sorted([d, s, u, su])
 
-    resources = await data.Resource.get_resources_for_version(env.id, 3, include_undefined=False)
-    assert len(resources) == 2
-    assert sorted([x.resource_version_id for x in resources]) == sorted([d, s])
+
+@pytest.mark.asyncio
+async def test_model_get_resources_for_version_optional_args(init_dataclasses_and_load_schema):
+    project = data.Project(name="test")
+    await project.insert()
+
+    env = data.Environment(name="dev", project=project.id, repo_url="", repo_branch="")
+    await env.insert()
+
+    version = int(time.time())
+    cm = data.ConfigurationModel(environment=env.id, version=version, date=datetime.datetime.now(), total=3, version_info={})
+    await cm.insert()
+
+    async def insert_resource(env_id, version, agent_name, path, status):
+        resource_version_id = f"std::File[{agent_name},path={path}],v={version}"
+        resource = data.Resource.new(environment=env_id,
+                                     resource_version_id=resource_version_id,
+                                     attributes={"path": path},
+                                     status=status)
+        await resource.insert()
+
+    await insert_resource(env.id, version, "agent1", "path1", const.ResourceState.deployed)
+    await insert_resource(env.id, version, "agent2", "path2", const.ResourceState.available)
+    await insert_resource(env.id, version, "agent1", "path3", const.ResourceState.undefined)
+
+    result = await data.Resource.get_resources_for_version(env.id, version)
+    assert len(result) == 3
+    assert sorted([r.agent for r in result]) == ["agent1", "agent1", "agent2"]
+    for r in result:
+        assert len(r.attributes) == 1
+
+    result = await data.Resource.get_resources_for_version(env.id, version, agent="agent2")
+    assert len(result) == 1
+    assert result[0].agent == "agent2"
+
+    result = await data.Resource.get_resources_for_version(env.id, version, no_obj=True)
+    assert len(result) == 3
+    assert sorted([r["agent"] for r in result]) == ["agent1", "agent1", "agent2"]
+    for r in result:
+        assert len(r["attributes"]) == 1
 
 
 @pytest.mark.asyncio