diff --git a/CHANGELOG.md b/CHANGELOG.md index a7a2e2223e..7fac0df4c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ v 2019.2 Changes in this release: - Various bugfixes (#1046, #968, #1045) -- Migration from mongodb to postgres (#1030) +- Migration from mongodb to postgres (#1023, #1024, #1025, #1030) - added metering using pyformance - Added influxdb reporter for protocol endpoint metrics - Remove the configuration option agent-run-at-start (#1055) diff --git a/src/inmanta/data.py b/src/inmanta/data.py index 930ef49e87..30a0218264 100644 --- a/src/inmanta/data.py +++ b/src/inmanta/data.py @@ -32,6 +32,7 @@ import pkgutil import inmanta.db.versions +from inmanta.resources import Id from inmanta import const import asyncpg @@ -1467,7 +1468,6 @@ class Resource(BaseDocument): resource_id = Field(field_type=str, required=True) resource_version_id = Field(field_type=str, required=True, part_of_primary_key=True) - resource_type = Field(field_type=str, required=True) agent = Field(field_type=str, required=True) # Field based on content from the resource actions @@ -1483,6 +1483,15 @@ class Resource(BaseDocument): # the list contains full rv id's provides = Field(field_type=list, default=[]) # List of resource versions + @property + def resource_type(self): + return self._resource_type + + def __init__(self, from_postgres=False, **kwargs): + super(Resource, self).__init__(from_postgres, **kwargs) + parsed_id = Id.parse_id(self.resource_version_id) + self._resource_type = parsed_id.entity_type + def make_hash(self): character = "|".join(sorted([str(k) + "||" + str(v) for k, v in self.attributes.items() if k not in ["requires", "provides", "version"]])) @@ -1594,8 +1603,9 @@ async def get_resources_report(cls, environment): else: deployed = latest + parsed_id = Id.parse_id(resource_id) result.append({"resource_id": resource_id, - "resource_type": latest["resource_type"], + "resource_type": parsed_id.entity_type, "agent": latest["agent"], "latest_version": latest["model"], "deployed_version": deployed["model"] if "last_deploy" in deployed else None, @@ -1608,21 +1618,13 @@ async def get_resources_for_version(cls, environment, version, agent=None, - include_attributes=True, - no_obj=False, - include_undefined=True): - projection = "*" - if not include_attributes: - projection = ','.join(["id", "environment", "model", "resource_id", "resource_version_id", - "resource_type", "agent", "last_deploy", "status", "provides"]) + no_obj=False): if agent: (filter_statement, values) = cls._get_composed_filter(environment=environment, model=version, agent=agent) else: (filter_statement, values) = cls._get_composed_filter(environment=environment, model=version) - if not include_undefined: - filter_statement += " AND status NOT IN ($" + str(len(values) + 1) + ",$" + str(len(values) + 2) + ")" - values += [cls._get_value(const.ResourceState.undefined), cls._get_value(const.ResourceState.skipped_for_undefined)] - query = "SELECT " + projection + " FROM " + cls.table_name() + " WHERE " + filter_statement + + query = f"SELECT * FROM {Resource.table_name()} WHERE {filter_statement}" resources = [] async with cls._connection_pool.acquire() as con: async with con.transaction(): @@ -1631,6 +1633,8 @@ async def get_resources_for_version(cls, record = dict(record) record["attributes"] = json.loads(record["attributes"]) record["id"] = record["resource_version_id"] + parsed_id = Id.parse_id(record["resource_version_id"]) + record["resource_type"] = parsed_id.entity_type resources.append(record) else: resources.append(cls(from_postgres=True, **record)) @@ -1682,11 +1686,11 @@ async def get_with_state(cls, environment, version): @classmethod def new(cls, environment, resource_version_id, **kwargs): - from inmanta.resources import Id vid = Id.parse_id(resource_version_id) attr = dict(environment=environment, model=vid.version, resource_id=vid.resource_str(), - resource_version_id=resource_version_id, resource_type=vid.entity_type, agent=vid.agent_name) + resource_version_id=resource_version_id, agent=vid.agent_name) + attr.update(kwargs) return cls(**attr) @@ -1788,6 +1792,7 @@ def to_dict(self): self.make_hash() dct = super(Resource, self).to_dict() dct["id"] = dct["resource_version_id"] + dct["resource_type"] = self._resource_type return dct diff --git a/src/inmanta/db/versions/v1.py b/src/inmanta/db/versions/v1.py index 7687f573dd..424820f634 100644 --- a/src/inmanta/db/versions/v1.py +++ b/src/inmanta/db/versions/v1.py @@ -48,7 +48,6 @@ async def update(connection): model integer NOT NULL, resource_id varchar NOT NULL, resource_version_id varchar NOT NULL, - resource_type varchar NOT NULL, agent varchar NOT NULL, last_deploy timestamp, attributes JSONB, diff --git a/src/inmanta/server/server.py b/src/inmanta/server/server.py index 6d9c358220..d46222ba9e 100644 --- a/src/inmanta/server/server.py +++ b/src/inmanta/server/server.py @@ -936,7 +936,7 @@ def get_version(self, env, version_id, include_logs=None, log_filter=None, limit if version is None: return 404, {"message": "The given configuration model does not exist yet."} - resources = yield data.Resource.get_resources_for_version(env.id, version_id, include_attributes=True, no_obj=True) + resources = yield data.Resource.get_resources_for_version(env.id, version_id, no_obj=True) if resources is None: return 404, {"message": "The given configuration model does not exist yet."} diff --git a/tests/test_data.py b/tests/test_data.py index 19eeed8a3f..50145c19cf 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -25,7 +25,6 @@ import types import pkgutil - from inmanta import data, const from inmanta.const import LogLevel from asyncpg import PostgresSyntaxError @@ -1048,7 +1047,7 @@ async def test_get_resources(init_dataclasses_and_load_schema): @pytest.mark.asyncio -async def test_get_resources_for_version(init_dataclasses_and_load_schema): +async def test_model_get_resources_for_version(init_dataclasses_and_load_schema): project = data.Project(name="test") await project.insert() @@ -1111,9 +1110,46 @@ async def make_with_status(i, status): assert len(resources) == 4 assert sorted([x.resource_version_id for x in resources]) == sorted([d, s, u, su]) - resources = await data.Resource.get_resources_for_version(env.id, 3, include_undefined=False) - assert len(resources) == 2 - assert sorted([x.resource_version_id for x in resources]) == sorted([d, s]) + +@pytest.mark.asyncio +async def test_model_get_resources_for_version_optional_args(init_dataclasses_and_load_schema): + project = data.Project(name="test") + await project.insert() + + env = data.Environment(name="dev", project=project.id, repo_url="", repo_branch="") + await env.insert() + + version = int(time.time()) + cm = data.ConfigurationModel(environment=env.id, version=version, date=datetime.datetime.now(), total=3, version_info={}) + await cm.insert() + + async def insert_resource(env_id, version, agent_name, path, status): + resource_version_id = f"std::File[{agent_name},path={path}],v={version}" + resource = data.Resource.new(environment=env_id, + resource_version_id=resource_version_id, + attributes={"path": path}, + status=status) + await resource.insert() + + await insert_resource(env.id, version, "agent1", "path1", const.ResourceState.deployed) + await insert_resource(env.id, version, "agent2", "path2", const.ResourceState.available) + await insert_resource(env.id, version, "agent1", "path3", const.ResourceState.undefined) + + result = await data.Resource.get_resources_for_version(env.id, version) + assert len(result) == 3 + assert sorted([r.agent for r in result]) == ["agent1", "agent1", "agent2"] + for r in result: + assert len(r.attributes) == 1 + + result = await data.Resource.get_resources_for_version(env.id, version, agent="agent2") + assert len(result) == 1 + assert result[0].agent == "agent2" + + result = await data.Resource.get_resources_for_version(env.id, version, no_obj=True) + assert len(result) == 3 + assert sorted([r["agent"] for r in result]) == ["agent1", "agent1", "agent2"] + for r in result: + assert len(r["attributes"]) == 1 @pytest.mark.asyncio