HJ-97 - Update systems endpoint to filter vendor deleted systems (#5553)

CHANGELOG.md

@@ -27,6 +27,7 @@ The types of changes are:
 - Adding hashes to system tab URLs [#5535](https://github.com/ethyca/fides/pull/5535)
 - Boolean inputs will now show as a select with true/false values in the connection form [#5555](https://github.com/ethyca/fides/pull/5555)
 - Updated Cookie House to be responsive [#5541](https://github.com/ethyca/fides/pull/5541)
+- Updated `/system` endpoint to filter vendor deleted systems [#5553](https://github.com/ethyca/fides/pull/5553)
 
 ### Developer Experience
 - Migrated remaining instances of Chakra's Select component to use Ant's Select component [#5502](https://github.com/ethyca/fides/pull/5502)

src/fides/api/api/v1/endpoints/system.py

@@ -1,3 +1,4 @@
+import datetime
 from typing import Annotated, Dict, List, Optional, Union
 
 from fastapi import Depends, HTTPException, Query, Response, Security
@@ -9,6 +10,7 @@
 from fideslang.validation import FidesKey
 from loguru import logger
 from pydantic import Field
+from sqlalchemy import or_
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
 from sqlalchemy.orm import Session
@@ -17,11 +19,7 @@
 
 from fides.api.api import deps
 from fides.api.api.v1.endpoints.saas_config_endpoints import instantiate_connection
-from fides.api.db.crud import (
-    get_resource,
-    get_resource_with_custom_fields,
-    list_resource,
-)
+from fides.api.db.crud import get_resource, get_resource_with_custom_fields
 from fides.api.db.ctl_session import get_async_db
 from fides.api.db.system import (
     create_system,
@@ -391,30 +389,23 @@ async def ls(  # pylint: disable=invalid-name
     data_subjects: Optional[List[FidesKey]] = Query(None),
     dnd_relevant: Optional[bool] = Query(None),
     show_hidden: Optional[bool] = Query(False),
+    show_deleted: Optional[bool] = Query(False),
 ) -> List:
     """Get a list of all of the Systems.
     If any parameters or filters are provided the response will be paginated and/or filtered.
     Otherwise all Systems will be returned (this may be a slow operation if there are many systems,
     so using the pagination parameters is recommended).
     """
-    if not (
-        size
-        or page
-        or search
-        or data_uses
-        or data_categories
-        or data_subjects
-        or dnd_relevant
-        or show_hidden
-    ):
-        return await list_resource(System, db)
+
+    query = select(System)
 
     pagination_params = Params(page=page or 1, size=size or 50)
     # Need to join with PrivacyDeclaration in order to be able to filter
     # by data use, data category, and data subject
-    query = select(System).outerjoin(
-        PrivacyDeclaration, System.id == PrivacyDeclaration.system_id
-    )
+    if any([data_uses, data_categories, data_subjects]):
+        query = query.outerjoin(
+            PrivacyDeclaration, System.id == PrivacyDeclaration.system_id
+        )
 
     # Fetch any system that is relevant for Detection and Discovery, ie any of the following:
     # - has connection configurations (has some integration for DnD or SaaS)
@@ -431,6 +422,15 @@ async def ls(  # pylint: disable=invalid-name
             System.hidden == False  # pylint: disable=singleton-comparison
         )
 
+    # Filter out any vendor deleted systems, unless explicitly asked for
+    if not show_deleted:
+        query = query.filter(
+            or_(
+                System.vendor_deleted_date.is_(None),
+                System.vendor_deleted_date >= datetime.datetime.now(),
+            )
+        )
+
     filter_params = FilterParams(
         search=search,
         data_uses=data_uses,
@@ -446,6 +446,20 @@ async def ls(  # pylint: disable=invalid-name
 
     # Add a distinct so we only get one row per system
     duplicates_removed = filtered_query.distinct(System.id)
+
+    if not (
+        size
+        or page
+        or search
+        or data_uses
+        or data_categories
+        or data_subjects
+        or dnd_relevant
+        or show_hidden
+    ):
+        result = await db.execute(duplicates_removed)
+        return result.scalars().all()
+
     return await async_paginate(db, duplicates_removed, pagination_params)
 
 

tests/ctl/core/test_api.py

@@ -2,7 +2,7 @@
 """Integration tests for the API module."""
 import json
 import typing
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from json import loads
 from typing import Dict, List, Tuple
 from uuid import uuid4
@@ -1665,6 +1665,40 @@ def test_list_with_pagination_and_multiple_filters_2(
 
         assert result_json["items"][0]["fides_key"] == tcf_system.fides_key
 
+    @pytest.mark.parametrize(
+        "vendor_deleted_date, expected_systems_count, show_deleted",
+        [
+            (datetime.now() - timedelta(days=1), 1, True),
+            (datetime.now() - timedelta(days=1), 0, None),
+            (datetime.now() + timedelta(days=1), 1, None),
+            (None, 1, None),
+        ],
+    )
+    def test_vendor_deleted_systems(
+        self,
+        db,
+        test_config,
+        system_with_cleanup,
+        vendor_deleted_date,
+        expected_systems_count,
+        show_deleted,
+    ):
+
+        system_with_cleanup.vendor_deleted_date = vendor_deleted_date
+        db.commit()
+
+        result = _api.ls(
+            url=test_config.cli.server_url,
+            headers=test_config.user.auth_header,
+            resource_type="system",
+            query_params={"show_deleted": True} if show_deleted else {},
+        )
+
+        assert result.status_code == 200
+        result_json = result.json()
+
+        assert len(result_json) == expected_systems_count
+
 
 @pytest.mark.unit
 class TestSystemUpdate: