WAFv2: Paginate all list_ methods (#8344)

docs/docs/services/wafv2.rst

@@ -59,10 +59,6 @@ wafv2
 - [ ] list_resources_for_web_acl
 - [X] list_rule_groups
 - [X] list_tags_for_resource
-
-        Pagination is not yet implemented
-
-
 - [X] list_web_acls
 - [X] put_logging_configuration
 - [ ] put_managed_rule_set_versions

moto/wafv2/models.py

@@ -9,6 +9,7 @@
     iso_8601_datetime_with_milliseconds,
 )
 from moto.moto_api._internal import mock_random
+from moto.utilities.paginator import paginate
 from moto.utilities.tagging_service import TaggingService
 from moto.utilities.utils import ARN_PARTITION_REGEX, PARTITION_NAMES
 
@@ -31,6 +32,39 @@
     + r":apigateway:[a-zA-Z0-9-]+::/restapis/[a-zA-Z0-9]+/stages/[a-zA-Z0-9]+"
 )
 
+PAGINATION_MODEL = {
+    "list_ip_sets": {
+        "input_token": "next_marker",
+        "limit_key": "limit",
+        "limit_default": 100,
+        "unique_attribute": "arn",
+    },
+    "list_logging_configurations": {
+        "input_token": "next_marker",
+        "limit_key": "limit",
+        "limit_default": 100,
+        "unique_attribute": "arn",
+    },
+    "list_rule_groups": {
+        "input_token": "next_marker",
+        "limit_key": "limit",
+        "limit_default": 100,
+        "unique_attribute": "arn",
+    },
+    "list_tags_for_resource": {
+        "input_token": "next_marker",
+        "limit_key": "limit",
+        "limit_default": 100,
+        "unique_attribute": "Key",
+    },
+    "list_web_acls": {
+        "input_token": "next_marker",
+        "limit_key": "limit",
+        "limit_default": 100,
+        "unique_attribute": "ARN",
+    },
+}
+
 
 class FakeRule(BaseModel):
     def __init__(
@@ -479,23 +513,23 @@ def get_web_acl(self, name: str, _id: str) -> FakeWebACL:
                 return wacl
         raise WAFNonexistentItemException
 
+    @paginate(PAGINATION_MODEL)  # type: ignore
     def list_web_acls(self) -> List[Dict[str, Any]]:
         return [wacl.to_short_dict() for wacl in self.wacls.values()]
 
     def _is_duplicate_name(self, name: str) -> bool:
         all_wacl_names = set(wacl.name for wacl in self.wacls.values())
         return name in all_wacl_names
 
+    @paginate(PAGINATION_MODEL)  # type: ignore
     def list_rule_groups(self, scope: str) -> List[Any]:
         rule_groups = [
             group for group in self.rule_groups.values() if group.scope == scope
         ]
         return rule_groups
 
+    @paginate(PAGINATION_MODEL)  # type: ignore
     def list_tags_for_resource(self, arn: str) -> List[Dict[str, str]]:
-        """
-        Pagination is not yet implemented
-        """
         return self.tagging_service.list_tags_for_resource(arn)["Tags"]
 
     def tag_resource(self, arn: str, tags: List[Dict[str, str]]) -> None:
@@ -591,6 +625,7 @@ def delete_ip_set(self, name: str, scope: str, _id: str, lock_token: str) -> Non
 
         self.ip_sets.pop(arn)
 
+    @paginate(PAGINATION_MODEL)  # type: ignore
     def list_ip_sets(self, scope: str) -> List[FakeIPSet]:
         ip_sets = [
             ip_set for arn, ip_set in self.ip_sets.items() if ip_set.scope == scope
@@ -665,6 +700,7 @@ def get_logging_configuration(self, arn: str) -> FakeLoggingConfiguration:
             raise WAFNonexistentItemException()
         return logging_configuration
 
+    @paginate(PAGINATION_MODEL)  # type: ignore
     def list_logging_configurations(self, scope: str) -> List[FakeLoggingConfiguration]:
         if scope == "CLOUDFRONT":
             scope = "global"

moto/wafv2/responses.py

@@ -3,7 +3,6 @@
 from moto.core.common_types import TYPE_RESPONSE
 from moto.core.responses import BaseResponse
 
-from ..moto_api._internal import mock_random
 from .models import GLOBAL_REGION, WAFV2Backend, wafv2_backends
 
 
@@ -98,25 +97,43 @@ def list_web_ac_ls(self) -> TYPE_RESPONSE:
         scope = self._get_param("Scope")
         if scope == "CLOUDFRONT":
             self.region = GLOBAL_REGION
-        all_web_acls = self.wafv2_backend.list_web_acls()
-        response = {"NextMarker": "Not Implemented", "WebACLs": all_web_acls}
+        limit = self._get_int_param("Limit")
+        next_marker = self._get_param("NextMarker")
+        web_acls, next_marker = self.wafv2_backend.list_web_acls(
+            limit=limit, next_marker=next_marker
+        )
+        response = {"NextMarker": next_marker, "WebACLs": web_acls}
         response_headers = {"Content-Type": "application/json"}
         return 200, response_headers, json.dumps(response)
 
     def list_rule_groups(self) -> TYPE_RESPONSE:
         scope = self._get_param("Scope")
         if scope == "CLOUDFRONT":
             self.region = GLOBAL_REGION
-        rule_groups = self.wafv2_backend.list_rule_groups(scope)
-        response = {"RuleGroups": [rg.to_short_dict() for rg in rule_groups]}
+        limit = self._get_int_param("Limit")
+        next_marker = self._get_param("NextMarker")
+        rule_groups, next_marker = self.wafv2_backend.list_rule_groups(
+            scope, limit=limit, next_marker=next_marker
+        )
+        response = {
+            "RuleGroups": [rg.to_short_dict() for rg in rule_groups],
+            "NextMarker": next_marker,
+        }
         response_headers = {"Content-Type": "application/json"}
         return 200, response_headers, json.dumps(response)
 
     def list_tags_for_resource(self) -> TYPE_RESPONSE:
         arn = self._get_param("ResourceARN")
         self.region = arn.split(":")[3]
-        tags = self.wafv2_backend.list_tags_for_resource(arn)
-        response = {"TagInfoForResource": {"ResourceARN": arn, "TagList": tags}}
+        limit = self._get_int_param("Limit")
+        next_marker = self._get_param("NextMarker")
+        tags, next_marker = self.wafv2_backend.list_tags_for_resource(
+            arn, limit=limit, next_marker=next_marker
+        )
+        response = {
+            "TagInfoForResource": {"ResourceARN": arn, "TagList": tags},
+            "NextMarker": next_marker,
+        }
         response_headers = {"Content-Type": "application/json"}
         return 200, response_headers, json.dumps(response)
 
@@ -217,11 +234,15 @@ def delete_ip_set(self) -> TYPE_RESPONSE:
 
         return 200, {}, "{}"
 
-    def list_ip_sets(self) -> TYPE_RESPONSE:
+    def list_ip_sets(self) -> str:
         scope = self._get_param("Scope")
         if scope == "CLOUDFRONT":
             self.region = GLOBAL_REGION
-        ip_sets = self.wafv2_backend.list_ip_sets(scope)
+        next_marker = self._get_param("NextMarker")
+        limit = self._get_int_param("Limit")
+        ip_sets, next_token = self.wafv2_backend.list_ip_sets(
+            scope, next_marker=next_marker, limit=limit
+        )
 
         formatted_ip_sets = [
             {
@@ -234,13 +255,7 @@ def list_ip_sets(self) -> TYPE_RESPONSE:
             for ip_set in ip_sets
         ]
 
-        return (
-            200,
-            {},
-            json.dumps(
-                {"NextMarker": str(mock_random.uuid4()), "IPSets": formatted_ip_sets}
-            ),
-        )
+        return json.dumps({"NextMarker": next_token, "IPSets": formatted_ip_sets})
 
     def get_ip_set(self) -> TYPE_RESPONSE:
         scope = self._get_param("Scope")
@@ -327,16 +342,22 @@ def get_logging_configuration(self) -> TYPE_RESPONSE:
             ),
         )
 
-    def list_logging_configurations(self) -> TYPE_RESPONSE:
+    def list_logging_configurations(self) -> str:
         body = json.loads(self.body)
         scope = body.get("Scope")
-        log_configs = self.wafv2_backend.list_logging_configurations(scope)
+        limit = self._get_int_param("Limit")
+        next_marker = self._get_param("NextMarker")
+        log_configs, next_marker = self.wafv2_backend.list_logging_configurations(
+            scope, limit=limit, next_marker=next_marker
+        )
 
         formatted = [
             {k: v for k, v in config.to_dict().items() if v is not None}
             for config in log_configs
         ]
-        return 200, {}, json.dumps({"LoggingConfigurations": formatted})
+        return json.dumps(
+            {"LoggingConfigurations": formatted, "NextMarker": next_marker}
+        )
 
     def delete_logging_configuration(self) -> TYPE_RESPONSE:
         body = json.loads(self.body)

tests/test_wafv2/test_helper_functions.py

@@ -9,7 +9,3 @@ def CREATE_WEB_ACL_BODY(name: str, scope: str) -> dict:
             "MetricName": "idk",
         },
     }
-
-
-def LIST_WEB_ACL_BODY(scope: str) -> dict:
-    return {"Scope": scope}

tests/test_wafv2/test_server.py

@@ -6,7 +6,7 @@
 from moto import mock_aws, settings
 from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
 
-from .test_helper_functions import CREATE_WEB_ACL_BODY, LIST_WEB_ACL_BODY
+from .test_helper_functions import CREATE_WEB_ACL_BODY
 
 CREATE_WEB_ACL_HEADERS = {
     "X-Amz-Target": "AWSWAF_20190729.CreateWebACL",
@@ -89,7 +89,7 @@ def test_list_web_ac_ls():
         json=CREATE_WEB_ACL_BODY("Sarah", "CLOUDFRONT"),
     )
     res = test_client.post(
-        "/", headers=LIST_WEB_ACL_HEADERS, json=LIST_WEB_ACL_BODY("REGIONAL")
+        "/", headers=LIST_WEB_ACL_HEADERS, json={"Scope": "REGIONAL"}
     )
     assert res.status_code == 200
 
@@ -99,7 +99,7 @@ def test_list_web_ac_ls():
     assert web_acls[1]["Name"] == "JohnSon"
 
     res = test_client.post(
-        "/", headers=LIST_WEB_ACL_HEADERS, json=LIST_WEB_ACL_BODY("CLOUDFRONT")
+        "/", headers=LIST_WEB_ACL_HEADERS, json={"Scope": "CLOUDFRONT"}
     )
     assert res.status_code == 200
     web_acls = res.json["WebACLs"]

tests/test_wafv2/test_wafv2.py

@@ -5,7 +5,7 @@
 from moto import mock_aws
 from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
 
-from .test_helper_functions import CREATE_WEB_ACL_BODY, LIST_WEB_ACL_BODY
+from .test_helper_functions import CREATE_WEB_ACL_BODY
 
 
 @mock_aws
@@ -163,18 +163,30 @@ def test_list_web_acl():
     conn = boto3.client("wafv2", region_name="us-east-1")
     conn.create_web_acl(**CREATE_WEB_ACL_BODY("Daphne", "REGIONAL"))
     conn.create_web_acl(**CREATE_WEB_ACL_BODY("Penelope", "CLOUDFRONT"))
-    conn.create_web_acl(**CREATE_WEB_ACL_BODY("Sarah", "REGIONAL"))
-    res = conn.list_web_acls(**LIST_WEB_ACL_BODY("REGIONAL"))
+    for idx in range(5):
+        conn.create_web_acl(**CREATE_WEB_ACL_BODY(f"Sarah {idx}", "REGIONAL"))
+    res = conn.list_web_acls(Scope="REGIONAL")
     web_acls = res["WebACLs"]
-    assert len(web_acls) == 2
+    assert len(web_acls) == 6
     assert web_acls[0]["Name"] == "Daphne"
-    assert web_acls[1]["Name"] == "Sarah"
+    assert web_acls[1]["Name"] == "Sarah 0"
 
-    res = conn.list_web_acls(**LIST_WEB_ACL_BODY("CLOUDFRONT"))
+    res = conn.list_web_acls(Scope="CLOUDFRONT")
     web_acls = res["WebACLs"]
     assert len(web_acls) == 1
     assert web_acls[0]["Name"] == "Penelope"
 
+    page1 = conn.list_web_acls(Scope="REGIONAL", Limit=2)
+    assert len(page1["WebACLs"]) == 2
+
+    page2 = conn.list_web_acls(
+        Scope="REGIONAL", Limit=1, NextMarker=page1["NextMarker"]
+    )
+    assert len(page2["WebACLs"]) == 1
+
+    page3 = conn.list_web_acls(Scope="REGIONAL", NextMarker=page2["NextMarker"])
+    assert len(page3["WebACLs"]) == 3
+
 
 @mock_aws
 def test_delete_web_acl():
@@ -191,7 +203,7 @@ def test_delete_web_acl():
         Name="Daphne", Id=wacl["Id"], Scope="REGIONAL", LockToken=wacl["LockToken"]
     )
 
-    res = conn.list_web_acls(**LIST_WEB_ACL_BODY("REGIONAL"))
+    res = conn.list_web_acls(Scope="REGIONAL")
     assert len(res["WebACLs"]) == 0
 
     with pytest.raises(ClientError) as exc:
@@ -347,7 +359,6 @@ def test_ip_set_crud():
             for key in ["ARN", "Description", "Id", "LockToken", "Name"]
         ]
     )
-    assert "NextMarker" in list_response
 
     client.delete_ip_set(
         Name=summary["Name"],
@@ -365,55 +376,29 @@ def test_ip_set_crud():
 
 
 @mock_aws
-def test_logging_configuration_crud():
-    wafv2_client = boto3.client("wafv2", region_name="us-east-1")
-    create_web_acl_response = wafv2_client.create_web_acl(
-        Name="TestWebACL",
-        Scope="REGIONAL",
-        DefaultAction={"Allow": {}},
-        VisibilityConfig={
-            "SampledRequestsEnabled": True,
-            "CloudWatchMetricsEnabled": True,
-            "MetricName": "TestWebACLMetric",
-        },
-        Rules=[],
-    )
-    web_acl_arn = create_web_acl_response["Summary"]["ARN"]
-
-    # Create log groups
-    logs_client = boto3.client("logs", region_name="us-east-1")
-    logs_client.create_log_group(logGroupName="aws-waf-logs-test")
-    log_group = logs_client.describe_log_groups(logGroupNamePrefix="aws-waf-logs-test")[
-        "logGroups"
-    ][0]
-
-    create_response = wafv2_client.put_logging_configuration(
-        LoggingConfiguration={
-            "ResourceArn": web_acl_arn,
-            "LogDestinationConfigs": [log_group["arn"]],
-        }
-    )
+def test_list_ip_sets_pagination():
+    client = boto3.client("wafv2", region_name="us-east-1")
 
-    assert "LoggingConfiguration" in create_response
-    logging_configuration = create_response["LoggingConfiguration"]
-    assert logging_configuration["ResourceArn"]
+    for idx in range(10):
+        client.create_ip_set(
+            Name=f"test-ip-set-{idx}",
+            Scope="CLOUDFRONT",
+            Description="Test IP set",
+            IPAddressVersion="IPV4",
+            Addresses=["192.168.0.1/32", "10.0.0.0/8"],
+            Tags=[{"Key": "Environment", "Value": "Test"}],
+        )
 
-    get_response = wafv2_client.get_logging_configuration(
-        ResourceArn=logging_configuration["ResourceArn"]
-    )
-    assert "LoggingConfiguration" in get_response
+    list_all = client.list_ip_sets(Scope="CLOUDFRONT")["IPSets"]
+    assert len(list_all) == 10
 
-    list_response = wafv2_client.list_logging_configurations(
-        Scope="REGIONAL",
-    )
-    assert len(list_response["LoggingConfigurations"]) > 0
+    page1 = client.list_ip_sets(Scope="CLOUDFRONT", Limit=2)
+    assert len(page1["IPSets"]) == 2
 
-    wafv2_client.delete_logging_configuration(
-        ResourceArn=logging_configuration["ResourceArn"],
+    page2 = client.list_ip_sets(
+        Scope="CLOUDFRONT", Limit=5, NextMarker=page1["NextMarker"]
     )
+    assert len(page2["IPSets"]) == 5
 
-    with pytest.raises(ClientError) as e:
-        wafv2_client.get_logging_configuration(
-            ResourceArn=logging_configuration["ResourceArn"]
-        )
-    assert e.value.response["Error"]["Code"] == "WAFNonexistentItemException"
+    page3 = client.list_ip_sets(Scope="CLOUDFRONT", NextMarker=page2["NextMarker"])
+    assert len(page3["IPSets"]) == 3