From 6aae8aee5b00b894b731a877ebbb3a1438039a5f Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 30 May 2017 23:24:21 -0400 Subject: [PATCH] Closes #1237: Enabled setting limit=0 to disable pagination in API requests; added MAX_PAGE_SIZE configuration setting --- docs/api/overview.md | 5 +++ docs/configuration/optional-settings.md | 8 ++++ netbox/netbox/configuration.example.py | 5 +++ netbox/netbox/settings.py | 3 +- netbox/utilities/api.py | 49 +++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 1 deletion(-) diff --git a/docs/api/overview.md b/docs/api/overview.md index 5f8e43973a..a9ad115f89 100644 --- a/docs/api/overview.md +++ b/docs/api/overview.md @@ -136,3 +136,8 @@ The response will return devices 1 through 100. The URL provided in the `next` a "results": [...] } ``` + +The maximum number of objects that can be returned is limited by the [`MAX_PAGE_SIZE`](../configuration/optional-settings/#max_page_size) setting, which is 1000 by default. Setting this to `0` or `None` will remove the maximum limit. An API consumer can then pass `?limit=0` to retrieve _all_ matching objects with a single request. + +!!! warning + Disabling the page size limit introduces a potential for very resource-intensive requests, since one API request can effectively retrieve an entire table from the database. diff --git a/docs/configuration/optional-settings.md b/docs/configuration/optional-settings.md index ed5d2c03c5..6c68ca386b 100644 --- a/docs/configuration/optional-settings.md +++ b/docs/configuration/optional-settings.md @@ -99,6 +99,14 @@ Setting this to True will display a "maintenance mode" banner at the top of ever --- +## MAX_PAGE_SIZE + +Default: 1000 + +An API consumer can request an arbitrary number of objects by appending the "limit" parameter to the URL (e.g. `?limit=1000`). This setting defines the maximum limit. Setting it to `0` or `None` will allow an API consumer to request all objects by specifying `?limit=0`. + +--- + ## NETBOX_USERNAME ## NETBOX_PASSWORD diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index f185a68c78..bc255bac36 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -79,6 +79,11 @@ # Setting this to True will display a "maintenance mode" banner at the top of every page. MAINTENANCE_MODE = False +# An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g. +# "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request +# all objects by specifying "?limit=0". +MAX_PAGE_SIZE = 1000 + # Credentials that NetBox will use to access live devices (future use). NETBOX_USERNAME = '' NETBOX_PASSWORD = '' diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 7eec504756..acdf8e38b8 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -48,6 +48,7 @@ BANNER_BOTTOM = getattr(configuration, 'BANNER_BOTTOM', False) PREFER_IPV4 = getattr(configuration, 'PREFER_IPV4', False) ENFORCE_GLOBAL_UNIQUE = getattr(configuration, 'ENFORCE_GLOBAL_UNIQUE', False) +MAX_PAGE_SIZE = getattr(configuration, 'MAX_PAGE_SIZE', 1000) CORS_ORIGIN_ALLOW_ALL = getattr(configuration, 'CORS_ORIGIN_ALLOW_ALL', False) CORS_ORIGIN_WHITELIST = getattr(configuration, 'CORS_ORIGIN_WHITELIST', []) CORS_ORIGIN_REGEX_WHITELIST = getattr(configuration, 'CORS_ORIGIN_REGEX_WHITELIST', []) @@ -208,7 +209,7 @@ 'DEFAULT_FILTER_BACKENDS': ( 'rest_framework.filters.DjangoFilterBackend', ), - 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', + 'DEFAULT_PAGINATION_CLASS': 'utilities.api.OptionalLimitOffsetPagination', 'DEFAULT_PERMISSION_CLASSES': ( 'utilities.api.TokenPermissions', ), diff --git a/netbox/utilities/api.py b/netbox/utilities/api.py index a587c67d1f..6fcfc69494 100644 --- a/netbox/utilities/api.py +++ b/netbox/utilities/api.py @@ -5,6 +5,7 @@ from rest_framework import authentication, exceptions from rest_framework.exceptions import APIException +from rest_framework.pagination import LimitOffsetPagination from rest_framework.permissions import DjangoModelPermissions, SAFE_METHODS from rest_framework.serializers import Field, ValidationError @@ -105,3 +106,51 @@ def get_serializer_class(self): if self.action in WRITE_OPERATIONS and hasattr(self, 'write_serializer_class'): return self.write_serializer_class return self.serializer_class + + +class OptionalLimitOffsetPagination(LimitOffsetPagination): + """ + Override the stock paginator to allow setting limit=0 to disable pagination for a request. This returns all objects + matching a query, but retains the same format as a paginated request. The limit can only be disabled if + MAX_PAGE_SIZE has been set to 0 or None. + """ + + def paginate_queryset(self, queryset, request, view=None): + + try: + self.count = queryset.count() + except (AttributeError, TypeError): + self.count = len(queryset) + self.limit = self.get_limit(request) + self.offset = self.get_offset(request) + self.request = request + + if self.limit and self.count > self.limit and self.template is not None: + self.display_page_controls = True + + if self.count == 0 or self.offset > self.count: + return list() + + if self.limit: + return list(queryset[self.offset:self.offset + self.limit]) + else: + return list(queryset[self.offset:]) + + def get_limit(self, request): + + if self.limit_query_param: + try: + limit = int(request.query_params[self.limit_query_param]) + if limit < 0: + raise ValueError() + # Enforce maximum page size, if defined + if settings.MAX_PAGE_SIZE: + if limit == 0: + return settings.MAX_PAGE_SIZE + else: + return min(limit, settings.MAX_PAGE_SIZE) + return limit + except (KeyError, ValueError): + pass + + return self.default_limit