Implement default and max page size for V2 API

Default to 20 items per page, with support for changing the default value, and
support for showing "max" results at once, whichd defaults to 1000 results.

Change-Id: Id358f50a696c848b59b1f339c6e0cb4817dc9c9f
Partial-Bug: 1434479
Closes-Bug: 1439457

13 files changed, 99 insertions, 57 deletions

diff --git a/designate/api/v2/__init__.py b/designate/api/v2/__init__.py
index 850cf66d..faeacf42 100644
--- a/designate/api/v2/__init__.py
+++ b/designate/api/v2/__init__.py
@@ -24,7 +24,12 @@ LOG = logging.getLogger(__name__)
 
 OPTS = [
     cfg.ListOpt('enabled-extensions-v2', default=[],
-                help='Enabled API Extensions'),
+                help='Enabled API Extensions for the V2 API'),
+    cfg.IntOpt('default-limit-v2', default=20,
+               help='Default per-page limit for the V2 API, a value of None '
+                    'means show all results by default'),
+    cfg.IntOpt('max-limit-v2', default=1000,
+               help='Max per-page limit for the V2 API'),
 ]
 
 cfg.CONF.register_opts(OPTS, group='service:api')
diff --git a/designate/api/v2/controllers/blacklists.py b/designate/api/v2/controllers/blacklists.py
index 23ff2342..36c340e0 100644
--- a/designate/api/v2/controllers/blacklists.py
+++ b/designate/api/v2/controllers/blacklists.py
@@ -49,7 +49,8 @@ class BlacklistsController(rest.RestController):
         context = request.environ['context']
 
         # Extract the pagination params
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params
         accepted_filters = ('pattern')
diff --git a/designate/api/v2/controllers/pools.py b/designate/api/v2/controllers/pools.py
index d532c850..e773e0e3 100644
--- a/designate/api/v2/controllers/pools.py
+++ b/designate/api/v2/controllers/pools.py
@@ -46,7 +46,8 @@ class PoolsController(rest.RestController):
         context = request.environ['context']
 
         # Extract the pagination params
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params.
         accepted_filters = ('name')
diff --git a/designate/api/v2/controllers/recordsets.py b/designate/api/v2/controllers/recordsets.py
index b24ea7d7..513a96a6 100644
--- a/designate/api/v2/controllers/recordsets.py
+++ b/designate/api/v2/controllers/recordsets.py
@@ -55,7 +55,8 @@ class RecordSetsController(rest.RestController):
         self.central_api.get_domain(context, zone_id)
 
         # Extract the pagination params
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params.
         accepted_filters = ('name', 'type', 'ttl', 'data', )
diff --git a/designate/api/v2/controllers/rest.py b/designate/api/v2/controllers/rest.py
index 494c1f46..110948a1 100644
--- a/designate/api/v2/controllers/rest.py
+++ b/designate/api/v2/controllers/rest.py
@@ -26,7 +26,6 @@
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 import inspect
 
-import six
 import pecan
 import pecan.rest
 import pecan.routing
@@ -55,44 +54,6 @@ class RestController(pecan.rest.RestController):
     def central_api(self):
         return central_rpcapi.CentralAPI.get_instance()
 
-    def _get_paging_params(self, params):
-        """
-        Extract any paging parameters
-        """
-        marker = params.pop('marker', None)
-        limit = params.pop('limit', None)
-        sort_key = params.pop('sort_key', None)
-        sort_dir = params.pop('sort_dir', None)
-
-        # Negative and zero limits are not caught in storage.
-        # With a number bigger than MAXSIZE, rpc throws an 'OverflowError long
-        #  too big to convert'.
-        # So the parameter 'limit' is checked here.
-        if limit:
-            try:
-                invalid_limit_message = _(str.format(
-                    'limit should be an integer between 1 and {0}',
-                    six.MAXSIZE))
-                int_limit = int(limit)
-                if int_limit <= 0 or int_limit > six.MAXSIZE:
-                    raise exceptions.InvalidLimit(invalid_limit_message)
-            # This exception is raised for non ints when int(limit) is called
-            except ValueError:
-                raise exceptions.InvalidLimit(invalid_limit_message)
-
-        # sort_dir is checked in paginate_query.
-        # We duplicate the sort_dir check here to throw a more specific
-        # exception than ValueError.
-        if sort_dir and sort_dir not in ['asc', 'desc']:
-            raise exceptions.InvalidSortDir(_("Unknown sort direction, "
-                                              "must be 'desc' or 'asc'"))
-
-        if sort_key and sort_key not in self.SORT_KEYS:
-            raise exceptions.InvalidSortKey(_(str.format(
-                'sort key must be one of {0}', str(self.SORT_KEYS))))
-
-        return marker, limit, sort_key, sort_dir
-
     def _apply_filter_params(self, params, accepted_filters, criterion):
 
         for k in accepted_filters:
diff --git a/designate/api/v2/controllers/tlds.py b/designate/api/v2/controllers/tlds.py
index 1dd5263d..99c575bd 100644
--- a/designate/api/v2/controllers/tlds.py
+++ b/designate/api/v2/controllers/tlds.py
@@ -47,7 +47,8 @@ class TldsController(rest.RestController):
         context = request.environ['context']
 
         # Extract the pagination params
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params.
         accepted_filters = ('name')
diff --git a/designate/api/v2/controllers/tsigkeys.py b/designate/api/v2/controllers/tsigkeys.py
index 05d49fda..e3c5a4fd 100644
--- a/designate/api/v2/controllers/tsigkeys.py
+++ b/designate/api/v2/controllers/tsigkeys.py
@@ -48,7 +48,8 @@ class TsigKeysController(rest.RestController):
         context = request.environ['context']
 
         # Extract the pagination params
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params
         accepted_filters = ('name', 'algorithm', 'scope')
diff --git a/designate/api/v2/controllers/zones/__init__.py b/designate/api/v2/controllers/zones/__init__.py
index 13cb2f21..0148f920 100644
--- a/designate/api/v2/controllers/zones/__init__.py
+++ b/designate/api/v2/controllers/zones/__init__.py
@@ -104,7 +104,8 @@ class ZonesController(rest.RestController):
         request = pecan.request
         context = request.environ['context']
 
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params.
         accepted_filters = ('name', 'email', 'status', )
diff --git a/designate/api/v2/controllers/zones/tasks/transfer_requests.py b/designate/api/v2/controllers/zones/tasks/transfer_requests.py
index 687a3e08..8a2bf19c 100644
--- a/designate/api/v2/controllers/zones/tasks/transfer_requests.py
+++ b/designate/api/v2/controllers/zones/tasks/transfer_requests.py
@@ -49,7 +49,8 @@ class TransferRequestsController(rest.RestController):
         context = request.environ['context']
 
         # Extract the pagination params
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params.
         criterion = self._apply_filter_params(params, ('status',), {})
diff --git a/designate/objects/adapters/api_v2/base.py b/designate/objects/adapters/api_v2/base.py
index d6afb4c9..72ad87dd 100644
--- a/designate/objects/adapters/api_v2/base.py
+++ b/designate/objects/adapters/api_v2/base.py
@@ -98,7 +98,16 @@ class APIv2Adapter(base.DesignateAdapter):
             'self': cls._get_collection_href(request)
         }
         params = request.GET
-        if 'limit' in params and int(params['limit']) == len(list):
+
+        limit = cfg.CONF['service:api'].default_limit_v2
+
+        if 'limit' in params and params['limit'] == 'max':
+            limit = cfg.CONF['service:api'].max_limit_v2
+
+        elif 'limit' in params:
+            limit = int(params['limit'])
+
+        if limit is not None and limit == len(list):
             links['next'] = cls._get_next_href(request, list)
 
         return links
diff --git a/designate/utils.py b/designate/utils.py
index 63582f20..e8671879 100644
--- a/designate/utils.py
+++ b/designate/utils.py
@@ -20,6 +20,7 @@ import inspect
 import os
 import uuid
 
+import six
 import pkg_resources
 from jinja2 import Template
 from oslo.config import cfg
@@ -28,6 +29,7 @@ from oslo_log import log as logging
 from oslo_utils import timeutils
 
 from designate import exceptions
+from designate.i18n import _
 from designate.openstack.common.report import guru_meditation_report as gmr
 from designate import version as designate_version
 
@@ -398,3 +400,51 @@ def split_host_port(string, default_port=53):
         port = default_port
 
     return (host, port)
+
+
+def get_paging_params(params, sort_keys):
+    """
+    Extract any paging parameters
+    """
+    marker = params.pop('marker', None)
+    limit = params.pop('limit', cfg.CONF['service:api'].default_limit_v2)
+    sort_key = params.pop('sort_key', None)
+    sort_dir = params.pop('sort_dir', None)
+    max_limit = cfg.CONF['service:api'].max_limit_v2
+
+    if isinstance(limit, six.string_types) and limit.lower() == "max":
+        # Support for retrieving the max results at once. If set to "max",
+        # the configured max limit will be used.
+        limit = max_limit
+
+    elif limit:
+        # Negative and zero limits are not caught in storage.
+        # With a number bigger than MAXSIZE, rpc throws an 'OverflowError long
+        # too big to convert'.
+        # So the parameter 'limit' is checked here.
+        invalid_limit_message = ('limit should be an integer between 1 and '
+                                 '%(max)s' % {'max': max_limit})
+        try:
+            int_limit = int(limit)
+            if int_limit <= 0 or int_limit > six.MAXSIZE:
+                raise exceptions.InvalidLimit(invalid_limit_message)
+        # This exception is raised for non ints when int(limit) is called
+        except ValueError:
+            raise exceptions.InvalidLimit(invalid_limit_message)
+
+    # sort_dir is checked in paginate_query.
+    # We duplicate the sort_dir check here to throw a more specific
+    # exception than ValueError.
+    if sort_dir and sort_dir not in ['asc', 'desc']:
+        raise exceptions.InvalidSortDir(_("Unknown sort direction, "
+                                          "must be 'desc' or 'asc'"))
+
+    if sort_keys is None:
+        sort_key = None
+        sort_dir = None
+
+    elif sort_key and sort_key not in sort_keys:
+        msg = 'sort key must be one of %(keys)s' % {'keys': sort_keys}
+        raise exceptions.InvalidSortKey(msg)
+
+    return marker, limit, sort_key, sort_dir
diff --git a/doc/source/rest/v2/collections.rst b/doc/source/rest/v2/collections.rst
index fae3187b..5df0a86a 100644
--- a/doc/source/rest/v2/collections.rst
+++ b/doc/source/rest/v2/collections.rst
@@ -91,7 +91,10 @@ Pagination and Sorting
     using a combination of four optional query paramaters:
 
     * `marker` - denotes the ID of the last item in the previous list.
-    * `limit` - use to set the maximum number of items per page.
+    * `limit` - use to set the maximum number of items per page, use
+                "max" to return the upper limit of results as defined
+                by the operator. If not suppied, the default per page
+                limit as defined by the operator will be used.
     * `sort_key` - sorts the results by the specified attribute
 
         * By default, elements will be sorted by their creation date.
@@ -106,6 +109,8 @@ Pagination and Sorting
     set in the URI (e.g.?limit=100&marker=<UUID>). Items are sorted, as
     a default, by create time in ascending order.
 
+
+
     Collection responses will include a `links` object containing absolute
     URLs for the current and next page. These links may be omitted, or
     null, at the edges of a paginated collection.
diff --git a/etc/designate/designate.conf.sample b/etc/designate/designate.conf.sample
index 43541cb1..4c8eb13e 100644
--- a/etc/designate/designate.conf.sample
+++ b/etc/designate/designate.conf.sample
@@ -81,9 +81,22 @@ debug = False
 # Enable Version 1 API
 #enable_api_v1 = True
 
+# Enabled API Version 1 extensions
+# Can be one or more of : diagnostics, quotas, reports, sync, touch
+#enabled_extensions_v1 =
+
 # Enable Version 2 API (experimental)
 #enable_api_v2 = False
 
+# Enabled API Version 2 extensions
+#enabled_extensions_v2 =
+
+# Default page size in the V2 API
+#default_limit_v2 = 20
+
+# Max page size in the V2 API
+#max_limit_v2 = 1000
+
 # Enable Admin API (experimental)
 #enable_api_admin = False
 
@@ -96,14 +109,6 @@ debug = False
 # if an error occurs
 #pecan_debug = False
 
-# Enabled API Version 1 extensions
-# Can be one or more of : diagnostics, quotas, reports, sync, touch
-#enabled_extensions_v1 =
-
-# Enabled API Version 2 extensions
-# Can be one or more of :
-#enabled_extensions_v2 =
-
 #-----------------------
 # Keystone Middleware
 #-----------------------