Merge "Implement default and max page size for V2 API"

13 files changed, 99 insertions, 57 deletions

diff --git a/designate/api/v2/__init__.py b/designate/api/v2/__init__.py
index 850cf66d..faeacf42 100644
--- a/designate/api/v2/__init__.py
+++ b/designate/api/v2/__init__.py
@@ -24,7 +24,12 @@ LOG = logging.getLogger(__name__)
 
 OPTS = [
     cfg.ListOpt('enabled-extensions-v2', default=[],
-                help='Enabled API Extensions'),
+                help='Enabled API Extensions for the V2 API'),
+    cfg.IntOpt('default-limit-v2', default=20,
+               help='Default per-page limit for the V2 API, a value of None '
+                    'means show all results by default'),
+    cfg.IntOpt('max-limit-v2', default=1000,
+               help='Max per-page limit for the V2 API'),
 ]
 
 cfg.CONF.register_opts(OPTS, group='service:api')
diff --git a/designate/api/v2/controllers/blacklists.py b/designate/api/v2/controllers/blacklists.py
index 23ff2342..36c340e0 100644
--- a/designate/api/v2/controllers/blacklists.py
+++ b/designate/api/v2/controllers/blacklists.py
@@ -49,7 +49,8 @@ class BlacklistsController(rest.RestController):
         context = request.environ['context']
 
         # Extract the pagination params
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params
         accepted_filters = ('pattern')
diff --git a/designate/api/v2/controllers/pools.py b/designate/api/v2/controllers/pools.py
index d532c850..e773e0e3 100644
--- a/designate/api/v2/controllers/pools.py
+++ b/designate/api/v2/controllers/pools.py
@@ -46,7 +46,8 @@ class PoolsController(rest.RestController):
         context = request.environ['context']
 
         # Extract the pagination params
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params.
         accepted_filters = ('name')
diff --git a/designate/api/v2/controllers/recordsets.py b/designate/api/v2/controllers/recordsets.py
index b24ea7d7..513a96a6 100644
--- a/designate/api/v2/controllers/recordsets.py
+++ b/designate/api/v2/controllers/recordsets.py
@@ -55,7 +55,8 @@ class RecordSetsController(rest.RestController):
         self.central_api.get_domain(context, zone_id)
 
         # Extract the pagination params
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params.
         accepted_filters = ('name', 'type', 'ttl', 'data', )
diff --git a/designate/api/v2/controllers/rest.py b/designate/api/v2/controllers/rest.py
index 494c1f46..110948a1 100644
--- a/designate/api/v2/controllers/rest.py
+++ b/designate/api/v2/controllers/rest.py
@@ -26,7 +26,6 @@
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 import inspect
 
-import six
 import pecan
 import pecan.rest
 import pecan.routing
@@ -55,44 +54,6 @@ class RestController(pecan.rest.RestController):
     def central_api(self):
         return central_rpcapi.CentralAPI.get_instance()
 
-    def _get_paging_params(self, params):
-        """
-        Extract any paging parameters
-        """
-        marker = params.pop('marker', None)
-        limit = params.pop('limit', None)
-        sort_key = params.pop('sort_key', None)
-        sort_dir = params.pop('sort_dir', None)
-
-        # Negative and zero limits are not caught in storage.
-        # With a number bigger than MAXSIZE, rpc throws an 'OverflowError long
-        #  too big to convert'.
-        # So the parameter 'limit' is checked here.
-        if limit:
-            try:
-                invalid_limit_message = _(str.format(
-                    'limit should be an integer between 1 and {0}',
-                    six.MAXSIZE))
-                int_limit = int(limit)
-                if int_limit <= 0 or int_limit > six.MAXSIZE:
-                    raise exceptions.InvalidLimit(invalid_limit_message)
-            # This exception is raised for non ints when int(limit) is called
-            except ValueError:
-                raise exceptions.InvalidLimit(invalid_limit_message)
-
-        # sort_dir is checked in paginate_query.
-        # We duplicate the sort_dir check here to throw a more specific
-        # exception than ValueError.
-        if sort_dir and sort_dir not in ['asc', 'desc']:
-            raise exceptions.InvalidSortDir(_("Unknown sort direction, "
-                                              "must be 'desc' or 'asc'"))
-
-        if sort_key and sort_key not in self.SORT_KEYS:
-            raise exceptions.InvalidSortKey(_(str.format(
-                'sort key must be one of {0}', str(self.SORT_KEYS))))
-
-        return marker, limit, sort_key, sort_dir
-
     def _apply_filter_params(self, params, accepted_filters, criterion):
 
         for k in accepted_filters:
diff --git a/designate/api/v2/controllers/tlds.py b/designate/api/v2/controllers/tlds.py
index 1dd5263d..99c575bd 100644
--- a/designate/api/v2/controllers/tlds.py
+++ b/designate/api/v2/controllers/tlds.py
@@ -47,7 +47,8 @@ class TldsController(rest.RestController):
         context = request.environ['context']
 
         # Extract the pagination params
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params.
         accepted_filters = ('name')
diff --git a/designate/api/v2/controllers/tsigkeys.py b/designate/api/v2/controllers/tsigkeys.py
index 05d49fda..e3c5a4fd 100644
--- a/designate/api/v2/controllers/tsigkeys.py
+++ b/designate/api/v2/controllers/tsigkeys.py
@@ -48,7 +48,8 @@ class TsigKeysController(rest.RestController):
         context = request.environ['context']
 
         # Extract the pagination params
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params
         accepted_filters = ('name', 'algorithm', 'scope')
diff --git a/designate/api/v2/controllers/zones/__init__.py b/designate/api/v2/controllers/zones/__init__.py
index 13cb2f21..0148f920 100644
--- a/designate/api/v2/controllers/zones/__init__.py
+++ b/designate/api/v2/controllers/zones/__init__.py
@@ -104,7 +104,8 @@ class ZonesController(rest.RestController):
         request = pecan.request
         context = request.environ['context']
 
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params.
         accepted_filters = ('name', 'email', 'status', )
diff --git a/designate/api/v2/controllers/zones/tasks/transfer_requests.py b/designate/api/v2/controllers/zones/tasks/transfer_requests.py
index 687a3e08..8a2bf19c 100644
--- a/designate/api/v2/controllers/zones/tasks/transfer_requests.py
+++ b/designate/api/v2/controllers/zones/tasks/transfer_requests.py
@@ -49,7 +49,8 @@ class TransferRequestsController(rest.RestController):
         context = request.environ['context']
 
         # Extract the pagination params
-        marker, limit, sort_key, sort_dir = self._get_paging_params(params)
+        marker, limit, sort_key, sort_dir = utils.get_paging_params(
+            params, self.SORT_KEYS)
 
         # Extract any filter params.
         criterion = self._apply_filter_params(params, ('status',), {})
diff --git a/designate/objects/adapters/api_v2/base.py b/designate/objects/adapters/api_v2/base.py
index d6afb4c9..72ad87dd 100644
--- a/designate/objects/adapters/api_v2/base.py
+++ b/designate/objects/adapters/api_v2/base.py
@@ -98,7 +98,16 @@ class APIv2Adapter(base.DesignateAdapter):
             'self': cls._get_collection_href(request)
         }
         params = request.GET
-        if 'limit' in params and int(params['limit']) == len(list):
+
+        limit = cfg.CONF['service:api'].default_limit_v2
+
+        if 'limit' in params and params['limit'] == 'max':
+            limit = cfg.CONF['service:api'].max_limit_v2
+
+        elif 'limit' in params:
+            limit = int(params['limit'])
+
+        if limit is not None and limit == len(list):
             links['next'] = cls._get_next_href(request, list)
 
         return links
diff --git a/designate/utils.py b/designate/utils.py
index 63582f20..e8671879 100644
--- a/designate/utils.py
+++ b/designate/utils.py
@@ -20,6 +20,7 @@ import inspect
 import os
 import uuid
 
+import six
 import pkg_resources
 from jinja2 import Template
 from oslo.config import cfg
@@ -28,6 +29,7 @@ from oslo_log import log as logging
 from oslo_utils import timeutils
 
 from designate import exceptions
+from designate.i18n import _
 from designate.openstack.common.report import guru_meditation_report as gmr
 from designate import version as designate_version
 
@@ -398,3 +400,51 @@ def split_host_port(string, default_port=53):
         port = default_port
 
     return (host, port)
+
+
+def get_paging_params(params, sort_keys):
+    """
+    Extract any paging parameters
+    """
+    marker = params.pop('marker', None)
+    limit = params.pop('limit', cfg.CONF['service:api'].default_limit_v2)
+    sort_key = params.pop('sort_key', None)
+    sort_dir = params.pop('sort_dir', None)
+    max_limit = cfg.CONF['service:api'].max_limit_v2
+
+    if isinstance(limit, six.string_types) and limit.lower() == "max":
+        # Support for retrieving the max results at once. If set to "max",
+        # the configured max limit will be used.
+        limit = max_limit
+
+    elif limit:
+        # Negative and zero limits are not caught in storage.
+        # With a number bigger than MAXSIZE, rpc throws an 'OverflowError long
+        # too big to convert'.
+        # So the parameter 'limit' is checked here.
+        invalid_limit_message = ('limit should be an integer between 1 and '
+                                 '%(max)s' % {'max': max_limit})
+        try:
+            int_limit = int(limit)
+            if int_limit <= 0 or int_limit > six.MAXSIZE:
+                raise exceptions.InvalidLimit(invalid_limit_message)
+        # This exception is raised for non ints when int(limit) is called
+        except ValueError:
+            raise exceptions.InvalidLimit(invalid_limit_message)
+
+    # sort_dir is checked in paginate_query.
+    # We duplicate the sort_dir check here to throw a more specific
+    # exception than ValueError.
+    if sort_dir and sort_dir not in ['asc', 'desc']:
+        raise exceptions.InvalidSortDir(_("Unknown sort direction, "
+                                          "must be 'desc' or 'asc'"))
+
+    if sort_keys is None:
+        sort_key = None
+        sort_dir = None
+
+    elif sort_key and sort_key not in sort_keys:
+        msg = 'sort key must be one of %(keys)s' % {'keys': sort_keys}
+        raise exceptions.InvalidSortKey(msg)
+
+    return marker, limit, sort_key, sort_dir
diff --git a/doc/source/rest/v2/collections.rst b/doc/source/rest/v2/collections.rst
index fae3187b..5df0a86a 100644
--- a/doc/source/rest/v2/collections.rst
+++ b/doc/source/rest/v2/collections.rst
@@ -91,7 +91,10 @@ Pagination and Sorting
     using a combination of four optional query paramaters:
 
     * `marker` - denotes the ID of the last item in the previous list.
-    * `limit` - use to set the maximum number of items per page.
+    * `limit` - use to set the maximum number of items per page, use
+                "max" to return the upper limit of results as defined
+                by the operator. If not suppied, the default per page
+                limit as defined by the operator will be used.
     * `sort_key` - sorts the results by the specified attribute
 
         * By default, elements will be sorted by their creation date.
@@ -106,6 +109,8 @@ Pagination and Sorting
     set in the URI (e.g.?limit=100&marker=<UUID>). Items are sorted, as
     a default, by create time in ascending order.
 
+
+
     Collection responses will include a `links` object containing absolute
     URLs for the current and next page. These links may be omitted, or
     null, at the edges of a paginated collection.
diff --git a/etc/designate/designate.conf.sample b/etc/designate/designate.conf.sample
index 43541cb1..4c8eb13e 100644
--- a/etc/designate/designate.conf.sample
+++ b/etc/designate/designate.conf.sample
@@ -81,9 +81,22 @@ debug = False
 # Enable Version 1 API
 #enable_api_v1 = True
 
+# Enabled API Version 1 extensions
+# Can be one or more of : diagnostics, quotas, reports, sync, touch
+#enabled_extensions_v1 =
+
 # Enable Version 2 API (experimental)
 #enable_api_v2 = False
 
+# Enabled API Version 2 extensions
+#enabled_extensions_v2 =
+
+# Default page size in the V2 API
+#default_limit_v2 = 20
+
+# Max page size in the V2 API
+#max_limit_v2 = 1000
+
 # Enable Admin API (experimental)
 #enable_api_admin = False
 
@@ -96,14 +109,6 @@ debug = False
 # if an error occurs
 #pecan_debug = False
 
-# Enabled API Version 1 extensions
-# Can be one or more of : diagnostics, quotas, reports, sync, touch
-#enabled_extensions_v1 =
-
-# Enabled API Version 2 extensions
-# Can be one or more of :
-#enabled_extensions_v2 =
-
 #-----------------------
 # Keystone Middleware
 #-----------------------