summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorStan Hu <stanhu@gmail.com>2018-11-05 15:37:21 -0800
committerStan Hu <stanhu@gmail.com>2018-11-07 11:37:46 -0800
commit5b6d5301d9589b694fea0820a2b6cf165642669b (patch)
tree50a3cfa2afc1b9fb6783cb15d9582d3a429480b0 /lib
parent4068d46078faaa97acbfbe33cc7663db6d1c831a (diff)
downloadgitlab-ce-5b6d5301d9589b694fea0820a2b6cf165642669b.tar.gz
Paginate Bitbucket Server importer projects
To prevent delays in loading the page and reduce memory usage, limit the number of projects shown at 25 per page. Part of https://gitlab.com/gitlab-org/gitlab-ce/issues/50021
Diffstat (limited to 'lib')
-rw-r--r--lib/bitbucket_server/client.rb8
-rw-r--r--lib/bitbucket_server/collection.rb24
-rw-r--r--lib/bitbucket_server/paginator.rb27
3 files changed, 49 insertions, 10 deletions
diff --git a/lib/bitbucket_server/client.rb b/lib/bitbucket_server/client.rb
index 15e59f93141..83e8808db07 100644
--- a/lib/bitbucket_server/client.rb
+++ b/lib/bitbucket_server/client.rb
@@ -35,9 +35,9 @@ module BitbucketServer
BitbucketServer::Representation::Repo.new(parsed_response)
end
- def repos
+ def repos(page_offset: 0, limit: nil)
path = "/repos"
- get_collection(path, :repo)
+ get_collection(path, :repo, page_offset: page_offset, limit: limit)
end
def create_branch(project_key, repo, branch_name, sha)
@@ -61,8 +61,8 @@ module BitbucketServer
private
- def get_collection(path, type)
- paginator = BitbucketServer::Paginator.new(connection, Addressable::URI.escape(path), type)
+ def get_collection(path, type, page_offset: 0, limit: nil)
+ paginator = BitbucketServer::Paginator.new(connection, Addressable::URI.escape(path), type, page_offset: page_offset, limit: limit)
BitbucketServer::Collection.new(paginator)
rescue *SERVER_ERRORS => e
raise ServerError, e
diff --git a/lib/bitbucket_server/collection.rb b/lib/bitbucket_server/collection.rb
index b50c5dde352..7e4b2277bbe 100644
--- a/lib/bitbucket_server/collection.rb
+++ b/lib/bitbucket_server/collection.rb
@@ -2,7 +2,13 @@
module BitbucketServer
class Collection < Enumerator
+ attr_reader :paginator
+
+ delegate :page_offset, :has_next_page?, to: :paginator
+
def initialize(paginator)
+ @paginator = paginator
+
super() do |yielder|
loop do
paginator.items.each { |item| yielder << item }
@@ -12,6 +18,24 @@ module BitbucketServer
lazy
end
+ def current_page
+ return 1 if page_offset <= 1
+
+ [1, page_offset].max
+ end
+
+ def prev_page
+ return nil unless current_page > 1
+
+ current_page - 1
+ end
+
+ def next_page
+ return nil unless has_next_page?
+
+ current_page + 1
+ end
+
def method_missing(method, *args)
return super unless self.respond_to?(method)
diff --git a/lib/bitbucket_server/paginator.rb b/lib/bitbucket_server/paginator.rb
index c351fb2f11f..aa5f84f44b3 100644
--- a/lib/bitbucket_server/paginator.rb
+++ b/lib/bitbucket_server/paginator.rb
@@ -4,34 +4,49 @@ module BitbucketServer
class Paginator
PAGE_LENGTH = 25
- def initialize(connection, url, type)
+ attr_reader :page_offset
+
+ def initialize(connection, url, type, page_offset: 0, limit: nil)
@connection = connection
@type = type
@url = url
@page = nil
+ @page_offset = page_offset
+ @limit = limit || PAGE_LENGTH
+ @total = 0
end
def items
raise StopIteration unless has_next_page?
+ raise StopIteration if over_limit?
@page = fetch_next_page
+ @total += @page.items.count
@page.items
end
+ def has_next_page?
+ page.nil? || page.next?
+ end
+
private
- attr_reader :connection, :page, :url, :type
+ attr_reader :connection, :page, :url, :type, :limit
- def has_next_page?
- page.nil? || page.next?
+ def over_limit?
+ @limit.positive? && @total >= @limit
end
def next_offset
- page.nil? ? 0 : page.next
+ page.nil? ? starting_offset : page.next
+ end
+
+ def starting_offset
+ [0, page_offset - 1].max * limit
end
def fetch_next_page
- parsed_response = connection.get(@url, start: next_offset, limit: PAGE_LENGTH)
+ parsed_response = connection.get(@url, start: next_offset, limit: @limit)
Page.new(parsed_response, type)
end
end