Fix bug where only the first 15 Bitbucket issues would be imported.

Also fix a number of issues where author/reporter names were not available.

5 files changed, 54 insertions, 13 deletions

diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb
index bb8bf7e3a98..d88a6eaac6b 100644
--- a/lib/gitlab/bitbucket_import/client.rb
+++ b/lib/gitlab/bitbucket_import/client.rb
@@ -52,11 +52,26 @@ module Gitlab
       end
 
       def issues(project_identifier)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on").body)
+        all_issues = []
+        offset = 0
+        per_page = 50  # Maximum number allowed by Bitbucket
+        index = 0
+
+        begin
+          issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body)
+          # Find out how many total issues are present
+          total = issues["count"] if index == 0
+          all_issues.concat(issues["issues"])
+          offset += issues["issues"].count
+          index += 1
+        end while all_issues.count < total
+
+        all_issues
       end
 
       def issue_comments(project_identifier, issue_id)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
+        comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
+        comments.sort_by { |comment| comment["utc_created_on"] }
       end
 
       def project(project_identifier)
@@ -100,6 +115,10 @@ module Gitlab
         response
       end
 
+      def issue_api_endpoint(project_identifier, per_page, offset)
+        "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}"
+      end
+
       def config
         Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"}
       end
diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index d8a7d29f1bf..2355b3c6ddc 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -20,8 +20,18 @@ module Gitlab
         #Issues && Comments
         issues = client.issues(project_identifier)
 
-        issues["issues"].each do |issue|
-          body = @formatter.author_line(issue["reported_by"]["username"], issue["content"])
+        issues.each do |issue|
+          body = ''
+          reporter = nil
+          author = 'Anonymous'
+
+          if issue["reported_by"] && issue["reported_by"]["username"]
+            reporter = issue["reported_by"]["username"]
+            author = reporter
+          end
+
+          body = @formatter.author_line(author)
+          body += issue["content"]
 
           comments = client.issue_comments(project_identifier, issue["local_id"])
 
@@ -30,14 +40,20 @@ module Gitlab
           end
 
           comments.each do |comment|
-            body += @formatter.comment(comment["author_info"]["username"], comment["utc_created_on"], comment["content"])
+            author = 'Anonymous'
+
+            if comment["author_info"] && comment["author_info"]["username"]
+              author = comment["author_info"]["username"]
+            end
+
+            body += @formatter.comment(author, comment["utc_created_on"], comment["content"])
           end
 
           project.issues.create!(
             description: body,
             title: issue["title"],
             state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened',
-            author_id: gl_user_id(project, issue["reported_by"]["username"])
+            author_id: gl_user_id(project, reporter)
           )
         end
 
@@ -47,9 +63,13 @@ module Gitlab
       private
 
       def gl_user_id(project, bitbucket_id)
-        user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
-        (user && user.id) || project.creator_id
-      end
+        if bitbucket_id
+          user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
+          (user && user.id) || project.creator_id
+        else
+          project.creator_id
+        end
+       end
     end
   end
 end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 8c106a61735..bd7340a80f1 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -18,7 +18,8 @@ module Gitlab
                                                   direction: :asc).each do |issue|
           if issue.pull_request.nil?
 
-            body = @formatter.author_line(issue.user.login, issue.body)
+            body = @formatter.author_line(issue.user.login)
+            body += issue.body
 
             if issue.comments > 0
               body += @formatter.comments_header
diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb
index 50594d2b24f..e24b94d6159 100644
--- a/lib/gitlab/gitlab_import/importer.rb
+++ b/lib/gitlab/gitlab_import/importer.rb
@@ -18,7 +18,8 @@ module Gitlab
         issues = client.issues(project_identifier)
 
         issues.each do |issue|
-          body = @formatter.author_line(issue["author"]["name"], issue["description"])
+          body = @formatter.author_line(issue["author"]["name"])
+          body += issue["description"]
 
           comments = client.issue_comments(project_identifier, issue["id"])
 
diff --git a/lib/gitlab/import_formatter.rb b/lib/gitlab/import_formatter.rb
index 72e041a90b1..3e54456e936 100644
--- a/lib/gitlab/import_formatter.rb
+++ b/lib/gitlab/import_formatter.rb
@@ -8,8 +8,8 @@ module Gitlab
       "\n\n\n**Imported comments:**\n"
     end
 
-    def author_line(author, body)
-      "*Created by: #{author}*\n\n#{body}"
+    def author_line(author)
+      "*Created by: #{author}*\n\n"
     end
   end
 end