Merge branch 'fix-bitbucket-import-issue-order' into 'master'

Import all Bitbucket issues and sort issues by creation date

Previously only the first 15 Bitbucket issues would be imported. Other issues in the API return call also calls import errors.

Tested by importing 400+ issues from this repo: https://bitbucket.org/ned/coveragepy

API reference here: https://bitbucket.org/site/master/issues/3571/api-issues-sorting-bb-3518

Closes https://github.com/gitlabhq/gitlabhq/issues/9519

See merge request !1226

5 files changed, 54 insertions, 13 deletions

diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb
index aec44b8c87b..d88a6eaac6b 100644
--- a/lib/gitlab/bitbucket_import/client.rb
+++ b/lib/gitlab/bitbucket_import/client.rb
@@ -52,11 +52,26 @@ module Gitlab
       end
 
       def issues(project_identifier)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues").body)
+        all_issues = []
+        offset = 0
+        per_page = 50  # Maximum number allowed by Bitbucket
+        index = 0
+
+        begin
+          issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body)
+          # Find out how many total issues are present
+          total = issues["count"] if index == 0
+          all_issues.concat(issues["issues"])
+          offset += issues["issues"].count
+          index += 1
+        end while all_issues.count < total
+
+        all_issues
       end
 
       def issue_comments(project_identifier, issue_id)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
+        comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
+        comments.sort_by { |comment| comment["utc_created_on"] }
       end
 
       def project(project_identifier)
@@ -100,6 +115,10 @@ module Gitlab
         response
       end
 
+      def issue_api_endpoint(project_identifier, per_page, offset)
+        "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}"
+      end
+
       def config
         Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"}
       end
diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index d8a7d29f1bf..2355b3c6ddc 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -20,8 +20,18 @@ module Gitlab
         #Issues && Comments
         issues = client.issues(project_identifier)
 
-        issues["issues"].each do |issue|
-          body = @formatter.author_line(issue["reported_by"]["username"], issue["content"])
+        issues.each do |issue|
+          body = ''
+          reporter = nil
+          author = 'Anonymous'
+
+          if issue["reported_by"] && issue["reported_by"]["username"]
+            reporter = issue["reported_by"]["username"]
+            author = reporter
+          end
+
+          body = @formatter.author_line(author)
+          body += issue["content"]
 
           comments = client.issue_comments(project_identifier, issue["local_id"])
 
@@ -30,14 +40,20 @@ module Gitlab
           end
 
           comments.each do |comment|
-            body += @formatter.comment(comment["author_info"]["username"], comment["utc_created_on"], comment["content"])
+            author = 'Anonymous'
+
+            if comment["author_info"] && comment["author_info"]["username"]
+              author = comment["author_info"]["username"]
+            end
+
+            body += @formatter.comment(author, comment["utc_created_on"], comment["content"])
           end
 
           project.issues.create!(
             description: body,
             title: issue["title"],
             state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened',
-            author_id: gl_user_id(project, issue["reported_by"]["username"])
+            author_id: gl_user_id(project, reporter)
           )
         end
 
@@ -47,9 +63,13 @@ module Gitlab
       private
 
       def gl_user_id(project, bitbucket_id)
-        user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
-        (user && user.id) || project.creator_id
-      end
+        if bitbucket_id
+          user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
+          (user && user.id) || project.creator_id
+        else
+          project.creator_id
+        end
+       end
     end
   end
 end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 8c106a61735..bd7340a80f1 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -18,7 +18,8 @@ module Gitlab
                                                   direction: :asc).each do |issue|
           if issue.pull_request.nil?
 
-            body = @formatter.author_line(issue.user.login, issue.body)
+            body = @formatter.author_line(issue.user.login)
+            body += issue.body
 
             if issue.comments > 0
               body += @formatter.comments_header
diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb
index 50594d2b24f..e24b94d6159 100644
--- a/lib/gitlab/gitlab_import/importer.rb
+++ b/lib/gitlab/gitlab_import/importer.rb
@@ -18,7 +18,8 @@ module Gitlab
         issues = client.issues(project_identifier)
 
         issues.each do |issue|
-          body = @formatter.author_line(issue["author"]["name"], issue["description"])
+          body = @formatter.author_line(issue["author"]["name"])
+          body += issue["description"]
 
           comments = client.issue_comments(project_identifier, issue["id"])
 
diff --git a/lib/gitlab/import_formatter.rb b/lib/gitlab/import_formatter.rb
index 72e041a90b1..3e54456e936 100644
--- a/lib/gitlab/import_formatter.rb
+++ b/lib/gitlab/import_formatter.rb
@@ -8,8 +8,8 @@ module Gitlab
       "\n\n\n**Imported comments:**\n"
     end
 
-    def author_line(author, body)
-      "*Created by: #{author}*\n\n#{body}"
+    def author_line(author)
+      "*Created by: #{author}*\n\n"
     end
   end
 end