Merge branch 'ab-49754-gh-importer-internal-ids' into 'master'

GitHub importer: Keep track of internal_ids

Closes #49754

See merge request gitlab-org/gitlab-ce!20926

4 files changed, 26 insertions, 4 deletions

diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb
index 147597289cf..da2f96b5c4b 100644
--- a/lib/gitlab/github_import/bulk_importing.rb
+++ b/lib/gitlab/github_import/bulk_importing.rb
@@ -15,10 +15,12 @@ module Gitlab
       end
 
       # Bulk inserts the given rows into the database.
-      def bulk_insert(model, rows, batch_size: 100)
+      def bulk_insert(model, rows, batch_size: 100, pre_hook: nil)
         rows.each_slice(batch_size) do |slice|
+          pre_hook.call(slice) if pre_hook
           Gitlab::Database.bulk_insert(model.table_name, slice)
         end
+        rows
       end
     end
   end
diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb
index 31fefebf787..ead4215810f 100644
--- a/lib/gitlab/github_import/importer/issue_importer.rb
+++ b/lib/gitlab/github_import/importer/issue_importer.rb
@@ -55,7 +55,11 @@ module Gitlab
             updated_at: issue.updated_at
           }
 
-          GithubImport.insert_and_return_id(attributes, project.issues)
+          GithubImport.insert_and_return_id(attributes, project.issues).tap do |id|
+            # We use .insert_and_return_id which effectively disables all callbacks.
+            # Trigger iid logic here to make sure we track internal id values consistently.
+            project.issues.find(id).ensure_project_iid!
+          end
         rescue ActiveRecord::InvalidForeignKey
           # It's possible the project has been deleted since scheduling this
           # job. In this case we'll just skip creating the issue.
diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb
index c53480e828a..94eb9136b9a 100644
--- a/lib/gitlab/github_import/importer/milestones_importer.rb
+++ b/lib/gitlab/github_import/importer/milestones_importer.rb
@@ -17,10 +17,20 @@ module Gitlab
         end
 
         def execute
-          bulk_insert(Milestone, build_milestones)
+          # We insert records in bulk, by-passing any standard model callbacks.
+          # The pre_hook here makes sure we track internal ids consistently.
+          # Note this has to be called before performing an insert of a batch
+          # because we're outside a transaction scope here.
+          bulk_insert(Milestone, build_milestones, pre_hook: method(:track_greatest_iid))
           build_milestones_cache
         end
 
+        def track_greatest_iid(slice)
+          greatest_iid = slice.max { |e| e[:iid] }[:iid]
+
+          InternalId.track_greatest(nil, { project: project }, :milestones, greatest_iid, ->(_) { project.milestones.maximum(:iid) })
+        end
+
         def build_milestones
           build_database_rows(each_milestone)
         end
diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb
index 6b3688c4381..e4b49d2143a 100644
--- a/lib/gitlab/github_import/importer/pull_request_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_request_importer.rb
@@ -76,7 +76,13 @@ module Gitlab
             merge_request_id = GithubImport
               .insert_and_return_id(attributes, project.merge_requests)
 
-            [project.merge_requests.find(merge_request_id), false]
+            merge_request = project.merge_requests.find(merge_request_id)
+
+            # We use .insert_and_return_id which effectively disables all callbacks.
+            # Trigger iid logic here to make sure we track internal id values consistently.
+            merge_request.ensure_target_project_iid!
+
+            [merge_request, false]
           end
         rescue ActiveRecord::InvalidForeignKey
           # It's possible the project has been deleted since scheduling this