Add latest changes from gitlab-org/gitlab@master

6 files changed, 197 insertions, 2 deletions

diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb
index b950bfb0f3a..60a50e97998 100644
--- a/lib/gitlab/application_context.rb
+++ b/lib/gitlab/application_context.rb
@@ -5,13 +5,14 @@ module Gitlab
   class ApplicationContext
     include Gitlab::Utils::LazyAttributes
 
-    Attribute = Struct.new(:name, :type, :evaluation)
+    Attribute = Struct.new(:name, :type)
 
     APPLICATION_ATTRIBUTES = [
       Attribute.new(:project, Project),
       Attribute.new(:namespace, Namespace),
       Attribute.new(:user, User),
-      Attribute.new(:caller_id, String)
+      Attribute.new(:caller_id, String),
+      Attribute.new(:related_class, String)
     ].freeze
 
     def self.with_context(args, &block)
@@ -39,6 +40,7 @@ module Gitlab
         hash[:project] = -> { project_path } if set_values.include?(:project)
         hash[:root_namespace] = -> { root_namespace_path } if include_namespace?
         hash[:caller_id] = caller_id if set_values.include?(:caller_id)
+        hash[:related_class] = related_class if set_values.include?(:related_class)
       end
     end
 
diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb
index ead94761ae7..7f2d2858149 100644
--- a/lib/gitlab/cache/import/caching.rb
+++ b/lib/gitlab/cache/import/caching.rb
@@ -70,6 +70,21 @@ module Gitlab
           value
         end
 
+        # Increment the integer value of a key by one.
+        # Sets the value to zero if missing before incrementing
+        #
+        # key - The cache key to increment.
+        # timeout - The time after which the cache key should expire.
+        # @return - the incremented value
+        def self.increment(raw_key, timeout: TIMEOUT)
+          key = cache_key_for(raw_key)
+
+          Redis::Cache.with do |redis|
+            redis.incr(key)
+            redis.expire(key, timeout)
+          end
+        end
+
         # Adds a value to a set.
         #
         # raw_key - The key of the set to add the value to.
diff --git a/lib/gitlab/jira_import.rb b/lib/gitlab/jira_import.rb
new file mode 100644
index 00000000000..1486c754caf
--- /dev/null
+++ b/lib/gitlab/jira_import.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    JIRA_IMPORT_CACHE_TIMEOUT = 10.seconds.to_i
+
+    FAILED_ISSUES_COUNTER_KEY = 'jira-import/failed/%{project_id}/%{collection_type}'
+    NEXT_ITEMS_START_AT_KEY = 'jira-import/paginator/%{project_id}/%{collection_type}'
+    ITEMS_MAPPER_CACHE_KEY = 'jira-import/items-mapper/%{project_id}/%{collection_type}/%{jira_isssue_id}'
+    ALREADY_IMPORTED_ITEMS_CACHE_KEY = 'jira-importer/already-imported/%{project}/%{collection_type}'
+
+    def self.jira_issue_cache_key(project_id, jira_issue_id)
+      ITEMS_MAPPER_CACHE_KEY % { project_id: project_id, collection_type: :issues, jira_isssue_id: jira_issue_id }
+    end
+
+    def self.already_imported_cache_key(collection_type, project_id)
+      ALREADY_IMPORTED_ITEMS_CACHE_KEY % { collection_type: collection_type, project: project_id }
+    end
+
+    def self.jira_issues_next_page_cache_key(project_id)
+      NEXT_ITEMS_START_AT_KEY % { project_id: project_id, collection_type: :issues }
+    end
+
+    def self.failed_issues_counter_cache_key(project_id)
+      FAILED_ISSUES_COUNTER_KEY % { project_id: project_id, collection_type: :issues }
+    end
+
+    def self.increment_issue_failures(project_id)
+      Gitlab::Cache::Import::Caching.increment(self.failed_issues_counter_cache_key(project_id))
+    end
+
+    def self.get_issues_next_start_at(project_id)
+      Gitlab::Cache::Import::Caching.read(self.jira_issues_next_page_cache_key(project_id)).to_i
+    end
+
+    def self.store_issues_next_started_at(project_id, value)
+      cache_key = self.jira_issues_next_page_cache_key(project_id)
+      Gitlab::Cache::Import::Caching.write(cache_key, value)
+    end
+
+    def self.cache_cleanup(project_id)
+      Gitlab::Cache::Import::Caching.expire(self.failed_issues_counter_cache_key(project_id), JIRA_IMPORT_CACHE_TIMEOUT)
+      Gitlab::Cache::Import::Caching.expire(self.jira_issues_next_page_cache_key(project_id), JIRA_IMPORT_CACHE_TIMEOUT)
+      Gitlab::Cache::Import::Caching.expire(self.already_imported_cache_key(:issues, project_id), JIRA_IMPORT_CACHE_TIMEOUT)
+    end
+  end
+end
diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb
new file mode 100644
index 00000000000..24158b8e734
--- /dev/null
+++ b/lib/gitlab/jira_import/base_importer.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    class BaseImporter
+      attr_reader :project, :client, :formatter, :jira_project_key
+
+      def initialize(project)
+        raise Projects::ImportService::Error, _('Jira import feature is disabled.') unless Feature.enabled?(:jira_issue_import, project)
+        raise Projects::ImportService::Error, _('Jira integration not configured.') unless project.jira_service&.active?
+
+        @jira_project_key = project&.import_data&.becomes(JiraImportData)&.current_project&.key
+        raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key
+
+        @project = project
+        @client = project.jira_service.client
+        @formatter = Gitlab::ImportFormatter.new
+      end
+
+      private
+
+      def imported_items_cache_key
+        raise NotImplementedError
+      end
+
+      def mark_as_imported(id)
+        Gitlab::Cache::Import::Caching.set_add(imported_items_cache_key, id)
+      end
+
+      def already_imported?(id)
+        Gitlab::Cache::Import::Caching.set_includes?(imported_items_cache_key, id)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/jira_import/issue_serializer.rb b/lib/gitlab/jira_import/issue_serializer.rb
new file mode 100644
index 00000000000..f00852a21a2
--- /dev/null
+++ b/lib/gitlab/jira_import/issue_serializer.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    class IssueSerializer
+      def initialize(project, jira_issue, params = {})
+      end
+
+      def execute
+        # this is going to be implemented in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/27201
+        {}
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb
new file mode 100644
index 00000000000..6543b633ddf
--- /dev/null
+++ b/lib/gitlab/jira_import/issues_importer.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    class IssuesImporter < BaseImporter
+      # Jira limits max items per request to be fetched to 100
+      # see https://jira.atlassian.com/browse/JRACLOUD-67570
+      # We set it to 1000 in case they change their mind.
+      BATCH_SIZE = 1000
+
+      attr_reader :imported_items_cache_key, :start_at, :job_waiter
+
+      def initialize(project)
+        super
+        # get cached start_at value, or zero if not cached yet
+        @start_at = Gitlab::JiraImport.get_issues_next_start_at(project.id)
+        @imported_items_cache_key = JiraImport.already_imported_cache_key(:issues, project.id)
+        @job_waiter = JobWaiter.new
+      end
+
+      def execute
+        import_issues
+      end
+
+      private
+
+      def import_issues
+        return job_waiter if jira_last_page_reached?
+
+        issues = fetch_issues(start_at)
+        update_start_at_with(issues)
+
+        schedule_issue_import_workers(issues)
+      end
+
+      def jira_last_page_reached?
+        start_at < 0
+      end
+
+      def update_start_at_with(issues)
+        @start_at += issues.size
+
+        # store -1 if this is the last page to be imported, so no more `ImportIssuesWorker` workers are scheduled
+        # from Gitlab::JiraImport::Stage::ImportIssuesWorker#perform
+        @start_at = -1 if issues.blank?
+        Gitlab::JiraImport.store_issues_next_started_at(project.id, start_at)
+      end
+
+      def schedule_issue_import_workers(issues)
+        next_iid = project.issues.maximum(:iid).to_i + 1
+
+        issues.each do |jira_issue|
+          # Technically it's possible that the same work is performed multiple
+          # times, as Sidekiq doesn't guarantee there will ever only be one
+          # instance of a job or if for some reason the paginated results
+          # returned from Jira include issues there were returned before.
+          # For such cases we exit early if issue was already imported.
+          next if already_imported?(jira_issue.id)
+
+          issue_attrs = IssueSerializer.new(project, jira_issue, { iid: next_iid }).execute
+          Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key)
+
+          job_waiter.jobs_remaining += 1
+          next_iid += 1
+
+          # Mark the issue as imported immediately so we don't end up
+          # importing it multiple times within same import.
+          # These ids are cleaned-up when import finishes.
+          # see Gitlab::JiraImport::Stage::FinishImportWorker
+          mark_as_imported(jira_issue.id)
+        end
+
+        job_waiter
+      end
+
+      def fetch_issues(start_at)
+        client.Issue.jql("PROJECT='#{jira_project_key}' ORDER BY created ASC", { max_results: BATCH_SIZE, start_at: start_at })
+      end
+    end
+  end
+end