diff options
Diffstat (limited to 'lib/gitlab/jira_import/issues_importer.rb')
-rw-r--r-- | lib/gitlab/jira_import/issues_importer.rb | 29 |
1 files changed, 29 insertions, 0 deletions
diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index f1ead57c911..25dffcbe0ee 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -7,6 +7,10 @@ module Gitlab # see https://jira.atlassian.com/browse/JRACLOUD-67570 # We set it to 1000 in case they change their mind. BATCH_SIZE = 1000 + JIRA_IMPORT_THRESHOLD = 100_000 + JIRA_IMPORT_PAUSE_LIMIT = 50_000 + + RetriesExceededError = Class.new(RuntimeError) attr_reader :imported_items_cache_key, :start_at, :job_waiter @@ -66,6 +70,10 @@ module Gitlab @issue_type_id, { iid: next_iid } ).execute + + # Pause the importer to allow the import to catch up and cache to drain + pause_jira_issue_importer if jira_import_issue_worker.queue_size > JIRA_IMPORT_THRESHOLD + Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key) job_waiter.jobs_remaining += 1 @@ -89,6 +97,27 @@ module Gitlab job_waiter end + def jira_import_issue_worker + @_jira_import_issue_worker ||= Gitlab::JiraImport::ImportIssueWorker + end + + def pause_jira_issue_importer + # Wait for import workers to drop below 50K in the iterations of the timeout + # timeout - Set to 5 seconds. + # Time to process 100K jobs is currently ~14 seconds. + # Source: https://github.com/mperham/sidekiq#performance + # retries - Set to 10 times to avoid indefinitely pause. + # Raises an error if the queue does not reduce below the limit after 10 tries. + + retries = 10 + while retries > 0 && jira_import_issue_worker.queue_size >= JIRA_IMPORT_PAUSE_LIMIT + job_waiter.wait(5) + retries -= 1 + end + + raise RetriesExceededError, 'Retry failed after 10 attempts' if retries == 0 + end + def fetch_issues(start_at) client.Issue.jql("PROJECT='#{jira_project_key}' ORDER BY created ASC", { max_results: BATCH_SIZE, start_at: start_at }) end |