summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Gemfile2
-rw-r--r--Gemfile.lock4
-rw-r--r--app/assets/javascripts/pages/projects/snippets/edit/index.js9
-rw-r--r--app/assets/javascripts/pages/projects/snippets/new/index.js9
-rw-r--r--app/assets/javascripts/pages/snippets/edit/index.js8
-rw-r--r--app/assets/javascripts/pages/snippets/new/index.js8
-rw-r--r--app/assets/javascripts/snippet/snippet_edit.js25
-rw-r--r--app/controllers/admin/concerns/authenticates_2fa_for_admin_mode.rb2
-rw-r--r--app/controllers/admin/sessions_controller.rb6
-rw-r--r--app/controllers/projects/merge_requests/diffs_controller.rb4
-rw-r--r--app/graphql/resolvers/projects/jira_imports_resolver.rb23
-rw-r--r--app/graphql/types/jira_import_type.rb27
-rw-r--r--app/graphql/types/project_type.rb11
-rw-r--r--app/models/ci/pipeline.rb17
-rw-r--r--app/models/jira_import_data.rb4
-rw-r--r--app/models/project.rb6
-rw-r--r--app/models/project_services/bamboo_service.rb41
-rw-r--r--app/models/user.rb17
-rw-r--r--app/services/ci/create_cross_project_pipeline_service.rb2
-rw-r--r--app/views/shared/snippets/_form.html.haml3
-rw-r--r--app/workers/all_queues.yml7
-rw-r--r--app/workers/gitlab/jira_import/import_issue_worker.rb36
-rw-r--r--app/workers/gitlab/jira_import/stage/finish_import_worker.rb1
-rw-r--r--app/workers/gitlab/jira_import/stage/import_issues_worker.rb15
-rw-r--r--app/workers/gitlab/jira_import/stage/start_import_worker.rb1
-rw-r--r--app/workers/reactive_caching_worker.rb5
-rw-r--r--changelogs/unreleased/jira-import-graphql-api.yml5
-rw-r--r--doc/api/graphql/reference/gitlab_schema.graphql89
-rw-r--r--doc/api/graphql/reference/gitlab_schema.json250
-rw-r--r--doc/api/graphql/reference/index.md11
-rw-r--r--doc/development/architecture.md25
-rw-r--r--doc/user/project/issues/design_management.md27
-rw-r--r--lib/gitlab/application_context.rb6
-rw-r--r--lib/gitlab/cache/import/caching.rb15
-rw-r--r--lib/gitlab/jira_import.rb47
-rw-r--r--lib/gitlab/jira_import/base_importer.rb35
-rw-r--r--lib/gitlab/jira_import/issue_serializer.rb15
-rw-r--r--lib/gitlab/jira_import/issues_importer.rb81
-rw-r--r--locale/gitlab.pot9
-rw-r--r--spec/controllers/admin/sessions_controller_spec.rb42
-rw-r--r--spec/graphql/resolvers/projects/jira_imports_resolver_spec.rb78
-rw-r--r--spec/graphql/types/jira_import_type_spec.rb11
-rw-r--r--spec/graphql/types/project_type_spec.rb2
-rw-r--r--spec/lib/gitlab/jira_import/base_importer_spec.rb89
-rw-r--r--spec/lib/gitlab/jira_import/issues_importer_spec.rb110
-rw-r--r--spec/lib/gitlab/jira_import_spec.rb67
-rw-r--r--spec/models/ci/pipeline_spec.rb24
-rw-r--r--spec/models/project_services/bamboo_service_spec.rb17
-rw-r--r--spec/models/project_spec.rb38
-rw-r--r--spec/requests/api/graphql/project/jira_import_spec.rb183
-rw-r--r--spec/support/shared_examples/graphql/jira_import/jira_import_resolved_shared_examples.rb15
-rw-r--r--spec/support/shared_examples/lib/gitlab/jira_import/base_importer_shared_examples.rb5
-rw-r--r--spec/support/shared_examples/workers/gitlab/jira_import/jira_import_workers_shared_examples.rb6
-rw-r--r--spec/workers/gitlab/jira_import/import_issue_worker_spec.rb44
-rw-r--r--spec/workers/gitlab/jira_import/stage/finish_import_worker_spec.rb6
-rw-r--r--spec/workers/gitlab/jira_import/stage/import_attachments_worker_spec.rb3
-rw-r--r--spec/workers/gitlab/jira_import/stage/import_issues_worker_spec.rb47
-rw-r--r--spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb3
-rw-r--r--spec/workers/gitlab/jira_import/stage/import_notes_worker_spec.rb3
-rw-r--r--spec/workers/gitlab/jira_import/stage/start_import_worker_spec.rb42
-rw-r--r--spec/workers/reactive_caching_worker_spec.rb10
61 files changed, 1592 insertions, 161 deletions
diff --git a/Gemfile b/Gemfile
index 414a8a67744..c3991a7b630 100644
--- a/Gemfile
+++ b/Gemfile
@@ -301,7 +301,7 @@ gem 'sentry-raven', '~> 2.9'
gem 'premailer-rails', '~> 1.10.3'
# LabKit: Tracing and Correlation
-gem 'gitlab-labkit', '0.11.0'
+gem 'gitlab-labkit', '0.12.0'
# I18n
gem 'ruby_parser', '~> 3.8', require: false
diff --git a/Gemfile.lock b/Gemfile.lock
index bb64fb09649..d050f628ab2 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -380,7 +380,7 @@ GEM
rake (> 10, < 14)
ruby-statistics (>= 2.1)
thor (>= 0.19, < 2)
- gitlab-labkit (0.11.0)
+ gitlab-labkit (0.12.0)
actionpack (>= 5.0.0, < 6.1.0)
activesupport (>= 5.0.0, < 6.1.0)
grpc (~> 1.19)
@@ -1232,7 +1232,7 @@ DEPENDENCIES
github-markup (~> 1.7.0)
gitlab-chronic (~> 0.10.5)
gitlab-derailed_benchmarks
- gitlab-labkit (= 0.11.0)
+ gitlab-labkit (= 0.12.0)
gitlab-license (~> 1.0)
gitlab-mail_room (~> 0.0.3)
gitlab-markup (~> 1.7.0)
diff --git a/app/assets/javascripts/pages/projects/snippets/edit/index.js b/app/assets/javascripts/pages/projects/snippets/edit/index.js
index 53606acc508..0a63f5c7172 100644
--- a/app/assets/javascripts/pages/projects/snippets/edit/index.js
+++ b/app/assets/javascripts/pages/projects/snippets/edit/index.js
@@ -1,8 +1 @@
-import $ from 'jquery';
-import initSnippet from '~/snippet/snippet_bundle';
-import initForm from '~/pages/projects/init_form';
-
-document.addEventListener('DOMContentLoaded', () => {
- initSnippet();
- initForm($('.snippet-form'));
-});
+import '~/snippet/snippet_edit';
diff --git a/app/assets/javascripts/pages/projects/snippets/new/index.js b/app/assets/javascripts/pages/projects/snippets/new/index.js
index 53606acc508..0a63f5c7172 100644
--- a/app/assets/javascripts/pages/projects/snippets/new/index.js
+++ b/app/assets/javascripts/pages/projects/snippets/new/index.js
@@ -1,8 +1 @@
-import $ from 'jquery';
-import initSnippet from '~/snippet/snippet_bundle';
-import initForm from '~/pages/projects/init_form';
-
-document.addEventListener('DOMContentLoaded', () => {
- initSnippet();
- initForm($('.snippet-form'));
-});
+import '~/snippet/snippet_edit';
diff --git a/app/assets/javascripts/pages/snippets/edit/index.js b/app/assets/javascripts/pages/snippets/edit/index.js
index d86e1632ae5..0a63f5c7172 100644
--- a/app/assets/javascripts/pages/snippets/edit/index.js
+++ b/app/assets/javascripts/pages/snippets/edit/index.js
@@ -1,7 +1 @@
-import initSnippet from '~/snippet/snippet_bundle';
-import form from '../form';
-
-document.addEventListener('DOMContentLoaded', () => {
- initSnippet();
- form();
-});
+import '~/snippet/snippet_edit';
diff --git a/app/assets/javascripts/pages/snippets/new/index.js b/app/assets/javascripts/pages/snippets/new/index.js
index d86e1632ae5..0a63f5c7172 100644
--- a/app/assets/javascripts/pages/snippets/new/index.js
+++ b/app/assets/javascripts/pages/snippets/new/index.js
@@ -1,7 +1 @@
-import initSnippet from '~/snippet/snippet_bundle';
-import form from '../form';
-
-document.addEventListener('DOMContentLoaded', () => {
- initSnippet();
- form();
-});
+import '~/snippet/snippet_edit';
diff --git a/app/assets/javascripts/snippet/snippet_edit.js b/app/assets/javascripts/snippet/snippet_edit.js
new file mode 100644
index 00000000000..a098d17a226
--- /dev/null
+++ b/app/assets/javascripts/snippet/snippet_edit.js
@@ -0,0 +1,25 @@
+import $ from 'jquery';
+import initSnippet from '~/snippet/snippet_bundle';
+import ZenMode from '~/zen_mode';
+import GLForm from '~/gl_form';
+
+document.addEventListener('DOMContentLoaded', () => {
+ const form = document.querySelector('.snippet-form');
+ const personalSnippetOptions = {
+ members: false,
+ issues: false,
+ mergeRequests: false,
+ epics: false,
+ milestones: false,
+ labels: false,
+ snippets: false,
+ };
+ const projectSnippetOptions = {};
+
+ const options =
+ form.dataset.snippetType === 'project' ? projectSnippetOptions : personalSnippetOptions;
+
+ initSnippet();
+ new ZenMode(); // eslint-disable-line no-new
+ new GLForm($(form), options); // eslint-disable-line no-new
+});
diff --git a/app/controllers/admin/concerns/authenticates_2fa_for_admin_mode.rb b/app/controllers/admin/concerns/authenticates_2fa_for_admin_mode.rb
index c6fd1d55e51..6014ed0dd13 100644
--- a/app/controllers/admin/concerns/authenticates_2fa_for_admin_mode.rb
+++ b/app/controllers/admin/concerns/authenticates_2fa_for_admin_mode.rb
@@ -37,7 +37,7 @@ module Authenticates2FAForAdminMode
# Remove any lingering user data from login
session.delete(:otp_user_id)
- user.save!
+ user.save! unless Gitlab::Database.read_only?
# The admin user has successfully passed 2fa, enable admin mode ignoring password
enable_admin_mode
diff --git a/app/controllers/admin/sessions_controller.rb b/app/controllers/admin/sessions_controller.rb
index 841ad46b47e..1dc1cd5fb82 100644
--- a/app/controllers/admin/sessions_controller.rb
+++ b/app/controllers/admin/sessions_controller.rb
@@ -64,7 +64,9 @@ class Admin::SessionsController < ApplicationController
end
def valid_otp_attempt?(user)
- user.validate_and_consume_otp!(user_params[:otp_attempt]) ||
- user.invalidate_otp_backup_code!(user_params[:otp_attempt])
+ valid_otp_attempt = user.validate_and_consume_otp!(user_params[:otp_attempt])
+ return valid_otp_attempt if Gitlab::Database.read_only?
+
+ valid_otp_attempt || user.invalidate_otp_backup_code!(user_params[:otp_attempt])
end
end
diff --git a/app/controllers/projects/merge_requests/diffs_controller.rb b/app/controllers/projects/merge_requests/diffs_controller.rb
index 1fb5e333aaa..51128ac1be4 100644
--- a/app/controllers/projects/merge_requests/diffs_controller.rb
+++ b/app/controllers/projects/merge_requests/diffs_controller.rb
@@ -92,8 +92,8 @@ class Projects::MergeRequests::DiffsController < Projects::MergeRequests::Applic
# Deprecated: https://gitlab.com/gitlab-org/gitlab/issues/37735
def find_merge_request_diff_compare
@merge_request_diff =
- if diff_id = params[:diff_id].presence
- @merge_request.merge_request_diffs.viewable.find_by(id: diff_id)
+ if params[:diff_id].present?
+ @merge_request.merge_request_diffs.viewable.find_by(id: params[:diff_id])
else
@merge_request.merge_request_diff
end
diff --git a/app/graphql/resolvers/projects/jira_imports_resolver.rb b/app/graphql/resolvers/projects/jira_imports_resolver.rb
new file mode 100644
index 00000000000..a9bfb7b612e
--- /dev/null
+++ b/app/graphql/resolvers/projects/jira_imports_resolver.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Resolvers
+ module Projects
+ class JiraImportsResolver < BaseResolver
+ include Gitlab::Graphql::Authorize::AuthorizeResource
+
+ alias_method :project, :object
+
+ def resolve(**args)
+ return JiraImportData.none unless project&.import_data.present?
+
+ authorize!(project)
+
+ project.import_data.becomes(JiraImportData).projects
+ end
+
+ def authorized_resource?(project)
+ Ability.allowed?(context[:current_user], :admin_project, project)
+ end
+ end
+ end
+end
diff --git a/app/graphql/types/jira_import_type.rb b/app/graphql/types/jira_import_type.rb
new file mode 100644
index 00000000000..01ec6184844
--- /dev/null
+++ b/app/graphql/types/jira_import_type.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Types
+ # rubocop: disable Graphql/AuthorizeTypes
+ # Authorization is at project level for owners or admins,
+ # so it is added directly to the Resolvers::JiraImportsResolver
+ class JiraImportType < BaseObject
+ graphql_name 'JiraImport'
+
+ field :scheduled_at, Types::TimeType, null: true,
+ description: 'Timestamp of when the Jira import was created/started'
+ field :scheduled_by, Types::UserType, null: true,
+ description: 'User that started the Jira import'
+ field :jira_project_key, GraphQL::STRING_TYPE, null: false,
+ description: 'Project key for the imported Jira project',
+ method: :key
+
+ def scheduled_at
+ DateTime.parse(object.scheduled_at)
+ end
+
+ def scheduled_by
+ ::Gitlab::Graphql::Loaders::BatchModelLoader.new(User, object.scheduled_by['user_id']).find
+ end
+ end
+ # rubocop: enable Graphql/AuthorizeTypes
+end
diff --git a/app/graphql/types/project_type.rb b/app/graphql/types/project_type.rb
index 5c0b9182ac5..b52266bc477 100644
--- a/app/graphql/types/project_type.rb
+++ b/app/graphql/types/project_type.rb
@@ -90,8 +90,9 @@ module Types
end
field :import_status, GraphQL::STRING_TYPE, null: true,
- description: 'Status of project import background job of the project'
-
+ description: 'Status of import background job of the project'
+ field :jira_import_status, GraphQL::STRING_TYPE, null: true,
+ description: 'Status of Jira import background job of the project'
field :only_allow_merge_if_pipeline_succeeds, GraphQL::BOOLEAN_TYPE, null: true,
description: 'Indicates if merge requests of the project can only be merged with successful jobs'
field :request_access_enabled, GraphQL::BOOLEAN_TYPE, null: true,
@@ -192,6 +193,12 @@ module Types
null: true,
description: 'A single board of the project',
resolver: Resolvers::BoardsResolver.single
+
+ field :jira_imports,
+ Types::JiraImportType.connection_type,
+ null: true,
+ description: 'Jira imports into the project',
+ resolver: Resolvers::Projects::JiraImportsResolver
end
end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index e9cd0d91bc0..4e9be86d3a9 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -231,14 +231,6 @@ module Ci
end
end
- after_transition created: :pending do |pipeline|
- next if Feature.enabled?(:ci_drop_bridge_on_downstream_errors, pipeline.project, default_enabled: true)
- next unless pipeline.bridge_triggered?
- next if pipeline.bridge_waiting?
-
- pipeline.update_bridge_status!
- end
-
after_transition any => [:success, :failed] do |pipeline|
pipeline.run_after_commit do
if Feature.enabled?(:ci_pipeline_fixed_notifications)
@@ -761,15 +753,6 @@ module Ci
end
end
- def update_bridge_status!
- raise ArgumentError unless bridge_triggered?
- raise BridgeStatusError unless source_bridge.active?
-
- source_bridge.success!
- rescue => e
- Gitlab::ErrorTracking.track_exception(e, pipeline_id: id)
- end
-
def bridge_triggered?
source_bridge.present?
end
diff --git a/app/models/jira_import_data.rb b/app/models/jira_import_data.rb
index 63be190aa0d..b39ca7290be 100644
--- a/app/models/jira_import_data.rb
+++ b/app/models/jira_import_data.rb
@@ -39,4 +39,8 @@ class JiraImportData < ProjectImportData
data['jira'].delete(FORCE_IMPORT_KEY)
end
+
+ def current_project
+ projects.last
+ end
end
diff --git a/app/models/project.rb b/app/models/project.rb
index 9d055cbd6c6..5aec64da4cf 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -857,6 +857,12 @@ class Project < ApplicationRecord
import_state&.status || 'none'
end
+ def jira_import_status
+ return import_status if jira_force_import?
+
+ import_data&.becomes(JiraImportData)&.projects.blank? ? 'none' : 'finished'
+ end
+
def human_import_status_name
import_state&.human_status_name || 'none'
end
diff --git a/app/models/project_services/bamboo_service.rb b/app/models/project_services/bamboo_service.rb
index dfeb21680a9..8c1f4fef09b 100644
--- a/app/models/project_services/bamboo_service.rb
+++ b/app/models/project_services/bamboo_service.rb
@@ -73,7 +73,7 @@ class BambooService < CiService
end
def calculate_reactive_cache(sha, ref)
- response = get_path("rest/api/latest/result/byChangeset/#{sha}")
+ response = try_get_path("rest/api/latest/result/byChangeset/#{sha}")
{ build_page: read_build_page(response), commit_status: read_commit_status(response) }
end
@@ -81,7 +81,7 @@ class BambooService < CiService
private
def get_build_result(response)
- return if response.code != 200
+ return if response&.code != 200
# May be nil if no result, a single result hash, or an array if multiple results for a given changeset.
result = response.dig('results', 'results', 'result')
@@ -107,7 +107,7 @@ class BambooService < CiService
end
def read_commit_status(response)
- return :error unless response.code == 200 || response.code == 404
+ return :error unless response && (response.code == 200 || response.code == 404)
result = get_build_result(response)
status =
@@ -130,24 +130,31 @@ class BambooService < CiService
end
end
+ def try_get_path(path, query_params = {})
+ params = build_get_params(query_params)
+ params[:extra_log_info] = { project_id: project_id }
+
+ Gitlab::HTTP.try_get(build_url(path), params)
+ end
+
+ def get_path(path, query_params = {})
+ Gitlab::HTTP.get(build_url(path), build_get_params(query_params))
+ end
+
def build_url(path)
Gitlab::Utils.append_path(bamboo_url, path)
end
- def get_path(path, query_params = {})
- url = build_url(path)
+ def build_get_params(query_params)
+ params = { verify: false, query: query_params }
+ return params if username.blank? && password.blank?
- if username.blank? && password.blank?
- Gitlab::HTTP.get(url, verify: false, query: query_params)
- else
- query_params[:os_authType] = 'basic'
- Gitlab::HTTP.get(url,
- verify: false,
- query: query_params,
- basic_auth: {
- username: username,
- password: password
- })
- end
+ query_params[:os_authType] = 'basic'
+ params[:basic_auth] = basic_auth
+ params
+ end
+
+ def basic_auth
+ { username: username, password: password }
end
end
diff --git a/app/models/user.rb b/app/models/user.rb
index 9fb3c47e143..65ff4c98b15 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -1715,6 +1715,23 @@ class User < ApplicationRecord
super
end
+ # This is copied from Devise::Models::TwoFactorAuthenticatable#consume_otp!
+ #
+ # An OTP cannot be used more than once in a given timestep
+ # Storing timestep of last valid OTP is sufficient to satisfy this requirement
+ #
+ # See:
+ # <https://github.com/tinfoil/devise-two-factor/blob/master/lib/devise_two_factor/models/two_factor_authenticatable.rb#L66>
+ #
+ def consume_otp!
+ if self.consumed_timestep != current_otp_timestep
+ self.consumed_timestep = current_otp_timestep
+ return Gitlab::Database.read_only? ? true : save(validate: false)
+ end
+
+ false
+ end
+
private
def default_private_profile_to_false
diff --git a/app/services/ci/create_cross_project_pipeline_service.rb b/app/services/ci/create_cross_project_pipeline_service.rb
index 3a2cc3f9d32..a73a2e2b471 100644
--- a/app/services/ci/create_cross_project_pipeline_service.rb
+++ b/app/services/ci/create_cross_project_pipeline_service.rb
@@ -34,8 +34,6 @@ module Ci
end
downstream_pipeline.tap do |pipeline|
- next if Feature.disabled?(:ci_drop_bridge_on_downstream_errors, project, default_enabled: true)
-
update_bridge_status!(@bridge, pipeline)
end
end
diff --git a/app/views/shared/snippets/_form.html.haml b/app/views/shared/snippets/_form.html.haml
index 828015d29f5..5ba6d52fefe 100644
--- a/app/views/shared/snippets/_form.html.haml
+++ b/app/views/shared/snippets/_form.html.haml
@@ -3,7 +3,8 @@
.snippet-form-holder
= form_for @snippet, url: url,
- html: { class: "snippet-form js-requires-input js-quick-submit common-note-form" } do |f|
+ html: { class: "snippet-form js-requires-input js-quick-submit common-note-form" },
+ data: { "snippet-type": @snippet.project_id ? 'project' : 'personal'} do |f|
= form_errors(@snippet)
.form-group
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 16520519a0b..7dc47c55a04 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -556,6 +556,13 @@
:resource_boundary: :unknown
:weight: 1
:idempotent:
+- :name: jira_importer:jira_import_import_issue
+ :feature_category: :importers
+ :has_external_dependencies:
+ :urgency: :low
+ :resource_boundary: :unknown
+ :weight: 1
+ :idempotent:
- :name: jira_importer:jira_import_stage_finish_import
:feature_category: :importers
:has_external_dependencies:
diff --git a/app/workers/gitlab/jira_import/import_issue_worker.rb b/app/workers/gitlab/jira_import/import_issue_worker.rb
new file mode 100644
index 00000000000..832916a03b6
--- /dev/null
+++ b/app/workers/gitlab/jira_import/import_issue_worker.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module JiraImport
+ class ImportIssueWorker # rubocop:disable Scalability/IdempotentWorker
+ include ApplicationWorker
+ include NotifyUponDeath
+ include Gitlab::JiraImport::QueueOptions
+ include Gitlab::Import::DatabaseHelpers
+
+ def perform(project_id, jira_issue_id, issue_attributes, waiter_key)
+ issue_id = insert_and_return_id(issue_attributes, Issue)
+ cache_issue_mapping(issue_id, jira_issue_id, project_id)
+ rescue => ex
+ # Todo: Record jira issue id(or better jira issue key),
+ # so that we can report the list of failed to import issues to the user
+ # see https://gitlab.com/gitlab-org/gitlab/-/issues/211653
+ #
+ # It's possible the project has been deleted since scheduling this
+ # job. In this case we'll just skip creating the issue.
+ Gitlab::ErrorTracking.track_exception(ex, project_id: project_id)
+ JiraImport.increment_issue_failures(project_id)
+ ensure
+ # ensure we notify job waiter that the job has finished
+ JobWaiter.notify(waiter_key, jid) if waiter_key
+ end
+
+ private
+
+ def cache_issue_mapping(issue_id, jira_issue_id, project_id)
+ cache_key = JiraImport.jira_issue_cache_key(project_id, jira_issue_id)
+ Gitlab::Cache::Import::Caching.write(cache_key, issue_id)
+ end
+ end
+ end
+end
diff --git a/app/workers/gitlab/jira_import/stage/finish_import_worker.rb b/app/workers/gitlab/jira_import/stage/finish_import_worker.rb
index 5b1661d68c6..f053037e78a 100644
--- a/app/workers/gitlab/jira_import/stage/finish_import_worker.rb
+++ b/app/workers/gitlab/jira_import/stage/finish_import_worker.rb
@@ -11,6 +11,7 @@ module Gitlab
def import(project)
project.after_import
ensure
+ JiraImport.cache_cleanup(project.id)
project.import_data.becomes(JiraImportData).finish_import!
project.import_data.save!
end
diff --git a/app/workers/gitlab/jira_import/stage/import_issues_worker.rb b/app/workers/gitlab/jira_import/stage/import_issues_worker.rb
index 79ed8e1f2da..7e257afc4d9 100644
--- a/app/workers/gitlab/jira_import/stage/import_issues_worker.rb
+++ b/app/workers/gitlab/jira_import/stage/import_issues_worker.rb
@@ -9,12 +9,19 @@ module Gitlab
private
def import(project)
- # fake issues import workers for now
- # new job waiter will have zero jobs_remaining by default, so it will just pass on to next stage
- jobs_waiter = JobWaiter.new
+ jobs_waiter = Gitlab::JiraImport::IssuesImporter.new(project).execute
+
project.import_state.refresh_jid_expiration
- Gitlab::JiraImport::AdvanceStageWorker.perform_async(project.id, { jobs_waiter.key => jobs_waiter.jobs_remaining }, :attachments)
+ Gitlab::JiraImport::AdvanceStageWorker.perform_async(
+ project.id,
+ { jobs_waiter.key => jobs_waiter.jobs_remaining },
+ next_stage(project)
+ )
+ end
+
+ def next_stage(project)
+ Gitlab::JiraImport.get_issues_next_start_at(project.id) < 0 ? :attachments : :issues
end
end
end
diff --git a/app/workers/gitlab/jira_import/stage/start_import_worker.rb b/app/workers/gitlab/jira_import/stage/start_import_worker.rb
index 8abbfab647b..ae4864064e4 100644
--- a/app/workers/gitlab/jira_import/stage/start_import_worker.rb
+++ b/app/workers/gitlab/jira_import/stage/start_import_worker.rb
@@ -26,6 +26,7 @@ module Gitlab
def start_import
return false unless project
return false if Feature.disabled?(:jira_issue_import, project)
+ return false unless project.jira_force_import?
return true if start(project.import_state)
Gitlab::Import::Logger.info(
diff --git a/app/workers/reactive_caching_worker.rb b/app/workers/reactive_caching_worker.rb
index 716b1de2bf5..1921ac6619b 100644
--- a/app/workers/reactive_caching_worker.rb
+++ b/app/workers/reactive_caching_worker.rb
@@ -13,6 +13,11 @@ class ReactiveCachingWorker # rubocop:disable Scalability/IdempotentWorker
urgency :high
worker_resource_boundary :cpu
+ def self.context_for_arguments(arguments)
+ class_name, *_other_args = arguments
+ Gitlab::ApplicationContext.new(related_class: class_name)
+ end
+
def perform(class_name, id, *args)
klass = begin
class_name.constantize
diff --git a/changelogs/unreleased/jira-import-graphql-api.yml b/changelogs/unreleased/jira-import-graphql-api.yml
new file mode 100644
index 00000000000..8cfdfda8e2a
--- /dev/null
+++ b/changelogs/unreleased/jira-import-graphql-api.yml
@@ -0,0 +1,5 @@
+---
+title: Allow querying of Jira imports and their status via GraphQL
+merge_request: 27587
+author:
+type: added
diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql
index 0fbd8c84e58..9766fc39e2e 100644
--- a/doc/api/graphql/reference/gitlab_schema.graphql
+++ b/doc/api/graphql/reference/gitlab_schema.graphql
@@ -63,6 +63,11 @@ input AdminSidekiqQueuesDeleteJobsInput {
queueName: String!
"""
+ Delete jobs matching related_class in the context metadata
+ """
+ relatedClass: String
+
+ """
Delete jobs matching root_namespace in the context metadata
"""
rootNamespace: String
@@ -4093,6 +4098,58 @@ Represents untyped JSON
"""
scalar JSON
+type JiraImport {
+ """
+ Project key for the imported Jira project
+ """
+ jiraProjectKey: String!
+
+ """
+ Timestamp of when the Jira import was created/started
+ """
+ scheduledAt: Time
+
+ """
+ User that started the Jira import
+ """
+ scheduledBy: User
+}
+
+"""
+The connection type for JiraImport.
+"""
+type JiraImportConnection {
+ """
+ A list of edges.
+ """
+ edges: [JiraImportEdge]
+
+ """
+ A list of nodes.
+ """
+ nodes: [JiraImport]
+
+ """
+ Information to aid in pagination.
+ """
+ pageInfo: PageInfo!
+}
+
+"""
+An edge in a connection.
+"""
+type JiraImportEdge {
+ """
+ A cursor for use in pagination.
+ """
+ cursor: String!
+
+ """
+ The item at the end of the edge.
+ """
+ node: JiraImport
+}
+
type Label {
"""
Background color of the label
@@ -5749,7 +5806,7 @@ type Project {
id: ID!
"""
- Status of project import background job of the project
+ Status of import background job of the project
"""
importStatus: String
@@ -5939,6 +5996,36 @@ type Project {
issuesEnabled: Boolean
"""
+ Status of Jira import background job of the project
+ """
+ jiraImportStatus: String
+
+ """
+ Jira imports into the project
+ """
+ jiraImports(
+ """
+ Returns the elements in the list that come after the specified cursor.
+ """
+ after: String
+
+ """
+ Returns the elements in the list that come before the specified cursor.
+ """
+ before: String
+
+ """
+ Returns the first _n_ elements from the list.
+ """
+ first: Int
+
+ """
+ Returns the last _n_ elements from the list.
+ """
+ last: Int
+ ): JiraImportConnection
+
+ """
(deprecated) Enable jobs for this project. Use `builds_access_level` instead
"""
jobsEnabled: Boolean
diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json
index bd78b51684f..7584e48b1e5 100644
--- a/doc/api/graphql/reference/gitlab_schema.json
+++ b/doc/api/graphql/reference/gitlab_schema.json
@@ -182,6 +182,16 @@
"defaultValue": null
},
{
+ "name": "relatedClass",
+ "description": "Delete jobs matching related_class in the context metadata",
+ "type": {
+ "kind": "SCALAR",
+ "name": "String",
+ "ofType": null
+ },
+ "defaultValue": null
+ },
+ {
"name": "queueName",
"description": "The name of the queue to delete jobs from",
"type": {
@@ -11613,6 +11623,177 @@
},
{
"kind": "OBJECT",
+ "name": "JiraImport",
+ "description": null,
+ "fields": [
+ {
+ "name": "jiraProjectKey",
+ "description": "Project key for the imported Jira project",
+ "args": [
+
+ ],
+ "type": {
+ "kind": "NON_NULL",
+ "name": null,
+ "ofType": {
+ "kind": "SCALAR",
+ "name": "String",
+ "ofType": null
+ }
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ },
+ {
+ "name": "scheduledAt",
+ "description": "Timestamp of when the Jira import was created/started",
+ "args": [
+
+ ],
+ "type": {
+ "kind": "SCALAR",
+ "name": "Time",
+ "ofType": null
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ },
+ {
+ "name": "scheduledBy",
+ "description": "User that started the Jira import",
+ "args": [
+
+ ],
+ "type": {
+ "kind": "OBJECT",
+ "name": "User",
+ "ofType": null
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ }
+ ],
+ "inputFields": null,
+ "interfaces": [
+
+ ],
+ "enumValues": null,
+ "possibleTypes": null
+ },
+ {
+ "kind": "OBJECT",
+ "name": "JiraImportConnection",
+ "description": "The connection type for JiraImport.",
+ "fields": [
+ {
+ "name": "edges",
+ "description": "A list of edges.",
+ "args": [
+
+ ],
+ "type": {
+ "kind": "LIST",
+ "name": null,
+ "ofType": {
+ "kind": "OBJECT",
+ "name": "JiraImportEdge",
+ "ofType": null
+ }
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ },
+ {
+ "name": "nodes",
+ "description": "A list of nodes.",
+ "args": [
+
+ ],
+ "type": {
+ "kind": "LIST",
+ "name": null,
+ "ofType": {
+ "kind": "OBJECT",
+ "name": "JiraImport",
+ "ofType": null
+ }
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ },
+ {
+ "name": "pageInfo",
+ "description": "Information to aid in pagination.",
+ "args": [
+
+ ],
+ "type": {
+ "kind": "NON_NULL",
+ "name": null,
+ "ofType": {
+ "kind": "OBJECT",
+ "name": "PageInfo",
+ "ofType": null
+ }
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ }
+ ],
+ "inputFields": null,
+ "interfaces": [
+
+ ],
+ "enumValues": null,
+ "possibleTypes": null
+ },
+ {
+ "kind": "OBJECT",
+ "name": "JiraImportEdge",
+ "description": "An edge in a connection.",
+ "fields": [
+ {
+ "name": "cursor",
+ "description": "A cursor for use in pagination.",
+ "args": [
+
+ ],
+ "type": {
+ "kind": "NON_NULL",
+ "name": null,
+ "ofType": {
+ "kind": "SCALAR",
+ "name": "String",
+ "ofType": null
+ }
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ },
+ {
+ "name": "node",
+ "description": "The item at the end of the edge.",
+ "args": [
+
+ ],
+ "type": {
+ "kind": "OBJECT",
+ "name": "JiraImport",
+ "ofType": null
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ }
+ ],
+ "inputFields": null,
+ "interfaces": [
+
+ ],
+ "enumValues": null,
+ "possibleTypes": null
+ },
+ {
+ "kind": "OBJECT",
"name": "Label",
"description": null,
"fields": [
@@ -17441,7 +17622,7 @@
},
{
"name": "importStatus",
- "description": "Status of project import background job of the project",
+ "description": "Status of import background job of the project",
"args": [
],
@@ -17866,6 +18047,73 @@
"deprecationReason": null
},
{
+ "name": "jiraImportStatus",
+ "description": "Status of Jira import background job of the project",
+ "args": [
+
+ ],
+ "type": {
+ "kind": "SCALAR",
+ "name": "String",
+ "ofType": null
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ },
+ {
+ "name": "jiraImports",
+ "description": "Jira imports into the project",
+ "args": [
+ {
+ "name": "after",
+ "description": "Returns the elements in the list that come after the specified cursor.",
+ "type": {
+ "kind": "SCALAR",
+ "name": "String",
+ "ofType": null
+ },
+ "defaultValue": null
+ },
+ {
+ "name": "before",
+ "description": "Returns the elements in the list that come before the specified cursor.",
+ "type": {
+ "kind": "SCALAR",
+ "name": "String",
+ "ofType": null
+ },
+ "defaultValue": null
+ },
+ {
+ "name": "first",
+ "description": "Returns the first _n_ elements from the list.",
+ "type": {
+ "kind": "SCALAR",
+ "name": "Int",
+ "ofType": null
+ },
+ "defaultValue": null
+ },
+ {
+ "name": "last",
+ "description": "Returns the last _n_ elements from the list.",
+ "type": {
+ "kind": "SCALAR",
+ "name": "Int",
+ "ofType": null
+ },
+ "defaultValue": null
+ }
+ ],
+ "type": {
+ "kind": "OBJECT",
+ "name": "JiraImportConnection",
+ "ofType": null
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ },
+ {
"name": "jobsEnabled",
"description": "(deprecated) Enable jobs for this project. Use `builds_access_level` instead",
"args": [
diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md
index dfbd08be898..2aee7d484d0 100644
--- a/doc/api/graphql/reference/index.md
+++ b/doc/api/graphql/reference/index.md
@@ -605,6 +605,14 @@ Autogenerated return type of IssueSetWeight
| `errors` | String! => Array | Reasons why the mutation failed. |
| `issue` | Issue | The issue after mutation |
+## JiraImport
+
+| Name | Type | Description |
+| --- | ---- | ---------- |
+| `jiraProjectKey` | String! | Project key for the imported Jira project |
+| `scheduledAt` | Time | Timestamp of when the Jira import was created/started |
+| `scheduledBy` | User | User that started the Jira import |
+
## Label
| Name | Type | Description |
@@ -879,9 +887,10 @@ Information about pagination in a connection.
| `group` | Group | Group of the project |
| `httpUrlToRepo` | String | URL to connect to the project via HTTPS |
| `id` | ID! | ID of the project |
-| `importStatus` | String | Status of project import background job of the project |
+| `importStatus` | String | Status of import background job of the project |
| `issue` | Issue | A single issue of the project |
| `issuesEnabled` | Boolean | (deprecated) Does this project have issues enabled?. Use `issues_access_level` instead |
+| `jiraImportStatus` | String | Status of Jira import background job of the project |
| `jobsEnabled` | Boolean | (deprecated) Enable jobs for this project. Use `builds_access_level` instead |
| `lastActivityAt` | Time | Timestamp of the project last activity |
| `lfsEnabled` | Boolean | Indicates if the project has Large File Storage (LFS) enabled |
diff --git a/doc/development/architecture.md b/doc/development/architecture.md
index 32626c19298..061e910a6d0 100644
--- a/doc/development/architecture.md
+++ b/doc/development/architecture.md
@@ -563,18 +563,23 @@ sequenceDiagram
participant Git on server
Note left of Git on client: git fetch
- Git on client->>SSH server: git fetch-pack
- SSH server-->>AuthorizedKeysCommand: gitlab-shell-authorized-keys-check git AAAA...
- AuthorizedKeysCommand-->>Rails: GET /internal/api/authorized_keys?key=AAAA...
+ Git on client->>+SSH server: ssh git fetch-pack request
+ SSH server->>+AuthorizedKeysCommand: gitlab-shell-authorized-keys-check git AAAA...
+ AuthorizedKeysCommand->>+Rails: GET /internal/api/authorized_keys?key=AAAA...
Note right of Rails: Lookup key ID
- Rails-->>SSH server: 200 OK, command="gitlab-shell upload-pack key_id=1"
- SSH server-->>GitLab Shell: gitlab-shell upload-pack key_id=1
- GitLab Shell-->>Rails: GET /internal/api/allowed?action=upload_pack&key_id=1
+ Rails-->>-AuthorizedKeysCommand: 200 OK, command="gitlab-shell upload-pack key_id=1"
+ AuthorizedKeysCommand-->>-SSH server: command="gitlab-shell upload-pack key_id=1"
+ SSH server->>+GitLab Shell: gitlab-shell upload-pack key_id=1
+ GitLab Shell->>+Rails: GET /internal/api/allowed?action=upload_pack&key_id=1
Note right of Rails: Auth check
- Rails-->>GitLab Shell: 200 OK, { gitaly: ... }
- GitLab Shell-->>Gitaly: SSHService.SSHUploadPack bidirectional request
- Gitaly-->>Git on server: git upload-pack
- Git on server->>Git on client: SSHService.SSHUploadPack bidirectional response
+ Rails-->>-GitLab Shell: 200 OK, { gitaly: ... }
+ GitLab Shell->>+Gitaly: SSHService.SSHUploadPack request
+ Gitaly->>+Git on server: git upload-pack request
+ Note over Git on client,Git on server: Bidirectional communication between Git client and server
+ Git on server-->>-Gitaly: git upload-pack response
+ Gitaly -->>-GitLab Shell: SSHService.SSHUploadPack response
+ GitLab Shell-->>-SSH server: gitlab-shell upload-pack response
+ SSH server-->>-Git on client: ssh git fetch-pack response
```
The `git push` operation is very similar, except `git receive-pack` is used
diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md
index add466a91b1..896ae8e7c51 100644
--- a/doc/user/project/issues/design_management.md
+++ b/doc/user/project/issues/design_management.md
@@ -94,7 +94,7 @@ Images on the Design Management page can be enlarged by clicking on them.
You can navigate through designs by clicking on the navigation buttons on the
top-right corner or with <kbd>Left</kbd>/<kbd>Right</kbd> keyboard buttons.
-The number of comments on a design — if any — is listed to the right
+The number of discussions on a design — if any — is listed to the right
of the design filename. Clicking on this number enlarges the design
just like clicking anywhere else on the design.
When a design is added or modified, an icon is displayed on the item
@@ -102,7 +102,7 @@ to help summarize changes between versions.
| Indicator | Example |
| --------- | ------- |
-| Comments | ![Comments Icon](img/design_comments_v12_3.png) |
+| Discussions | ![Discussions Icon](img/design_comments_v12_3.png) |
| Modified (in the selected version) | ![Design Modified](img/design_modified_v12_3.png) |
| Added (in the selected version) | ![Design Added](img/design_added_v12_3.png) |
@@ -110,7 +110,9 @@ to help summarize changes between versions.
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13217) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.7.
-Designs can be explored in greater detail by zooming in and out of the image. Control the amount of zoom with the `+` and `-` buttons at the bottom of the image. While zoomed, you can still [add new annotations](#adding-annotations-to-designs) to the image, and see any existing ones.
+Designs can be explored in greater detail by zooming in and out of the image.
+Control the amount of zoom with the `+` and `-` buttons at the bottom of the image.
+While zoomed, you can still [start new discussions](#starting-discussions-on-designs) on the image, and see any existing ones.
![Design zooming](img/design_zooming_v12_7.png)
@@ -142,23 +144,22 @@ Only the latest version of the designs can be deleted.
Deleted designs are not permanently lost; they can be
viewed by browsing previous versions.
-## Adding annotations to designs
+## Starting discussions on designs
-When a design is uploaded, you can add annotations by clicking on
-the image on the exact location you'd like to add the note to.
-A badge is added to the image identifying the annotation, from
-which you can start a new discussion:
+When a design is uploaded, you can start a discussion by clicking on
+the image on the exact location you would like the discussion to be focused on.
+A pin is added to the image, identifying the discussion's location.
![Starting a new discussion on design](img/adding_note_to_design_1.png)
[Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34353) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8,
-you can adjust the badge's position by dragging it around the image. This is useful
+you can adjust a pin's position by dragging it around the image. This is useful
for when your design layout has changed between revisions, or if you need to move an
-existing badge to add a new one in its place.
+existing pin to add a new one in its place.
-Different discussions have different badge numbers:
+Different discussions have different pin numbers:
-![Discussions on design annotations](img/adding_note_to_design_2.png)
+![Discussions on designs](img/adding_note_to_design_2.png)
-From GitLab 12.5 on, new annotations will be outputted to the issue activity,
+From GitLab 12.5 on, new discussions will be outputted to the issue activity,
so that everyone involved can participate in the discussion.
diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb
index b950bfb0f3a..60a50e97998 100644
--- a/lib/gitlab/application_context.rb
+++ b/lib/gitlab/application_context.rb
@@ -5,13 +5,14 @@ module Gitlab
class ApplicationContext
include Gitlab::Utils::LazyAttributes
- Attribute = Struct.new(:name, :type, :evaluation)
+ Attribute = Struct.new(:name, :type)
APPLICATION_ATTRIBUTES = [
Attribute.new(:project, Project),
Attribute.new(:namespace, Namespace),
Attribute.new(:user, User),
- Attribute.new(:caller_id, String)
+ Attribute.new(:caller_id, String),
+ Attribute.new(:related_class, String)
].freeze
def self.with_context(args, &block)
@@ -39,6 +40,7 @@ module Gitlab
hash[:project] = -> { project_path } if set_values.include?(:project)
hash[:root_namespace] = -> { root_namespace_path } if include_namespace?
hash[:caller_id] = caller_id if set_values.include?(:caller_id)
+ hash[:related_class] = related_class if set_values.include?(:related_class)
end
end
diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb
index ead94761ae7..7f2d2858149 100644
--- a/lib/gitlab/cache/import/caching.rb
+++ b/lib/gitlab/cache/import/caching.rb
@@ -70,6 +70,21 @@ module Gitlab
value
end
+ # Increment the integer value of a key by one.
+ # Sets the value to zero if missing before incrementing
+ #
+ # key - The cache key to increment.
+ # timeout - The time after which the cache key should expire.
+ # @return - the incremented value
+ def self.increment(raw_key, timeout: TIMEOUT)
+ key = cache_key_for(raw_key)
+
+ Redis::Cache.with do |redis|
+ redis.incr(key)
+ redis.expire(key, timeout)
+ end
+ end
+
# Adds a value to a set.
#
# raw_key - The key of the set to add the value to.
diff --git a/lib/gitlab/jira_import.rb b/lib/gitlab/jira_import.rb
new file mode 100644
index 00000000000..1486c754caf
--- /dev/null
+++ b/lib/gitlab/jira_import.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module JiraImport
+ JIRA_IMPORT_CACHE_TIMEOUT = 10.seconds.to_i
+
+ FAILED_ISSUES_COUNTER_KEY = 'jira-import/failed/%{project_id}/%{collection_type}'
+ NEXT_ITEMS_START_AT_KEY = 'jira-import/paginator/%{project_id}/%{collection_type}'
+ ITEMS_MAPPER_CACHE_KEY = 'jira-import/items-mapper/%{project_id}/%{collection_type}/%{jira_isssue_id}'
+ ALREADY_IMPORTED_ITEMS_CACHE_KEY = 'jira-importer/already-imported/%{project}/%{collection_type}'
+
+ def self.jira_issue_cache_key(project_id, jira_issue_id)
+ ITEMS_MAPPER_CACHE_KEY % { project_id: project_id, collection_type: :issues, jira_isssue_id: jira_issue_id }
+ end
+
+ def self.already_imported_cache_key(collection_type, project_id)
+ ALREADY_IMPORTED_ITEMS_CACHE_KEY % { collection_type: collection_type, project: project_id }
+ end
+
+ def self.jira_issues_next_page_cache_key(project_id)
+ NEXT_ITEMS_START_AT_KEY % { project_id: project_id, collection_type: :issues }
+ end
+
+ def self.failed_issues_counter_cache_key(project_id)
+ FAILED_ISSUES_COUNTER_KEY % { project_id: project_id, collection_type: :issues }
+ end
+
+ def self.increment_issue_failures(project_id)
+ Gitlab::Cache::Import::Caching.increment(self.failed_issues_counter_cache_key(project_id))
+ end
+
+ def self.get_issues_next_start_at(project_id)
+ Gitlab::Cache::Import::Caching.read(self.jira_issues_next_page_cache_key(project_id)).to_i
+ end
+
+ def self.store_issues_next_started_at(project_id, value)
+ cache_key = self.jira_issues_next_page_cache_key(project_id)
+ Gitlab::Cache::Import::Caching.write(cache_key, value)
+ end
+
+ def self.cache_cleanup(project_id)
+ Gitlab::Cache::Import::Caching.expire(self.failed_issues_counter_cache_key(project_id), JIRA_IMPORT_CACHE_TIMEOUT)
+ Gitlab::Cache::Import::Caching.expire(self.jira_issues_next_page_cache_key(project_id), JIRA_IMPORT_CACHE_TIMEOUT)
+ Gitlab::Cache::Import::Caching.expire(self.already_imported_cache_key(:issues, project_id), JIRA_IMPORT_CACHE_TIMEOUT)
+ end
+ end
+end
diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb
new file mode 100644
index 00000000000..24158b8e734
--- /dev/null
+++ b/lib/gitlab/jira_import/base_importer.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module JiraImport
+ class BaseImporter
+ attr_reader :project, :client, :formatter, :jira_project_key
+
+ def initialize(project)
+ raise Projects::ImportService::Error, _('Jira import feature is disabled.') unless Feature.enabled?(:jira_issue_import, project)
+ raise Projects::ImportService::Error, _('Jira integration not configured.') unless project.jira_service&.active?
+
+ @jira_project_key = project&.import_data&.becomes(JiraImportData)&.current_project&.key
+ raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key
+
+ @project = project
+ @client = project.jira_service.client
+ @formatter = Gitlab::ImportFormatter.new
+ end
+
+ private
+
+ def imported_items_cache_key
+ raise NotImplementedError
+ end
+
+ def mark_as_imported(id)
+ Gitlab::Cache::Import::Caching.set_add(imported_items_cache_key, id)
+ end
+
+ def already_imported?(id)
+ Gitlab::Cache::Import::Caching.set_includes?(imported_items_cache_key, id)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/jira_import/issue_serializer.rb b/lib/gitlab/jira_import/issue_serializer.rb
new file mode 100644
index 00000000000..f00852a21a2
--- /dev/null
+++ b/lib/gitlab/jira_import/issue_serializer.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module JiraImport
+ class IssueSerializer
+ def initialize(project, jira_issue, params = {})
+ end
+
+ def execute
+ # this is going to be implemented in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/27201
+ {}
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb
new file mode 100644
index 00000000000..6543b633ddf
--- /dev/null
+++ b/lib/gitlab/jira_import/issues_importer.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module JiraImport
+ class IssuesImporter < BaseImporter
+ # Jira limits max items per request to be fetched to 100
+ # see https://jira.atlassian.com/browse/JRACLOUD-67570
+ # We set it to 1000 in case they change their mind.
+ BATCH_SIZE = 1000
+
+ attr_reader :imported_items_cache_key, :start_at, :job_waiter
+
+ def initialize(project)
+ super
+ # get cached start_at value, or zero if not cached yet
+ @start_at = Gitlab::JiraImport.get_issues_next_start_at(project.id)
+ @imported_items_cache_key = JiraImport.already_imported_cache_key(:issues, project.id)
+ @job_waiter = JobWaiter.new
+ end
+
+ def execute
+ import_issues
+ end
+
+ private
+
+ def import_issues
+ return job_waiter if jira_last_page_reached?
+
+ issues = fetch_issues(start_at)
+ update_start_at_with(issues)
+
+ schedule_issue_import_workers(issues)
+ end
+
+ def jira_last_page_reached?
+ start_at < 0
+ end
+
+ def update_start_at_with(issues)
+ @start_at += issues.size
+
+ # store -1 if this is the last page to be imported, so no more `ImportIssuesWorker` workers are scheduled
+ # from Gitlab::JiraImport::Stage::ImportIssuesWorker#perform
+ @start_at = -1 if issues.blank?
+ Gitlab::JiraImport.store_issues_next_started_at(project.id, start_at)
+ end
+
+ def schedule_issue_import_workers(issues)
+ next_iid = project.issues.maximum(:iid).to_i + 1
+
+ issues.each do |jira_issue|
+ # Technically it's possible that the same work is performed multiple
+ # times, as Sidekiq doesn't guarantee there will ever only be one
+ # instance of a job or if for some reason the paginated results
+ # returned from Jira include issues there were returned before.
+ # For such cases we exit early if issue was already imported.
+ next if already_imported?(jira_issue.id)
+
+ issue_attrs = IssueSerializer.new(project, jira_issue, { iid: next_iid }).execute
+ Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key)
+
+ job_waiter.jobs_remaining += 1
+ next_iid += 1
+
+ # Mark the issue as imported immediately so we don't end up
+ # importing it multiple times within same import.
+ # These ids are cleaned-up when import finishes.
+ # see Gitlab::JiraImport::Stage::FinishImportWorker
+ mark_as_imported(jira_issue.id)
+ end
+
+ job_waiter
+ end
+
+ def fetch_issues(start_at)
+ client.Issue.jql("PROJECT='#{jira_project_key}' ORDER BY created ASC", { max_results: BATCH_SIZE, start_at: start_at })
+ end
+ end
+ end
+end
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 19fe4f77c85..80f16fd3e1d 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -11287,6 +11287,12 @@ msgstr ""
msgid "Jira Issue Import"
msgstr ""
+msgid "Jira import feature is disabled."
+msgstr ""
+
+msgid "Jira integration not configured."
+msgstr ""
+
msgid "JiraService|Events for %{noteable_model_name} are disabled."
msgstr ""
@@ -21398,6 +21404,9 @@ msgstr ""
msgid "Unable to fetch vulnerable projects"
msgstr ""
+msgid "Unable to find Jira project to import data from."
+msgstr ""
+
msgid "Unable to generate new instance ID"
msgstr ""
diff --git a/spec/controllers/admin/sessions_controller_spec.rb b/spec/controllers/admin/sessions_controller_spec.rb
index fabd79133ec..351148dbc65 100644
--- a/spec/controllers/admin/sessions_controller_spec.rb
+++ b/spec/controllers/admin/sessions_controller_spec.rb
@@ -176,6 +176,48 @@ describe Admin::SessionsController, :do_not_mock_admin_mode do
expect(controller.current_user_mode.admin_mode?).to be(true)
end
end
+
+ context 'on a read-only instance' do
+ before do
+ allow(Gitlab::Database).to receive(:read_only?).and_return(true)
+ end
+
+ it 'does not attempt to write to the database with valid otp' do
+ expect_any_instance_of(User).not_to receive(:save)
+ expect_any_instance_of(User).not_to receive(:save!)
+
+ controller.store_location_for(:redirect, admin_root_path)
+ controller.current_user_mode.request_admin_mode!
+
+ authenticate_2fa(otp_attempt: user.current_otp)
+
+ expect(response).to redirect_to admin_root_path
+ end
+
+ it 'does not attempt to write to the database with invalid otp' do
+ expect_any_instance_of(User).not_to receive(:save)
+ expect_any_instance_of(User).not_to receive(:save!)
+
+ controller.current_user_mode.request_admin_mode!
+
+ authenticate_2fa(otp_attempt: 'invalid')
+
+ expect(response).to render_template('admin/sessions/two_factor')
+ expect(controller.current_user_mode.admin_mode?).to be(false)
+ end
+
+ it 'does not attempt to write to the database with backup code' do
+ expect_any_instance_of(User).not_to receive(:save)
+ expect_any_instance_of(User).not_to receive(:save!)
+
+ controller.current_user_mode.request_admin_mode!
+
+ authenticate_2fa(otp_attempt: user.otp_backup_codes.first)
+
+ expect(response).to render_template('admin/sessions/two_factor')
+ expect(controller.current_user_mode.admin_mode?).to be(false)
+ end
+ end
end
context 'when using two-factor authentication via U2F' do
diff --git a/spec/graphql/resolvers/projects/jira_imports_resolver_spec.rb b/spec/graphql/resolvers/projects/jira_imports_resolver_spec.rb
new file mode 100644
index 00000000000..54d51292919
--- /dev/null
+++ b/spec/graphql/resolvers/projects/jira_imports_resolver_spec.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Resolvers::Projects::JiraImportsResolver do
+ include GraphqlHelpers
+
+ describe '#resolve' do
+ let_it_be(:user) { create(:user) }
+ let_it_be(:jira_import_data) do
+ data = JiraImportData.new
+ data << JiraImportData::JiraProjectDetails.new('AA', 2.days.ago.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name })
+ data << JiraImportData::JiraProjectDetails.new('BB', 5.days.ago.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name })
+ data
+ end
+
+ context 'when project does not have Jira import data' do
+ let_it_be(:project) { create(:project, :private, import_data: nil) }
+
+ context 'when user cannot read Jira import data' do
+ context 'when anonymous user' do
+ it_behaves_like 'no jira import data present'
+ end
+
+ context 'when user developer' do
+ before do
+ project.add_developer(user)
+ end
+
+ it_behaves_like 'no jira import data present'
+ end
+ end
+
+ context 'when user can read Jira import data' do
+ before do
+ project.add_maintainer(user)
+ end
+
+ it_behaves_like 'no jira import data present'
+ end
+ end
+
+ context 'when project has Jira import data' do
+ let_it_be(:project) { create(:project, :private, import_data: jira_import_data) }
+
+ context 'when user cannot read Jira import data' do
+ context 'when anonymous user' do
+ it_behaves_like 'no jira import access'
+ end
+
+ context 'when user developer' do
+ before do
+ project.add_developer(user)
+ end
+
+ it_behaves_like 'no jira import access'
+ end
+ end
+
+ context 'when user can access Jira import data' do
+ before do
+ project.add_maintainer(user)
+ end
+
+ it 'returns Jira imports sorted ascending by scheduledAt time' do
+ imports = resolve_imports
+
+ expect(imports.size).to eq 2
+ expect(imports.map(&:key)).to eq %w(BB AA)
+ end
+ end
+ end
+ end
+
+ def resolve_imports(args = {}, context = { current_user: user })
+ resolve(described_class, obj: project, args: args, ctx: context)
+ end
+end
diff --git a/spec/graphql/types/jira_import_type_spec.rb b/spec/graphql/types/jira_import_type_spec.rb
new file mode 100644
index 00000000000..8448a120682
--- /dev/null
+++ b/spec/graphql/types/jira_import_type_spec.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe GitlabSchema.types['JiraImport'] do
+ it { expect(described_class.graphql_name).to eq('JiraImport') }
+
+ it 'has the expected fields' do
+ expect(described_class).to have_graphql_fields(:jira_project_key, :scheduled_at, :scheduled_by)
+ end
+end
diff --git a/spec/graphql/types/project_type_spec.rb b/spec/graphql/types/project_type_spec.rb
index 475ae9ff5f6..0c8be50ed90 100644
--- a/spec/graphql/types/project_type_spec.rb
+++ b/spec/graphql/types/project_type_spec.rb
@@ -24,7 +24,7 @@ describe GitlabSchema.types['Project'] do
namespace group statistics repository merge_requests merge_request issues
issue pipelines removeSourceBranchAfterMerge sentryDetailedError snippets
grafanaIntegration autocloseReferencedIssues suggestion_commit_message environments
- boards
+ boards jira_import_status jira_imports
]
expect(described_class).to include_graphql_fields(*expected_fields)
diff --git a/spec/lib/gitlab/jira_import/base_importer_spec.rb b/spec/lib/gitlab/jira_import/base_importer_spec.rb
new file mode 100644
index 00000000000..8bc43feb356
--- /dev/null
+++ b/spec/lib/gitlab/jira_import/base_importer_spec.rb
@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::JiraImport::BaseImporter do
+ let(:project) { create(:project) }
+
+ describe 'with any inheriting class' do
+ context 'when feature flag disabled' do
+ before do
+ stub_feature_flags(jira_issue_import: false)
+ end
+
+ it 'raises exception' do
+ expect { described_class.new(project) }.to raise_error(Projects::ImportService::Error, 'Jira import feature is disabled.')
+ end
+ end
+
+ context 'when feature flag enabled' do
+ before do
+ stub_feature_flags(jira_issue_import: true)
+ end
+
+ context 'when Jira service was not setup' do
+ it 'raises exception' do
+ expect { described_class.new(project) }.to raise_error(Projects::ImportService::Error, 'Jira integration not configured.')
+ end
+ end
+
+ context 'when Jira service exists' do
+ let!(:jira_service) { create(:jira_service, project: project) }
+
+ context 'when Jira import data is not present' do
+ it 'raises exception' do
+ expect { described_class.new(project) }.to raise_error(Projects::ImportService::Error, 'Unable to find Jira project to import data from.')
+ end
+ end
+
+ context 'when import data exists' do
+ let(:jira_import_data) do
+ data = JiraImportData.new
+ data << JiraImportData::JiraProjectDetails.new('xx', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: 1, name: 'root' })
+ data
+ end
+ let(:project) { create(:project, import_data: jira_import_data) }
+ let(:subject) { described_class.new(project) }
+
+ context 'when #imported_items_cache_key is not implemented' do
+ it { expect { subject.send(:imported_items_cache_key) }.to raise_error(NotImplementedError) }
+ end
+
+ context 'when #imported_items_cache_key is implemented' do
+ before do
+ allow(subject).to receive(:imported_items_cache_key).and_return('dumb-importer-key')
+ end
+
+ describe '#imported_items_cache_key' do
+ it { expect(subject.send(:imported_items_cache_key)).to eq('dumb-importer-key') }
+ end
+
+ describe '#mark_as_imported', :clean_gitlab_redis_cache do
+ it 'stores id in redis cache' do
+ expect(Gitlab::Cache::Import::Caching).to receive(:set_add).once.and_call_original
+
+ subject.send(:mark_as_imported, 'some-id')
+
+ expect(Gitlab::Cache::Import::Caching.set_includes?(subject.send(:imported_items_cache_key), 'some-id')).to be true
+ end
+ end
+
+ describe '#already_imported?', :clean_gitlab_redis_cache do
+ it 'returns false if value is not in cache' do
+ expect(Gitlab::Cache::Import::Caching).to receive(:set_includes?).once.and_call_original
+
+ expect(subject.send(:already_imported?, 'some-id')).to be false
+ end
+
+ it 'returns true if value already stored in cache' do
+ Gitlab::Cache::Import::Caching.set_add(subject.send(:imported_items_cache_key), 'some-id')
+
+ expect(subject.send(:already_imported?, 'some-id')).to be true
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/spec/lib/gitlab/jira_import/issues_importer_spec.rb b/spec/lib/gitlab/jira_import/issues_importer_spec.rb
new file mode 100644
index 00000000000..88e8b195dbe
--- /dev/null
+++ b/spec/lib/gitlab/jira_import/issues_importer_spec.rb
@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::JiraImport::IssuesImporter do
+ let(:user) { create(:user) }
+ let(:jira_import_data) do
+ data = JiraImportData.new
+ data << JiraImportData::JiraProjectDetails.new('XX', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name })
+ data
+ end
+ let(:project) { create(:project, import_data: jira_import_data) }
+ let!(:jira_service) { create(:jira_service, project: project) }
+
+ subject { described_class.new(project) }
+
+ before do
+ stub_feature_flags(jira_issue_import: true)
+ end
+
+ describe '#imported_items_cache_key' do
+ it_behaves_like 'raise exception if not implemented'
+ it { expect(subject.imported_items_cache_key).to eq("jira-importer/already-imported/#{project.id}/issues") }
+ end
+
+ describe '#execute', :clean_gitlab_redis_cache do
+ context 'when no returned issues' do
+ it 'does not schedule any import jobs' do
+ expect(subject).to receive(:fetch_issues).with(0).and_return([])
+ expect(subject).not_to receive(:already_imported?)
+ expect(subject).not_to receive(:mark_as_imported)
+ expect(Gitlab::JiraImport::ImportIssueWorker).not_to receive(:perform_async)
+
+ job_waiter = subject.execute
+
+ expect(job_waiter.jobs_remaining).to eq(0)
+ expect(Gitlab::JiraImport.get_issues_next_start_at(project.id)).to eq(-1)
+ end
+ end
+
+ context 'with results returned' do
+ JiraIssue = Struct.new(:id)
+ let_it_be(:jira_issue1) { JiraIssue.new(1) }
+ let_it_be(:jira_issue2) { JiraIssue.new(2) }
+
+ context 'when single page of results is returned' do
+ before do
+ stub_const("#{described_class.name}::BATCH_SIZE", 3)
+ end
+
+ it 'schedules 2 import jobs' do
+ expect(subject).to receive(:fetch_issues).and_return([jira_issue1, jira_issue2])
+ expect(Gitlab::JiraImport::ImportIssueWorker).to receive(:perform_async).twice
+ expect(Gitlab::Cache::Import::Caching).to receive(:set_add).twice.and_call_original
+ expect(Gitlab::Cache::Import::Caching).to receive(:set_includes?).twice.and_call_original
+ allow_next_instance_of(Gitlab::JiraImport::IssueSerializer) do |instance|
+ allow(instance).to receive(:execute).and_return({ key: 'data' })
+ end
+
+ job_waiter = subject.execute
+
+ expect(job_waiter.jobs_remaining).to eq(2)
+ expect(Gitlab::JiraImport.get_issues_next_start_at(project.id)).to eq(2)
+ end
+ end
+
+ context 'when there is more than one page of results' do
+ before do
+ stub_const("#{described_class.name}::BATCH_SIZE", 2)
+ end
+
+ it 'schedules 3 import jobs' do
+ expect(subject).to receive(:fetch_issues).with(0).and_return([jira_issue1, jira_issue2])
+ expect(Gitlab::JiraImport::ImportIssueWorker).to receive(:perform_async).twice.times
+ expect(Gitlab::Cache::Import::Caching).to receive(:set_add).twice.times.and_call_original
+ expect(Gitlab::Cache::Import::Caching).to receive(:set_includes?).twice.times.and_call_original
+ allow_next_instance_of(Gitlab::JiraImport::IssueSerializer) do |instance|
+ allow(instance).to receive(:execute).and_return({ key: 'data' })
+ end
+
+ job_waiter = subject.execute
+
+ expect(job_waiter.jobs_remaining).to eq(2)
+ expect(Gitlab::JiraImport.get_issues_next_start_at(project.id)).to eq(2)
+ end
+ end
+
+ context 'when duplicate results are returned' do
+ before do
+ stub_const("#{described_class.name}::BATCH_SIZE", 2)
+ end
+
+ it 'schedules 2 import jobs' do
+ expect(subject).to receive(:fetch_issues).with(0).and_return([jira_issue1, jira_issue1])
+ expect(Gitlab::JiraImport::ImportIssueWorker).to receive(:perform_async).once
+ expect(Gitlab::Cache::Import::Caching).to receive(:set_add).once.and_call_original
+ expect(Gitlab::Cache::Import::Caching).to receive(:set_includes?).twice.times.and_call_original
+ allow_next_instance_of(Gitlab::JiraImport::IssueSerializer) do |instance|
+ allow(instance).to receive(:execute).and_return({ key: 'data' })
+ end
+
+ job_waiter = subject.execute
+
+ expect(job_waiter.jobs_remaining).to eq(1)
+ expect(Gitlab::JiraImport.get_issues_next_start_at(project.id)).to eq(2)
+ end
+ end
+ end
+ end
+end
diff --git a/spec/lib/gitlab/jira_import_spec.rb b/spec/lib/gitlab/jira_import_spec.rb
new file mode 100644
index 00000000000..c5c3d6ef4b9
--- /dev/null
+++ b/spec/lib/gitlab/jira_import_spec.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::JiraImport do
+ let(:project_id) { 321 }
+
+ describe '.jira_issue_cache_key' do
+ it 'returns cache key for Jira issue imported to given project' do
+ expect(described_class.jira_issue_cache_key(project_id, 'DEMO-123')).to eq("jira-import/items-mapper/#{project_id}/issues/DEMO-123")
+ end
+ end
+
+ describe '.already_imported_cache_key' do
+ it 'returns cache key for already imported items' do
+ expect(described_class.already_imported_cache_key(:issues, project_id)).to eq("jira-importer/already-imported/#{project_id}/issues")
+ end
+ end
+
+ describe '.jira_issues_next_page_cache_key' do
+ it 'returns cache key for next issues' do
+ expect(described_class.jira_issues_next_page_cache_key(project_id)).to eq("jira-import/paginator/#{project_id}/issues")
+ end
+ end
+
+ describe '.get_issues_next_start_at', :clean_gitlab_redis_cache do
+ it 'returns zero when not defined' do
+ expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues")).to be nil
+ expect(described_class.get_issues_next_start_at(project_id)).to eq(0)
+ end
+
+ it 'returns negative value for next issues to be imported starting point' do
+ Gitlab::Cache::Import::Caching.write("jira-import/paginator/#{project_id}/issues", -10)
+
+ expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues")).to eq('-10')
+ expect(described_class.get_issues_next_start_at(project_id)).to eq(-10)
+ end
+
+ it 'returns cached value for next issues to be imported starting point' do
+ Gitlab::Cache::Import::Caching.write("jira-import/paginator/#{project_id}/issues", 10)
+
+ expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues")).to eq('10')
+ expect(described_class.get_issues_next_start_at(project_id)).to eq(10)
+ end
+ end
+
+ describe '.store_issues_next_started_at', :clean_gitlab_redis_cache do
+ it 'stores nil value' do
+ described_class.store_issues_next_started_at(project_id, nil)
+
+ expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues")).to eq ''
+ expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues").to_i).to eq(0)
+ end
+
+ it 'stores positive value' do
+ described_class.store_issues_next_started_at(project_id, 10)
+
+ expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues").to_i).to eq(10)
+ end
+
+ it 'stores negative value' do
+ described_class.store_issues_next_started_at(project_id, -10)
+
+ expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues").to_i).to eq(-10)
+ end
+ end
+end
diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb
index c3f2e3aebdd..425194ba0e3 100644
--- a/spec/models/ci/pipeline_spec.rb
+++ b/spec/models/ci/pipeline_spec.rb
@@ -3061,20 +3061,6 @@ describe Ci::Pipeline, :mailer do
expect(pipeline.source_bridge).to eq bridge
end
end
-
- describe '#update_bridge_status!' do
- it 'can update bridge status if it is running' do
- pipeline.update_bridge_status!
-
- expect(bridge.reload).to be_success
- end
-
- it 'can not update bridge status if is not active' do
- bridge.success!
-
- expect { pipeline.update_bridge_status! }.not_to change { bridge.status }
- end
- end
end
context 'when an upstream status is a build' do
@@ -3101,16 +3087,6 @@ describe Ci::Pipeline, :mailer do
expect(pipeline.source_bridge).to be_nil
end
end
-
- describe '#update_bridge_status!' do
- it 'tracks an ArgumentError and does not update upstream job status' do
- expect(Gitlab::ErrorTracking)
- .to receive(:track_exception)
- .with(instance_of(ArgumentError), pipeline_id: pipeline.id)
-
- pipeline.update_bridge_status!
- end
- end
end
end
end
diff --git a/spec/models/project_services/bamboo_service_spec.rb b/spec/models/project_services/bamboo_service_spec.rb
index 1b946278790..c1efa3a4348 100644
--- a/spec/models/project_services/bamboo_service_spec.rb
+++ b/spec/models/project_services/bamboo_service_spec.rb
@@ -8,9 +8,11 @@ describe BambooService, :use_clean_rails_memory_store_caching do
let(:bamboo_url) { 'http://gitlab.com/bamboo' }
+ let_it_be(:project) { create(:project) }
+
subject(:service) do
described_class.create(
- project: create(:project),
+ project: project,
properties: {
bamboo_url: bamboo_url,
username: 'mic',
@@ -224,6 +226,19 @@ describe BambooService, :use_clean_rails_memory_store_caching do
is_expected.to eq(:error)
end
+
+ Gitlab::HTTP::HTTP_ERRORS.each do |http_error|
+ it "sets commit status to :error with a #{http_error.name} error" do
+ WebMock.stub_request(:get, 'http://gitlab.com/bamboo/rest/api/latest/result/byChangeset/123?os_authType=basic')
+ .to_raise(http_error)
+
+ expect(Gitlab::ErrorTracking)
+ .to receive(:log_exception)
+ .with(instance_of(http_error), project_id: project.id)
+
+ is_expected.to eq(:error)
+ end
+ end
end
end
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index 51fcee29485..694c6935c1d 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -2282,6 +2282,44 @@ describe Project do
end
end
+ describe '#jira_import_status' do
+ let(:project) { create(:project, :import_started, import_type: 'jira') }
+
+ context 'when import_data is nil' do
+ it 'returns none' do
+ expect(project.import_data).to be nil
+ expect(project.jira_import_status).to eq('none')
+ end
+ end
+
+ context 'when import_data is set' do
+ let(:jira_import_data) { JiraImportData.new }
+ let(:project) { create(:project, :import_started, import_data: jira_import_data, import_type: 'jira') }
+
+ it 'returns none' do
+ expect(project.import_data.becomes(JiraImportData).force_import?).to be false
+ expect(project.jira_import_status).to eq('none')
+ end
+
+ context 'when jira_force_import is true' do
+ let(:imported_jira_project) do
+ JiraImportData::JiraProjectDetails.new('xx', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: 1, name: 'root' })
+ end
+
+ before do
+ jira_import_data = project.import_data.becomes(JiraImportData)
+ jira_import_data << imported_jira_project
+ jira_import_data.force_import!
+ end
+
+ it 'returns started' do
+ expect(project.import_data.becomes(JiraImportData).force_import?).to be true
+ expect(project.jira_import_status).to eq('started')
+ end
+ end
+ end
+ end
+
describe '#human_import_status_name' do
context 'with import_state' do
it 'returns the right human import status' do
diff --git a/spec/requests/api/graphql/project/jira_import_spec.rb b/spec/requests/api/graphql/project/jira_import_spec.rb
new file mode 100644
index 00000000000..beebc63a3c6
--- /dev/null
+++ b/spec/requests/api/graphql/project/jira_import_spec.rb
@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 'query jira import data' do
+ include GraphqlHelpers
+
+ let_it_be(:current_user) { create(:user) }
+ let_it_be(:jira_import_data) do
+ data = JiraImportData.new
+ data << JiraImportData::JiraProjectDetails.new(
+ 'AA', 2.days.ago.strftime('%Y-%m-%d %H:%M:%S'),
+ { user_id: current_user.id, name: current_user.name }
+ )
+ data << JiraImportData::JiraProjectDetails.new(
+ 'BB', 5.days.ago.strftime('%Y-%m-%d %H:%M:%S'),
+ { user_id: current_user.id, name: current_user.name }
+ )
+ data
+ end
+ let_it_be(:project) { create(:project, :private, :import_started, import_data: jira_import_data, import_type: 'jira') }
+ let(:query) do
+ %(
+ query {
+ project(fullPath: "#{project.full_path}") {
+ jiraImportStatus
+ jiraImports {
+ nodes {
+ jiraProjectKey
+ scheduledAt
+ scheduledBy {
+ username
+ }
+ }
+ }
+ }
+ }
+ )
+ end
+ let(:jira_imports) { graphql_data.dig('project', 'jiraImports', 'nodes')}
+ let(:jira_import_status) { graphql_data.dig('project', 'jiraImportStatus')}
+
+ context 'when user cannot read Jira import data' do
+ before do
+ post_graphql(query, current_user: current_user)
+ end
+
+ context 'when anonymous user' do
+ let(:current_user) { nil }
+
+ it { expect(jira_imports).to be nil }
+ end
+
+ context 'when user developer' do
+ before do
+ project.add_developer(current_user)
+ end
+
+ it { expect(jira_imports).to be nil }
+ end
+ end
+
+ context 'when user can access Jira import data' do
+ before do
+ project.add_maintainer(current_user)
+ post_graphql(query, current_user: current_user)
+ end
+
+ it_behaves_like 'a working graphql query'
+
+ context 'list of jira imports sorted ascending by scheduledAt time' do
+ it 'retuns list of jira imports' do
+ jira_proket_keys = jira_imports.map {|ji| ji['jiraProjectKey']}
+ usernames = jira_imports.map {|ji| ji.dig('scheduledBy', 'username')}
+
+ expect(jira_imports.size).to eq 2
+ expect(jira_proket_keys).to eq %w(BB AA)
+ expect(usernames).to eq [current_user.username, current_user.username]
+ end
+ end
+
+ describe 'jira imports pagination' do
+ context 'first jira import' do
+ let(:query) do
+ %(
+ query {
+ project(fullPath:"#{project.full_path}") {
+ jiraImports(first: 1) {
+ nodes {
+ jiraProjectKey
+ scheduledBy {
+ username
+ }
+ }
+ }
+ }
+ }
+ )
+ end
+
+ it 'returns latest jira import data' do
+ first_jira_import = jira_imports.first
+
+ expect(first_jira_import['jiraProjectKey']).to eq 'BB'
+ expect(first_jira_import.dig('scheduledBy', 'username')).to eq current_user.username
+ end
+ end
+
+ context 'lastest jira import' do
+ let(:query) do
+ %(
+ query {
+ project(fullPath:"#{project.full_path}") {
+ jiraImports(last: 1) {
+ nodes {
+ jiraProjectKey
+ scheduledBy {
+ username
+ }
+ }
+ }
+ }
+ }
+ )
+ end
+
+ it 'returns latest jira import data' do
+ latest_jira_import = jira_imports.first
+
+ expect(latest_jira_import['jiraProjectKey']).to eq 'AA'
+ expect(latest_jira_import.dig('scheduledBy', 'username')).to eq current_user.username
+ end
+ end
+ end
+ end
+
+ context 'jira import status' do
+ context 'when user cannot access project' do
+ it 'does not return import status' do
+ post_graphql(query, current_user: current_user)
+
+ expect(graphql_data['project']).to be nil
+ end
+ end
+
+ context 'when user can access project' do
+ before do
+ project.add_guest(current_user)
+ end
+
+ context 'when import never ran' do
+ let(:project) { create(:project) }
+
+ it 'returns import status' do
+ post_graphql(query, current_user: current_user)
+
+ expect(jira_import_status).to eq('none')
+ end
+ end
+
+ context 'when import finished' do
+ it 'returns import status' do
+ post_graphql(query, current_user: current_user)
+
+ expect(jira_import_status).to eq('finished')
+ end
+ end
+
+ context 'when import running, i.e. force-import: true' do
+ before do
+ project.import_data.becomes(JiraImportData).force_import!
+ project.save!
+ end
+
+ it 'returns import status' do
+ post_graphql(query, current_user: current_user)
+
+ expect(jira_import_status).to eq('started')
+ end
+ end
+ end
+ end
+end
diff --git a/spec/support/shared_examples/graphql/jira_import/jira_import_resolved_shared_examples.rb b/spec/support/shared_examples/graphql/jira_import/jira_import_resolved_shared_examples.rb
new file mode 100644
index 00000000000..b1d178521bb
--- /dev/null
+++ b/spec/support/shared_examples/graphql/jira_import/jira_import_resolved_shared_examples.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+shared_examples 'no jira import data present' do
+ it 'returns none' do
+ expect(resolve_imports).to eq JiraImportData.none
+ end
+end
+
+shared_examples 'no jira import access' do
+ it 'raises error' do
+ expect do
+ resolve_imports
+ end.to raise_error(Gitlab::Graphql::Errors::ResourceNotAvailable)
+ end
+end
diff --git a/spec/support/shared_examples/lib/gitlab/jira_import/base_importer_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/jira_import/base_importer_shared_examples.rb
new file mode 100644
index 00000000000..85dcc053447
--- /dev/null
+++ b/spec/support/shared_examples/lib/gitlab/jira_import/base_importer_shared_examples.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+shared_examples 'raise exception if not implemented' do
+ it { expect { described_class.new(project).imported_items_cache_key }.not_to raise_error }
+end
diff --git a/spec/support/shared_examples/workers/gitlab/jira_import/jira_import_workers_shared_examples.rb b/spec/support/shared_examples/workers/gitlab/jira_import/jira_import_workers_shared_examples.rb
index 5448526f954..71ec1ea6a74 100644
--- a/spec/support/shared_examples/workers/gitlab/jira_import/jira_import_workers_shared_examples.rb
+++ b/spec/support/shared_examples/workers/gitlab/jira_import/jira_import_workers_shared_examples.rb
@@ -16,7 +16,7 @@ shared_examples 'exit import not started' do
it 'does nothing, and exits' do
expect(Gitlab::JiraImport::AdvanceStageWorker).not_to receive(:perform_async)
- worker.perform(project.id)
+ described_class.new.perform(project.id)
end
end
@@ -25,8 +25,8 @@ shared_examples 'advance to next stage' do |next_stage|
it "advances to #{next_stage} stage" do
expect(Gitlab::JobWaiter).to receive(:new).and_return(job_waiter)
- expect(Gitlab::JiraImport::AdvanceStageWorker).to receive(:perform_async).with(project.id, { job_waiter.key => job_waiter.jobs_remaining }, next_stage.to_sym)
+ expect(Gitlab::JiraImport::AdvanceStageWorker).to receive(:perform_async).with(project.id, { job_waiter.key => job_waiter.jobs_remaining }, next_stage.to_sym).and_return([])
- worker.perform(project.id)
+ described_class.new.perform(project.id)
end
end
diff --git a/spec/workers/gitlab/jira_import/import_issue_worker_spec.rb b/spec/workers/gitlab/jira_import/import_issue_worker_spec.rb
new file mode 100644
index 00000000000..c09492efcae
--- /dev/null
+++ b/spec/workers/gitlab/jira_import/import_issue_worker_spec.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::JiraImport::ImportIssueWorker do
+ let_it_be(:user) { create(:user) }
+ let_it_be(:project) { create(:project) }
+
+ describe 'modules' do
+ it { expect(described_class).to include_module(ApplicationWorker) }
+ it { expect(described_class).to include_module(Gitlab::NotifyUponDeath) }
+ it { expect(described_class).to include_module(Gitlab::JiraImport::QueueOptions) }
+ it { expect(described_class).to include_module(Gitlab::Import::DatabaseHelpers) }
+ end
+
+ subject { described_class.new }
+
+ describe '#perform', :clean_gitlab_redis_cache do
+ let(:issue_attrs) { build(:issue, project_id: project.id).as_json.compact }
+
+ context 'when any exception raised while inserting to DB' do
+ before do
+ allow(subject).to receive(:insert_and_return_id).and_raise(StandardError)
+ expect(Gitlab::JobWaiter).to receive(:notify)
+
+ subject.perform(project.id, 123, issue_attrs, 'some-key')
+ end
+
+ it 'record a failed to import issue' do
+ expect(Gitlab::Cache::Import::Caching.read(Gitlab::JiraImport.failed_issues_counter_cache_key(project.id)).to_i).to eq(1)
+ end
+ end
+
+ context 'when record is successfully inserted' do
+ before do
+ subject.perform(project.id, 123, issue_attrs, 'some-key')
+ end
+
+ it 'does not record import failure' do
+ expect(Gitlab::Cache::Import::Caching.read(Gitlab::JiraImport.failed_issues_counter_cache_key(project.id)).to_i).to eq(0)
+ end
+ end
+ end
+end
diff --git a/spec/workers/gitlab/jira_import/stage/finish_import_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/finish_import_worker_spec.rb
index fa0c7d83851..00505226212 100644
--- a/spec/workers/gitlab/jira_import/stage/finish_import_worker_spec.rb
+++ b/spec/workers/gitlab/jira_import/stage/finish_import_worker_spec.rb
@@ -3,8 +3,8 @@
require 'spec_helper'
describe Gitlab::JiraImport::Stage::FinishImportWorker do
- let(:project) { create(:project) }
- let(:worker) { described_class.new }
+ let_it_be(:project) { create(:project) }
+ let_it_be(:worker) { described_class.new }
describe 'modules' do
it_behaves_like 'include import workers modules'
@@ -46,7 +46,7 @@ describe Gitlab::JiraImport::Stage::FinishImportWorker do
it 'changes import state to finished' do
worker.perform(project.id)
- expect(project.reload.import_state.status).to eq "finished"
+ expect(project.reload.import_state.status).to eq("finished")
end
it 'removes force-import flag' do
diff --git a/spec/workers/gitlab/jira_import/stage/import_attachments_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/import_attachments_worker_spec.rb
index fa2f3501973..513925507a1 100644
--- a/spec/workers/gitlab/jira_import/stage/import_attachments_worker_spec.rb
+++ b/spec/workers/gitlab/jira_import/stage/import_attachments_worker_spec.rb
@@ -3,8 +3,7 @@
require 'spec_helper'
describe Gitlab::JiraImport::Stage::ImportAttachmentsWorker do
- let(:project) { create(:project) }
- let(:worker) { described_class.new }
+ let_it_be(:project) { create(:project) }
describe 'modules' do
it_behaves_like 'include import workers modules'
diff --git a/spec/workers/gitlab/jira_import/stage/import_issues_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/import_issues_worker_spec.rb
index b43519a3e5d..dca748a6ebc 100644
--- a/spec/workers/gitlab/jira_import/stage/import_issues_worker_spec.rb
+++ b/spec/workers/gitlab/jira_import/stage/import_issues_worker_spec.rb
@@ -3,8 +3,8 @@
require 'spec_helper'
describe Gitlab::JiraImport::Stage::ImportIssuesWorker do
- let(:project) { create(:project) }
- let(:worker) { described_class.new }
+ let_it_be(:user) { create(:user) }
+ let_it_be(:project) { create(:project) }
describe 'modules' do
it_behaves_like 'include import workers modules'
@@ -30,10 +30,49 @@ describe Gitlab::JiraImport::Stage::ImportIssuesWorker do
it_behaves_like 'exit import not started'
end
- context 'when import started' do
+ context 'when import started', :clean_gitlab_redis_cache do
+ let(:jira_import_data) do
+ data = JiraImportData.new
+ data << JiraImportData::JiraProjectDetails.new('XX', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name })
+ data
+ end
+ let(:project) { create(:project, import_data: jira_import_data) }
+ let!(:jira_service) { create(:jira_service, project: project) }
let!(:import_state) { create(:import_state, status: :started, project: project) }
- it_behaves_like 'advance to next stage', :attachments
+ before do
+ allow_next_instance_of(Gitlab::JiraImport::IssuesImporter) do |instance|
+ allow(instance).to receive(:fetch_issues).and_return([])
+ end
+ end
+
+ context 'when start_at is nil' do
+ it_behaves_like 'advance to next stage', :attachments
+ end
+
+ context 'when start_at is zero' do
+ before do
+ allow(Gitlab::Cache::Import::Caching).to receive(:read).and_return(0)
+ end
+
+ it_behaves_like 'advance to next stage', :issues
+ end
+
+ context 'when start_at is greater than zero' do
+ before do
+ allow(Gitlab::Cache::Import::Caching).to receive(:read).and_return(25)
+ end
+
+ it_behaves_like 'advance to next stage', :issues
+ end
+
+ context 'when start_at is below zero' do
+ before do
+ allow(Gitlab::Cache::Import::Caching).to receive(:read).and_return(-1)
+ end
+
+ it_behaves_like 'advance to next stage', :attachments
+ end
end
end
end
diff --git a/spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb
index 827efb85a17..2b156e0f489 100644
--- a/spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb
+++ b/spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb
@@ -3,8 +3,7 @@
require 'spec_helper'
describe Gitlab::JiraImport::Stage::ImportLabelsWorker do
- let(:project) { create(:project) }
- let(:worker) { described_class.new }
+ let_it_be(:project) { create(:project) }
describe 'modules' do
it_behaves_like 'include import workers modules'
diff --git a/spec/workers/gitlab/jira_import/stage/import_notes_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/import_notes_worker_spec.rb
index bd6b36613cc..7d1c29614e4 100644
--- a/spec/workers/gitlab/jira_import/stage/import_notes_worker_spec.rb
+++ b/spec/workers/gitlab/jira_import/stage/import_notes_worker_spec.rb
@@ -3,8 +3,7 @@
require 'spec_helper'
describe Gitlab::JiraImport::Stage::ImportNotesWorker do
- let(:project) { create(:project) }
- let(:worker) { described_class.new }
+ let_it_be(:project) { create(:project) }
describe 'modules' do
it_behaves_like 'include import workers modules'
diff --git a/spec/workers/gitlab/jira_import/stage/start_import_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/start_import_worker_spec.rb
index cc70277384d..d5e10a950bb 100644
--- a/spec/workers/gitlab/jira_import/stage/start_import_worker_spec.rb
+++ b/spec/workers/gitlab/jira_import/stage/start_import_worker_spec.rb
@@ -3,7 +3,7 @@
require 'spec_helper'
describe Gitlab::JiraImport::Stage::StartImportWorker do
- let(:project) { create(:project) }
+ let(:project) { create(:project, import_type: 'jira') }
let(:worker) { described_class.new }
let(:jid) { '12345678' }
@@ -24,13 +24,19 @@ describe Gitlab::JiraImport::Stage::StartImportWorker do
end
end
- context 'when feature flag not enabled' do
+ context 'when feature flag enabled' do
+ let(:symbol_keys_project) do
+ { key: 'AA', scheduled_at: 2.days.ago.strftime('%Y-%m-%d %H:%M:%S'), scheduled_by: { 'user_id' => 1, 'name' => 'tester1' } }
+ end
+ let(:import_data) { JiraImportData.new( data: { 'jira' => { JiraImportData::FORCE_IMPORT_KEY => true, projects: [symbol_keys_project] } }) }
+
before do
stub_feature_flags(jira_issue_import: true)
end
- context 'when import is not scheudled' do
- let!(:import_state) { create(:import_state, project: project, status: :none, jid: jid) }
+ context 'when import is not scheduled' do
+ let(:project) { create(:project, import_type: 'jira') }
+ let(:import_state) { create(:import_state, project: project, status: :none, jid: jid) }
it 'exits because import not started' do
expect(Gitlab::JiraImport::Stage::ImportLabelsWorker).not_to receive(:perform_async)
@@ -40,17 +46,32 @@ describe Gitlab::JiraImport::Stage::StartImportWorker do
end
context 'when import is scheduled' do
- let!(:import_state) { create(:import_state, project: project, status: :scheduled, jid: jid) }
+ let(:import_state) { create(:import_state, status: :scheduled, jid: jid) }
+ let(:project) { create(:project, import_type: 'jira', import_state: import_state) }
- it 'advances to importing labels' do
- expect(Gitlab::JiraImport::Stage::ImportLabelsWorker).to receive(:perform_async)
+ context 'when this is a mirror sync in a jira imported project' do
+ it 'exits early' do
+ expect(Gitlab::Import::SetAsyncJid).not_to receive(:set_jid)
+ expect(Gitlab::JiraImport::Stage::ImportLabelsWorker).not_to receive(:perform_async)
- worker.perform(project.id)
+ worker.perform(project.id)
+ end
+ end
+
+ context 'when scheduled import is a hard triggered jira import and not a mirror' do
+ let!(:project) { create(:project, import_type: 'jira', import_data: import_data, import_state: import_state) }
+
+ it 'advances to importing labels' do
+ expect(Gitlab::JiraImport::Stage::ImportLabelsWorker).to receive(:perform_async)
+
+ worker.perform(project.id)
+ end
end
end
context 'when import is started' do
- let!(:import_state) { create(:import_state, project: project, status: :started, jid: jid) }
+ let!(:import_state) { create(:import_state, status: :started, jid: jid) }
+ let!(:project) { create(:project, import_type: 'jira', import_data: import_data, import_state: import_state) }
context 'when this is the same worker that stated import' do
it 'advances to importing labels' do
@@ -72,7 +93,8 @@ describe Gitlab::JiraImport::Stage::StartImportWorker do
end
context 'when import is finished' do
- let!(:import_state) { create(:import_state, project: project, status: :finished, jid: jid) }
+ let!(:import_state) { create(:import_state, status: :finished, jid: jid) }
+ let!(:project) { create(:project, import_type: 'jira', import_data: import_data, import_state: import_state) }
it 'advances to importing labels' do
allow(worker).to receive(:jid).and_return(jid)
diff --git a/spec/workers/reactive_caching_worker_spec.rb b/spec/workers/reactive_caching_worker_spec.rb
index 6c74c4ea072..c39a97b4eee 100644
--- a/spec/workers/reactive_caching_worker_spec.rb
+++ b/spec/workers/reactive_caching_worker_spec.rb
@@ -28,4 +28,14 @@ describe ReactiveCachingWorker do
end
end
end
+
+ describe 'worker context' do
+ it 'sets the related class on the job' do
+ described_class.perform_async('Environment', 1, 'other', 'argument')
+
+ scheduled_job = described_class.jobs.first
+
+ expect(scheduled_job).to include('meta.related_class' => 'Environment')
+ end
+ end
end