diff options
61 files changed, 1592 insertions, 161 deletions
@@ -301,7 +301,7 @@ gem 'sentry-raven', '~> 2.9' gem 'premailer-rails', '~> 1.10.3' # LabKit: Tracing and Correlation -gem 'gitlab-labkit', '0.11.0' +gem 'gitlab-labkit', '0.12.0' # I18n gem 'ruby_parser', '~> 3.8', require: false diff --git a/Gemfile.lock b/Gemfile.lock index bb64fb09649..d050f628ab2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -380,7 +380,7 @@ GEM rake (> 10, < 14) ruby-statistics (>= 2.1) thor (>= 0.19, < 2) - gitlab-labkit (0.11.0) + gitlab-labkit (0.12.0) actionpack (>= 5.0.0, < 6.1.0) activesupport (>= 5.0.0, < 6.1.0) grpc (~> 1.19) @@ -1232,7 +1232,7 @@ DEPENDENCIES github-markup (~> 1.7.0) gitlab-chronic (~> 0.10.5) gitlab-derailed_benchmarks - gitlab-labkit (= 0.11.0) + gitlab-labkit (= 0.12.0) gitlab-license (~> 1.0) gitlab-mail_room (~> 0.0.3) gitlab-markup (~> 1.7.0) diff --git a/app/assets/javascripts/pages/projects/snippets/edit/index.js b/app/assets/javascripts/pages/projects/snippets/edit/index.js index 53606acc508..0a63f5c7172 100644 --- a/app/assets/javascripts/pages/projects/snippets/edit/index.js +++ b/app/assets/javascripts/pages/projects/snippets/edit/index.js @@ -1,8 +1 @@ -import $ from 'jquery'; -import initSnippet from '~/snippet/snippet_bundle'; -import initForm from '~/pages/projects/init_form'; - -document.addEventListener('DOMContentLoaded', () => { - initSnippet(); - initForm($('.snippet-form')); -}); +import '~/snippet/snippet_edit'; diff --git a/app/assets/javascripts/pages/projects/snippets/new/index.js b/app/assets/javascripts/pages/projects/snippets/new/index.js index 53606acc508..0a63f5c7172 100644 --- a/app/assets/javascripts/pages/projects/snippets/new/index.js +++ b/app/assets/javascripts/pages/projects/snippets/new/index.js @@ -1,8 +1 @@ -import $ from 'jquery'; -import initSnippet from '~/snippet/snippet_bundle'; -import initForm from '~/pages/projects/init_form'; - -document.addEventListener('DOMContentLoaded', () => { - initSnippet(); - initForm($('.snippet-form')); -}); +import '~/snippet/snippet_edit'; diff --git a/app/assets/javascripts/pages/snippets/edit/index.js b/app/assets/javascripts/pages/snippets/edit/index.js index d86e1632ae5..0a63f5c7172 100644 --- a/app/assets/javascripts/pages/snippets/edit/index.js +++ b/app/assets/javascripts/pages/snippets/edit/index.js @@ -1,7 +1 @@ -import initSnippet from '~/snippet/snippet_bundle'; -import form from '../form'; - -document.addEventListener('DOMContentLoaded', () => { - initSnippet(); - form(); -}); +import '~/snippet/snippet_edit'; diff --git a/app/assets/javascripts/pages/snippets/new/index.js b/app/assets/javascripts/pages/snippets/new/index.js index d86e1632ae5..0a63f5c7172 100644 --- a/app/assets/javascripts/pages/snippets/new/index.js +++ b/app/assets/javascripts/pages/snippets/new/index.js @@ -1,7 +1 @@ -import initSnippet from '~/snippet/snippet_bundle'; -import form from '../form'; - -document.addEventListener('DOMContentLoaded', () => { - initSnippet(); - form(); -}); +import '~/snippet/snippet_edit'; diff --git a/app/assets/javascripts/snippet/snippet_edit.js b/app/assets/javascripts/snippet/snippet_edit.js new file mode 100644 index 00000000000..a098d17a226 --- /dev/null +++ b/app/assets/javascripts/snippet/snippet_edit.js @@ -0,0 +1,25 @@ +import $ from 'jquery'; +import initSnippet from '~/snippet/snippet_bundle'; +import ZenMode from '~/zen_mode'; +import GLForm from '~/gl_form'; + +document.addEventListener('DOMContentLoaded', () => { + const form = document.querySelector('.snippet-form'); + const personalSnippetOptions = { + members: false, + issues: false, + mergeRequests: false, + epics: false, + milestones: false, + labels: false, + snippets: false, + }; + const projectSnippetOptions = {}; + + const options = + form.dataset.snippetType === 'project' ? projectSnippetOptions : personalSnippetOptions; + + initSnippet(); + new ZenMode(); // eslint-disable-line no-new + new GLForm($(form), options); // eslint-disable-line no-new +}); diff --git a/app/controllers/admin/concerns/authenticates_2fa_for_admin_mode.rb b/app/controllers/admin/concerns/authenticates_2fa_for_admin_mode.rb index c6fd1d55e51..6014ed0dd13 100644 --- a/app/controllers/admin/concerns/authenticates_2fa_for_admin_mode.rb +++ b/app/controllers/admin/concerns/authenticates_2fa_for_admin_mode.rb @@ -37,7 +37,7 @@ module Authenticates2FAForAdminMode # Remove any lingering user data from login session.delete(:otp_user_id) - user.save! + user.save! unless Gitlab::Database.read_only? # The admin user has successfully passed 2fa, enable admin mode ignoring password enable_admin_mode diff --git a/app/controllers/admin/sessions_controller.rb b/app/controllers/admin/sessions_controller.rb index 841ad46b47e..1dc1cd5fb82 100644 --- a/app/controllers/admin/sessions_controller.rb +++ b/app/controllers/admin/sessions_controller.rb @@ -64,7 +64,9 @@ class Admin::SessionsController < ApplicationController end def valid_otp_attempt?(user) - user.validate_and_consume_otp!(user_params[:otp_attempt]) || - user.invalidate_otp_backup_code!(user_params[:otp_attempt]) + valid_otp_attempt = user.validate_and_consume_otp!(user_params[:otp_attempt]) + return valid_otp_attempt if Gitlab::Database.read_only? + + valid_otp_attempt || user.invalidate_otp_backup_code!(user_params[:otp_attempt]) end end diff --git a/app/controllers/projects/merge_requests/diffs_controller.rb b/app/controllers/projects/merge_requests/diffs_controller.rb index 1fb5e333aaa..51128ac1be4 100644 --- a/app/controllers/projects/merge_requests/diffs_controller.rb +++ b/app/controllers/projects/merge_requests/diffs_controller.rb @@ -92,8 +92,8 @@ class Projects::MergeRequests::DiffsController < Projects::MergeRequests::Applic # Deprecated: https://gitlab.com/gitlab-org/gitlab/issues/37735 def find_merge_request_diff_compare @merge_request_diff = - if diff_id = params[:diff_id].presence - @merge_request.merge_request_diffs.viewable.find_by(id: diff_id) + if params[:diff_id].present? + @merge_request.merge_request_diffs.viewable.find_by(id: params[:diff_id]) else @merge_request.merge_request_diff end diff --git a/app/graphql/resolvers/projects/jira_imports_resolver.rb b/app/graphql/resolvers/projects/jira_imports_resolver.rb new file mode 100644 index 00000000000..a9bfb7b612e --- /dev/null +++ b/app/graphql/resolvers/projects/jira_imports_resolver.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Resolvers + module Projects + class JiraImportsResolver < BaseResolver + include Gitlab::Graphql::Authorize::AuthorizeResource + + alias_method :project, :object + + def resolve(**args) + return JiraImportData.none unless project&.import_data.present? + + authorize!(project) + + project.import_data.becomes(JiraImportData).projects + end + + def authorized_resource?(project) + Ability.allowed?(context[:current_user], :admin_project, project) + end + end + end +end diff --git a/app/graphql/types/jira_import_type.rb b/app/graphql/types/jira_import_type.rb new file mode 100644 index 00000000000..01ec6184844 --- /dev/null +++ b/app/graphql/types/jira_import_type.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Types + # rubocop: disable Graphql/AuthorizeTypes + # Authorization is at project level for owners or admins, + # so it is added directly to the Resolvers::JiraImportsResolver + class JiraImportType < BaseObject + graphql_name 'JiraImport' + + field :scheduled_at, Types::TimeType, null: true, + description: 'Timestamp of when the Jira import was created/started' + field :scheduled_by, Types::UserType, null: true, + description: 'User that started the Jira import' + field :jira_project_key, GraphQL::STRING_TYPE, null: false, + description: 'Project key for the imported Jira project', + method: :key + + def scheduled_at + DateTime.parse(object.scheduled_at) + end + + def scheduled_by + ::Gitlab::Graphql::Loaders::BatchModelLoader.new(User, object.scheduled_by['user_id']).find + end + end + # rubocop: enable Graphql/AuthorizeTypes +end diff --git a/app/graphql/types/project_type.rb b/app/graphql/types/project_type.rb index 5c0b9182ac5..b52266bc477 100644 --- a/app/graphql/types/project_type.rb +++ b/app/graphql/types/project_type.rb @@ -90,8 +90,9 @@ module Types end field :import_status, GraphQL::STRING_TYPE, null: true, - description: 'Status of project import background job of the project' - + description: 'Status of import background job of the project' + field :jira_import_status, GraphQL::STRING_TYPE, null: true, + description: 'Status of Jira import background job of the project' field :only_allow_merge_if_pipeline_succeeds, GraphQL::BOOLEAN_TYPE, null: true, description: 'Indicates if merge requests of the project can only be merged with successful jobs' field :request_access_enabled, GraphQL::BOOLEAN_TYPE, null: true, @@ -192,6 +193,12 @@ module Types null: true, description: 'A single board of the project', resolver: Resolvers::BoardsResolver.single + + field :jira_imports, + Types::JiraImportType.connection_type, + null: true, + description: 'Jira imports into the project', + resolver: Resolvers::Projects::JiraImportsResolver end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index e9cd0d91bc0..4e9be86d3a9 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -231,14 +231,6 @@ module Ci end end - after_transition created: :pending do |pipeline| - next if Feature.enabled?(:ci_drop_bridge_on_downstream_errors, pipeline.project, default_enabled: true) - next unless pipeline.bridge_triggered? - next if pipeline.bridge_waiting? - - pipeline.update_bridge_status! - end - after_transition any => [:success, :failed] do |pipeline| pipeline.run_after_commit do if Feature.enabled?(:ci_pipeline_fixed_notifications) @@ -761,15 +753,6 @@ module Ci end end - def update_bridge_status! - raise ArgumentError unless bridge_triggered? - raise BridgeStatusError unless source_bridge.active? - - source_bridge.success! - rescue => e - Gitlab::ErrorTracking.track_exception(e, pipeline_id: id) - end - def bridge_triggered? source_bridge.present? end diff --git a/app/models/jira_import_data.rb b/app/models/jira_import_data.rb index 63be190aa0d..b39ca7290be 100644 --- a/app/models/jira_import_data.rb +++ b/app/models/jira_import_data.rb @@ -39,4 +39,8 @@ class JiraImportData < ProjectImportData data['jira'].delete(FORCE_IMPORT_KEY) end + + def current_project + projects.last + end end diff --git a/app/models/project.rb b/app/models/project.rb index 9d055cbd6c6..5aec64da4cf 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -857,6 +857,12 @@ class Project < ApplicationRecord import_state&.status || 'none' end + def jira_import_status + return import_status if jira_force_import? + + import_data&.becomes(JiraImportData)&.projects.blank? ? 'none' : 'finished' + end + def human_import_status_name import_state&.human_status_name || 'none' end diff --git a/app/models/project_services/bamboo_service.rb b/app/models/project_services/bamboo_service.rb index dfeb21680a9..8c1f4fef09b 100644 --- a/app/models/project_services/bamboo_service.rb +++ b/app/models/project_services/bamboo_service.rb @@ -73,7 +73,7 @@ class BambooService < CiService end def calculate_reactive_cache(sha, ref) - response = get_path("rest/api/latest/result/byChangeset/#{sha}") + response = try_get_path("rest/api/latest/result/byChangeset/#{sha}") { build_page: read_build_page(response), commit_status: read_commit_status(response) } end @@ -81,7 +81,7 @@ class BambooService < CiService private def get_build_result(response) - return if response.code != 200 + return if response&.code != 200 # May be nil if no result, a single result hash, or an array if multiple results for a given changeset. result = response.dig('results', 'results', 'result') @@ -107,7 +107,7 @@ class BambooService < CiService end def read_commit_status(response) - return :error unless response.code == 200 || response.code == 404 + return :error unless response && (response.code == 200 || response.code == 404) result = get_build_result(response) status = @@ -130,24 +130,31 @@ class BambooService < CiService end end + def try_get_path(path, query_params = {}) + params = build_get_params(query_params) + params[:extra_log_info] = { project_id: project_id } + + Gitlab::HTTP.try_get(build_url(path), params) + end + + def get_path(path, query_params = {}) + Gitlab::HTTP.get(build_url(path), build_get_params(query_params)) + end + def build_url(path) Gitlab::Utils.append_path(bamboo_url, path) end - def get_path(path, query_params = {}) - url = build_url(path) + def build_get_params(query_params) + params = { verify: false, query: query_params } + return params if username.blank? && password.blank? - if username.blank? && password.blank? - Gitlab::HTTP.get(url, verify: false, query: query_params) - else - query_params[:os_authType] = 'basic' - Gitlab::HTTP.get(url, - verify: false, - query: query_params, - basic_auth: { - username: username, - password: password - }) - end + query_params[:os_authType] = 'basic' + params[:basic_auth] = basic_auth + params + end + + def basic_auth + { username: username, password: password } end end diff --git a/app/models/user.rb b/app/models/user.rb index 9fb3c47e143..65ff4c98b15 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1715,6 +1715,23 @@ class User < ApplicationRecord super end + # This is copied from Devise::Models::TwoFactorAuthenticatable#consume_otp! + # + # An OTP cannot be used more than once in a given timestep + # Storing timestep of last valid OTP is sufficient to satisfy this requirement + # + # See: + # <https://github.com/tinfoil/devise-two-factor/blob/master/lib/devise_two_factor/models/two_factor_authenticatable.rb#L66> + # + def consume_otp! + if self.consumed_timestep != current_otp_timestep + self.consumed_timestep = current_otp_timestep + return Gitlab::Database.read_only? ? true : save(validate: false) + end + + false + end + private def default_private_profile_to_false diff --git a/app/services/ci/create_cross_project_pipeline_service.rb b/app/services/ci/create_cross_project_pipeline_service.rb index 3a2cc3f9d32..a73a2e2b471 100644 --- a/app/services/ci/create_cross_project_pipeline_service.rb +++ b/app/services/ci/create_cross_project_pipeline_service.rb @@ -34,8 +34,6 @@ module Ci end downstream_pipeline.tap do |pipeline| - next if Feature.disabled?(:ci_drop_bridge_on_downstream_errors, project, default_enabled: true) - update_bridge_status!(@bridge, pipeline) end end diff --git a/app/views/shared/snippets/_form.html.haml b/app/views/shared/snippets/_form.html.haml index 828015d29f5..5ba6d52fefe 100644 --- a/app/views/shared/snippets/_form.html.haml +++ b/app/views/shared/snippets/_form.html.haml @@ -3,7 +3,8 @@ .snippet-form-holder = form_for @snippet, url: url, - html: { class: "snippet-form js-requires-input js-quick-submit common-note-form" } do |f| + html: { class: "snippet-form js-requires-input js-quick-submit common-note-form" }, + data: { "snippet-type": @snippet.project_id ? 'project' : 'personal'} do |f| = form_errors(@snippet) .form-group diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 16520519a0b..7dc47c55a04 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -556,6 +556,13 @@ :resource_boundary: :unknown :weight: 1 :idempotent: +- :name: jira_importer:jira_import_import_issue + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: - :name: jira_importer:jira_import_stage_finish_import :feature_category: :importers :has_external_dependencies: diff --git a/app/workers/gitlab/jira_import/import_issue_worker.rb b/app/workers/gitlab/jira_import/import_issue_worker.rb new file mode 100644 index 00000000000..832916a03b6 --- /dev/null +++ b/app/workers/gitlab/jira_import/import_issue_worker.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + class ImportIssueWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include NotifyUponDeath + include Gitlab::JiraImport::QueueOptions + include Gitlab::Import::DatabaseHelpers + + def perform(project_id, jira_issue_id, issue_attributes, waiter_key) + issue_id = insert_and_return_id(issue_attributes, Issue) + cache_issue_mapping(issue_id, jira_issue_id, project_id) + rescue => ex + # Todo: Record jira issue id(or better jira issue key), + # so that we can report the list of failed to import issues to the user + # see https://gitlab.com/gitlab-org/gitlab/-/issues/211653 + # + # It's possible the project has been deleted since scheduling this + # job. In this case we'll just skip creating the issue. + Gitlab::ErrorTracking.track_exception(ex, project_id: project_id) + JiraImport.increment_issue_failures(project_id) + ensure + # ensure we notify job waiter that the job has finished + JobWaiter.notify(waiter_key, jid) if waiter_key + end + + private + + def cache_issue_mapping(issue_id, jira_issue_id, project_id) + cache_key = JiraImport.jira_issue_cache_key(project_id, jira_issue_id) + Gitlab::Cache::Import::Caching.write(cache_key, issue_id) + end + end + end +end diff --git a/app/workers/gitlab/jira_import/stage/finish_import_worker.rb b/app/workers/gitlab/jira_import/stage/finish_import_worker.rb index 5b1661d68c6..f053037e78a 100644 --- a/app/workers/gitlab/jira_import/stage/finish_import_worker.rb +++ b/app/workers/gitlab/jira_import/stage/finish_import_worker.rb @@ -11,6 +11,7 @@ module Gitlab def import(project) project.after_import ensure + JiraImport.cache_cleanup(project.id) project.import_data.becomes(JiraImportData).finish_import! project.import_data.save! end diff --git a/app/workers/gitlab/jira_import/stage/import_issues_worker.rb b/app/workers/gitlab/jira_import/stage/import_issues_worker.rb index 79ed8e1f2da..7e257afc4d9 100644 --- a/app/workers/gitlab/jira_import/stage/import_issues_worker.rb +++ b/app/workers/gitlab/jira_import/stage/import_issues_worker.rb @@ -9,12 +9,19 @@ module Gitlab private def import(project) - # fake issues import workers for now - # new job waiter will have zero jobs_remaining by default, so it will just pass on to next stage - jobs_waiter = JobWaiter.new + jobs_waiter = Gitlab::JiraImport::IssuesImporter.new(project).execute + project.import_state.refresh_jid_expiration - Gitlab::JiraImport::AdvanceStageWorker.perform_async(project.id, { jobs_waiter.key => jobs_waiter.jobs_remaining }, :attachments) + Gitlab::JiraImport::AdvanceStageWorker.perform_async( + project.id, + { jobs_waiter.key => jobs_waiter.jobs_remaining }, + next_stage(project) + ) + end + + def next_stage(project) + Gitlab::JiraImport.get_issues_next_start_at(project.id) < 0 ? :attachments : :issues end end end diff --git a/app/workers/gitlab/jira_import/stage/start_import_worker.rb b/app/workers/gitlab/jira_import/stage/start_import_worker.rb index 8abbfab647b..ae4864064e4 100644 --- a/app/workers/gitlab/jira_import/stage/start_import_worker.rb +++ b/app/workers/gitlab/jira_import/stage/start_import_worker.rb @@ -26,6 +26,7 @@ module Gitlab def start_import return false unless project return false if Feature.disabled?(:jira_issue_import, project) + return false unless project.jira_force_import? return true if start(project.import_state) Gitlab::Import::Logger.info( diff --git a/app/workers/reactive_caching_worker.rb b/app/workers/reactive_caching_worker.rb index 716b1de2bf5..1921ac6619b 100644 --- a/app/workers/reactive_caching_worker.rb +++ b/app/workers/reactive_caching_worker.rb @@ -13,6 +13,11 @@ class ReactiveCachingWorker # rubocop:disable Scalability/IdempotentWorker urgency :high worker_resource_boundary :cpu + def self.context_for_arguments(arguments) + class_name, *_other_args = arguments + Gitlab::ApplicationContext.new(related_class: class_name) + end + def perform(class_name, id, *args) klass = begin class_name.constantize diff --git a/changelogs/unreleased/jira-import-graphql-api.yml b/changelogs/unreleased/jira-import-graphql-api.yml new file mode 100644 index 00000000000..8cfdfda8e2a --- /dev/null +++ b/changelogs/unreleased/jira-import-graphql-api.yml @@ -0,0 +1,5 @@ +--- +title: Allow querying of Jira imports and their status via GraphQL +merge_request: 27587 +author: +type: added diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index 0fbd8c84e58..9766fc39e2e 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -63,6 +63,11 @@ input AdminSidekiqQueuesDeleteJobsInput { queueName: String! """ + Delete jobs matching related_class in the context metadata + """ + relatedClass: String + + """ Delete jobs matching root_namespace in the context metadata """ rootNamespace: String @@ -4093,6 +4098,58 @@ Represents untyped JSON """ scalar JSON +type JiraImport { + """ + Project key for the imported Jira project + """ + jiraProjectKey: String! + + """ + Timestamp of when the Jira import was created/started + """ + scheduledAt: Time + + """ + User that started the Jira import + """ + scheduledBy: User +} + +""" +The connection type for JiraImport. +""" +type JiraImportConnection { + """ + A list of edges. + """ + edges: [JiraImportEdge] + + """ + A list of nodes. + """ + nodes: [JiraImport] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type JiraImportEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: JiraImport +} + type Label { """ Background color of the label @@ -5749,7 +5806,7 @@ type Project { id: ID! """ - Status of project import background job of the project + Status of import background job of the project """ importStatus: String @@ -5939,6 +5996,36 @@ type Project { issuesEnabled: Boolean """ + Status of Jira import background job of the project + """ + jiraImportStatus: String + + """ + Jira imports into the project + """ + jiraImports( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): JiraImportConnection + + """ (deprecated) Enable jobs for this project. Use `builds_access_level` instead """ jobsEnabled: Boolean diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index bd78b51684f..7584e48b1e5 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -182,6 +182,16 @@ "defaultValue": null }, { + "name": "relatedClass", + "description": "Delete jobs matching related_class in the context metadata", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { "name": "queueName", "description": "The name of the queue to delete jobs from", "type": { @@ -11613,6 +11623,177 @@ }, { "kind": "OBJECT", + "name": "JiraImport", + "description": null, + "fields": [ + { + "name": "jiraProjectKey", + "description": "Project key for the imported Jira project", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "scheduledAt", + "description": "Timestamp of when the Jira import was created/started", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "scheduledBy", + "description": "User that started the Jira import", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "JiraImportConnection", + "description": "The connection type for JiraImport.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "JiraImportEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "JiraImport", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "JiraImportEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "JiraImport", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "Label", "description": null, "fields": [ @@ -17441,7 +17622,7 @@ }, { "name": "importStatus", - "description": "Status of project import background job of the project", + "description": "Status of import background job of the project", "args": [ ], @@ -17866,6 +18047,73 @@ "deprecationReason": null }, { + "name": "jiraImportStatus", + "description": "Status of Jira import background job of the project", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "jiraImports", + "description": "Jira imports into the project", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "JiraImportConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "jobsEnabled", "description": "(deprecated) Enable jobs for this project. Use `builds_access_level` instead", "args": [ diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index dfbd08be898..2aee7d484d0 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -605,6 +605,14 @@ Autogenerated return type of IssueSetWeight | `errors` | String! => Array | Reasons why the mutation failed. | | `issue` | Issue | The issue after mutation | +## JiraImport + +| Name | Type | Description | +| --- | ---- | ---------- | +| `jiraProjectKey` | String! | Project key for the imported Jira project | +| `scheduledAt` | Time | Timestamp of when the Jira import was created/started | +| `scheduledBy` | User | User that started the Jira import | + ## Label | Name | Type | Description | @@ -879,9 +887,10 @@ Information about pagination in a connection. | `group` | Group | Group of the project | | `httpUrlToRepo` | String | URL to connect to the project via HTTPS | | `id` | ID! | ID of the project | -| `importStatus` | String | Status of project import background job of the project | +| `importStatus` | String | Status of import background job of the project | | `issue` | Issue | A single issue of the project | | `issuesEnabled` | Boolean | (deprecated) Does this project have issues enabled?. Use `issues_access_level` instead | +| `jiraImportStatus` | String | Status of Jira import background job of the project | | `jobsEnabled` | Boolean | (deprecated) Enable jobs for this project. Use `builds_access_level` instead | | `lastActivityAt` | Time | Timestamp of the project last activity | | `lfsEnabled` | Boolean | Indicates if the project has Large File Storage (LFS) enabled | diff --git a/doc/development/architecture.md b/doc/development/architecture.md index 32626c19298..061e910a6d0 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -563,18 +563,23 @@ sequenceDiagram participant Git on server Note left of Git on client: git fetch - Git on client->>SSH server: git fetch-pack - SSH server-->>AuthorizedKeysCommand: gitlab-shell-authorized-keys-check git AAAA... - AuthorizedKeysCommand-->>Rails: GET /internal/api/authorized_keys?key=AAAA... + Git on client->>+SSH server: ssh git fetch-pack request + SSH server->>+AuthorizedKeysCommand: gitlab-shell-authorized-keys-check git AAAA... + AuthorizedKeysCommand->>+Rails: GET /internal/api/authorized_keys?key=AAAA... Note right of Rails: Lookup key ID - Rails-->>SSH server: 200 OK, command="gitlab-shell upload-pack key_id=1" - SSH server-->>GitLab Shell: gitlab-shell upload-pack key_id=1 - GitLab Shell-->>Rails: GET /internal/api/allowed?action=upload_pack&key_id=1 + Rails-->>-AuthorizedKeysCommand: 200 OK, command="gitlab-shell upload-pack key_id=1" + AuthorizedKeysCommand-->>-SSH server: command="gitlab-shell upload-pack key_id=1" + SSH server->>+GitLab Shell: gitlab-shell upload-pack key_id=1 + GitLab Shell->>+Rails: GET /internal/api/allowed?action=upload_pack&key_id=1 Note right of Rails: Auth check - Rails-->>GitLab Shell: 200 OK, { gitaly: ... } - GitLab Shell-->>Gitaly: SSHService.SSHUploadPack bidirectional request - Gitaly-->>Git on server: git upload-pack - Git on server->>Git on client: SSHService.SSHUploadPack bidirectional response + Rails-->>-GitLab Shell: 200 OK, { gitaly: ... } + GitLab Shell->>+Gitaly: SSHService.SSHUploadPack request + Gitaly->>+Git on server: git upload-pack request + Note over Git on client,Git on server: Bidirectional communication between Git client and server + Git on server-->>-Gitaly: git upload-pack response + Gitaly -->>-GitLab Shell: SSHService.SSHUploadPack response + GitLab Shell-->>-SSH server: gitlab-shell upload-pack response + SSH server-->>-Git on client: ssh git fetch-pack response ``` The `git push` operation is very similar, except `git receive-pack` is used diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md index add466a91b1..896ae8e7c51 100644 --- a/doc/user/project/issues/design_management.md +++ b/doc/user/project/issues/design_management.md @@ -94,7 +94,7 @@ Images on the Design Management page can be enlarged by clicking on them. You can navigate through designs by clicking on the navigation buttons on the top-right corner or with <kbd>Left</kbd>/<kbd>Right</kbd> keyboard buttons. -The number of comments on a design — if any — is listed to the right +The number of discussions on a design — if any — is listed to the right of the design filename. Clicking on this number enlarges the design just like clicking anywhere else on the design. When a design is added or modified, an icon is displayed on the item @@ -102,7 +102,7 @@ to help summarize changes between versions. | Indicator | Example | | --------- | ------- | -| Comments |  | +| Discussions |  | | Modified (in the selected version) |  | | Added (in the selected version) |  | @@ -110,7 +110,9 @@ to help summarize changes between versions. > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13217) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.7. -Designs can be explored in greater detail by zooming in and out of the image. Control the amount of zoom with the `+` and `-` buttons at the bottom of the image. While zoomed, you can still [add new annotations](#adding-annotations-to-designs) to the image, and see any existing ones. +Designs can be explored in greater detail by zooming in and out of the image. +Control the amount of zoom with the `+` and `-` buttons at the bottom of the image. +While zoomed, you can still [start new discussions](#starting-discussions-on-designs) on the image, and see any existing ones.  @@ -142,23 +144,22 @@ Only the latest version of the designs can be deleted. Deleted designs are not permanently lost; they can be viewed by browsing previous versions. -## Adding annotations to designs +## Starting discussions on designs -When a design is uploaded, you can add annotations by clicking on -the image on the exact location you'd like to add the note to. -A badge is added to the image identifying the annotation, from -which you can start a new discussion: +When a design is uploaded, you can start a discussion by clicking on +the image on the exact location you would like the discussion to be focused on. +A pin is added to the image, identifying the discussion's location.  [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34353) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8, -you can adjust the badge's position by dragging it around the image. This is useful +you can adjust a pin's position by dragging it around the image. This is useful for when your design layout has changed between revisions, or if you need to move an -existing badge to add a new one in its place. +existing pin to add a new one in its place. -Different discussions have different badge numbers: +Different discussions have different pin numbers: - + -From GitLab 12.5 on, new annotations will be outputted to the issue activity, +From GitLab 12.5 on, new discussions will be outputted to the issue activity, so that everyone involved can participate in the discussion. diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index b950bfb0f3a..60a50e97998 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -5,13 +5,14 @@ module Gitlab class ApplicationContext include Gitlab::Utils::LazyAttributes - Attribute = Struct.new(:name, :type, :evaluation) + Attribute = Struct.new(:name, :type) APPLICATION_ATTRIBUTES = [ Attribute.new(:project, Project), Attribute.new(:namespace, Namespace), Attribute.new(:user, User), - Attribute.new(:caller_id, String) + Attribute.new(:caller_id, String), + Attribute.new(:related_class, String) ].freeze def self.with_context(args, &block) @@ -39,6 +40,7 @@ module Gitlab hash[:project] = -> { project_path } if set_values.include?(:project) hash[:root_namespace] = -> { root_namespace_path } if include_namespace? hash[:caller_id] = caller_id if set_values.include?(:caller_id) + hash[:related_class] = related_class if set_values.include?(:related_class) end end diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index ead94761ae7..7f2d2858149 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -70,6 +70,21 @@ module Gitlab value end + # Increment the integer value of a key by one. + # Sets the value to zero if missing before incrementing + # + # key - The cache key to increment. + # timeout - The time after which the cache key should expire. + # @return - the incremented value + def self.increment(raw_key, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.incr(key) + redis.expire(key, timeout) + end + end + # Adds a value to a set. # # raw_key - The key of the set to add the value to. diff --git a/lib/gitlab/jira_import.rb b/lib/gitlab/jira_import.rb new file mode 100644 index 00000000000..1486c754caf --- /dev/null +++ b/lib/gitlab/jira_import.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + JIRA_IMPORT_CACHE_TIMEOUT = 10.seconds.to_i + + FAILED_ISSUES_COUNTER_KEY = 'jira-import/failed/%{project_id}/%{collection_type}' + NEXT_ITEMS_START_AT_KEY = 'jira-import/paginator/%{project_id}/%{collection_type}' + ITEMS_MAPPER_CACHE_KEY = 'jira-import/items-mapper/%{project_id}/%{collection_type}/%{jira_isssue_id}' + ALREADY_IMPORTED_ITEMS_CACHE_KEY = 'jira-importer/already-imported/%{project}/%{collection_type}' + + def self.jira_issue_cache_key(project_id, jira_issue_id) + ITEMS_MAPPER_CACHE_KEY % { project_id: project_id, collection_type: :issues, jira_isssue_id: jira_issue_id } + end + + def self.already_imported_cache_key(collection_type, project_id) + ALREADY_IMPORTED_ITEMS_CACHE_KEY % { collection_type: collection_type, project: project_id } + end + + def self.jira_issues_next_page_cache_key(project_id) + NEXT_ITEMS_START_AT_KEY % { project_id: project_id, collection_type: :issues } + end + + def self.failed_issues_counter_cache_key(project_id) + FAILED_ISSUES_COUNTER_KEY % { project_id: project_id, collection_type: :issues } + end + + def self.increment_issue_failures(project_id) + Gitlab::Cache::Import::Caching.increment(self.failed_issues_counter_cache_key(project_id)) + end + + def self.get_issues_next_start_at(project_id) + Gitlab::Cache::Import::Caching.read(self.jira_issues_next_page_cache_key(project_id)).to_i + end + + def self.store_issues_next_started_at(project_id, value) + cache_key = self.jira_issues_next_page_cache_key(project_id) + Gitlab::Cache::Import::Caching.write(cache_key, value) + end + + def self.cache_cleanup(project_id) + Gitlab::Cache::Import::Caching.expire(self.failed_issues_counter_cache_key(project_id), JIRA_IMPORT_CACHE_TIMEOUT) + Gitlab::Cache::Import::Caching.expire(self.jira_issues_next_page_cache_key(project_id), JIRA_IMPORT_CACHE_TIMEOUT) + Gitlab::Cache::Import::Caching.expire(self.already_imported_cache_key(:issues, project_id), JIRA_IMPORT_CACHE_TIMEOUT) + end + end +end diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb new file mode 100644 index 00000000000..24158b8e734 --- /dev/null +++ b/lib/gitlab/jira_import/base_importer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + class BaseImporter + attr_reader :project, :client, :formatter, :jira_project_key + + def initialize(project) + raise Projects::ImportService::Error, _('Jira import feature is disabled.') unless Feature.enabled?(:jira_issue_import, project) + raise Projects::ImportService::Error, _('Jira integration not configured.') unless project.jira_service&.active? + + @jira_project_key = project&.import_data&.becomes(JiraImportData)&.current_project&.key + raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key + + @project = project + @client = project.jira_service.client + @formatter = Gitlab::ImportFormatter.new + end + + private + + def imported_items_cache_key + raise NotImplementedError + end + + def mark_as_imported(id) + Gitlab::Cache::Import::Caching.set_add(imported_items_cache_key, id) + end + + def already_imported?(id) + Gitlab::Cache::Import::Caching.set_includes?(imported_items_cache_key, id) + end + end + end +end diff --git a/lib/gitlab/jira_import/issue_serializer.rb b/lib/gitlab/jira_import/issue_serializer.rb new file mode 100644 index 00000000000..f00852a21a2 --- /dev/null +++ b/lib/gitlab/jira_import/issue_serializer.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + class IssueSerializer + def initialize(project, jira_issue, params = {}) + end + + def execute + # this is going to be implemented in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/27201 + {} + end + end + end +end diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb new file mode 100644 index 00000000000..6543b633ddf --- /dev/null +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + class IssuesImporter < BaseImporter + # Jira limits max items per request to be fetched to 100 + # see https://jira.atlassian.com/browse/JRACLOUD-67570 + # We set it to 1000 in case they change their mind. + BATCH_SIZE = 1000 + + attr_reader :imported_items_cache_key, :start_at, :job_waiter + + def initialize(project) + super + # get cached start_at value, or zero if not cached yet + @start_at = Gitlab::JiraImport.get_issues_next_start_at(project.id) + @imported_items_cache_key = JiraImport.already_imported_cache_key(:issues, project.id) + @job_waiter = JobWaiter.new + end + + def execute + import_issues + end + + private + + def import_issues + return job_waiter if jira_last_page_reached? + + issues = fetch_issues(start_at) + update_start_at_with(issues) + + schedule_issue_import_workers(issues) + end + + def jira_last_page_reached? + start_at < 0 + end + + def update_start_at_with(issues) + @start_at += issues.size + + # store -1 if this is the last page to be imported, so no more `ImportIssuesWorker` workers are scheduled + # from Gitlab::JiraImport::Stage::ImportIssuesWorker#perform + @start_at = -1 if issues.blank? + Gitlab::JiraImport.store_issues_next_started_at(project.id, start_at) + end + + def schedule_issue_import_workers(issues) + next_iid = project.issues.maximum(:iid).to_i + 1 + + issues.each do |jira_issue| + # Technically it's possible that the same work is performed multiple + # times, as Sidekiq doesn't guarantee there will ever only be one + # instance of a job or if for some reason the paginated results + # returned from Jira include issues there were returned before. + # For such cases we exit early if issue was already imported. + next if already_imported?(jira_issue.id) + + issue_attrs = IssueSerializer.new(project, jira_issue, { iid: next_iid }).execute + Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key) + + job_waiter.jobs_remaining += 1 + next_iid += 1 + + # Mark the issue as imported immediately so we don't end up + # importing it multiple times within same import. + # These ids are cleaned-up when import finishes. + # see Gitlab::JiraImport::Stage::FinishImportWorker + mark_as_imported(jira_issue.id) + end + + job_waiter + end + + def fetch_issues(start_at) + client.Issue.jql("PROJECT='#{jira_project_key}' ORDER BY created ASC", { max_results: BATCH_SIZE, start_at: start_at }) + end + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 19fe4f77c85..80f16fd3e1d 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -11287,6 +11287,12 @@ msgstr "" msgid "Jira Issue Import" msgstr "" +msgid "Jira import feature is disabled." +msgstr "" + +msgid "Jira integration not configured." +msgstr "" + msgid "JiraService|Events for %{noteable_model_name} are disabled." msgstr "" @@ -21398,6 +21404,9 @@ msgstr "" msgid "Unable to fetch vulnerable projects" msgstr "" +msgid "Unable to find Jira project to import data from." +msgstr "" + msgid "Unable to generate new instance ID" msgstr "" diff --git a/spec/controllers/admin/sessions_controller_spec.rb b/spec/controllers/admin/sessions_controller_spec.rb index fabd79133ec..351148dbc65 100644 --- a/spec/controllers/admin/sessions_controller_spec.rb +++ b/spec/controllers/admin/sessions_controller_spec.rb @@ -176,6 +176,48 @@ describe Admin::SessionsController, :do_not_mock_admin_mode do expect(controller.current_user_mode.admin_mode?).to be(true) end end + + context 'on a read-only instance' do + before do + allow(Gitlab::Database).to receive(:read_only?).and_return(true) + end + + it 'does not attempt to write to the database with valid otp' do + expect_any_instance_of(User).not_to receive(:save) + expect_any_instance_of(User).not_to receive(:save!) + + controller.store_location_for(:redirect, admin_root_path) + controller.current_user_mode.request_admin_mode! + + authenticate_2fa(otp_attempt: user.current_otp) + + expect(response).to redirect_to admin_root_path + end + + it 'does not attempt to write to the database with invalid otp' do + expect_any_instance_of(User).not_to receive(:save) + expect_any_instance_of(User).not_to receive(:save!) + + controller.current_user_mode.request_admin_mode! + + authenticate_2fa(otp_attempt: 'invalid') + + expect(response).to render_template('admin/sessions/two_factor') + expect(controller.current_user_mode.admin_mode?).to be(false) + end + + it 'does not attempt to write to the database with backup code' do + expect_any_instance_of(User).not_to receive(:save) + expect_any_instance_of(User).not_to receive(:save!) + + controller.current_user_mode.request_admin_mode! + + authenticate_2fa(otp_attempt: user.otp_backup_codes.first) + + expect(response).to render_template('admin/sessions/two_factor') + expect(controller.current_user_mode.admin_mode?).to be(false) + end + end end context 'when using two-factor authentication via U2F' do diff --git a/spec/graphql/resolvers/projects/jira_imports_resolver_spec.rb b/spec/graphql/resolvers/projects/jira_imports_resolver_spec.rb new file mode 100644 index 00000000000..54d51292919 --- /dev/null +++ b/spec/graphql/resolvers/projects/jira_imports_resolver_spec.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Resolvers::Projects::JiraImportsResolver do + include GraphqlHelpers + + describe '#resolve' do + let_it_be(:user) { create(:user) } + let_it_be(:jira_import_data) do + data = JiraImportData.new + data << JiraImportData::JiraProjectDetails.new('AA', 2.days.ago.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name }) + data << JiraImportData::JiraProjectDetails.new('BB', 5.days.ago.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name }) + data + end + + context 'when project does not have Jira import data' do + let_it_be(:project) { create(:project, :private, import_data: nil) } + + context 'when user cannot read Jira import data' do + context 'when anonymous user' do + it_behaves_like 'no jira import data present' + end + + context 'when user developer' do + before do + project.add_developer(user) + end + + it_behaves_like 'no jira import data present' + end + end + + context 'when user can read Jira import data' do + before do + project.add_maintainer(user) + end + + it_behaves_like 'no jira import data present' + end + end + + context 'when project has Jira import data' do + let_it_be(:project) { create(:project, :private, import_data: jira_import_data) } + + context 'when user cannot read Jira import data' do + context 'when anonymous user' do + it_behaves_like 'no jira import access' + end + + context 'when user developer' do + before do + project.add_developer(user) + end + + it_behaves_like 'no jira import access' + end + end + + context 'when user can access Jira import data' do + before do + project.add_maintainer(user) + end + + it 'returns Jira imports sorted ascending by scheduledAt time' do + imports = resolve_imports + + expect(imports.size).to eq 2 + expect(imports.map(&:key)).to eq %w(BB AA) + end + end + end + end + + def resolve_imports(args = {}, context = { current_user: user }) + resolve(described_class, obj: project, args: args, ctx: context) + end +end diff --git a/spec/graphql/types/jira_import_type_spec.rb b/spec/graphql/types/jira_import_type_spec.rb new file mode 100644 index 00000000000..8448a120682 --- /dev/null +++ b/spec/graphql/types/jira_import_type_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe GitlabSchema.types['JiraImport'] do + it { expect(described_class.graphql_name).to eq('JiraImport') } + + it 'has the expected fields' do + expect(described_class).to have_graphql_fields(:jira_project_key, :scheduled_at, :scheduled_by) + end +end diff --git a/spec/graphql/types/project_type_spec.rb b/spec/graphql/types/project_type_spec.rb index 475ae9ff5f6..0c8be50ed90 100644 --- a/spec/graphql/types/project_type_spec.rb +++ b/spec/graphql/types/project_type_spec.rb @@ -24,7 +24,7 @@ describe GitlabSchema.types['Project'] do namespace group statistics repository merge_requests merge_request issues issue pipelines removeSourceBranchAfterMerge sentryDetailedError snippets grafanaIntegration autocloseReferencedIssues suggestion_commit_message environments - boards + boards jira_import_status jira_imports ] expect(described_class).to include_graphql_fields(*expected_fields) diff --git a/spec/lib/gitlab/jira_import/base_importer_spec.rb b/spec/lib/gitlab/jira_import/base_importer_spec.rb new file mode 100644 index 00000000000..8bc43feb356 --- /dev/null +++ b/spec/lib/gitlab/jira_import/base_importer_spec.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::JiraImport::BaseImporter do + let(:project) { create(:project) } + + describe 'with any inheriting class' do + context 'when feature flag disabled' do + before do + stub_feature_flags(jira_issue_import: false) + end + + it 'raises exception' do + expect { described_class.new(project) }.to raise_error(Projects::ImportService::Error, 'Jira import feature is disabled.') + end + end + + context 'when feature flag enabled' do + before do + stub_feature_flags(jira_issue_import: true) + end + + context 'when Jira service was not setup' do + it 'raises exception' do + expect { described_class.new(project) }.to raise_error(Projects::ImportService::Error, 'Jira integration not configured.') + end + end + + context 'when Jira service exists' do + let!(:jira_service) { create(:jira_service, project: project) } + + context 'when Jira import data is not present' do + it 'raises exception' do + expect { described_class.new(project) }.to raise_error(Projects::ImportService::Error, 'Unable to find Jira project to import data from.') + end + end + + context 'when import data exists' do + let(:jira_import_data) do + data = JiraImportData.new + data << JiraImportData::JiraProjectDetails.new('xx', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: 1, name: 'root' }) + data + end + let(:project) { create(:project, import_data: jira_import_data) } + let(:subject) { described_class.new(project) } + + context 'when #imported_items_cache_key is not implemented' do + it { expect { subject.send(:imported_items_cache_key) }.to raise_error(NotImplementedError) } + end + + context 'when #imported_items_cache_key is implemented' do + before do + allow(subject).to receive(:imported_items_cache_key).and_return('dumb-importer-key') + end + + describe '#imported_items_cache_key' do + it { expect(subject.send(:imported_items_cache_key)).to eq('dumb-importer-key') } + end + + describe '#mark_as_imported', :clean_gitlab_redis_cache do + it 'stores id in redis cache' do + expect(Gitlab::Cache::Import::Caching).to receive(:set_add).once.and_call_original + + subject.send(:mark_as_imported, 'some-id') + + expect(Gitlab::Cache::Import::Caching.set_includes?(subject.send(:imported_items_cache_key), 'some-id')).to be true + end + end + + describe '#already_imported?', :clean_gitlab_redis_cache do + it 'returns false if value is not in cache' do + expect(Gitlab::Cache::Import::Caching).to receive(:set_includes?).once.and_call_original + + expect(subject.send(:already_imported?, 'some-id')).to be false + end + + it 'returns true if value already stored in cache' do + Gitlab::Cache::Import::Caching.set_add(subject.send(:imported_items_cache_key), 'some-id') + + expect(subject.send(:already_imported?, 'some-id')).to be true + end + end + end + end + end + end + end +end diff --git a/spec/lib/gitlab/jira_import/issues_importer_spec.rb b/spec/lib/gitlab/jira_import/issues_importer_spec.rb new file mode 100644 index 00000000000..88e8b195dbe --- /dev/null +++ b/spec/lib/gitlab/jira_import/issues_importer_spec.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::JiraImport::IssuesImporter do + let(:user) { create(:user) } + let(:jira_import_data) do + data = JiraImportData.new + data << JiraImportData::JiraProjectDetails.new('XX', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name }) + data + end + let(:project) { create(:project, import_data: jira_import_data) } + let!(:jira_service) { create(:jira_service, project: project) } + + subject { described_class.new(project) } + + before do + stub_feature_flags(jira_issue_import: true) + end + + describe '#imported_items_cache_key' do + it_behaves_like 'raise exception if not implemented' + it { expect(subject.imported_items_cache_key).to eq("jira-importer/already-imported/#{project.id}/issues") } + end + + describe '#execute', :clean_gitlab_redis_cache do + context 'when no returned issues' do + it 'does not schedule any import jobs' do + expect(subject).to receive(:fetch_issues).with(0).and_return([]) + expect(subject).not_to receive(:already_imported?) + expect(subject).not_to receive(:mark_as_imported) + expect(Gitlab::JiraImport::ImportIssueWorker).not_to receive(:perform_async) + + job_waiter = subject.execute + + expect(job_waiter.jobs_remaining).to eq(0) + expect(Gitlab::JiraImport.get_issues_next_start_at(project.id)).to eq(-1) + end + end + + context 'with results returned' do + JiraIssue = Struct.new(:id) + let_it_be(:jira_issue1) { JiraIssue.new(1) } + let_it_be(:jira_issue2) { JiraIssue.new(2) } + + context 'when single page of results is returned' do + before do + stub_const("#{described_class.name}::BATCH_SIZE", 3) + end + + it 'schedules 2 import jobs' do + expect(subject).to receive(:fetch_issues).and_return([jira_issue1, jira_issue2]) + expect(Gitlab::JiraImport::ImportIssueWorker).to receive(:perform_async).twice + expect(Gitlab::Cache::Import::Caching).to receive(:set_add).twice.and_call_original + expect(Gitlab::Cache::Import::Caching).to receive(:set_includes?).twice.and_call_original + allow_next_instance_of(Gitlab::JiraImport::IssueSerializer) do |instance| + allow(instance).to receive(:execute).and_return({ key: 'data' }) + end + + job_waiter = subject.execute + + expect(job_waiter.jobs_remaining).to eq(2) + expect(Gitlab::JiraImport.get_issues_next_start_at(project.id)).to eq(2) + end + end + + context 'when there is more than one page of results' do + before do + stub_const("#{described_class.name}::BATCH_SIZE", 2) + end + + it 'schedules 3 import jobs' do + expect(subject).to receive(:fetch_issues).with(0).and_return([jira_issue1, jira_issue2]) + expect(Gitlab::JiraImport::ImportIssueWorker).to receive(:perform_async).twice.times + expect(Gitlab::Cache::Import::Caching).to receive(:set_add).twice.times.and_call_original + expect(Gitlab::Cache::Import::Caching).to receive(:set_includes?).twice.times.and_call_original + allow_next_instance_of(Gitlab::JiraImport::IssueSerializer) do |instance| + allow(instance).to receive(:execute).and_return({ key: 'data' }) + end + + job_waiter = subject.execute + + expect(job_waiter.jobs_remaining).to eq(2) + expect(Gitlab::JiraImport.get_issues_next_start_at(project.id)).to eq(2) + end + end + + context 'when duplicate results are returned' do + before do + stub_const("#{described_class.name}::BATCH_SIZE", 2) + end + + it 'schedules 2 import jobs' do + expect(subject).to receive(:fetch_issues).with(0).and_return([jira_issue1, jira_issue1]) + expect(Gitlab::JiraImport::ImportIssueWorker).to receive(:perform_async).once + expect(Gitlab::Cache::Import::Caching).to receive(:set_add).once.and_call_original + expect(Gitlab::Cache::Import::Caching).to receive(:set_includes?).twice.times.and_call_original + allow_next_instance_of(Gitlab::JiraImport::IssueSerializer) do |instance| + allow(instance).to receive(:execute).and_return({ key: 'data' }) + end + + job_waiter = subject.execute + + expect(job_waiter.jobs_remaining).to eq(1) + expect(Gitlab::JiraImport.get_issues_next_start_at(project.id)).to eq(2) + end + end + end + end +end diff --git a/spec/lib/gitlab/jira_import_spec.rb b/spec/lib/gitlab/jira_import_spec.rb new file mode 100644 index 00000000000..c5c3d6ef4b9 --- /dev/null +++ b/spec/lib/gitlab/jira_import_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::JiraImport do + let(:project_id) { 321 } + + describe '.jira_issue_cache_key' do + it 'returns cache key for Jira issue imported to given project' do + expect(described_class.jira_issue_cache_key(project_id, 'DEMO-123')).to eq("jira-import/items-mapper/#{project_id}/issues/DEMO-123") + end + end + + describe '.already_imported_cache_key' do + it 'returns cache key for already imported items' do + expect(described_class.already_imported_cache_key(:issues, project_id)).to eq("jira-importer/already-imported/#{project_id}/issues") + end + end + + describe '.jira_issues_next_page_cache_key' do + it 'returns cache key for next issues' do + expect(described_class.jira_issues_next_page_cache_key(project_id)).to eq("jira-import/paginator/#{project_id}/issues") + end + end + + describe '.get_issues_next_start_at', :clean_gitlab_redis_cache do + it 'returns zero when not defined' do + expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues")).to be nil + expect(described_class.get_issues_next_start_at(project_id)).to eq(0) + end + + it 'returns negative value for next issues to be imported starting point' do + Gitlab::Cache::Import::Caching.write("jira-import/paginator/#{project_id}/issues", -10) + + expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues")).to eq('-10') + expect(described_class.get_issues_next_start_at(project_id)).to eq(-10) + end + + it 'returns cached value for next issues to be imported starting point' do + Gitlab::Cache::Import::Caching.write("jira-import/paginator/#{project_id}/issues", 10) + + expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues")).to eq('10') + expect(described_class.get_issues_next_start_at(project_id)).to eq(10) + end + end + + describe '.store_issues_next_started_at', :clean_gitlab_redis_cache do + it 'stores nil value' do + described_class.store_issues_next_started_at(project_id, nil) + + expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues")).to eq '' + expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues").to_i).to eq(0) + end + + it 'stores positive value' do + described_class.store_issues_next_started_at(project_id, 10) + + expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues").to_i).to eq(10) + end + + it 'stores negative value' do + described_class.store_issues_next_started_at(project_id, -10) + + expect(Gitlab::Cache::Import::Caching.read("jira-import/paginator/#{project_id}/issues").to_i).to eq(-10) + end + end +end diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index c3f2e3aebdd..425194ba0e3 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -3061,20 +3061,6 @@ describe Ci::Pipeline, :mailer do expect(pipeline.source_bridge).to eq bridge end end - - describe '#update_bridge_status!' do - it 'can update bridge status if it is running' do - pipeline.update_bridge_status! - - expect(bridge.reload).to be_success - end - - it 'can not update bridge status if is not active' do - bridge.success! - - expect { pipeline.update_bridge_status! }.not_to change { bridge.status } - end - end end context 'when an upstream status is a build' do @@ -3101,16 +3087,6 @@ describe Ci::Pipeline, :mailer do expect(pipeline.source_bridge).to be_nil end end - - describe '#update_bridge_status!' do - it 'tracks an ArgumentError and does not update upstream job status' do - expect(Gitlab::ErrorTracking) - .to receive(:track_exception) - .with(instance_of(ArgumentError), pipeline_id: pipeline.id) - - pipeline.update_bridge_status! - end - end end end end diff --git a/spec/models/project_services/bamboo_service_spec.rb b/spec/models/project_services/bamboo_service_spec.rb index 1b946278790..c1efa3a4348 100644 --- a/spec/models/project_services/bamboo_service_spec.rb +++ b/spec/models/project_services/bamboo_service_spec.rb @@ -8,9 +8,11 @@ describe BambooService, :use_clean_rails_memory_store_caching do let(:bamboo_url) { 'http://gitlab.com/bamboo' } + let_it_be(:project) { create(:project) } + subject(:service) do described_class.create( - project: create(:project), + project: project, properties: { bamboo_url: bamboo_url, username: 'mic', @@ -224,6 +226,19 @@ describe BambooService, :use_clean_rails_memory_store_caching do is_expected.to eq(:error) end + + Gitlab::HTTP::HTTP_ERRORS.each do |http_error| + it "sets commit status to :error with a #{http_error.name} error" do + WebMock.stub_request(:get, 'http://gitlab.com/bamboo/rest/api/latest/result/byChangeset/123?os_authType=basic') + .to_raise(http_error) + + expect(Gitlab::ErrorTracking) + .to receive(:log_exception) + .with(instance_of(http_error), project_id: project.id) + + is_expected.to eq(:error) + end + end end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 51fcee29485..694c6935c1d 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2282,6 +2282,44 @@ describe Project do end end + describe '#jira_import_status' do + let(:project) { create(:project, :import_started, import_type: 'jira') } + + context 'when import_data is nil' do + it 'returns none' do + expect(project.import_data).to be nil + expect(project.jira_import_status).to eq('none') + end + end + + context 'when import_data is set' do + let(:jira_import_data) { JiraImportData.new } + let(:project) { create(:project, :import_started, import_data: jira_import_data, import_type: 'jira') } + + it 'returns none' do + expect(project.import_data.becomes(JiraImportData).force_import?).to be false + expect(project.jira_import_status).to eq('none') + end + + context 'when jira_force_import is true' do + let(:imported_jira_project) do + JiraImportData::JiraProjectDetails.new('xx', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: 1, name: 'root' }) + end + + before do + jira_import_data = project.import_data.becomes(JiraImportData) + jira_import_data << imported_jira_project + jira_import_data.force_import! + end + + it 'returns started' do + expect(project.import_data.becomes(JiraImportData).force_import?).to be true + expect(project.jira_import_status).to eq('started') + end + end + end + end + describe '#human_import_status_name' do context 'with import_state' do it 'returns the right human import status' do diff --git a/spec/requests/api/graphql/project/jira_import_spec.rb b/spec/requests/api/graphql/project/jira_import_spec.rb new file mode 100644 index 00000000000..beebc63a3c6 --- /dev/null +++ b/spec/requests/api/graphql/project/jira_import_spec.rb @@ -0,0 +1,183 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'query jira import data' do + include GraphqlHelpers + + let_it_be(:current_user) { create(:user) } + let_it_be(:jira_import_data) do + data = JiraImportData.new + data << JiraImportData::JiraProjectDetails.new( + 'AA', 2.days.ago.strftime('%Y-%m-%d %H:%M:%S'), + { user_id: current_user.id, name: current_user.name } + ) + data << JiraImportData::JiraProjectDetails.new( + 'BB', 5.days.ago.strftime('%Y-%m-%d %H:%M:%S'), + { user_id: current_user.id, name: current_user.name } + ) + data + end + let_it_be(:project) { create(:project, :private, :import_started, import_data: jira_import_data, import_type: 'jira') } + let(:query) do + %( + query { + project(fullPath: "#{project.full_path}") { + jiraImportStatus + jiraImports { + nodes { + jiraProjectKey + scheduledAt + scheduledBy { + username + } + } + } + } + } + ) + end + let(:jira_imports) { graphql_data.dig('project', 'jiraImports', 'nodes')} + let(:jira_import_status) { graphql_data.dig('project', 'jiraImportStatus')} + + context 'when user cannot read Jira import data' do + before do + post_graphql(query, current_user: current_user) + end + + context 'when anonymous user' do + let(:current_user) { nil } + + it { expect(jira_imports).to be nil } + end + + context 'when user developer' do + before do + project.add_developer(current_user) + end + + it { expect(jira_imports).to be nil } + end + end + + context 'when user can access Jira import data' do + before do + project.add_maintainer(current_user) + post_graphql(query, current_user: current_user) + end + + it_behaves_like 'a working graphql query' + + context 'list of jira imports sorted ascending by scheduledAt time' do + it 'retuns list of jira imports' do + jira_proket_keys = jira_imports.map {|ji| ji['jiraProjectKey']} + usernames = jira_imports.map {|ji| ji.dig('scheduledBy', 'username')} + + expect(jira_imports.size).to eq 2 + expect(jira_proket_keys).to eq %w(BB AA) + expect(usernames).to eq [current_user.username, current_user.username] + end + end + + describe 'jira imports pagination' do + context 'first jira import' do + let(:query) do + %( + query { + project(fullPath:"#{project.full_path}") { + jiraImports(first: 1) { + nodes { + jiraProjectKey + scheduledBy { + username + } + } + } + } + } + ) + end + + it 'returns latest jira import data' do + first_jira_import = jira_imports.first + + expect(first_jira_import['jiraProjectKey']).to eq 'BB' + expect(first_jira_import.dig('scheduledBy', 'username')).to eq current_user.username + end + end + + context 'lastest jira import' do + let(:query) do + %( + query { + project(fullPath:"#{project.full_path}") { + jiraImports(last: 1) { + nodes { + jiraProjectKey + scheduledBy { + username + } + } + } + } + } + ) + end + + it 'returns latest jira import data' do + latest_jira_import = jira_imports.first + + expect(latest_jira_import['jiraProjectKey']).to eq 'AA' + expect(latest_jira_import.dig('scheduledBy', 'username')).to eq current_user.username + end + end + end + end + + context 'jira import status' do + context 'when user cannot access project' do + it 'does not return import status' do + post_graphql(query, current_user: current_user) + + expect(graphql_data['project']).to be nil + end + end + + context 'when user can access project' do + before do + project.add_guest(current_user) + end + + context 'when import never ran' do + let(:project) { create(:project) } + + it 'returns import status' do + post_graphql(query, current_user: current_user) + + expect(jira_import_status).to eq('none') + end + end + + context 'when import finished' do + it 'returns import status' do + post_graphql(query, current_user: current_user) + + expect(jira_import_status).to eq('finished') + end + end + + context 'when import running, i.e. force-import: true' do + before do + project.import_data.becomes(JiraImportData).force_import! + project.save! + end + + it 'returns import status' do + post_graphql(query, current_user: current_user) + + expect(jira_import_status).to eq('started') + end + end + end + end +end diff --git a/spec/support/shared_examples/graphql/jira_import/jira_import_resolved_shared_examples.rb b/spec/support/shared_examples/graphql/jira_import/jira_import_resolved_shared_examples.rb new file mode 100644 index 00000000000..b1d178521bb --- /dev/null +++ b/spec/support/shared_examples/graphql/jira_import/jira_import_resolved_shared_examples.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +shared_examples 'no jira import data present' do + it 'returns none' do + expect(resolve_imports).to eq JiraImportData.none + end +end + +shared_examples 'no jira import access' do + it 'raises error' do + expect do + resolve_imports + end.to raise_error(Gitlab::Graphql::Errors::ResourceNotAvailable) + end +end diff --git a/spec/support/shared_examples/lib/gitlab/jira_import/base_importer_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/jira_import/base_importer_shared_examples.rb new file mode 100644 index 00000000000..85dcc053447 --- /dev/null +++ b/spec/support/shared_examples/lib/gitlab/jira_import/base_importer_shared_examples.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +shared_examples 'raise exception if not implemented' do + it { expect { described_class.new(project).imported_items_cache_key }.not_to raise_error } +end diff --git a/spec/support/shared_examples/workers/gitlab/jira_import/jira_import_workers_shared_examples.rb b/spec/support/shared_examples/workers/gitlab/jira_import/jira_import_workers_shared_examples.rb index 5448526f954..71ec1ea6a74 100644 --- a/spec/support/shared_examples/workers/gitlab/jira_import/jira_import_workers_shared_examples.rb +++ b/spec/support/shared_examples/workers/gitlab/jira_import/jira_import_workers_shared_examples.rb @@ -16,7 +16,7 @@ shared_examples 'exit import not started' do it 'does nothing, and exits' do expect(Gitlab::JiraImport::AdvanceStageWorker).not_to receive(:perform_async) - worker.perform(project.id) + described_class.new.perform(project.id) end end @@ -25,8 +25,8 @@ shared_examples 'advance to next stage' do |next_stage| it "advances to #{next_stage} stage" do expect(Gitlab::JobWaiter).to receive(:new).and_return(job_waiter) - expect(Gitlab::JiraImport::AdvanceStageWorker).to receive(:perform_async).with(project.id, { job_waiter.key => job_waiter.jobs_remaining }, next_stage.to_sym) + expect(Gitlab::JiraImport::AdvanceStageWorker).to receive(:perform_async).with(project.id, { job_waiter.key => job_waiter.jobs_remaining }, next_stage.to_sym).and_return([]) - worker.perform(project.id) + described_class.new.perform(project.id) end end diff --git a/spec/workers/gitlab/jira_import/import_issue_worker_spec.rb b/spec/workers/gitlab/jira_import/import_issue_worker_spec.rb new file mode 100644 index 00000000000..c09492efcae --- /dev/null +++ b/spec/workers/gitlab/jira_import/import_issue_worker_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::JiraImport::ImportIssueWorker do + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project) } + + describe 'modules' do + it { expect(described_class).to include_module(ApplicationWorker) } + it { expect(described_class).to include_module(Gitlab::NotifyUponDeath) } + it { expect(described_class).to include_module(Gitlab::JiraImport::QueueOptions) } + it { expect(described_class).to include_module(Gitlab::Import::DatabaseHelpers) } + end + + subject { described_class.new } + + describe '#perform', :clean_gitlab_redis_cache do + let(:issue_attrs) { build(:issue, project_id: project.id).as_json.compact } + + context 'when any exception raised while inserting to DB' do + before do + allow(subject).to receive(:insert_and_return_id).and_raise(StandardError) + expect(Gitlab::JobWaiter).to receive(:notify) + + subject.perform(project.id, 123, issue_attrs, 'some-key') + end + + it 'record a failed to import issue' do + expect(Gitlab::Cache::Import::Caching.read(Gitlab::JiraImport.failed_issues_counter_cache_key(project.id)).to_i).to eq(1) + end + end + + context 'when record is successfully inserted' do + before do + subject.perform(project.id, 123, issue_attrs, 'some-key') + end + + it 'does not record import failure' do + expect(Gitlab::Cache::Import::Caching.read(Gitlab::JiraImport.failed_issues_counter_cache_key(project.id)).to_i).to eq(0) + end + end + end +end diff --git a/spec/workers/gitlab/jira_import/stage/finish_import_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/finish_import_worker_spec.rb index fa0c7d83851..00505226212 100644 --- a/spec/workers/gitlab/jira_import/stage/finish_import_worker_spec.rb +++ b/spec/workers/gitlab/jira_import/stage/finish_import_worker_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' describe Gitlab::JiraImport::Stage::FinishImportWorker do - let(:project) { create(:project) } - let(:worker) { described_class.new } + let_it_be(:project) { create(:project) } + let_it_be(:worker) { described_class.new } describe 'modules' do it_behaves_like 'include import workers modules' @@ -46,7 +46,7 @@ describe Gitlab::JiraImport::Stage::FinishImportWorker do it 'changes import state to finished' do worker.perform(project.id) - expect(project.reload.import_state.status).to eq "finished" + expect(project.reload.import_state.status).to eq("finished") end it 'removes force-import flag' do diff --git a/spec/workers/gitlab/jira_import/stage/import_attachments_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/import_attachments_worker_spec.rb index fa2f3501973..513925507a1 100644 --- a/spec/workers/gitlab/jira_import/stage/import_attachments_worker_spec.rb +++ b/spec/workers/gitlab/jira_import/stage/import_attachments_worker_spec.rb @@ -3,8 +3,7 @@ require 'spec_helper' describe Gitlab::JiraImport::Stage::ImportAttachmentsWorker do - let(:project) { create(:project) } - let(:worker) { described_class.new } + let_it_be(:project) { create(:project) } describe 'modules' do it_behaves_like 'include import workers modules' diff --git a/spec/workers/gitlab/jira_import/stage/import_issues_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/import_issues_worker_spec.rb index b43519a3e5d..dca748a6ebc 100644 --- a/spec/workers/gitlab/jira_import/stage/import_issues_worker_spec.rb +++ b/spec/workers/gitlab/jira_import/stage/import_issues_worker_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' describe Gitlab::JiraImport::Stage::ImportIssuesWorker do - let(:project) { create(:project) } - let(:worker) { described_class.new } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project) } describe 'modules' do it_behaves_like 'include import workers modules' @@ -30,10 +30,49 @@ describe Gitlab::JiraImport::Stage::ImportIssuesWorker do it_behaves_like 'exit import not started' end - context 'when import started' do + context 'when import started', :clean_gitlab_redis_cache do + let(:jira_import_data) do + data = JiraImportData.new + data << JiraImportData::JiraProjectDetails.new('XX', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name }) + data + end + let(:project) { create(:project, import_data: jira_import_data) } + let!(:jira_service) { create(:jira_service, project: project) } let!(:import_state) { create(:import_state, status: :started, project: project) } - it_behaves_like 'advance to next stage', :attachments + before do + allow_next_instance_of(Gitlab::JiraImport::IssuesImporter) do |instance| + allow(instance).to receive(:fetch_issues).and_return([]) + end + end + + context 'when start_at is nil' do + it_behaves_like 'advance to next stage', :attachments + end + + context 'when start_at is zero' do + before do + allow(Gitlab::Cache::Import::Caching).to receive(:read).and_return(0) + end + + it_behaves_like 'advance to next stage', :issues + end + + context 'when start_at is greater than zero' do + before do + allow(Gitlab::Cache::Import::Caching).to receive(:read).and_return(25) + end + + it_behaves_like 'advance to next stage', :issues + end + + context 'when start_at is below zero' do + before do + allow(Gitlab::Cache::Import::Caching).to receive(:read).and_return(-1) + end + + it_behaves_like 'advance to next stage', :attachments + end end end end diff --git a/spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb index 827efb85a17..2b156e0f489 100644 --- a/spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb +++ b/spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb @@ -3,8 +3,7 @@ require 'spec_helper' describe Gitlab::JiraImport::Stage::ImportLabelsWorker do - let(:project) { create(:project) } - let(:worker) { described_class.new } + let_it_be(:project) { create(:project) } describe 'modules' do it_behaves_like 'include import workers modules' diff --git a/spec/workers/gitlab/jira_import/stage/import_notes_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/import_notes_worker_spec.rb index bd6b36613cc..7d1c29614e4 100644 --- a/spec/workers/gitlab/jira_import/stage/import_notes_worker_spec.rb +++ b/spec/workers/gitlab/jira_import/stage/import_notes_worker_spec.rb @@ -3,8 +3,7 @@ require 'spec_helper' describe Gitlab::JiraImport::Stage::ImportNotesWorker do - let(:project) { create(:project) } - let(:worker) { described_class.new } + let_it_be(:project) { create(:project) } describe 'modules' do it_behaves_like 'include import workers modules' diff --git a/spec/workers/gitlab/jira_import/stage/start_import_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/start_import_worker_spec.rb index cc70277384d..d5e10a950bb 100644 --- a/spec/workers/gitlab/jira_import/stage/start_import_worker_spec.rb +++ b/spec/workers/gitlab/jira_import/stage/start_import_worker_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe Gitlab::JiraImport::Stage::StartImportWorker do - let(:project) { create(:project) } + let(:project) { create(:project, import_type: 'jira') } let(:worker) { described_class.new } let(:jid) { '12345678' } @@ -24,13 +24,19 @@ describe Gitlab::JiraImport::Stage::StartImportWorker do end end - context 'when feature flag not enabled' do + context 'when feature flag enabled' do + let(:symbol_keys_project) do + { key: 'AA', scheduled_at: 2.days.ago.strftime('%Y-%m-%d %H:%M:%S'), scheduled_by: { 'user_id' => 1, 'name' => 'tester1' } } + end + let(:import_data) { JiraImportData.new( data: { 'jira' => { JiraImportData::FORCE_IMPORT_KEY => true, projects: [symbol_keys_project] } }) } + before do stub_feature_flags(jira_issue_import: true) end - context 'when import is not scheudled' do - let!(:import_state) { create(:import_state, project: project, status: :none, jid: jid) } + context 'when import is not scheduled' do + let(:project) { create(:project, import_type: 'jira') } + let(:import_state) { create(:import_state, project: project, status: :none, jid: jid) } it 'exits because import not started' do expect(Gitlab::JiraImport::Stage::ImportLabelsWorker).not_to receive(:perform_async) @@ -40,17 +46,32 @@ describe Gitlab::JiraImport::Stage::StartImportWorker do end context 'when import is scheduled' do - let!(:import_state) { create(:import_state, project: project, status: :scheduled, jid: jid) } + let(:import_state) { create(:import_state, status: :scheduled, jid: jid) } + let(:project) { create(:project, import_type: 'jira', import_state: import_state) } - it 'advances to importing labels' do - expect(Gitlab::JiraImport::Stage::ImportLabelsWorker).to receive(:perform_async) + context 'when this is a mirror sync in a jira imported project' do + it 'exits early' do + expect(Gitlab::Import::SetAsyncJid).not_to receive(:set_jid) + expect(Gitlab::JiraImport::Stage::ImportLabelsWorker).not_to receive(:perform_async) - worker.perform(project.id) + worker.perform(project.id) + end + end + + context 'when scheduled import is a hard triggered jira import and not a mirror' do + let!(:project) { create(:project, import_type: 'jira', import_data: import_data, import_state: import_state) } + + it 'advances to importing labels' do + expect(Gitlab::JiraImport::Stage::ImportLabelsWorker).to receive(:perform_async) + + worker.perform(project.id) + end end end context 'when import is started' do - let!(:import_state) { create(:import_state, project: project, status: :started, jid: jid) } + let!(:import_state) { create(:import_state, status: :started, jid: jid) } + let!(:project) { create(:project, import_type: 'jira', import_data: import_data, import_state: import_state) } context 'when this is the same worker that stated import' do it 'advances to importing labels' do @@ -72,7 +93,8 @@ describe Gitlab::JiraImport::Stage::StartImportWorker do end context 'when import is finished' do - let!(:import_state) { create(:import_state, project: project, status: :finished, jid: jid) } + let!(:import_state) { create(:import_state, status: :finished, jid: jid) } + let!(:project) { create(:project, import_type: 'jira', import_data: import_data, import_state: import_state) } it 'advances to importing labels' do allow(worker).to receive(:jid).and_return(jid) diff --git a/spec/workers/reactive_caching_worker_spec.rb b/spec/workers/reactive_caching_worker_spec.rb index 6c74c4ea072..c39a97b4eee 100644 --- a/spec/workers/reactive_caching_worker_spec.rb +++ b/spec/workers/reactive_caching_worker_spec.rb @@ -28,4 +28,14 @@ describe ReactiveCachingWorker do end end end + + describe 'worker context' do + it 'sets the related class on the job' do + described_class.perform_async('Environment', 1, 'other', 'argument') + + scheduled_job = described_class.jobs.first + + expect(scheduled_job).to include('meta.related_class' => 'Environment') + end + end end |