diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-04-09 12:09:24 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-04-09 12:09:24 +0000 |
| commit | a9ced7da447785c57477b3d8dbccc73a78cface1 (patch) | |
| tree | 5179d27ab9d801748ee4ed1c64c985974e799812 | |
| parent | ad0265eead72a624ce7a020847db4f0f0c877e57 (diff) | |
| download | gitlab-ce-a9ced7da447785c57477b3d8dbccc73a78cface1.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
76 files changed, 1127 insertions, 636 deletions
diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 0ca27c52083..4a13a92598e 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -20,12 +20,11 @@ build-qa-image: - time docker build --cache-from "${QA_MASTER_IMAGE}" --tag ${QA_IMAGE} --file ./qa/Dockerfile ./ - time docker push ${QA_IMAGE} -review-cleanup: +.review-cleanup-base: extends: - .default-retry - .review:rules:review-cleanup stage: prepare - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base allow_failure: true environment: name: review/auto-cleanup @@ -36,6 +35,18 @@ review-cleanup: script: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb +review-cleanup: + extends: + - .review-cleanup-base + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base + +review-cleanup-helm3: + extends: + - .review-cleanup-base + variables: + HELM_3: 1 + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3-kubectl1.14 + review-gcp-cleanup: extends: - .review:rules:review-gcp-cleanup diff --git a/app/assets/javascripts/boards/components/board_card.vue b/app/assets/javascripts/boards/components/board_card.vue index 5735c8ded3d..246d3b9dcd1 100644 --- a/app/assets/javascripts/boards/components/board_card.vue +++ b/app/assets/javascripts/boards/components/board_card.vue @@ -2,6 +2,7 @@ /* eslint-disable vue/require-default-prop */ import IssueCardInner from './issue_card_inner.vue'; import eventHub from '../eventhub'; +import sidebarEventHub from '~/sidebar/event_hub'; import boardsStore from '../stores/boards_store'; export default { @@ -73,6 +74,11 @@ export default { showIssue(e) { if (e.target.classList.contains('js-no-trigger')) return; + // If no issues are opened, close all sidebars first + if (!boardsStore.detail?.issue?.id) { + sidebarEventHub.$emit('sidebar.closeAll'); + } + // If CMD or CTRL is clicked const isMultiSelect = this.canMultiSelect && (e.ctrlKey || e.metaKey); diff --git a/app/assets/javascripts/boards/components/board_sidebar.js b/app/assets/javascripts/boards/components/board_sidebar.js index a3a9753f1b5..66a5e134205 100644 --- a/app/assets/javascripts/boards/components/board_sidebar.js +++ b/app/assets/javascripts/boards/components/board_sidebar.js @@ -103,12 +103,14 @@ export default Vue.extend({ eventHub.$on('sidebar.addAssignee', this.addAssignee); eventHub.$on('sidebar.removeAllAssignees', this.removeAllAssignees); eventHub.$on('sidebar.saveAssignees', this.saveAssignees); + eventHub.$on('sidebar.closeAll', this.closeSidebar); }, beforeDestroy() { eventHub.$off('sidebar.removeAssignee', this.removeAssignee); eventHub.$off('sidebar.addAssignee', this.addAssignee); eventHub.$off('sidebar.removeAllAssignees', this.removeAllAssignees); eventHub.$off('sidebar.saveAssignees', this.saveAssignees); + eventHub.$off('sidebar.closeAll', this.closeSidebar); }, mounted() { new IssuableContext(this.currentUser); diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb index 7175eefcde7..44120fda17c 100644 --- a/app/controllers/groups_controller.rb +++ b/app/controllers/groups_controller.rb @@ -6,18 +6,20 @@ class GroupsController < Groups::ApplicationController include ParamsBackwardCompatibility include PreviewMarkdown include RecordUserLastActivity + include SendFileUpload extend ::Gitlab::Utils::Override respond_to :html prepend_before_action(only: [:show, :issues]) { authenticate_sessionless_user!(:rss) } prepend_before_action(only: [:issues_calendar]) { authenticate_sessionless_user!(:ics) } + prepend_before_action :ensure_export_enabled, only: [:export, :download_export] before_action :authenticate_user!, only: [:new, :create] before_action :group, except: [:index, :new, :create] # Authorize - before_action :authorize_admin_group!, only: [:edit, :update, :destroy, :projects, :transfer] + before_action :authorize_admin_group!, only: [:edit, :update, :destroy, :projects, :transfer, :export, :download_export] before_action :authorize_create_group!, only: [:new] before_action :group_projects, only: [:projects, :activity, :issues, :merge_requests] @@ -29,6 +31,8 @@ class GroupsController < Groups::ApplicationController push_frontend_feature_flag(:vue_issuables_list, @group) end + before_action :export_rate_limit, only: [:export, :download_export] + skip_cross_project_access_check :index, :new, :create, :edit, :update, :destroy, :projects # When loading show as an atom feed, we render events that could leak cross @@ -134,6 +138,25 @@ class GroupsController < Groups::ApplicationController end # rubocop: enable CodeReuse/ActiveRecord + def export + export_service = Groups::ImportExport::ExportService.new(group: @group, user: current_user) + + if export_service.async_execute + redirect_to edit_group_path(@group), notice: _('Group export started.') + else + redirect_to edit_group_path(@group), alert: _('Group export could not be started.') + end + end + + def download_export + if @group.export_file_exists? + send_upload(@group.export_file, attachment: @group.export_file.filename) + else + redirect_to edit_group_path(@group), + alert: _('Group export link has expired. Please generate a new export from your group settings.') + end + end + protected def render_show_html @@ -234,6 +257,21 @@ class GroupsController < Groups::ApplicationController url_for(safe_params) end + def export_rate_limit + prefixed_action = "group_#{params[:action]}".to_sym + + if Gitlab::ApplicationRateLimiter.throttled?(prefixed_action, scope: [current_user, prefixed_action, @group]) + Gitlab::ApplicationRateLimiter.log_request(request, "#{prefixed_action}_request_limit".to_sym, current_user) + + flash[:alert] = _('This endpoint has been requested too many times. Try again later.') + redirect_to edit_group_path(@group) + end + end + + def ensure_export_enabled + render_404 unless Feature.enabled?(:group_import_export, @group, default_enabled: true) + end + private def groups diff --git a/app/controllers/projects/import/jira_controller.rb b/app/controllers/projects/import/jira_controller.rb index c8f53cef5b2..b5adef399c7 100644 --- a/app/controllers/projects/import/jira_controller.rb +++ b/app/controllers/projects/import/jira_controller.rb @@ -9,7 +9,7 @@ module Projects def show return if Feature.enabled?(:jira_issue_import_vue, @project) - unless @project.import_state&.in_progress? + unless @project.latest_jira_import&.in_progress? jira_client = @project.jira_service.client jira_projects = jira_client.Project.all @@ -20,7 +20,7 @@ module Projects end end - flash[:notice] = _("Import %{status}") % { status: @project.import_state.status } if @project.import_state.present? && !@project.import_state.none? + flash[:notice] = _("Import %{status}") % { status: @project.jira_import_status } unless @project.latest_jira_import&.initial? end def import diff --git a/app/graphql/mutations/jira_import/start.rb b/app/graphql/mutations/jira_import/start.rb index ffd3ce53b57..6b80c9f8ca4 100644 --- a/app/graphql/mutations/jira_import/start.rb +++ b/app/graphql/mutations/jira_import/start.rb @@ -30,11 +30,11 @@ module Mutations service_response = ::JiraImport::StartImportService .new(context[:current_user], project, jira_project_key) .execute - import_data = service_response.payload[:import_data] - + jira_import = service_response.success? ? service_response.payload[:import_data] : nil + errors = service_response.error? ? [service_response.message] : [] { - jira_import: import_data.errors.blank? ? import_data.projects.last : nil, - errors: errors_on_object(import_data) + jira_import: jira_import, + errors: errors } end diff --git a/app/graphql/resolvers/projects/jira_imports_resolver.rb b/app/graphql/resolvers/projects/jira_imports_resolver.rb index e7403745bea..9f71d4f187e 100644 --- a/app/graphql/resolvers/projects/jira_imports_resolver.rb +++ b/app/graphql/resolvers/projects/jira_imports_resolver.rb @@ -8,11 +8,9 @@ module Resolvers alias_method :project, :object def resolve(**args) - return JiraImportData.none unless project&.import_data.present? - authorize!(project) - project.import_data.becomes(JiraImportData).projects + project.jira_imports end def authorized_resource?(project) diff --git a/app/graphql/types/jira_import_type.rb b/app/graphql/types/jira_import_type.rb index 01ec6184844..ccd463370b6 100644 --- a/app/graphql/types/jira_import_type.rb +++ b/app/graphql/types/jira_import_type.rb @@ -8,20 +8,12 @@ module Types graphql_name 'JiraImport' field :scheduled_at, Types::TimeType, null: true, - description: 'Timestamp of when the Jira import was created/started' + method: :created_at, + description: 'Timestamp of when the Jira import was created' field :scheduled_by, Types::UserType, null: true, description: 'User that started the Jira import' field :jira_project_key, GraphQL::STRING_TYPE, null: false, - description: 'Project key for the imported Jira project', - method: :key - - def scheduled_at - DateTime.parse(object.scheduled_at) - end - - def scheduled_by - ::Gitlab::Graphql::Loaders::BatchModelLoader.new(User, object.scheduled_by['user_id']).find - end + description: 'Project key for the imported Jira project' end # rubocop: enable Graphql/AuthorizeTypes end diff --git a/app/models/concerns/import_state/sidekiq_job_tracker.rb b/app/models/concerns/import_state/sidekiq_job_tracker.rb index 6bb07b7c06a..55f171d158d 100644 --- a/app/models/concerns/import_state/sidekiq_job_tracker.rb +++ b/app/models/concerns/import_state/sidekiq_job_tracker.rb @@ -16,7 +16,7 @@ module ImportState end def self.jid_by(project_id:, status:) - select(:jid).with_status(status).find_by(project_id: project_id) + select(:jid).where(status: status).find_by(project_id: project_id) end end end diff --git a/app/models/jira_import_data.rb b/app/models/jira_import_data.rb deleted file mode 100644 index b39ca7290be..00000000000 --- a/app/models/jira_import_data.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -class JiraImportData < ProjectImportData - JiraProjectDetails = Struct.new(:key, :scheduled_at, :scheduled_by) - - FORCE_IMPORT_KEY = 'force-import' - - def projects - return [] unless data - - projects = data.dig('jira', 'projects')&.map do |p| - JiraProjectDetails.new(p['key'], p['scheduled_at'], p['scheduled_by']) - end - - projects&.sort_by { |jp| jp.scheduled_at } || [] - end - - def <<(project) - self.data ||= { 'jira' => { 'projects' => [] } } - self.data['jira'] ||= { 'projects' => [] } - self.data['jira']['projects'] = [] if data['jira']['projects'].blank? || !data['jira']['projects'].is_a?(Array) - - self.data['jira']['projects'] << project.to_h - self.data.deep_stringify_keys! - end - - def force_import! - self.data ||= {} - self.data.deep_merge!({ 'jira' => { FORCE_IMPORT_KEY => true } }) - self.data.deep_stringify_keys! - end - - def force_import? - !!data&.dig('jira', FORCE_IMPORT_KEY) && !projects.blank? - end - - def finish_import! - return if data&.dig('jira', FORCE_IMPORT_KEY).nil? - - data['jira'].delete(FORCE_IMPORT_KEY) - end - - def current_project - projects.last - end -end diff --git a/app/models/jira_import_state.rb b/app/models/jira_import_state.rb index 713feec013f..ec1b8f03d36 100644 --- a/app/models/jira_import_state.rb +++ b/app/models/jira_import_state.rb @@ -22,6 +22,8 @@ class JiraImportState < ApplicationRecord message: _('Cannot have multiple Jira imports running at the same time') } + alias_method :scheduled_by, :user + state_machine :status, initial: :initial do event :schedule do transition initial: :scheduled @@ -46,6 +48,11 @@ class JiraImportState < ApplicationRecord end end + before_transition any => :finished do |state, _| + InternalId.flush_records!(project: state.project) + state.project.update_project_counter_caches + end + after_transition any => :finished do |state, _| if state.jid.present? Gitlab::SidekiqStatus.unset(state.jid) @@ -67,4 +74,8 @@ class JiraImportState < ApplicationRecord def in_progress? scheduled? || started? end + + def non_initial? + !initial? + end end diff --git a/app/models/project.rb b/app/models/project.rb index 15b8d5db214..1f968cdfad1 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -859,9 +859,7 @@ class Project < ApplicationRecord end def jira_import_status - return import_status if jira_force_import? - - import_data&.becomes(JiraImportData)&.projects.blank? ? 'none' : 'finished' + latest_jira_import&.status || 'initial' end def human_import_status_name @@ -875,8 +873,6 @@ class Project < ApplicationRecord elsif gitlab_project_import? # Do not retry on Import/Export until https://gitlab.com/gitlab-org/gitlab-foss/issues/26189 is solved. RepositoryImportWorker.set(retry: false).perform_async(self.id) - elsif jira_import? - Gitlab::JiraImport::Stage::StartImportWorker.perform_async(self.id) else RepositoryImportWorker.perform_async(self.id) end @@ -909,7 +905,7 @@ class Project < ApplicationRecord # This method is overridden in EE::Project model def remove_import_data - import_data&.destroy unless jira_import? + import_data&.destroy end def ci_config_path=(value) @@ -972,11 +968,7 @@ class Project < ApplicationRecord end def jira_import? - import_type == 'jira' && Feature.enabled?(:jira_issue_import, self) - end - - def jira_force_import? - jira_import? && import_data&.becomes(JiraImportData)&.force_import? + import_type == 'jira' && latest_jira_import.present? && Feature.enabled?(:jira_issue_import, self) end def gitlab_project_import? diff --git a/app/services/groups/import_export/export_service.rb b/app/services/groups/import_export/export_service.rb index 0bf54844430..86e2eeda21f 100644 --- a/app/services/groups/import_export/export_service.rb +++ b/app/services/groups/import_export/export_service.rb @@ -10,9 +10,15 @@ module Groups @shared = @params[:shared] || Gitlab::ImportExport::Shared.new(@group) end + def async_execute + GroupExportWorker.perform_async(@current_user.id, @group.id, @params) + end + def execute validate_user_permissions + remove_existing_export! if @group.export_file_exists? + save! ensure cleanup @@ -30,6 +36,13 @@ module Groups end end + def remove_existing_export! + import_export_upload = @group.import_export_upload + + import_export_upload.remove_export_file! + import_export_upload.save + end + def save! if savers.all?(&:save) notify_success diff --git a/app/services/jira_import/start_import_service.rb b/app/services/jira_import/start_import_service.rb index 91a7956e585..fbbd2d883f0 100644 --- a/app/services/jira_import/start_import_service.rb +++ b/app/services/jira_import/start_import_service.rb @@ -20,23 +20,27 @@ module JiraImport private def create_and_schedule_import - import_data = project.create_or_update_import_data(data: {}).becomes(JiraImportData) - jira_project_details = JiraImportData::JiraProjectDetails.new( - jira_project_key, - Time.now.strftime('%Y-%m-%d %H:%M:%S'), - { user_id: user.id, name: user.name } - ) - import_data << jira_project_details - import_data.force_import! - + jira_import = build_jira_import project.import_type = 'jira' - project.import_state.schedule if project.save! + project.save! && jira_import.schedule! - ServiceResponse.success(payload: { import_data: import_data } ) + ServiceResponse.success(payload: { import_data: jira_import } ) rescue => ex # in case project.save! raises an erorr Gitlab::ErrorTracking.track_exception(ex, project_id: project.id) build_error_response(ex.message) + jira_import.do_fail! + end + + def build_jira_import + project.jira_imports.build( + user: user, + jira_project_key: jira_project_key, + # we do not have the jira_project_name or jira_project_xid yet so just set a mock value, + # we will once https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28190 + jira_project_name: jira_project_key, + jira_project_xid: 0 + ) end def validate @@ -48,18 +52,11 @@ module JiraImport end def build_error_response(message) - import_data = JiraImportData.new(project: project) - import_data.errors.add(:base, message) - ServiceResponse.error( - message: import_data.errors.full_messages.to_sentence, - http_status: 400, - payload: { import_data: import_data } - ) + ServiceResponse.error(message: message, http_status: 400) end def import_in_progress? - import_state = project.import_state || project.create_import_state - import_state.in_progress? + project.latest_jira_import&.in_progress? end end end diff --git a/app/views/layouts/nav/_dashboard.html.haml b/app/views/layouts/nav/_dashboard.html.haml index 2efb304b397..6fc06030d7a 100644 --- a/app/views/layouts/nav/_dashboard.html.haml +++ b/app/views/layouts/nav/_dashboard.html.haml @@ -30,7 +30,7 @@ = _('Groups') - if dashboard_nav_link?(:activity) = nav_link(path: 'dashboard#activity') do - = link_to activity_dashboard_path do + = link_to activity_dashboard_path, class: 'dashboard-shortcuts-activity' do = _('Activity') - if dashboard_nav_link?(:milestones) diff --git a/app/workers/concerns/gitlab/jira_import/import_worker.rb b/app/workers/concerns/gitlab/jira_import/import_worker.rb index 7cc650bfc29..169d3797b88 100644 --- a/app/workers/concerns/gitlab/jira_import/import_worker.rb +++ b/app/workers/concerns/gitlab/jira_import/import_worker.rb @@ -28,7 +28,7 @@ module Gitlab return false unless project return false if Feature.disabled?(:jira_issue_import, project) - project.import_state.started? + project.latest_jira_import&.started? end end end diff --git a/app/workers/gitlab/jira_import/advance_stage_worker.rb b/app/workers/gitlab/jira_import/advance_stage_worker.rb index c83a57bcd83..c3a64669c60 100644 --- a/app/workers/gitlab/jira_import/advance_stage_worker.rb +++ b/app/workers/gitlab/jira_import/advance_stage_worker.rb @@ -17,7 +17,7 @@ module Gitlab }.freeze def find_import_state(project_id) - ProjectImportState.jid_by(project_id: project_id, status: :started) + JiraImportState.jid_by(project_id: project_id, status: :started) end private diff --git a/app/workers/gitlab/jira_import/stage/finish_import_worker.rb b/app/workers/gitlab/jira_import/stage/finish_import_worker.rb index f053037e78a..1d57b77ac7e 100644 --- a/app/workers/gitlab/jira_import/stage/finish_import_worker.rb +++ b/app/workers/gitlab/jira_import/stage/finish_import_worker.rb @@ -9,11 +9,8 @@ module Gitlab private def import(project) - project.after_import - ensure JiraImport.cache_cleanup(project.id) - project.import_data.becomes(JiraImportData).finish_import! - project.import_data.save! + project.latest_jira_import&.finish! end end end diff --git a/app/workers/gitlab/jira_import/stage/import_attachments_worker.rb b/app/workers/gitlab/jira_import/stage/import_attachments_worker.rb index 3b209a279b5..905ef3efa67 100644 --- a/app/workers/gitlab/jira_import/stage/import_attachments_worker.rb +++ b/app/workers/gitlab/jira_import/stage/import_attachments_worker.rb @@ -13,7 +13,7 @@ module Gitlab # new job waiter will have zero jobs_remaining by default, so it will just pass on to next stage fake_waiter = JobWaiter.new - project.import_state.refresh_jid_expiration + project.latest_jira_import.refresh_jid_expiration Gitlab::JiraImport::AdvanceStageWorker.perform_async(project.id, { fake_waiter.key => fake_waiter.jobs_remaining }, :notes) end end diff --git a/app/workers/gitlab/jira_import/stage/import_issues_worker.rb b/app/workers/gitlab/jira_import/stage/import_issues_worker.rb index 7e257afc4d9..7a5eb6c1e3a 100644 --- a/app/workers/gitlab/jira_import/stage/import_issues_worker.rb +++ b/app/workers/gitlab/jira_import/stage/import_issues_worker.rb @@ -11,7 +11,7 @@ module Gitlab def import(project) jobs_waiter = Gitlab::JiraImport::IssuesImporter.new(project).execute - project.import_state.refresh_jid_expiration + project.latest_jira_import.refresh_jid_expiration Gitlab::JiraImport::AdvanceStageWorker.perform_async( project.id, diff --git a/app/workers/gitlab/jira_import/stage/import_notes_worker.rb b/app/workers/gitlab/jira_import/stage/import_notes_worker.rb index 9eef0d31a8c..b34e64b203b 100644 --- a/app/workers/gitlab/jira_import/stage/import_notes_worker.rb +++ b/app/workers/gitlab/jira_import/stage/import_notes_worker.rb @@ -12,7 +12,7 @@ module Gitlab # fake notes import workers for now # new job waiter will have zero jobs_remaining by default, so it will just pass on to next stage jobs_waiter = JobWaiter.new - project.import_state.refresh_jid_expiration + project.latest_jira_import.refresh_jid_expiration Gitlab::JiraImport::AdvanceStageWorker.perform_async(project.id, { jobs_waiter.key => jobs_waiter.jobs_remaining }, :finish) end diff --git a/app/workers/gitlab/jira_import/stage/start_import_worker.rb b/app/workers/gitlab/jira_import/stage/start_import_worker.rb index 80f0221c53d..1561ad90cc1 100644 --- a/app/workers/gitlab/jira_import/stage/start_import_worker.rb +++ b/app/workers/gitlab/jira_import/stage/start_import_worker.rb @@ -16,7 +16,7 @@ module Gitlab return unless start_import - Gitlab::Import::SetAsyncJid.set_jid(project.import_state) + Gitlab::Import::SetAsyncJid.set_jid(project.latest_jira_import) Gitlab::JiraImport::Stage::ImportLabelsWorker.perform_async(project.id) end @@ -26,14 +26,13 @@ module Gitlab def start_import return false unless project return false if Feature.disabled?(:jira_issue_import, project) - return false unless project.jira_force_import? - return true if start(project.import_state) + return true if start(project.latest_jira_import) Gitlab::Import::Logger.info( { project_id: project.id, project_path: project.full_path, - state: project&.import_status, + state: project&.jira_import_status, message: 'inconsistent state while importing' } ) diff --git a/changelogs/unreleased/213126-refactor-issues_with_embedded_grafana_charts_approx.yml b/changelogs/unreleased/213126-refactor-issues_with_embedded_grafana_charts_approx.yml new file mode 100644 index 00000000000..a0da66118a5 --- /dev/null +++ b/changelogs/unreleased/213126-refactor-issues_with_embedded_grafana_charts_approx.yml @@ -0,0 +1,5 @@ +--- +title: Optimize issues with embedded grafana charts usage counter +merge_request: 28936 +author: +type: performance diff --git a/changelogs/unreleased/fix-keyboard-shortcut-nav-to-dashboard-activity.yml b/changelogs/unreleased/fix-keyboard-shortcut-nav-to-dashboard-activity.yml new file mode 100644 index 00000000000..bcc7dc48228 --- /dev/null +++ b/changelogs/unreleased/fix-keyboard-shortcut-nav-to-dashboard-activity.yml @@ -0,0 +1,5 @@ +--- +title: Fix keyboard shortcut to navigate to dashboard activity +merge_request: 28985 +author: Victor Wu +type: other diff --git a/config/routes/group.rb b/config/routes/group.rb index 97d339fea98..3186a01d073 100644 --- a/config/routes/group.rb +++ b/config/routes/group.rb @@ -13,6 +13,9 @@ constraints(::Constraints::GroupUrlConstrainer.new) do get :details, as: :details_group get :activity, as: :activity_group put :transfer, as: :transfer_group + post :export, as: :export_group + get :download_export, as: :download_export_group + # TODO: Remove as part of refactor in https://gitlab.com/gitlab-org/gitlab-foss/issues/49693 get 'shared', action: :show, as: :group_shared get 'archived', action: :show, as: :group_archived diff --git a/db/migrate/20200312053852_populate_canonical_emails.rb b/db/migrate/20200312053852_populate_canonical_emails.rb new file mode 100644 index 00000000000..10efffab59c --- /dev/null +++ b/db/migrate/20200312053852_populate_canonical_emails.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class PopulateCanonicalEmails < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class User < ActiveRecord::Base + include EachBatch + + self.table_name = 'users' + + scope :with_gmail, -> { select(:id, :email).where("email ILIKE '%gmail.com'") } + end + + # Limited to *@gmail.com addresses only as a first iteration, because we know + # Gmail ignores `.` appearing in the Agent name, as well as anything after `+` + + def up + # batch size is the default, 1000 + migration = Gitlab::BackgroundMigration::PopulateCanonicalEmails + migration_name = migration.to_s.demodulize + + queue_background_migration_jobs_by_range_at_intervals( + User.with_gmail, + migration_name, + 1.minute) + end + + def down + # no-op + end +end diff --git a/db/structure.sql b/db/structure.sql index 459d04570cc..d179c52ee6b 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -13007,6 +13007,7 @@ COPY "schema_migrations" (version) FROM STDIN; 20200311165635 20200311192351 20200311214912 +20200312053852 20200312125121 20200312160532 20200312163407 diff --git a/doc/api/epics.md b/doc/api/epics.md index 014de1602ee..a0261aab605 100644 --- a/doc/api/epics.md +++ b/doc/api/epics.md @@ -77,6 +77,7 @@ Example response: "id": 29, "iid": 4, "group_id": 7, + "parent_id": 23, "title": "Accusamus iste et ullam ratione voluptatem omnis debitis dolor est.", "description": "Molestias dolorem eos vitae expedita impedit necessitatibus quo voluptatum.", "state": "opened", @@ -117,6 +118,7 @@ Example response: "id": 50, "iid": 35, "group_id": 17, + "parent_id": 19, "title": "Accusamus iste et ullam ratione voluptatem omnis debitis dolor est.", "description": "Molestias dolorem eos vitae expedita impedit necessitatibus quo voluptatum.", "state": "opened", @@ -245,7 +247,7 @@ POST /groups/:id/epics | `parent_id` | integer/string | no | The id of a parent epic (since 11.11) | ```shell -curl --header POST "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics?title=Epic&description=Epic%20description +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics?title=Epic&description=Epic%20description ``` Example response: @@ -320,7 +322,7 @@ PUT /groups/:id/epics/:epic_iid | `state_event` | string | no | State event for an epic. Set `close` to close the epic and `reopen` to reopen it (since 11.4) | ```shell -curl --header PUT "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5?title=New%20Title +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5?title=New%20Title ``` Example response: @@ -382,7 +384,7 @@ DELETE /groups/:id/epics/:epic_iid | `epic_iid` | integer/string | yes | The internal ID of the epic. | ```shell -curl --header DELETE "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5 ``` ## Create a todo diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index 0eebc74cc6c..583e27f3301 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -4265,7 +4265,7 @@ type JiraImport { jiraProjectKey: String! """ - Timestamp of when the Jira import was created/started + Timestamp of when the Jira import was created """ scheduledAt: Time diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index 9abc312da33..79d4088e566 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -12108,7 +12108,7 @@ }, { "name": "scheduledAt", - "description": "Timestamp of when the Jira import was created/started", + "description": "Timestamp of when the Jira import was created", "args": [ ], diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 6948f361a14..43e8677f384 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -637,7 +637,7 @@ Autogenerated return type of IssueSetWeight | Name | Type | Description | | --- | ---- | ---------- | | `jiraProjectKey` | String! | Project key for the imported Jira project | -| `scheduledAt` | Time | Timestamp of when the Jira import was created/started | +| `scheduledAt` | Time | Timestamp of when the Jira import was created | | `scheduledBy` | User | User that started the Jira import | ## JiraImportStartPayload diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index 6fe72458da2..8ca5dfa082e 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -27,9 +27,13 @@ module API detail 'This feature was introduced in GitLab 12.5.' end post ':id/export' do - GroupExportWorker.perform_async(current_user.id, user_group.id, params) # rubocop:disable CodeReuse/Worker + export_service = ::Groups::ImportExport::ExportService.new(group: user_group, user: current_user) - accepted! + if export_service.async_execute + accepted! + else + render_api_error!(message: 'Group export could not be started.') + end end end end diff --git a/lib/banzai/filter/epic_reference_filter.rb b/lib/banzai/filter/epic_reference_filter.rb index d1e1a56424d..70a6cb0a6dc 100644 --- a/lib/banzai/filter/epic_reference_filter.rb +++ b/lib/banzai/filter/epic_reference_filter.rb @@ -4,8 +4,6 @@ module Banzai module Filter # The actual filter is implemented in the EE mixin class EpicReferenceFilter < IssuableReferenceFilter - prepend_if_ee('EE::Banzai::Filter::EpicReferenceFilter') # rubocop: disable Cop/InjectEnterpriseEditionModule - self.reference_type = :epic def self.object_class @@ -20,3 +18,5 @@ module Banzai end end end + +Banzai::Filter::EpicReferenceFilter.prepend_if_ee('EE::Banzai::Filter::EpicReferenceFilter') diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index d1e8587314c..8994cdbed60 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -9,8 +9,6 @@ module Banzai # so we can avoid N+1 queries problem class IssuableExtractor - prepend_if_ee('EE::Banzai::IssuableExtractor') # rubocop: disable Cop/InjectEnterpriseEditionModule - attr_reader :context ISSUE_REFERENCE_TYPE = '@data-reference-type="issue"' @@ -59,3 +57,5 @@ module Banzai end end end + +Banzai::IssuableExtractor.prepend_if_ee('EE::Banzai::IssuableExtractor') diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index b6238dfe7f0..329bbb270bd 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -3,8 +3,6 @@ module Banzai module Pipeline class GfmPipeline < BasePipeline - prepend_if_ee('EE::Banzai::Pipeline::GfmPipeline') # rubocop: disable Cop/InjectEnterpriseEditionModule - # These filters transform GitLab Flavored Markdown (GFM) to HTML. # The nodes and marks referenced in app/assets/javascripts/behaviors/markdown/editor_extensions.js # consequently transform that same HTML to GFM to be copied to the clipboard. @@ -77,3 +75,5 @@ module Banzai end end end + +Banzai::Pipeline::GfmPipeline.prepend_if_ee('EE::Banzai::Pipeline::GfmPipeline') diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index 8236b702147..32d7126c97d 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -3,8 +3,6 @@ module Banzai module Pipeline class PostProcessPipeline < BasePipeline - prepend_if_ee('EE::Banzai::Pipeline::PostProcessPipeline') # rubocop: disable Cop/InjectEnterpriseEditionModule - def self.filters @filters ||= FilterArray[ *internal_link_filters, @@ -34,3 +32,5 @@ module Banzai end end end + +Banzai::Pipeline::PostProcessPipeline.prepend_if_ee('EE::Banzai::Pipeline::PostProcessPipeline') diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index a16ab04b792..7fe13100ec2 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -3,8 +3,6 @@ module Banzai module Pipeline class SingleLinePipeline < GfmPipeline - prepend_if_ee('EE::Banzai::Pipeline::SingleLinePipeline') # rubocop: disable Cop/InjectEnterpriseEditionModule - def self.filters @filters ||= FilterArray[ Filter::HtmlEntityFilter, @@ -41,3 +39,5 @@ module Banzai end end end + +Banzai::Pipeline::SingleLinePipeline.prepend_if_ee('EE::Banzai::Pipeline::SingleLinePipeline') diff --git a/lib/banzai/reference_parser/epic_parser.rb b/lib/banzai/reference_parser/epic_parser.rb index b5fbf7accc4..7e72a260839 100644 --- a/lib/banzai/reference_parser/epic_parser.rb +++ b/lib/banzai/reference_parser/epic_parser.rb @@ -4,8 +4,6 @@ module Banzai module ReferenceParser # The actual parser is implemented in the EE mixin class EpicParser < IssuableParser - prepend_if_ee('::EE::Banzai::ReferenceParser::EpicParser') # rubocop: disable Cop/InjectEnterpriseEditionModule - self.reference_type = :epic def records_for_nodes(_nodes) @@ -14,3 +12,5 @@ module Banzai end end end + +Banzai::ReferenceParser::EpicParser.prepend_if_ee('::EE::Banzai::ReferenceParser::EpicParser') diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 211c59fe841..c1066d8fa62 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -25,7 +25,9 @@ module Gitlab project_generate_new_export: { threshold: 1, interval: 5.minutes }, project_import: { threshold: 30, interval: 5.minutes }, play_pipeline_schedule: { threshold: 1, interval: 1.minute }, - show_raw_controller: { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute } + show_raw_controller: { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute }, + group_export: { threshold: 1, interval: 5.minutes }, + group_download_export: { threshold: 10, interval: 10.minutes } }.freeze end diff --git a/lib/gitlab/background_migration/populate_canonical_emails.rb b/lib/gitlab/background_migration/populate_canonical_emails.rb new file mode 100644 index 00000000000..052e75c5655 --- /dev/null +++ b/lib/gitlab/background_migration/populate_canonical_emails.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class to populate new rows of UserCanonicalEmail based on existing email addresses + class PopulateCanonicalEmails + def perform(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO + user_canonical_emails ( + user_id, + canonical_email, + created_at, + updated_at + ) + SELECT users.id AS user_id, + concat(translate(split_part(split_part(users.email, '@', 1), '+', 1), '.', ''), '@gmail.com') AS canonical_email, + NOW() AS created_at, + NOW() AS updated_at + FROM users + WHERE users.email ILIKE '%@gmail.com' + AND users.id BETWEEN #{start_id} AND #{stop_id} + ON CONFLICT DO NOTHING; + SQL + end + end + end +end diff --git a/lib/gitlab/grafana_embed_usage_data.rb b/lib/gitlab/grafana_embed_usage_data.rb deleted file mode 100644 index 78a87623e1f..00000000000 --- a/lib/gitlab/grafana_embed_usage_data.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class GrafanaEmbedUsageData - class << self - def issue_count - # rubocop:disable CodeReuse/ActiveRecord - Issue.joins('JOIN grafana_integrations USING (project_id)') - .where("issues.description LIKE '%' || grafana_integrations.grafana_url || '%'") - .where(grafana_integrations: { enabled: true }) - .count - # rubocop:enable CodeReuse/ActiveRecord - end - end - end -end diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb index 24158b8e734..afb443020b7 100644 --- a/lib/gitlab/jira_import/base_importer.rb +++ b/lib/gitlab/jira_import/base_importer.rb @@ -9,7 +9,7 @@ module Gitlab raise Projects::ImportService::Error, _('Jira import feature is disabled.') unless Feature.enabled?(:jira_issue_import, project) raise Projects::ImportService::Error, _('Jira integration not configured.') unless project.jira_service&.active? - @jira_project_key = project&.import_data&.becomes(JiraImportData)&.current_project&.key + @jira_project_key = project.latest_jira_import&.jira_project_key raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key @project = project diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 6333e7923c6..6c98f8f5585 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -90,7 +90,7 @@ module Gitlab issues_created_from_gitlab_error_tracking_ui: count(SentryIssue), issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue), issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), - issues_with_embedded_grafana_charts_approx: ::Gitlab::GrafanaEmbedUsageData.issue_count, + issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data, incident_issues: count(::Issue.authored(::User.alert_bot)), keys: count(Key), label_lists: count(List.label), @@ -133,6 +133,14 @@ module Gitlab { avg_cycle_analytics: {} } end + # rubocop:disable CodeReuse/ActiveRecord + def grafana_embed_usage_data + count(Issue.joins('JOIN grafana_integrations USING (project_id)') + .where("issues.description LIKE '%' || grafana_integrations.grafana_url || '%'") + .where(grafana_integrations: { enabled: true })) + end + # rubocop: enable CodeReuse/ActiveRecord + def features_usage_data features_usage_data_ce end diff --git a/lib/quality/helm3_client.rb b/lib/quality/helm3_client.rb new file mode 100644 index 00000000000..f5eb0834386 --- /dev/null +++ b/lib/quality/helm3_client.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require 'time' +require_relative '../gitlab/popen' unless defined?(Gitlab::Popen) + +module Quality + class Helm3Client + CommandFailedError = Class.new(StandardError) + + attr_reader :namespace + + RELEASE_JSON_ATTRIBUTES = %w[name revision updated status chart app_version namespace].freeze + PAGINATION_SIZE = 256 # Default helm list pagination size + + Release = Struct.new(:name, :revision, :last_update, :status, :chart, :app_version, :namespace) do + def revision + @revision ||= self[:revision].to_i + end + + def status + @status ||= self[:status].downcase + end + + def last_update + @last_update ||= Time.parse(self[:last_update]) + end + end + + # A single page of data and the corresponding page number. + Page = Struct.new(:releases, :number) + + def initialize(namespace:, tiller_namespace: nil) + @namespace = namespace + end + + def releases(args: []) + each_release(args) + end + + def delete(release_name:) + run_command([ + 'uninstall', + %(--namespace "#{namespace}"), + release_name + ]) + end + + private + + def run_command(command) + final_command = ['helm', *command].join(' ') + puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output + + result = Gitlab::Popen.popen_with_detail([final_command]) + + if result.status.success? + result.stdout.chomp.freeze + else + raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}" + end + end + + def raw_releases(page, args = []) + command = [ + 'list', + %(--namespace "#{namespace}"), + %(--max #{PAGINATION_SIZE}), + %(--offset #{PAGINATION_SIZE * page}), + %(--output json), + *args + ] + releases = JSON.parse(run_command(command)) + + releases.map do |release| + Release.new(*release.values_at(*RELEASE_JSON_ATTRIBUTES)) + end + rescue JSON::ParserError => ex + puts "Ignoring this JSON parsing error: #{ex}" # rubocop:disable Rails/Output + [] + end + + # Fetches data from Helm and yields a Page object for every page + # of data, without loading all of them into memory. + # + # method - The Octokit method to use for getting the data. + # args - Arguments to pass to the `helm list` command. + def each_releases_page(args, &block) + return to_enum(__method__, args) unless block_given? + + page = 0 + final_args = args.dup + + begin + collection = raw_releases(page, final_args) + + yield Page.new(collection, page += 1) + end while collection.any? + end + + # Iterates over all of the releases. + # + # args - Any arguments to pass to the `helm list` command. + def each_release(args, &block) + return to_enum(__method__, args) unless block_given? + + each_releases_page(args) do |page| + page.releases.each do |release| + yield release + end + end + end + end +end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 69d542a4f02..506027aa866 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -80,16 +80,19 @@ namespace :gitlab do end desc 'This adjusts and cleans db/structure.sql - it runs after db:structure:dump' - task :clean_structure_sql do + task :clean_structure_sql do |task_name| structure_file = 'db/structure.sql' schema = File.read(structure_file) File.open(structure_file, 'wb+') do |io| Gitlab::Database::SchemaCleaner.new(schema).clean(io) end + + # Allow this task to be called multiple times, as happens when running db:migrate:redo + Rake::Task[task_name].reenable end - # Inform Rake that gitlab:schema:fix_structure_sql should be run every time rake db:structure:dump is run + # Inform Rake that gitlab:schema:clean_structure_sql should be run every time rake db:structure:dump is run Rake::Task['db:structure:dump'].enhance do Rake::Task['gitlab:db:clean_structure_sql'].invoke end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index cadbce443c7..4ef30825233 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -10112,6 +10112,15 @@ msgstr "" msgid "Group details" msgstr "" +msgid "Group export could not be started." +msgstr "" + +msgid "Group export link has expired. Please generate a new export from your group settings." +msgstr "" + +msgid "Group export started." +msgstr "" + msgid "Group has been already marked for deletion" msgstr "" diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb index 0a073a28bf3..e3ed7143ea2 100755 --- a/scripts/review_apps/automated_cleanup.rb +++ b/scripts/review_apps/automated_cleanup.rb @@ -2,6 +2,7 @@ require 'gitlab' require_relative File.expand_path('../../lib/quality/helm_client.rb', __dir__) +require_relative File.expand_path('../../lib/quality/helm3_client.rb', __dir__) require_relative File.expand_path('../../lib/quality/kubernetes_client.rb', __dir__) class AutomatedCleanup @@ -11,7 +12,8 @@ class AutomatedCleanup HELM_RELEASES_BATCH_SIZE = 5 IGNORED_HELM_ERRORS = [ 'transport is closing', - 'error upgrading connection' + 'error upgrading connection', + 'not found' ].freeze IGNORED_KUBERNETES_ERRORS = [ 'NotFound' @@ -43,8 +45,16 @@ class AutomatedCleanup self.class.ee? ? 'review-apps-ee' : 'review-apps-ce' end + def helm3? + !ENV['HELM_3'].nil? + end + + def helm_client_class + helm3? ? Quality::Helm3Client : Quality::HelmClient + end + def helm - @helm ||= Quality::HelmClient.new( + @helm ||= helm_client_class.new( tiller_namespace: review_apps_namespace, namespace: review_apps_namespace) end @@ -78,7 +88,7 @@ class AutomatedCleanup if deployed_at < delete_threshold deleted_environment = delete_environment(environment, deployment) if deleted_environment - release = Quality::HelmClient::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace) + release = helm_client_class::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace) releases_to_delete << release end else @@ -97,7 +107,7 @@ class AutomatedCleanup end def perform_helm_releases_cleanup!(days:) - puts "Checking for Helm releases that are FAILED or not updated in the last #{days} days..." + puts "Checking for Helm releases that are failed or not updated in the last #{days} days..." threshold = threshold_time(days: days) @@ -107,7 +117,7 @@ class AutomatedCleanup # Prevents deleting `dns-gitlab-review-app` releases or other unrelated releases next unless release.name.start_with?('review-') - if release.status == 'FAILED' || release.last_update < threshold + if release.status.casecmp('failed') == 0 || release.last_update < threshold releases_to_delete << release else print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'leaving') @@ -143,7 +153,8 @@ class AutomatedCleanup end def helm_releases - args = ['--all', '--date', "--max #{HELM_RELEASES_BATCH_SIZE}"] + args = ['--all', '--date'] + args << "--max #{HELM_RELEASES_BATCH_SIZE}" unless helm3? helm.releases(args: args) end @@ -159,7 +170,7 @@ class AutomatedCleanup helm.delete(release_name: releases_names) kubernetes.cleanup(release_name: releases_names, wait: false) - rescue Quality::HelmClient::CommandFailedError => ex + rescue helm_client_class::CommandFailedError => ex raise ex unless ignore_exception?(ex.message, IGNORED_HELM_ERRORS) puts "Ignoring the following Helm error:\n#{ex}\n" diff --git a/spec/controllers/groups_controller_spec.rb b/spec/controllers/groups_controller_spec.rb index 7bd0f6bd6b8..93478bbff1d 100644 --- a/spec/controllers/groups_controller_spec.rb +++ b/spec/controllers/groups_controller_spec.rb @@ -764,6 +764,136 @@ describe GroupsController do end end + describe 'POST #export' do + context 'when the group export feature flag is not enabled' do + before do + sign_in(admin) + stub_feature_flags(group_import_export: false) + end + + it 'returns a not found error' do + post :export, params: { id: group.to_param } + + expect(response).to have_gitlab_http_status(:not_found) + end + end + + context 'when the user does not have permission to export the group' do + before do + sign_in(guest) + end + + it 'returns an error' do + post :export, params: { id: group.to_param } + + expect(response).to have_gitlab_http_status(:not_found) + end + end + + context 'when supplied valid params' do + before do + sign_in(admin) + end + + it 'triggers the export job' do + expect(GroupExportWorker).to receive(:perform_async).with(admin.id, group.id, {}) + + post :export, params: { id: group.to_param } + end + + it 'redirects to the edit page' do + post :export, params: { id: group.to_param } + + expect(response).to have_gitlab_http_status(:found) + end + end + + context 'when the endpoint receives requests above the rate limit' do + before do + sign_in(admin) + allow(Gitlab::ApplicationRateLimiter).to receive(:throttled?).and_return(true) + end + + it 'throttles the endpoint' do + post :export, params: { id: group.to_param } + + expect(flash[:alert]).to eq('This endpoint has been requested too many times. Try again later.') + expect(response).to have_gitlab_http_status(:found) + end + end + end + + describe 'GET #download_export' do + context 'when there is a file available to download' do + let(:export_file) { fixture_file_upload('spec/fixtures/group_export.tar.gz') } + + before do + sign_in(admin) + create(:import_export_upload, group: group, export_file: export_file) + end + + it 'sends the file' do + get :download_export, params: { id: group.to_param } + + expect(response.body).to eq export_file.tempfile.read + end + end + + context 'when there is no file available to download' do + before do + sign_in(admin) + end + + it 'returns not found' do + get :download_export, params: { id: group.to_param } + + expect(flash[:alert]) + .to eq 'Group export link has expired. Please generate a new export from your group settings.' + + expect(response).to redirect_to(edit_group_path(group)) + end + end + + context 'when the group export feature flag is not enabled' do + before do + sign_in(admin) + stub_feature_flags(group_import_export: false) + end + + it 'returns a not found error' do + post :export, params: { id: group.to_param } + + expect(response).to have_gitlab_http_status(:not_found) + end + end + + context 'when the user does not have the required permissions' do + before do + sign_in(guest) + end + + it 'returns not_found' do + get :download_export, params: { id: group.to_param } + + expect(response).to have_gitlab_http_status(:not_found) + end + end + + context 'when the endpoint receives requests above the rate limit' do + before do + sign_in(admin) + allow(Gitlab::ApplicationRateLimiter).to receive(:throttled?).and_return(true) + end + + it 'throttles the endpoint' do + get :download_export, params: { id: group.to_param } + + expect(flash[:alert]).to eq('This endpoint has been requested too many times. Try again later.') + expect(response).to have_gitlab_http_status(:found) + end + end + end + context 'token authentication' do it_behaves_like 'authenticates sessionless user', :show, :atom, public: true do before do diff --git a/spec/controllers/projects/import/jira_controller_spec.rb b/spec/controllers/projects/import/jira_controller_spec.rb index 57e0aa098c0..8e0d506e5e4 100644 --- a/spec/controllers/projects/import/jira_controller_spec.rb +++ b/spec/controllers/projects/import/jira_controller_spec.rb @@ -105,16 +105,16 @@ describe Projects::Import::JiraController do context 'when everything is ok' do it 'creates import state' do - expect(project.import_state).to be_nil + expect(project.latest_jira_import).to be_nil post :import, params: { namespace_id: project.namespace, project_id: project, jira_project_key: 'Test' } project.reload - jira_project = project.import_data.data.dig('jira', 'projects').first + jira_import = project.latest_jira_import expect(project.import_type).to eq 'jira' - expect(project.import_state.status).to eq 'scheduled' - expect(jira_project['key']).to eq 'Test' + expect(jira_import.status).to eq 'scheduled' + expect(jira_import.jira_project_key).to eq 'Test' expect(response).to redirect_to(project_import_jira_path(project)) end end @@ -122,29 +122,19 @@ describe Projects::Import::JiraController do end context 'when import state is scheduled' do - let_it_be(:import_state) { create(:import_state, project: project, status: :scheduled) } + let_it_be(:jira_import_state) { create(:jira_import_state, :scheduled, project: project) } context 'get show' do it 'renders import status' do get :show, params: { namespace_id: project.namespace.to_param, project_id: project } - expect(project.import_state.status).to eq 'scheduled' + jira_import = project.latest_jira_import + expect(jira_import.status).to eq 'scheduled' expect(flash.now[:notice]).to eq 'Import scheduled' end end context 'post import' do - before do - project.reload - project.create_import_data( - data: { - 'jira': { - 'projects': [{ 'key': 'Test', scheduled_at: 5.days.ago, scheduled_by: { user_id: user.id, name: user.name } }] - } - } - ) - end - it 'uses the existing import data' do post :import, params: { namespace_id: project.namespace, project_id: project, jira_project_key: 'New Project' } @@ -155,39 +145,27 @@ describe Projects::Import::JiraController do end context 'when jira import ran before' do - let_it_be(:import_state) { create(:import_state, project: project, status: :finished) } + let_it_be(:jira_import_state) { create(:jira_import_state, :finished, project: project, jira_project_key: 'Test') } context 'get show' do it 'renders import status' do allow(JIRA::Resource::Project).to receive(:all).and_return([]) get :show, params: { namespace_id: project.namespace.to_param, project_id: project } - expect(project.import_state.status).to eq 'finished' + expect(project.latest_jira_import.status).to eq 'finished' expect(flash.now[:notice]).to eq 'Import finished' end end context 'post import' do - before do - project.reload - project.create_import_data( - data: { - 'jira': { - 'projects': [{ 'key': 'Test', scheduled_at: 5.days.ago, scheduled_by: { user_id: user.id, name: user.name } }] - } - } - ) - end - it 'uses the existing import data' do post :import, params: { namespace_id: project.namespace, project_id: project, jira_project_key: 'New Project' } project.reload - expect(project.import_state.status).to eq 'scheduled' - jira_imported_projects = project.import_data.data.dig('jira', 'projects') - expect(jira_imported_projects.size).to eq 2 - expect(jira_imported_projects.first['key']).to eq 'Test' - expect(jira_imported_projects.last['key']).to eq 'New Project' + expect(project.latest_jira_import.status).to eq 'scheduled' + expect(project.jira_imports.size).to eq 2 + expect(project.jira_imports.first.jira_project_key).to eq 'Test' + expect(project.jira_imports.last.jira_project_key).to eq 'New Project' expect(response).to redirect_to(project_import_jira_path(project)) end end diff --git a/spec/features/dashboard/shortcuts_spec.rb b/spec/features/dashboard/shortcuts_spec.rb index cf74b2cc8ce..2cd9cbc4471 100644 --- a/spec/features/dashboard/shortcuts_spec.rb +++ b/spec/features/dashboard/shortcuts_spec.rb @@ -29,6 +29,10 @@ describe 'Dashboard shortcuts', :js do find('body').send_keys([:shift, 'P']) check_page_title('Projects') + + find('body').send_keys([:shift, 'A']) + + check_page_title('Activity') end end diff --git a/spec/frontend/boards/board_card_spec.js b/spec/frontend/boards/board_card_spec.js index 2524af21826..959c71d05ca 100644 --- a/spec/frontend/boards/board_card_spec.js +++ b/spec/frontend/boards/board_card_spec.js @@ -9,6 +9,7 @@ import axios from '~/lib/utils/axios_utils'; import waitForPromises from 'helpers/wait_for_promises'; import eventHub from '~/boards/eventhub'; +import sidebarEventHub from '~/sidebar/event_hub'; import '~/boards/models/label'; import '~/boards/models/assignee'; import '~/boards/models/list'; @@ -201,7 +202,8 @@ describe('Board card', () => { it('resets detail issue to empty if already set', () => { jest.spyOn(eventHub, '$emit').mockImplementation(() => {}); - [boardsStore.detail.issue] = list.issues; + const [issue] = list.issues; + boardsStore.detail.issue = issue; mountComponent(); wrapper.trigger('mousedown'); @@ -210,4 +212,27 @@ describe('Board card', () => { expect(eventHub.$emit).toHaveBeenCalledWith('clearDetailIssue', undefined); }); }); + + describe('sidebarHub events', () => { + it('closes all sidebars before showing an issue if no issues are opened', () => { + jest.spyOn(sidebarEventHub, '$emit').mockImplementation(() => {}); + boardsStore.detail.issue = {}; + mountComponent(); + + wrapper.trigger('mouseup'); + + expect(sidebarEventHub.$emit).toHaveBeenCalledWith('sidebar.closeAll'); + }); + + it('it does not closes all sidebars before showing an issue if an issue is opened', () => { + jest.spyOn(sidebarEventHub, '$emit').mockImplementation(() => {}); + const [issue] = list.issues; + boardsStore.detail.issue = issue; + mountComponent(); + + wrapper.trigger('mousedown'); + + expect(sidebarEventHub.$emit).not.toHaveBeenCalledWith('sidebar.closeAll'); + }); + }); }); diff --git a/spec/graphql/resolvers/projects/jira_imports_resolver_spec.rb b/spec/graphql/resolvers/projects/jira_imports_resolver_spec.rb index a5f17acce91..4d060d213ed 100644 --- a/spec/graphql/resolvers/projects/jira_imports_resolver_spec.rb +++ b/spec/graphql/resolvers/projects/jira_imports_resolver_spec.rb @@ -7,29 +7,16 @@ describe Resolvers::Projects::JiraImportsResolver do describe '#resolve' do let_it_be(:user) { create(:user) } - let_it_be(:jira_import_data) do - data = JiraImportData.new - data << JiraImportData::JiraProjectDetails.new('AA', 2.days.ago.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name }) - data << JiraImportData::JiraProjectDetails.new('BB', 5.days.ago.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name }) - data - end - - context 'when feature flag disabled' do - let_it_be(:project) { create(:project, :private, import_data: jira_import_data) } - - before do - stub_feature_flags(jira_issue_import: false) - end + let_it_be(:project, reload: true) { create(:project, :public) } - it_behaves_like 'no jira import access' - end - - context 'when project does not have Jira import data' do - let_it_be(:project) { create(:project, :private, import_data: nil) } + context 'when project does not have Jira imports' do + let(:current_user) { user } - context 'when user cannot read Jira import data' do + context 'when user cannot read Jira imports' do context 'when anonymous user' do - it_behaves_like 'no jira import data present' + let(:current_user) { nil } + + it_behaves_like 'no jira import access' end context 'when user developer' do @@ -37,7 +24,7 @@ describe Resolvers::Projects::JiraImportsResolver do project.add_developer(user) end - it_behaves_like 'no jira import data present' + it_behaves_like 'no jira import access' end end @@ -50,11 +37,25 @@ describe Resolvers::Projects::JiraImportsResolver do end end - context 'when project has Jira import data' do - let_it_be(:project) { create(:project, :private, import_data: jira_import_data) } + context 'when project has Jira imports' do + let_it_be(:current_user) { user } + let_it_be(:jira_import1) { create(:jira_import_state, :finished, project: project, jira_project_key: 'AA', created_at: 2.days.ago) } + let_it_be(:jira_import2) { create(:jira_import_state, :finished, project: project, jira_project_key: 'BB', created_at: 5.days.ago) } + + context 'when feature flag disabled' do + let(:current_user) { user } + + before do + stub_feature_flags(jira_issue_import: false) + end + + it_behaves_like 'no jira import access' + end - context 'when user cannot read Jira import data' do + context 'when user cannot read Jira imports' do context 'when anonymous user' do + let(:current_user) { nil } + it_behaves_like 'no jira import access' end @@ -67,22 +68,22 @@ describe Resolvers::Projects::JiraImportsResolver do end end - context 'when user can access Jira import data' do + context 'when user can access Jira imports' do before do project.add_maintainer(user) end - it 'returns Jira imports sorted ascending by scheduledAt time' do + it 'returns Jira imports sorted ascending by created_at time' do imports = resolve_imports expect(imports.size).to eq 2 - expect(imports.map(&:key)).to eq %w(BB AA) + expect(imports.map(&:jira_project_key)).to eq %w(BB AA) end end end end - def resolve_imports(args = {}, context = { current_user: user }) + def resolve_imports(args = {}, context = { current_user: current_user }) resolve(described_class, obj: project, args: args, ctx: context) end end diff --git a/spec/lib/gitlab/background_migration/populate_canonical_emails_spec.rb b/spec/lib/gitlab/background_migration/populate_canonical_emails_spec.rb new file mode 100644 index 00000000000..37ddb8b569d --- /dev/null +++ b/spec/lib/gitlab/background_migration/populate_canonical_emails_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::BackgroundMigration::PopulateCanonicalEmails, :migration, schema: 20200312053852 do + let(:migration) { described_class.new } + + let_it_be(:users_table) { table(:users) } + let_it_be(:user_canonical_emails_table) { table(:user_canonical_emails) } + + let_it_be(:users) { users_table.all } + let_it_be(:user_canonical_emails) { user_canonical_emails_table.all } + + subject { migration.perform(1, 1) } + + describe 'gmail users' do + using RSpec::Parameterized::TableSyntax + + where(:original_email, :expected_result) do + 'legitimateuser@gmail.com' | 'legitimateuser@gmail.com' + 'userwithplus+somestuff@gmail.com' | 'userwithplus@gmail.com' + 'user.with.periods@gmail.com' | 'userwithperiods@gmail.com' + 'user.with.periods.and.plus+someotherstuff@gmail.com' | 'userwithperiodsandplus@gmail.com' + end + + with_them do + it 'generates the correct canonical email' do + create_user(email: original_email, id: 1) + + subject + + result = canonical_emails + expect(result.count).to eq 1 + expect(result.first).to match({ + 'user_id' => 1, + 'canonical_email' => expected_result + }) + end + end + end + + describe 'non gmail.com domain users' do + %w[ + legitimateuser@somedomain.com + userwithplus+somestuff@other.com + user.with.periods@gmail.org + user.with.periods.and.plus+someotherstuff@orangmail.com + ].each do |non_gmail_address| + it 'does not generate a canonical email' do + create_user(email: non_gmail_address, id: 1) + + subject + + expect(canonical_emails(user_id: 1).count).to eq 0 + end + end + end + + describe 'gracefully handles missing records' do + specify { expect { subject }.not_to raise_error } + end + + describe 'gracefully handles existing records, some of which may have an already-existing identical canonical_email field' do + let_it_be(:user_one) { create_user(email: "example.user@gmail.com", id: 1) } + let_it_be(:user_two) { create_user(email: "exampleuser@gmail.com", id: 2) } + let_it_be(:user_email_one) { user_canonical_emails.create(canonical_email: "exampleuser@gmail.com", user_id: user_one.id) } + + subject { migration.perform(1, 2) } + + it 'only creates one record' do + subject + + expect(canonical_emails.count).not_to be_nil + end + end + + def create_user(attributes) + default_attributes = { + projects_limit: 0 + } + + users.create(default_attributes.merge!(attributes)) + end + + def canonical_emails(user_id: nil) + filter_by_id = user_id ? "WHERE user_id = #{user_id}" : "" + + ApplicationRecord.connection.execute <<~SQL + SELECT canonical_email, user_id + FROM user_canonical_emails + #{filter_by_id}; + SQL + end +end diff --git a/spec/lib/gitlab/grafana_embed_usage_data_spec.rb b/spec/lib/gitlab/grafana_embed_usage_data_spec.rb deleted file mode 100644 index 162db46719b..00000000000 --- a/spec/lib/gitlab/grafana_embed_usage_data_spec.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Gitlab::GrafanaEmbedUsageData do - describe '#issue_count' do - subject { described_class.issue_count } - - let(:project) { create(:project) } - let(:description_with_embed) { "Some comment\n\nhttps://grafana.example.com/d/xvAk4q0Wk/go-processes?orgId=1&from=1573238522762&to=1573240322762&var-job=prometheus&var-interval=10m&panelId=1&fullscreen" } - let(:description_with_unintegrated_embed) { "Some comment\n\nhttps://grafana.exp.com/d/xvAk4q0Wk/go-processes?orgId=1&from=1573238522762&to=1573240322762&var-job=prometheus&var-interval=10m&panelId=1&fullscreen" } - let(:description_with_non_grafana_inline_metric) { "Some comment\n\n#{Gitlab::Routing.url_helpers.metrics_namespace_project_environment_url(*['foo', 'bar', 12])}" } - - shared_examples "zero count" do - it "does not count the issue" do - expect(subject).to eq(0) - end - end - - context 'with project grafana integration enabled' do - before do - create(:grafana_integration, project: project, enabled: true) - end - - context 'with valid and invalid embeds' do - before do - # Valid - create(:issue, project: project, description: description_with_embed) - create(:issue, project: project, description: description_with_embed) - # In-Valid - create(:issue, project: project, description: description_with_unintegrated_embed) - create(:issue, project: project, description: description_with_non_grafana_inline_metric) - create(:issue, project: project, description: nil) - create(:issue, project: project, description: '') - create(:issue, project: project) - end - - it 'counts only the issues with embeds' do - expect(subject).to eq(2) - end - end - end - - context 'with project grafana integration disabled' do - before do - create(:grafana_integration, project: project, enabled: false) - end - - context 'with one issue having a grafana link in the description and one without' do - before do - create(:issue, project: project, description: description_with_embed) - create(:issue, project: project) - end - - it_behaves_like('zero count') - end - end - - context 'with an un-integrated project' do - context 'with one issue having a grafana link in the description and one without' do - before do - create(:issue, project: project, description: description_with_embed) - create(:issue, project: project) - end - - it_behaves_like('zero count') - end - end - end -end diff --git a/spec/lib/gitlab/jira_import/base_importer_spec.rb b/spec/lib/gitlab/jira_import/base_importer_spec.rb index 8bc43feb356..f22efcb8743 100644 --- a/spec/lib/gitlab/jira_import/base_importer_spec.rb +++ b/spec/lib/gitlab/jira_import/base_importer_spec.rb @@ -37,12 +37,8 @@ describe Gitlab::JiraImport::BaseImporter do end context 'when import data exists' do - let(:jira_import_data) do - data = JiraImportData.new - data << JiraImportData::JiraProjectDetails.new('xx', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: 1, name: 'root' }) - data - end - let(:project) { create(:project, import_data: jira_import_data) } + let_it_be(:project) { create(:project) } + let_it_be(:jira_import) { create(:jira_import_state, project: project) } let(:subject) { described_class.new(project) } context 'when #imported_items_cache_key is not implemented' do diff --git a/spec/lib/gitlab/jira_import/issues_importer_spec.rb b/spec/lib/gitlab/jira_import/issues_importer_spec.rb index 88e8b195dbe..8e16fd3e978 100644 --- a/spec/lib/gitlab/jira_import/issues_importer_spec.rb +++ b/spec/lib/gitlab/jira_import/issues_importer_spec.rb @@ -3,14 +3,10 @@ require 'spec_helper' describe Gitlab::JiraImport::IssuesImporter do - let(:user) { create(:user) } - let(:jira_import_data) do - data = JiraImportData.new - data << JiraImportData::JiraProjectDetails.new('XX', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name }) - data - end - let(:project) { create(:project, import_data: jira_import_data) } - let!(:jira_service) { create(:jira_service, project: project) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project) } + let_it_be(:jira_import) { create(:jira_import_state, project: project) } + let_it_be(:jira_service) { create(:jira_service, project: project) } subject { described_class.new(project) } diff --git a/spec/lib/gitlab/jira_import/labels_importer_spec.rb b/spec/lib/gitlab/jira_import/labels_importer_spec.rb index eaa13d9ed32..2d0e2bc6b53 100644 --- a/spec/lib/gitlab/jira_import/labels_importer_spec.rb +++ b/spec/lib/gitlab/jira_import/labels_importer_spec.rb @@ -3,14 +3,10 @@ require 'spec_helper' describe Gitlab::JiraImport::LabelsImporter do - let(:user) { create(:user) } - let(:jira_import_data) do - data = JiraImportData.new - data << JiraImportData::JiraProjectDetails.new('XX', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name }) - data - end - let(:project) { create(:project, import_data: jira_import_data) } - let!(:jira_service) { create(:jira_service, project: project) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project) } + let_it_be(:jira_import) { create(:jira_import_state, project: project) } + let_it_be(:jira_service) { create(:jira_service, project: project) } subject { described_class.new(project).execute } diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index 37d9c5389dd..12199bc6d5a 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -14,7 +14,7 @@ describe Gitlab::UsageData, :aggregate_failures do let!(:ud) { build(:usage_data) } before do - allow(Gitlab::GrafanaEmbedUsageData).to receive(:issue_count).and_return(2) + allow(described_class).to receive(:grafana_embed_usage_data).and_return(2) end subject { described_class.data } @@ -220,6 +220,71 @@ describe Gitlab::UsageData, :aggregate_failures do end end + describe '#grafana_embed_usage_data' do + subject { described_class.grafana_embed_usage_data } + + let(:project) { create(:project) } + let(:description_with_embed) { "Some comment\n\nhttps://grafana.example.com/d/xvAk4q0Wk/go-processes?orgId=1&from=1573238522762&to=1573240322762&var-job=prometheus&var-interval=10m&panelId=1&fullscreen" } + let(:description_with_unintegrated_embed) { "Some comment\n\nhttps://grafana.exp.com/d/xvAk4q0Wk/go-processes?orgId=1&from=1573238522762&to=1573240322762&var-job=prometheus&var-interval=10m&panelId=1&fullscreen" } + let(:description_with_non_grafana_inline_metric) { "Some comment\n\n#{Gitlab::Routing.url_helpers.metrics_namespace_project_environment_url(*['foo', 'bar', 12])}" } + + shared_examples "zero count" do + it "does not count the issue" do + expect(subject).to eq(0) + end + end + + context 'with project grafana integration enabled' do + before do + create(:grafana_integration, project: project, enabled: true) + end + + context 'with valid and invalid embeds' do + before do + # Valid + create(:issue, project: project, description: description_with_embed) + create(:issue, project: project, description: description_with_embed) + # In-Valid + create(:issue, project: project, description: description_with_unintegrated_embed) + create(:issue, project: project, description: description_with_non_grafana_inline_metric) + create(:issue, project: project, description: nil) + create(:issue, project: project, description: '') + create(:issue, project: project) + end + + it 'counts only the issues with embeds' do + expect(subject).to eq(2) + end + end + end + + context 'with project grafana integration disabled' do + before do + create(:grafana_integration, project: project, enabled: false) + end + + context 'with one issue having a grafana link in the description and one without' do + before do + create(:issue, project: project, description: description_with_embed) + create(:issue, project: project) + end + + it_behaves_like('zero count') + end + end + + context 'with an un-integrated project' do + context 'with one issue having a grafana link in the description and one without' do + before do + create(:issue, project: project, description: description_with_embed) + create(:issue, project: project) + end + + it_behaves_like('zero count') + end + end + end + describe '#count' do let(:relation) { double(:relation) } diff --git a/spec/lib/quality/helm3_client_spec.rb b/spec/lib/quality/helm3_client_spec.rb new file mode 100644 index 00000000000..1144ee9369d --- /dev/null +++ b/spec/lib/quality/helm3_client_spec.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Quality::Helm3Client do + let(:namespace) { 'review-apps-ee' } + let(:release_name) { 'my-release' } + let(:raw_helm_list_page1) do + <<~OUTPUT + [ + {"name":"review-qa-60-reor-1mugd1","namespace":"#{namespace}","revision":1,"updated":"2020-04-03 17:27:10.245952 +0800 +08","status":"failed","chart":"gitlab-1.1.3","app_version":"12.9.2"}, + {"name":"review-7846-fix-s-261vd6","namespace":"#{namespace}","revision":2,"updated":"2020-04-02 17:27:12.245952 +0800 +08","status":"deployed","chart":"gitlab-1.1.3","app_version":"12.9.2"}, + {"name":"review-7867-snowp-lzo3iy","namespace":"#{namespace}","revision":1,"updated":"2020-04-02 15:27:12.245952 +0800 +08","status":"deployed","chart":"gitlab-1.1.3","app_version":"12.9.1"}, + {"name":"review-6709-group-2pzeec","namespace":"#{namespace}","revision":2,"updated":"2020-04-01 21:27:12.245952 +0800 +08","status":"failed","chart":"gitlab-1.1.3","app_version":"12.9.1"} + ] + OUTPUT + end + let(:raw_helm_list_page2) do + <<~OUTPUT + [ + {"name":"review-6709-group-t40qbv","namespace":"#{namespace}","revision":2,"updated":"2020-04-01 11:27:12.245952 +0800 +08","status":"deployed","chart":"gitlab-1.1.3","app_version":"12.9.1"} + ] + OUTPUT + end + let(:raw_helm_list_empty) do + <<~OUTPUT + [] + OUTPUT + end + + subject { described_class.new(namespace: namespace) } + + describe '#releases' do + it 'raises an error if the Helm command fails' do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with([%(helm list --namespace "#{namespace}" --max 256 --offset 0 --output json)]) + .and_return(Gitlab::Popen::Result.new([], '', '', double(success?: false))) + + expect { subject.releases.to_a }.to raise_error(described_class::CommandFailedError) + end + + it 'calls helm list with default arguments' do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with([%(helm list --namespace "#{namespace}" --max 256 --offset 0 --output json)]) + .and_return(Gitlab::Popen::Result.new([], '', '', double(success?: true))) + + subject.releases.to_a + end + + it 'calls helm list with extra arguments' do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with([%(helm list --namespace "#{namespace}" --max 256 --offset 0 --output json --deployed)]) + .and_return(Gitlab::Popen::Result.new([], '', '', double(success?: true))) + + subject.releases(args: ['--deployed']).to_a + end + + it 'returns a list of Release objects' do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with([%(helm list --namespace "#{namespace}" --max 256 --offset 0 --output json --deployed)]) + .and_return(Gitlab::Popen::Result.new([], raw_helm_list_page2, '', double(success?: true))) + expect(Gitlab::Popen).to receive(:popen_with_detail).ordered + .and_return(Gitlab::Popen::Result.new([], raw_helm_list_empty, '', double(success?: true))) + + releases = subject.releases(args: ['--deployed']).to_a + + expect(releases.size).to eq(1) + expect(releases[0]).to have_attributes( + name: 'review-6709-group-t40qbv', + revision: 2, + last_update: Time.parse('2020-04-01 11:27:12.245952 +0800 +08'), + status: 'deployed', + chart: 'gitlab-1.1.3', + app_version: '12.9.1', + namespace: namespace + ) + end + + it 'automatically paginates releases' do + expect(Gitlab::Popen).to receive(:popen_with_detail).ordered + .with([%(helm list --namespace "#{namespace}" --max 256 --offset 0 --output json)]) + .and_return(Gitlab::Popen::Result.new([], raw_helm_list_page1, '', double(success?: true))) + expect(Gitlab::Popen).to receive(:popen_with_detail).ordered + .with([%(helm list --namespace "#{namespace}" --max 256 --offset 256 --output json)]) + .and_return(Gitlab::Popen::Result.new([], raw_helm_list_page2, '', double(success?: true))) + expect(Gitlab::Popen).to receive(:popen_with_detail).ordered + .with([%(helm list --namespace "#{namespace}" --max 256 --offset 512 --output json)]) + .and_return(Gitlab::Popen::Result.new([], raw_helm_list_empty, '', double(success?: true))) + releases = subject.releases.to_a + + expect(releases.size).to eq(5) + expect(releases.last.name).to eq('review-6709-group-t40qbv') + end + end + + describe '#delete' do + it 'raises an error if the Helm command fails' do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with([%(helm uninstall --namespace "#{namespace}" #{release_name})]) + .and_return(Gitlab::Popen::Result.new([], '', '', double(success?: false))) + + expect { subject.delete(release_name: release_name) }.to raise_error(described_class::CommandFailedError) + end + + it 'calls helm uninstall with default arguments' do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with([%(helm uninstall --namespace "#{namespace}" #{release_name})]) + .and_return(Gitlab::Popen::Result.new([], '', '', double(success?: true))) + + expect(subject.delete(release_name: release_name)).to eq('') + end + + context 'with multiple release names' do + let(:release_name) { %w[my-release my-release-2] } + + it 'raises an error if the Helm command fails' do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with([%(helm uninstall --namespace "#{namespace}" #{release_name.join(' ')})]) + .and_return(Gitlab::Popen::Result.new([], '', '', double(success?: false))) + + expect { subject.delete(release_name: release_name) }.to raise_error(described_class::CommandFailedError) + end + + it 'calls helm uninstall with multiple release names' do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with([%(helm uninstall --namespace "#{namespace}" #{release_name.join(' ')})]) + .and_return(Gitlab::Popen::Result.new([], '', '', double(success?: true))) + + expect(subject.delete(release_name: release_name)).to eq('') + end + end + end +end diff --git a/spec/models/jira_import_data_spec.rb b/spec/models/jira_import_data_spec.rb deleted file mode 100644 index ad7a704236b..00000000000 --- a/spec/models/jira_import_data_spec.rb +++ /dev/null @@ -1,134 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe JiraImportData do - let(:symbol_keys_project) do - { key: 'AA', scheduled_at: 2.days.ago.strftime('%Y-%m-%d %H:%M:%S'), scheduled_by: { 'user_id' => 1, 'name' => 'tester1' } } - end - - let(:string_keys_project) do - { 'key': 'BB', 'scheduled_at': 1.hour.ago.strftime('%Y-%m-%d %H:%M:%S'), 'scheduled_by': { 'user_id': 2, 'name': 'tester2' } } - end - - let(:jira_project_details) do - JiraImportData::JiraProjectDetails.new('CC', 1.day.ago.strftime('%Y-%m-%d %H:%M:%S'), { user_id: 3, name: 'tester3' }) - end - - describe '#projects' do - it 'returns empty array if no data' do - expect(described_class.new.projects).to eq([]) - end - - it 'returns empty array if no projects' do - import_data = described_class.new(data: { 'some-key' => 10 }) - expect(import_data.projects).to eq([]) - end - - it 'returns JiraProjectDetails sorted by scheduled_at time' do - import_data = described_class.new(data: { jira: { projects: [symbol_keys_project, string_keys_project, jira_project_details] } }) - - expect(import_data.projects.size).to eq 3 - expect(import_data.projects.map(&:key)).to eq(%w(AA CC BB)) - expect(import_data.projects.map(&:scheduled_by).map {|e| e['name']}).to eq %w(tester1 tester3 tester2) - expect(import_data.projects.map(&:scheduled_by).map {|e| e['user_id']}).to eq [1, 3, 2] - end - end - - describe 'add projects' do - it 'adds project when data is nil' do - import_data = described_class.new - expect(import_data.data).to be nil - - import_data << string_keys_project - - expect(import_data.data).to eq({ 'jira' => { 'projects' => [string_keys_project] } }) - end - - it 'adds project when data has some random info' do - import_data = described_class.new(data: { 'one-key': 10 }) - expect(import_data.data).to eq({ 'one-key' => 10 }) - - import_data << string_keys_project - - expect(import_data.data).to eq({ 'one-key' => 10, 'jira' => { 'projects' => [string_keys_project] } }) - end - - it 'adds project when data already has some jira projects' do - import_data = described_class.new(data: { jira: { projects: [symbol_keys_project] } }) - expect(import_data.projects.map(&:to_h)).to eq [symbol_keys_project] - - import_data << string_keys_project - - expect(import_data.data['jira']['projects'].size).to eq 2 - expect(import_data.projects.map(&:key)).to eq(%w(AA BB)) - expect(import_data.projects.map(&:scheduled_by).map {|e| e['name']}).to eq %w(tester1 tester2) - expect(import_data.projects.map(&:scheduled_by).map {|e| e['user_id']}).to eq [1, 2] - end - end - - describe '#force_import!' do - it 'sets force import when data is nil' do - import_data = described_class.new - - import_data.force_import! - - expect(import_data.data['jira'][JiraImportData::FORCE_IMPORT_KEY]).to be true - expect(import_data.force_import?).to be false - end - - it 'sets force import when data is present but no jira key' do - import_data = described_class.new(data: { 'some-key': 'some-data' }) - - import_data.force_import! - - expect(import_data.data['jira'][JiraImportData::FORCE_IMPORT_KEY]).to be true - expect(import_data.data).to eq({ 'some-key' => 'some-data', 'jira' => { JiraImportData::FORCE_IMPORT_KEY => true } }) - expect(import_data.force_import?).to be false - end - - it 'sets force import when data and jira keys exist' do - import_data = described_class.new(data: { 'some-key': 'some-data', 'jira': {} }) - - import_data.force_import! - - expect(import_data.data['jira'][JiraImportData::FORCE_IMPORT_KEY]).to be true - expect(import_data.data).to eq({ 'some-key' => 'some-data', 'jira' => { JiraImportData::FORCE_IMPORT_KEY => true } }) - expect(import_data.force_import?).to be false - end - - it 'sets force import when data and jira project data exist' do - import_data = described_class.new(data: { jira: { projects: [symbol_keys_project], JiraImportData::FORCE_IMPORT_KEY => false }, 'some-key': 'some-data' }) - - import_data.force_import! - - expect(import_data.data['jira'][JiraImportData::FORCE_IMPORT_KEY]).to be true - expect(import_data.data).to eq({ 'some-key' => 'some-data', 'jira' => { 'projects' => [symbol_keys_project.deep_stringify_keys!], JiraImportData::FORCE_IMPORT_KEY => true } }) - expect(import_data.force_import?).to be true - end - end - - describe '#force_import?' do - it 'returns false when data blank' do - expect(described_class.new.force_import?).to be false - end - - it 'returns false if there is no project data present' do - import_data = described_class.new(data: { jira: { JiraImportData::FORCE_IMPORT_KEY => true }, 'one-key': 10 }) - - expect(import_data.force_import?).to be false - end - - it 'returns false when force import set to false' do - import_data = described_class.new(data: { jira: { projects: [symbol_keys_project], JiraImportData::FORCE_IMPORT_KEY => false }, 'one-key': 10 }) - - expect(import_data.force_import?).to be false - end - - it 'returns true when force import set to true' do - import_data = described_class.new(data: { jira: { projects: [symbol_keys_project], JiraImportData::FORCE_IMPORT_KEY => true } }) - - expect(import_data.force_import?).to be true - end - end -end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 21c074cdce2..3c8afee4466 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2281,38 +2281,35 @@ describe Project do end describe '#jira_import_status' do - let(:project) { create(:project, :import_started, import_type: 'jira') } + let(:project) { create(:project, import_type: 'jira') } - context 'when import_data is nil' do + context 'when no jira imports' do it 'returns none' do - expect(project.import_data).to be nil - expect(project.jira_import_status).to eq('none') + expect(project.jira_import_status).to eq('initial') end end - context 'when import_data is set' do - let(:jira_import_data) { JiraImportData.new } - let(:project) { create(:project, :import_started, import_data: jira_import_data, import_type: 'jira') } + context 'when there are jira imports' do + let(:jira_import1) { build(:jira_import_state, :finished, project: project) } + let(:jira_import2) { build(:jira_import_state, project: project) } - it 'returns none' do - expect(project.import_data.becomes(JiraImportData).force_import?).to be false - expect(project.jira_import_status).to eq('none') + before do + expect(project).to receive(:latest_jira_import).and_return(jira_import2) end - context 'when jira_force_import is true' do - let(:imported_jira_project) do - JiraImportData::JiraProjectDetails.new('xx', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: 1, name: 'root' }) + context 'when latest import status is initial or jira imports are mising' do + it 'returns initial' do + expect(project.jira_import_status).to eq('initial') end + end + context 'when latest import status is scheduled' do before do - jira_import_data = project.import_data.becomes(JiraImportData) - jira_import_data << imported_jira_project - jira_import_data.force_import! + jira_import2.schedule! end - it 'returns started' do - expect(project.import_data.becomes(JiraImportData).force_import?).to be true - expect(project.jira_import_status).to eq('started') + it 'returns scheduled' do + expect(project.jira_import_status).to eq('scheduled') end end end @@ -2375,52 +2372,46 @@ describe Project do context 'jira import' do it 'schedules a jira import job' do project = create(:project, import_type: 'jira') + jira_import = create(:jira_import_state, project: project) expect(Gitlab::JiraImport::Stage::StartImportWorker).to receive(:perform_async).with(project.id).and_return(import_jid) - expect(project.add_import_job).to eq(import_jid) + + jira_import.schedule! + + expect(jira_import.jid).to eq(import_jid) end end end describe '#jira_import?' do - subject(:project) { build(:project, import_type: 'jira') } + let_it_be(:project) { build(:project, import_type: 'jira') } + let_it_be(:jira_import) { build(:jira_import_state, project: project) } - it { expect(project.jira_import?).to be true } - it { expect(project.import?).to be true } - end - - describe '#jira_force_import?' do - let(:imported_jira_project) do - JiraImportData::JiraProjectDetails.new('xx', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: 1, name: 'root' }) - end - let(:jira_import_data) do - data = JiraImportData.new - data << imported_jira_project - data.force_import! - data + before do + expect(project).to receive(:jira_imports).and_return([jira_import]) end - subject(:project) { build(:project, import_type: 'jira', import_data: jira_import_data) } - - it { expect(project.jira_force_import?).to be true } + it { expect(project.jira_import?).to be true } + it { expect(project.import?).to be true } end describe '#remove_import_data' do - let(:import_data) { ProjectImportData.new(data: { 'test' => 'some data' }) } + let_it_be(:import_data) { ProjectImportData.new(data: { 'test' => 'some data' }) } context 'when jira import' do - let!(:project) { create(:project, import_type: 'jira', import_data: import_data) } + let_it_be(:project, reload: true) { create(:project, import_type: 'jira', import_data: import_data) } + let_it_be(:jira_import) { create(:jira_import_state, project: project) } - it 'does not remove import data' do + it 'does remove import data' do expect(project.mirror?).to be false expect(project.jira_import?).to be true - expect { project.remove_import_data }.not_to change { ProjectImportData.count } + expect { project.remove_import_data }.to change { ProjectImportData.count }.by(-1) end end - context 'when not mirror neither jira import' do - let(:user) { create(:user) } - let!(:project) { create(:project, import_type: 'github', import_data: import_data) } + context 'when neither a mirror nor a jira import' do + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project, import_type: 'github', import_data: import_data) } it 'removes import data' do expect(project.mirror?).to be false diff --git a/spec/requests/api/graphql/mutations/jira_import/start_spec.rb b/spec/requests/api/graphql/mutations/jira_import/start_spec.rb index 3eb2ca6909b..7e213d3adb0 100644 --- a/spec/requests/api/graphql/mutations/jira_import/start_spec.rb +++ b/spec/requests/api/graphql/mutations/jira_import/start_spec.rb @@ -118,9 +118,8 @@ describe 'Starting a Jira Import' do expect(jira_import['jiraProjectKey']).to eq 'AA' expect(jira_import['scheduledBy']['username']).to eq current_user.username - expect(project.import_state).not_to be nil - expect(project.import_state.status).to eq 'scheduled' - expect(project.import_data.becomes(JiraImportData).projects.last.scheduled_by['user_id']).to eq current_user.id + expect(project.latest_jira_import).not_to be_nil + expect(project.latest_jira_import).to be_scheduled end end end diff --git a/spec/requests/api/graphql/project/jira_import_spec.rb b/spec/requests/api/graphql/project/jira_import_spec.rb index beebc63a3c6..43e1bb13342 100644 --- a/spec/requests/api/graphql/project/jira_import_spec.rb +++ b/spec/requests/api/graphql/project/jira_import_spec.rb @@ -6,19 +6,10 @@ describe 'query jira import data' do include GraphqlHelpers let_it_be(:current_user) { create(:user) } - let_it_be(:jira_import_data) do - data = JiraImportData.new - data << JiraImportData::JiraProjectDetails.new( - 'AA', 2.days.ago.strftime('%Y-%m-%d %H:%M:%S'), - { user_id: current_user.id, name: current_user.name } - ) - data << JiraImportData::JiraProjectDetails.new( - 'BB', 5.days.ago.strftime('%Y-%m-%d %H:%M:%S'), - { user_id: current_user.id, name: current_user.name } - ) - data - end - let_it_be(:project) { create(:project, :private, :import_started, import_data: jira_import_data, import_type: 'jira') } + let_it_be(:project) { create(:project, :private, :import_started, import_type: 'jira') } + let_it_be(:jira_import1) { create(:jira_import_state, :finished, project: project, jira_project_key: 'AA', user: current_user, created_at: 2.days.ago) } + let_it_be(:jira_import2) { create(:jira_import_state, :finished, project: project, jira_project_key: 'BB', user: current_user, created_at: 5.days.ago) } + let(:query) do %( query { @@ -48,7 +39,7 @@ describe 'query jira import data' do context 'when anonymous user' do let(:current_user) { nil } - it { expect(jira_imports).to be nil } + it { expect(jira_imports).to be_nil } end context 'when user developer' do @@ -56,7 +47,7 @@ describe 'query jira import data' do project.add_developer(current_user) end - it { expect(jira_imports).to be nil } + it { expect(jira_imports).to be_nil } end end @@ -139,7 +130,7 @@ describe 'query jira import data' do it 'does not return import status' do post_graphql(query, current_user: current_user) - expect(graphql_data['project']).to be nil + expect(graphql_data['project']).to be_nil end end @@ -149,12 +140,12 @@ describe 'query jira import data' do end context 'when import never ran' do - let(:project) { create(:project) } + let_it_be(:initial_jira_import) { create(:jira_import_state, project: project, jira_project_key: 'BB', user: current_user) } it 'returns import status' do post_graphql(query, current_user: current_user) - expect(jira_import_status).to eq('none') + expect(jira_import_status).to eq('initial') end end @@ -166,11 +157,8 @@ describe 'query jira import data' do end end - context 'when import running, i.e. force-import: true' do - before do - project.import_data.becomes(JiraImportData).force_import! - project.save! - end + context 'when import running' do + let_it_be(:started_jira_import) { create(:jira_import_state, :started, project: project, jira_project_key: 'BB', user: current_user) } it 'returns import status' do post_graphql(query, current_user: current_user) diff --git a/spec/requests/api/group_export_spec.rb b/spec/requests/api/group_export_spec.rb index 47193591cf1..02ae9d71702 100644 --- a/spec/requests/api/group_export_spec.rb +++ b/spec/requests/api/group_export_spec.rb @@ -102,6 +102,19 @@ describe API::GroupExport do end end + context 'when the export cannot be started' do + before do + group.add_owner(user) + allow(GroupExportWorker).to receive(:perform_async).and_return(nil) + end + + it 'returns an error' do + post api(path, user) + + expect(response).to have_gitlab_http_status(:error) + end + end + context 'when user is not a group owner' do before do group.add_developer(user) diff --git a/spec/services/groups/import_export/export_service_spec.rb b/spec/services/groups/import_export/export_service_spec.rb index 0d7fa98e16b..e026d2166d6 100644 --- a/spec/services/groups/import_export/export_service_spec.rb +++ b/spec/services/groups/import_export/export_service_spec.rb @@ -3,6 +3,37 @@ require 'spec_helper' describe Groups::ImportExport::ExportService do + describe '#async_execute' do + let(:user) { create(:user) } + let(:group) { create(:group) } + + context 'when the job can be successfully scheduled' do + let(:export_service) { described_class.new(group: group, user: user) } + + it 'enqueues an export job' do + expect(GroupExportWorker).to receive(:perform_async).with(user.id, group.id, {}) + + export_service.async_execute + end + + it 'returns truthy' do + expect(export_service.async_execute).to be_present + end + end + + context 'when the job cannot be scheduled' do + let(:export_service) { described_class.new(group: group, user: user) } + + before do + allow(GroupExportWorker).to receive(:perform_async).and_return(nil) + end + + it 'returns falsey' do + expect(export_service.async_execute).to be_falsey + end + end + end + describe '#execute' do let!(:user) { create(:user) } let(:group) { create(:group) } @@ -103,5 +134,23 @@ describe Groups::ImportExport::ExportService do end end end + + context 'when there is an existing export file' do + subject(:export_service) { described_class.new(group: group, user: user) } + + let(:import_export_upload) do + create( + :import_export_upload, + group: group, + export_file: fixture_file_upload('spec/fixtures/group_export.tar.gz') + ) + end + + it 'removes it' do + existing_file = import_export_upload.export_file + + expect { export_service.execute }.to change { existing_file.file }.to(be_nil) + end + end end end diff --git a/spec/services/jira_import/start_import_service_spec.rb b/spec/services/jira_import/start_import_service_spec.rb index 038c53b2b22..8d9ba5ac692 100644 --- a/spec/services/jira_import/start_import_service_spec.rb +++ b/spec/services/jira_import/start_import_service_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' describe JiraImport::StartImportService do let_it_be(:user) { create(:user) } - let(:project) { create(:project) } + let_it_be(:project, reload: true) { create(:project) } subject { described_class.new(user, project, '').execute } @@ -46,10 +46,12 @@ describe JiraImport::StartImportService do end context 'when correct data provided' do - subject { described_class.new(user, project, 'some-key').execute } + let(:fake_key) { 'some-key' } + + subject { described_class.new(user, project, fake_key).execute } context 'when import is already running' do - let!(:import_state) { create(:import_state, project: project, status: :started) } + let_it_be(:jira_import_state) { create(:jira_import_state, :started, project: project) } it_behaves_like 'responds with error', 'Jira import is already running.' end @@ -62,17 +64,16 @@ describe JiraImport::StartImportService do it 'schedules jira import' do subject - expect(project.import_state.status).to eq('scheduled') + expect(project.latest_jira_import).to be_scheduled end it 'creates jira import data' do - subject + jira_import = subject.payload[:import_data] - jira_import_data = project.import_data.becomes(JiraImportData) - expect(jira_import_data.force_import?).to be true - imported_project_data = jira_import_data.projects.last - expect(imported_project_data.key).to eq('some-key') - expect(imported_project_data.scheduled_by['user_id']).to eq(user.id) + expect(jira_import.jira_project_xid).to eq(0) + expect(jira_import.jira_project_name).to eq(fake_key) + expect(jira_import.jira_project_key).to eq(fake_key) + expect(jira_import.user).to eq(user) end end end diff --git a/spec/support/shared_examples/graphql/jira_import/jira_import_resolved_shared_examples.rb b/spec/support/shared_examples/graphql/jira_import/jira_import_resolver_shared_examples.rb index b1d178521bb..3d97fe10a47 100644 --- a/spec/support/shared_examples/graphql/jira_import/jira_import_resolved_shared_examples.rb +++ b/spec/support/shared_examples/graphql/jira_import/jira_import_resolver_shared_examples.rb @@ -2,7 +2,7 @@ shared_examples 'no jira import data present' do it 'returns none' do - expect(resolve_imports).to eq JiraImportData.none + expect(resolve_imports).to eq JiraImportState.none end end diff --git a/spec/support/shared_examples/workers/gitlab/jira_import/jira_import_workers_shared_examples.rb b/spec/support/shared_examples/workers/gitlab/jira_import/jira_import_workers_shared_examples.rb index 71ec1ea6a74..c0d17d6853d 100644 --- a/spec/support/shared_examples/workers/gitlab/jira_import/jira_import_workers_shared_examples.rb +++ b/spec/support/shared_examples/workers/gitlab/jira_import/jira_import_workers_shared_examples.rb @@ -12,14 +12,23 @@ shared_examples 'include import workers modules' do end end -shared_examples 'exit import not started' do - it 'does nothing, and exits' do +shared_examples 'does not advance to next stage' do + it 'does not advance to next stage' do expect(Gitlab::JiraImport::AdvanceStageWorker).not_to receive(:perform_async) described_class.new.perform(project.id) end end +shared_examples 'cannot do jira import' do + it 'does not advance to next stage' do + worker = described_class.new + expect(worker).not_to receive(:import) + + worker.perform(project.id) + end +end + shared_examples 'advance to next stage' do |next_stage| let(:job_waiter) { Gitlab::JobWaiter.new(2, 'some-job-key') } diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb index 2e96f42a251..1d2341bb46f 100644 --- a/spec/tasks/gitlab/db_rake_spec.rb +++ b/spec/tasks/gitlab/db_rake_spec.rb @@ -98,6 +98,39 @@ describe 'gitlab:db namespace rake task' do end end + describe 'clean_structure_sql' do + let_it_be(:clean_rake_task) { 'gitlab:db:clean_structure_sql' } + let_it_be(:test_task_name) { 'gitlab:db:_test_multiple_structure_cleans' } + let_it_be(:structure_file) { 'db/structure.sql' } + let_it_be(:input) { 'this is structure data' } + let(:output) { StringIO.new } + + before do + allow(File).to receive(:read).with(structure_file).and_return(input) + allow(File).to receive(:open).with(structure_file, any_args).and_yield(output) + end + + after do + Rake::Task[test_task_name].clear if Rake::Task.task_defined?(test_task_name) + end + + it 'can be executed multiple times within another rake task' do + Rake::Task.define_task(test_task_name => :environment) do + expect_next_instance_of(Gitlab::Database::SchemaCleaner) do |cleaner| + expect(cleaner).to receive(:clean).with(output) + end + Rake::Task[clean_rake_task].invoke + + expect_next_instance_of(Gitlab::Database::SchemaCleaner) do |cleaner| + expect(cleaner).to receive(:clean).with(output) + end + Rake::Task[clean_rake_task].invoke + end + + run_rake_task(test_task_name) + end + end + def run_rake_task(task_name) Rake::Task[task_name].reenable Rake.application.invoke_task task_name diff --git a/spec/workers/gitlab/jira_import/import_issue_worker_spec.rb b/spec/workers/gitlab/jira_import/import_issue_worker_spec.rb index b6db803ddf5..80629cb875e 100644 --- a/spec/workers/gitlab/jira_import/import_issue_worker_spec.rb +++ b/spec/workers/gitlab/jira_import/import_issue_worker_spec.rb @@ -5,6 +5,7 @@ require 'spec_helper' describe Gitlab::JiraImport::ImportIssueWorker do let_it_be(:user) { create(:user) } let_it_be(:project) { create(:project) } + let(:some_key) { 'some-key' } describe 'modules' do it { expect(described_class).to include_module(ApplicationWorker) } @@ -23,7 +24,7 @@ describe Gitlab::JiraImport::ImportIssueWorker do allow(subject).to receive(:insert_and_return_id).and_raise(StandardError) expect(Gitlab::JobWaiter).to receive(:notify) - subject.perform(project.id, 123, issue_attrs, 'some-key') + subject.perform(project.id, 123, issue_attrs, some_key) end it 'record a failed to import issue' do @@ -36,7 +37,7 @@ describe Gitlab::JiraImport::ImportIssueWorker do context 'when import label does not exist' do it 'does not record import failure' do - subject.perform(project.id, 123, issue_attrs, 'some-key') + subject.perform(project.id, 123, issue_attrs, some_key) expect(label.issues.count).to eq(0) expect(Gitlab::Cache::Import::Caching.read(Gitlab::JiraImport.failed_issues_counter_cache_key(project.id)).to_i).to eq(0) @@ -49,7 +50,7 @@ describe Gitlab::JiraImport::ImportIssueWorker do end it 'does not record import failure' do - subject.perform(project.id, 123, issue_attrs, 'some-key') + subject.perform(project.id, 123, issue_attrs, some_key) expect(label.issues.count).to eq(1) expect(Gitlab::Cache::Import::Caching.read(Gitlab::JiraImport.failed_issues_counter_cache_key(project.id)).to_i).to eq(0) diff --git a/spec/workers/gitlab/jira_import/stage/finish_import_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/finish_import_worker_spec.rb index 00505226212..93e2a44223b 100644 --- a/spec/workers/gitlab/jira_import/stage/finish_import_worker_spec.rb +++ b/spec/workers/gitlab/jira_import/stage/finish_import_worker_spec.rb @@ -16,46 +16,29 @@ describe Gitlab::JiraImport::Stage::FinishImportWorker do stub_feature_flags(jira_issue_import: false) end - it_behaves_like 'exit import not started' + it_behaves_like 'cannot do jira import' end context 'when feature flag enabled' do + let_it_be(:jira_import) { create(:jira_import_state, :scheduled, project: project) } + before do stub_feature_flags(jira_issue_import: true) end context 'when import did not start' do - let!(:import_state) { create(:import_state, project: project) } - - it_behaves_like 'exit import not started' + it_behaves_like 'cannot do jira import' end context 'when import started' do - let(:imported_jira_project) do - JiraImportData::JiraProjectDetails.new('xx', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: 1, name: 'root' }) + before do + jira_import.start! end - let(:jira_import_data) do - data = JiraImportData.new - data << imported_jira_project - data.force_import! - data - end - let(:import_state) { create(:import_state, status: :started) } - let(:project) { create(:project, import_type: 'jira', import_data: jira_import_data, import_state: import_state) } it 'changes import state to finished' do worker.perform(project.id) - expect(project.reload.import_state.status).to eq("finished") - end - - it 'removes force-import flag' do - expect(project.reload.import_data.data['jira'][JiraImportData::FORCE_IMPORT_KEY]).to be true - - worker.perform(project.id) - - expect(project.reload.import_data.data['jira'][JiraImportData::FORCE_IMPORT_KEY]).to be nil - expect(project.reload.import_data.data['jira']).not_to be nil + expect(project.jira_import_status).to eq('finished') end end end diff --git a/spec/workers/gitlab/jira_import/stage/import_attachments_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/import_attachments_worker_spec.rb index 513925507a1..478cb447dc5 100644 --- a/spec/workers/gitlab/jira_import/stage/import_attachments_worker_spec.rb +++ b/spec/workers/gitlab/jira_import/stage/import_attachments_worker_spec.rb @@ -3,34 +3,38 @@ require 'spec_helper' describe Gitlab::JiraImport::Stage::ImportAttachmentsWorker do - let_it_be(:project) { create(:project) } + let_it_be(:project) { create(:project, import_type: 'jira') } describe 'modules' do it_behaves_like 'include import workers modules' end describe '#perform' do - context 'when feature flag enabled' do + context 'when feature flag disabled' do before do stub_feature_flags(jira_issue_import: false) end - it_behaves_like 'exit import not started' + it_behaves_like 'cannot do jira import' + it_behaves_like 'does not advance to next stage' end context 'when feature flag enabled' do + let_it_be(:jira_import) { create(:jira_import_state, :scheduled, project: project) } + before do stub_feature_flags(jira_issue_import: true) end context 'when import did not start' do - let!(:import_state) { create(:import_state, project: project) } - - it_behaves_like 'exit import not started' + it_behaves_like 'cannot do jira import' + it_behaves_like 'does not advance to next stage' end context 'when import started' do - let!(:import_state) { create(:import_state, status: :started, project: project) } + before do + jira_import.start! + end it_behaves_like 'advance to next stage', :notes end diff --git a/spec/workers/gitlab/jira_import/stage/import_issues_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/import_issues_worker_spec.rb index dca748a6ebc..6470a293461 100644 --- a/spec/workers/gitlab/jira_import/stage/import_issues_worker_spec.rb +++ b/spec/workers/gitlab/jira_import/stage/import_issues_worker_spec.rb @@ -4,43 +4,39 @@ require 'spec_helper' describe Gitlab::JiraImport::Stage::ImportIssuesWorker do let_it_be(:user) { create(:user) } - let_it_be(:project) { create(:project) } + let_it_be(:project) { create(:project, import_type: 'jira') } describe 'modules' do it_behaves_like 'include import workers modules' end describe '#perform' do - context 'when feature flag enabled' do + context 'when feature flag disabled' do before do stub_feature_flags(jira_issue_import: false) end - it_behaves_like 'exit import not started' + it_behaves_like 'cannot do jira import' + it_behaves_like 'does not advance to next stage' end context 'when feature flag enabled' do + let_it_be(:jira_import, reload: true) { create(:jira_import_state, :scheduled, project: project) } + before do stub_feature_flags(jira_issue_import: true) end context 'when import did not start' do - let!(:import_state) { create(:import_state, project: project) } - - it_behaves_like 'exit import not started' + it_behaves_like 'cannot do jira import' + it_behaves_like 'does not advance to next stage' end context 'when import started', :clean_gitlab_redis_cache do - let(:jira_import_data) do - data = JiraImportData.new - data << JiraImportData::JiraProjectDetails.new('XX', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name }) - data - end - let(:project) { create(:project, import_data: jira_import_data) } - let!(:jira_service) { create(:jira_service, project: project) } - let!(:import_state) { create(:import_state, status: :started, project: project) } + let_it_be(:jira_service) { create(:jira_service, project: project) } before do + jira_import.start! allow_next_instance_of(Gitlab::JiraImport::IssuesImporter) do |instance| allow(instance).to receive(:fetch_issues).and_return([]) end diff --git a/spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb index a3e38cba115..f1562395546 100644 --- a/spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb +++ b/spec/workers/gitlab/jira_import/stage/import_labels_worker_spec.rb @@ -4,41 +4,40 @@ require 'spec_helper' describe Gitlab::JiraImport::Stage::ImportLabelsWorker do let_it_be(:user) { create(:user) } - let_it_be(:project) { create(:project) } + let_it_be(:project) { create(:project, import_type: 'jira') } describe 'modules' do it_behaves_like 'include import workers modules' end describe '#perform' do - context 'when feature flag enabled' do + context 'when feature flag disabled' do before do stub_feature_flags(jira_issue_import: false) end - it_behaves_like 'exit import not started' + it_behaves_like 'cannot do jira import' + it_behaves_like 'does not advance to next stage' end context 'when feature flag enabled' do + let_it_be(:jira_import, reload: true) { create(:jira_import_state, :scheduled, project: project) } + before do stub_feature_flags(jira_issue_import: true) end context 'when import did not start' do - let!(:import_state) { create(:import_state, project: project) } - - it_behaves_like 'exit import not started' + it_behaves_like 'cannot do jira import' + it_behaves_like 'does not advance to next stage' end context 'when import started' do - let(:jira_import_data) do - data = JiraImportData.new - data << JiraImportData::JiraProjectDetails.new('XX', Time.now.strftime('%Y-%m-%d %H:%M:%S'), { user_id: user.id, name: user.name }) - data - end - let(:project) { create(:project, import_data: jira_import_data) } let!(:jira_service) { create(:jira_service, project: project) } - let!(:import_state) { create(:import_state, status: :started, project: project) } + + before do + jira_import.start! + end it_behaves_like 'advance to next stage', :issues diff --git a/spec/workers/gitlab/jira_import/stage/import_notes_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/import_notes_worker_spec.rb index 7d1c29614e4..956898c1abc 100644 --- a/spec/workers/gitlab/jira_import/stage/import_notes_worker_spec.rb +++ b/spec/workers/gitlab/jira_import/stage/import_notes_worker_spec.rb @@ -3,34 +3,38 @@ require 'spec_helper' describe Gitlab::JiraImport::Stage::ImportNotesWorker do - let_it_be(:project) { create(:project) } + let_it_be(:project) { create(:project, import_type: 'jira') } describe 'modules' do it_behaves_like 'include import workers modules' end describe '#perform' do - context 'when feature flag enabled' do + context 'when feature flag disabled' do before do stub_feature_flags(jira_issue_import: false) end - it_behaves_like 'exit import not started' + it_behaves_like 'cannot do jira import' + it_behaves_like 'does not advance to next stage' end context 'when feature flag enabled' do + let_it_be(:jira_import) { create(:jira_import_state, :scheduled, project: project) } + before do stub_feature_flags(jira_issue_import: true) end context 'when import did not start' do - let!(:import_state) { create(:import_state, project: project) } - - it_behaves_like 'exit import not started' + it_behaves_like 'cannot do jira import' + it_behaves_like 'does not advance to next stage' end context 'when import started' do - let!(:import_state) { create(:import_state, status: :started, project: project) } + before do + jira_import.start! + end it_behaves_like 'advance to next stage', :finish end diff --git a/spec/workers/gitlab/jira_import/stage/start_import_worker_spec.rb b/spec/workers/gitlab/jira_import/stage/start_import_worker_spec.rb index d5e10a950bb..9cffe6e4ff7 100644 --- a/spec/workers/gitlab/jira_import/stage/start_import_worker_spec.rb +++ b/spec/workers/gitlab/jira_import/stage/start_import_worker_spec.rb @@ -3,16 +3,16 @@ require 'spec_helper' describe Gitlab::JiraImport::Stage::StartImportWorker do - let(:project) { create(:project, import_type: 'jira') } + let_it_be(:project) { create(:project, import_type: 'jira') } + let_it_be(:jid) { '12345678' } let(:worker) { described_class.new } - let(:jid) { '12345678' } describe 'modules' do it_behaves_like 'include import workers modules' end describe '#perform' do - context 'when feature flag not enabled' do + context 'when feature flag not disabled' do before do stub_feature_flags(jira_issue_import: false) end @@ -25,19 +25,13 @@ describe Gitlab::JiraImport::Stage::StartImportWorker do end context 'when feature flag enabled' do - let(:symbol_keys_project) do - { key: 'AA', scheduled_at: 2.days.ago.strftime('%Y-%m-%d %H:%M:%S'), scheduled_by: { 'user_id' => 1, 'name' => 'tester1' } } - end - let(:import_data) { JiraImportData.new( data: { 'jira' => { JiraImportData::FORCE_IMPORT_KEY => true, projects: [symbol_keys_project] } }) } + let_it_be(:jira_import, reload: true) { create(:jira_import_state, project: project, jid: jid) } before do stub_feature_flags(jira_issue_import: true) end context 'when import is not scheduled' do - let(:project) { create(:project, import_type: 'jira') } - let(:import_state) { create(:import_state, project: project, status: :none, jid: jid) } - it 'exits because import not started' do expect(Gitlab::JiraImport::Stage::ImportLabelsWorker).not_to receive(:perform_async) @@ -46,32 +40,21 @@ describe Gitlab::JiraImport::Stage::StartImportWorker do end context 'when import is scheduled' do - let(:import_state) { create(:import_state, status: :scheduled, jid: jid) } - let(:project) { create(:project, import_type: 'jira', import_state: import_state) } - - context 'when this is a mirror sync in a jira imported project' do - it 'exits early' do - expect(Gitlab::Import::SetAsyncJid).not_to receive(:set_jid) - expect(Gitlab::JiraImport::Stage::ImportLabelsWorker).not_to receive(:perform_async) - - worker.perform(project.id) - end + before do + jira_import.schedule! end - context 'when scheduled import is a hard triggered jira import and not a mirror' do - let!(:project) { create(:project, import_type: 'jira', import_data: import_data, import_state: import_state) } - - it 'advances to importing labels' do - expect(Gitlab::JiraImport::Stage::ImportLabelsWorker).to receive(:perform_async) + it 'advances to importing labels' do + expect(Gitlab::JiraImport::Stage::ImportLabelsWorker).to receive(:perform_async) - worker.perform(project.id) - end + worker.perform(project.id) end end context 'when import is started' do - let!(:import_state) { create(:import_state, status: :started, jid: jid) } - let!(:project) { create(:project, import_type: 'jira', import_data: import_data, import_state: import_state) } + before do + jira_import.update!(status: :started) + end context 'when this is the same worker that stated import' do it 'advances to importing labels' do @@ -93,8 +76,9 @@ describe Gitlab::JiraImport::Stage::StartImportWorker do end context 'when import is finished' do - let!(:import_state) { create(:import_state, status: :finished, jid: jid) } - let!(:project) { create(:project, import_type: 'jira', import_data: import_data, import_state: import_state) } + before do + jira_import.update!(status: :finished) + end it 'advances to importing labels' do allow(worker).to receive(:jid).and_return(jid) |