diff options
249 files changed, 3050 insertions, 1371 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 60a2b5d5b5b..19540e4331e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -586,6 +586,7 @@ codequality: paths: [codeclimate.json] qa:internal: + <<: *except-docs stage: test variables: SETUP_DB: "false" diff --git a/app/assets/javascripts/clusters/clusters_bundle.js b/app/assets/javascripts/clusters/clusters_bundle.js index dc443475952..cdb5c430aa9 100644 --- a/app/assets/javascripts/clusters/clusters_bundle.js +++ b/app/assets/javascripts/clusters/clusters_bundle.js @@ -48,6 +48,7 @@ export default class Clusters { this.toggle = this.toggle.bind(this); this.installApplication = this.installApplication.bind(this); + this.showToken = this.showToken.bind(this); this.toggleButton = document.querySelector('.js-toggle-cluster'); this.toggleInput = document.querySelector('.js-toggle-input'); @@ -56,6 +57,8 @@ export default class Clusters { this.creatingContainer = document.querySelector('.js-cluster-creating'); this.errorReasonContainer = this.errorContainer.querySelector('.js-error-reason'); this.successApplicationContainer = document.querySelector('.js-cluster-application-notice'); + this.showTokenButton = document.querySelector('.js-show-cluster-token'); + this.tokenField = document.querySelector('.js-cluster-token'); initSettingsPanels(); this.initApplications(); @@ -97,11 +100,13 @@ export default class Clusters { addListeners() { this.toggleButton.addEventListener('click', this.toggle); + if (this.showTokenButton) this.showTokenButton.addEventListener('click', this.showToken); eventHub.$on('installApplication', this.installApplication); } removeListeners() { this.toggleButton.removeEventListener('click', this.toggle); + if (this.showTokenButton) this.showTokenButton.removeEventListener('click', this.showToken); eventHub.$off('installApplication', this.installApplication); } @@ -149,6 +154,16 @@ export default class Clusters { this.toggleInput.setAttribute('value', this.toggleButton.classList.contains('checked').toString()); } + showToken() { + const type = this.tokenField.getAttribute('type'); + + if (type === 'password') { + this.tokenField.setAttribute('type', 'text'); + } else { + this.tokenField.setAttribute('type', 'password'); + } + } + hideAll() { this.errorContainer.classList.add('hidden'); this.successContainer.classList.add('hidden'); diff --git a/app/assets/javascripts/issuable_index.js b/app/assets/javascripts/issuable_index.js index 0b123a11a3b..c3e0acdff66 100644 --- a/app/assets/javascripts/issuable_index.js +++ b/app/assets/javascripts/issuable_index.js @@ -28,7 +28,7 @@ export default class IssuableIndex { url: $('.incoming-email-token-reset').attr('href'), dataType: 'json', success(response) { - $('#issue_email').val(response.new_issue_address).focus(); + $('#issuable_email').val(response.new_address).focus(); }, beforeSend() { $('.incoming-email-token-reset').text('resetting...'); diff --git a/app/assets/stylesheets/pages/clusters.scss b/app/assets/stylesheets/pages/clusters.scss index e5b9e1f2de6..83e211d6086 100644 --- a/app/assets/stylesheets/pages/clusters.scss +++ b/app/assets/stylesheets/pages/clusters.scss @@ -8,3 +8,9 @@ // Wait for the Vue to kick-in and render the applications block min-height: 302px; } + +.clusters-dropdown-menu { + max-width: 100%; +} + +@include new-style-dropdown('.clusters-dropdown '); diff --git a/app/assets/stylesheets/pages/issues.scss b/app/assets/stylesheets/pages/issues.scss index d3dda2e7d25..8218326ae8f 100644 --- a/app/assets/stylesheets/pages/issues.scss +++ b/app/assets/stylesheets/pages/issues.scss @@ -164,12 +164,7 @@ ul.related-merge-requests > li { } } -.issues-footer { - padding-top: $gl-padding; - padding-bottom: 37px; -} - -.issue-email-modal-btn { +.issuable-email-modal-btn { padding: 0; color: $gl-link-color; background-color: transparent; diff --git a/app/assets/stylesheets/pages/projects.scss b/app/assets/stylesheets/pages/projects.scss index 4c1e6d46242..9345177a4dc 100644 --- a/app/assets/stylesheets/pages/projects.scss +++ b/app/assets/stylesheets/pages/projects.scss @@ -1210,3 +1210,8 @@ pre.light-well { border-color: $border-color; } } + +.issuable-footer { + padding-top: $gl-padding; + padding-bottom: 37px; +} diff --git a/app/controllers/admin/appearances_controller.rb b/app/controllers/admin/appearances_controller.rb index 92df1c8dff0..dd0b38970bd 100644 --- a/app/controllers/admin/appearances_controller.rb +++ b/app/controllers/admin/appearances_controller.rb @@ -4,8 +4,8 @@ class Admin::AppearancesController < Admin::ApplicationController def show end - def preview - render 'preview', layout: 'devise' + def preview_sign_in + render 'preview_sign_in', layout: 'devise' end def create @@ -52,7 +52,7 @@ class Admin::AppearancesController < Admin::ApplicationController def appearance_params params.require(:appearance).permit( :title, :description, :logo, :logo_cache, :header_logo, :header_logo_cache, - :updated_by + :new_project_guidelines, :updated_by ) end end diff --git a/app/controllers/projects/clusters/gcp_controller.rb b/app/controllers/projects/clusters/gcp_controller.rb new file mode 100644 index 00000000000..b64f7a2a6bd --- /dev/null +++ b/app/controllers/projects/clusters/gcp_controller.rb @@ -0,0 +1,75 @@ +class Projects::Clusters::GcpController < Projects::ApplicationController + before_action :authorize_read_cluster! + before_action :authorize_google_api, except: [:login] + before_action :authorize_create_cluster!, only: [:new, :create] + + def login + begin + state = generate_session_key_redirect(gcp_new_namespace_project_clusters_path.to_s) + + @authorize_url = GoogleApi::CloudPlatform::Client.new( + nil, callback_google_api_auth_url, + state: state).authorize_url + rescue GoogleApi::Auth::ConfigMissingError + # no-op + end + end + + def new + @cluster = ::Clusters::Cluster.new.tap do |cluster| + cluster.build_provider_gcp + end + end + + def create + @cluster = ::Clusters::CreateService + .new(project, current_user, create_params) + .execute(token_in_session) + + if @cluster.persisted? + redirect_to project_cluster_path(project, @cluster) + else + render :new + end + end + + private + + def create_params + params.require(:cluster).permit( + :enabled, + :name, + provider_gcp_attributes: [ + :gcp_project_id, + :zone, + :num_nodes, + :machine_type + ]).merge( + provider_type: :gcp, + platform_type: :kubernetes + ) + end + + def authorize_google_api + unless GoogleApi::CloudPlatform::Client.new(token_in_session, nil) + .validate_token(expires_at_in_session) + redirect_to action: 'login' + end + end + + def token_in_session + @token_in_session ||= + session[GoogleApi::CloudPlatform::Client.session_key_for_token] + end + + def expires_at_in_session + @expires_at_in_session ||= + session[GoogleApi::CloudPlatform::Client.session_key_for_expires_at] + end + + def generate_session_key_redirect(uri) + GoogleApi::CloudPlatform::Client.new_session_key_for_redirect_uri do |key| + session[key] = uri + end + end +end diff --git a/app/controllers/projects/clusters/user_controller.rb b/app/controllers/projects/clusters/user_controller.rb new file mode 100644 index 00000000000..d7678512073 --- /dev/null +++ b/app/controllers/projects/clusters/user_controller.rb @@ -0,0 +1,39 @@ +class Projects::Clusters::UserController < Projects::ApplicationController + before_action :authorize_read_cluster! + before_action :authorize_create_cluster!, only: [:new, :create] + + def new + @cluster = ::Clusters::Cluster.new.tap do |cluster| + cluster.build_platform_kubernetes + end + end + + def create + @cluster = ::Clusters::CreateService + .new(project, current_user, create_params) + .execute + + if @cluster.persisted? + redirect_to project_cluster_path(project, @cluster) + else + render :new + end + end + + private + + def create_params + params.require(:cluster).permit( + :enabled, + :name, + platform_kubernetes_attributes: [ + :namespace, + :api_url, + :token, + :ca_cert + ]).merge( + provider_type: :user, + platform_type: :kubernetes + ) + end +end diff --git a/app/controllers/projects/clusters_controller.rb b/app/controllers/projects/clusters_controller.rb index 9a56c9de858..d18b6d4b78c 100644 --- a/app/controllers/projects/clusters_controller.rb +++ b/app/controllers/projects/clusters_controller.rb @@ -1,11 +1,12 @@ class Projects::ClustersController < Projects::ApplicationController - before_action :cluster, except: [:login, :index, :new, :new_gcp, :create] + before_action :cluster, except: [:index, :new] before_action :authorize_read_cluster! - before_action :authorize_create_cluster!, only: [:new, :new_gcp, :create] - before_action :authorize_google_api, only: [:new_gcp, :create] + before_action :authorize_create_cluster!, only: [:new] before_action :authorize_update_cluster!, only: [:update] before_action :authorize_admin_cluster!, only: [:destroy] + STATUS_POLLING_INTERVAL = 10_000 + def index if project.cluster redirect_to project_cluster_path(project, project.cluster) @@ -14,43 +15,13 @@ class Projects::ClustersController < Projects::ApplicationController end end - def login - begin - state = generate_session_key_redirect(providers_gcp_new_namespace_project_clusters_url.to_s) - - @authorize_url = GoogleApi::CloudPlatform::Client.new( - nil, callback_google_api_auth_url, - state: state).authorize_url - rescue GoogleApi::Auth::ConfigMissingError - # no-op - end - end - def new end - def new_gcp - @cluster = Clusters::Cluster.new.tap do |cluster| - cluster.build_provider_gcp - end - end - - def create - @cluster = Clusters::CreateService - .new(project, current_user, create_params) - .execute(token_in_session) - - if @cluster.persisted? - redirect_to project_cluster_path(project, @cluster) - else - render :new_gcp - end - end - def status respond_to do |format| format.json do - Gitlab::PollingInterval.set_header(response, interval: 10_000) + Gitlab::PollingInterval.set_header(response, interval: STATUS_POLLING_INTERVAL) render json: ClusterSerializer .new(project: @project, current_user: @current_user) @@ -88,46 +59,29 @@ class Projects::ClustersController < Projects::ApplicationController private def cluster - @cluster ||= project.cluster.present(current_user: current_user) - end - - def create_params - params.require(:cluster).permit( - :enabled, - :name, - :provider_type, - provider_gcp_attributes: [ - :gcp_project_id, - :zone, - :num_nodes, - :machine_type - ]) + @cluster ||= project.clusters.find(params[:id]) + .present(current_user: current_user) end def update_params - params.require(:cluster).permit(:enabled) - end - - def authorize_google_api - unless GoogleApi::CloudPlatform::Client.new(token_in_session, nil) - .validate_token(expires_at_in_session) - redirect_to action: 'login' - end - end - - def token_in_session - @token_in_session ||= - session[GoogleApi::CloudPlatform::Client.session_key_for_token] - end - - def expires_at_in_session - @expires_at_in_session ||= - session[GoogleApi::CloudPlatform::Client.session_key_for_expires_at] - end - - def generate_session_key_redirect(uri) - GoogleApi::CloudPlatform::Client.new_session_key_for_redirect_uri do |key| - session[key] = uri + if cluster.managed? + params.require(:cluster).permit( + :enabled, + platform_kubernetes_attributes: [ + :namespace + ] + ) + else + params.require(:cluster).permit( + :enabled, + :name, + platform_kubernetes_attributes: [ + :api_url, + :token, + :ca_cert, + :namespace + ] + ) end end diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index a784c6f402a..3882fa4791d 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -133,11 +133,11 @@ class ProjectsController < Projects::ApplicationController redirect_to edit_project_path(@project), status: 302, alert: ex.message end - def new_issue_address + def new_issuable_address return render_404 unless Gitlab::IncomingEmail.supports_issue_creation? current_user.reset_incoming_email_token! - render json: { new_issue_address: @project.new_issue_address(current_user) } + render json: { new_address: @project.new_issuable_address(current_user, params[:issuable_type]) } end def archive diff --git a/app/helpers/appearances_helper.rb b/app/helpers/appearances_helper.rb index df590cf47c8..c037de33c22 100644 --- a/app/helpers/appearances_helper.rb +++ b/app/helpers/appearances_helper.rb @@ -1,30 +1,26 @@ module AppearancesHelper def brand_title - if brand_item && brand_item.title - brand_item.title - else - 'GitLab Community Edition' - end + brand_item&.title.presence || 'GitLab Community Edition' end def brand_image - if brand_item.logo? - image_tag brand_item.logo - else - nil - end + image_tag(brand_item.logo) if brand_item&.logo? end def brand_text markdown_field(brand_item, :description) end + def brand_new_project_guidelines + markdown_field(brand_item, :new_project_guidelines) + end + def brand_item @appearance ||= Appearance.current end def brand_header_logo - if brand_item && brand_item.header_logo? + if brand_item&.header_logo? image_tag brand_item.header_logo else render 'shared/logo.svg' @@ -33,7 +29,7 @@ module AppearancesHelper # Skip the 'GitLab' type logo when custom brand logo is set def brand_header_logo_type - unless brand_item && brand_item.header_logo? + unless brand_item&.header_logo? render 'shared/logo_type.svg' end end diff --git a/app/models/appearance.rb b/app/models/appearance.rb index ff15689ecac..76cfe28742a 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -2,9 +2,8 @@ class Appearance < ActiveRecord::Base include CacheMarkdownField cache_markdown_field :description + cache_markdown_field :new_project_guidelines - validates :title, presence: true - validates :description, presence: true validates :logo, file_size: { maximum: 1.megabyte } validates :header_logo, file_size: { maximum: 1.megabyte } diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 4ea040dfad5..8738e094510 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -1,5 +1,6 @@ module Ci class Build < CommitStatus + prepend ArtifactMigratable include TokenAuthenticatable include AfterCommitQueue include Presentable @@ -10,9 +11,14 @@ module Ci belongs_to :erased_by, class_name: 'User' has_many :deployments, as: :deployable + has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment' has_many :trace_sections, class_name: 'Ci::BuildTraceSection' + has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id + has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id + # The "environment" field for builds is a String, and is the unexpanded name def persisted_environment @persisted_environment ||= Environment.find_by( @@ -31,15 +37,18 @@ module Ci scope :unstarted, ->() { where(runner_id: nil) } scope :ignore_failures, ->() { where(allow_failure: false) } - scope :with_artifacts, ->() { where.not(artifacts_file: [nil, '']) } + scope :with_artifacts, ->() do + where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)', + '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id')) + end scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :ref_protected, -> { where(protected: true) } - mount_uploader :artifacts_file, ArtifactUploader - mount_uploader :artifacts_metadata, ArtifactUploader + mount_uploader :legacy_artifacts_file, LegacyArtifactUploader, mount_on: :artifacts_file + mount_uploader :legacy_artifacts_metadata, LegacyArtifactUploader, mount_on: :artifacts_metadata acts_as_taggable @@ -326,14 +335,6 @@ module Ci project.running_or_pending_build_count(force: true) end - def artifacts? - !artifacts_expired? && artifacts_file.exists? - end - - def artifacts_metadata? - artifacts? && artifacts_metadata.exists? - end - def artifacts_metadata_entry(path, **options) metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( artifacts_metadata.path, @@ -386,6 +387,7 @@ module Ci def keep_artifacts! self.update(artifacts_expire_at: nil) + self.job_artifacts.update_all(expire_at: nil) end def coverage_regex @@ -473,11 +475,7 @@ module Ci private def update_artifacts_size - self.artifacts_size = if artifacts_file.exists? - artifacts_file.size - else - nil - end + self.artifacts_size = legacy_artifacts_file&.size end def erase_trace! diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb new file mode 100644 index 00000000000..84fc6863567 --- /dev/null +++ b/app/models/ci/job_artifact.rb @@ -0,0 +1,36 @@ +module Ci + class JobArtifact < ActiveRecord::Base + extend Gitlab::Ci::Model + + belongs_to :project + belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + + before_save :set_size, if: :file_changed? + + mount_uploader :file, JobArtifactUploader + + enum file_type: { + archive: 1, + metadata: 2 + } + + def self.artifacts_size_for(project) + self.where(project: project).sum(:size) + end + + def set_size + self.size = file.size + end + + def expire_in + expire_at - Time.now if expire_at + end + + def expire_in=(value) + self.expire_at = + if value + ChronicDuration.parse(value)&.seconds&.from_now + end + end + end +end diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 6d7fb4b7dbf..45beced1427 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -17,7 +17,7 @@ module Clusters # we force autosave to happen when we save `Cluster` model has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true - has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes' + has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true has_one :application_helm, class_name: 'Clusters::Applications::Helm' has_one :application_ingress, class_name: 'Clusters::Applications::Ingress' @@ -70,6 +70,10 @@ module Clusters return platform_kubernetes if kubernetes? end + def managed? + !user? + end + def first_project return @first_project if defined?(@first_project) diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 7ab670cf1ef..9160a169452 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -34,12 +34,15 @@ module Clusters validates :api_url, url: true, presence: true validates :token, presence: true + validate :prevent_modification, on: :update + after_save :clear_reactive_cache! alias_attribute :ca_pem, :ca_cert delegate :project, to: :cluster, allow_nil: true delegate :enabled?, to: :cluster, allow_nil: true + delegate :managed?, to: :cluster, allow_nil: true alias_method :active?, :enabled? @@ -173,6 +176,17 @@ module Clusters def enforce_namespace_to_lower_case self.namespace = self.namespace&.downcase end + + def prevent_modification + return unless managed? + + if api_url_changed? || token_changed? || ca_pem_changed? + errors.add(:base, "cannot modify managed cluster") + return false + end + + true + end end end end diff --git a/app/models/concerns/artifact_migratable.rb b/app/models/concerns/artifact_migratable.rb new file mode 100644 index 00000000000..0460439e9e6 --- /dev/null +++ b/app/models/concerns/artifact_migratable.rb @@ -0,0 +1,45 @@ +# Adapter class to unify the interface between mounted uploaders and the +# Ci::Artifact model +# Meant to be prepended so the interface can stay the same +module ArtifactMigratable + def artifacts_file + job_artifacts_archive&.file || legacy_artifacts_file + end + + def artifacts_metadata + job_artifacts_metadata&.file || legacy_artifacts_metadata + end + + def artifacts? + !artifacts_expired? && artifacts_file.exists? + end + + def artifacts_metadata? + artifacts? && artifacts_metadata.exists? + end + + def artifacts_file_changed? + job_artifacts_archive&.file_changed? || attribute_changed?(:artifacts_file) + end + + def remove_artifacts_file! + if job_artifacts_archive + job_artifacts_archive.destroy + else + remove_legacy_artifacts_file! + end + end + + def remove_artifacts_metadata! + if job_artifacts_metadata + job_artifacts_metadata.destroy + else + remove_legacy_artifacts_metadata! + end + end + + def artifacts_size + read_attribute(:artifacts_size).to_i + + job_artifacts_archive&.size.to_i + job_artifacts_metadata&.size.to_i + end +end diff --git a/app/models/group.rb b/app/models/group.rb index 27287f079a4..505e943e464 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -2,6 +2,7 @@ require 'carrierwave/orm/activerecord' class Group < Namespace include Gitlab::ConfigHelper + include AfterCommitQueue include AccessRequestable include Avatarable include Referable diff --git a/app/models/key.rb b/app/models/key.rb index 815fd1de909..a3f8a5d6dc7 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -2,6 +2,7 @@ require 'digest/md5' class Key < ActiveRecord::Base include Gitlab::CurrentSettings + include AfterCommitQueue include Sortable belongs_to :user diff --git a/app/models/member.rb b/app/models/member.rb index cbbd58f2eaf..2fe5fda985f 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -1,4 +1,5 @@ class Member < ActiveRecord::Base + include AfterCommitQueue include Sortable include Importable include Expirable diff --git a/app/models/namespace.rb b/app/models/namespace.rb index fa76729a702..901dbf2ba69 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -139,7 +139,17 @@ class Namespace < ActiveRecord::Base def find_fork_of(project) return nil unless project.fork_network - project.fork_network.find_forks_in(projects).first + if RequestStore.active? + forks_in_namespace = RequestStore.fetch("namespaces:#{id}:forked_projects") do + Hash.new do |found_forks, project| + found_forks[project] = project.fork_network.find_forks_in(projects).first + end + end + + forks_in_namespace[project] + else + project.fork_network.find_forks_in(projects).first + end end def lfs_enabled? diff --git a/app/models/project.rb b/app/models/project.rb index 0a7e7617d9b..cc530076bf7 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -752,13 +752,14 @@ class Project < ActiveRecord::Base Gitlab::Routing.url_helpers.project_url(self) end - def new_issue_address(author) + def new_issuable_address(author, address_type) return unless Gitlab::IncomingEmail.supports_issue_creation? && author author.ensure_incoming_email_token! + suffix = address_type == 'merge_request' ? '+merge-request' : '' Gitlab::IncomingEmail.reply_address( - "#{full_path}+#{author.incoming_email_token}") + "#{full_path}#{suffix}+#{author.incoming_email_token}") end def build_commit_note(commit) diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 715b215d1db..17b9d2cf7b4 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -35,7 +35,9 @@ class ProjectStatistics < ActiveRecord::Base end def update_build_artifacts_size - self.build_artifacts_size = project.builds.sum(:artifacts_size) + self.build_artifacts_size = + project.builds.sum(:artifacts_size) + + Ci::JobArtifact.artifacts_size_for(self) end def update_storage_size diff --git a/app/models/service.rb b/app/models/service.rb index fdd2605e3e3..3c4f1885dd0 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -211,7 +211,7 @@ class Service < ActiveRecord::Base def async_execute(data) return unless supported_events.include?(data[:object_kind]) - Sidekiq::Client.enqueue(ProjectServiceWorker, id, data) + ProjectServiceWorker.perform_async(id, data) end def issue_tracker? diff --git a/app/models/user.rb b/app/models/user.rb index 76fd395be9a..38ee4ed50c1 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -7,6 +7,7 @@ class User < ActiveRecord::Base include Gitlab::ConfigHelper include Gitlab::CurrentSettings include Gitlab::SQL::Pattern + include AfterCommitQueue include Avatarable include Referable include Sortable @@ -903,6 +904,7 @@ class User < ActiveRecord::Base def post_destroy_hook log_info("User \"#{name}\" (#{email}) was removed") + system_hook_service.execute_hooks_for(self, :destroy) end diff --git a/app/services/clusters/create_service.rb b/app/services/clusters/create_service.rb index 1d407739b21..7b697f6d807 100644 --- a/app/services/clusters/create_service.rb +++ b/app/services/clusters/create_service.rb @@ -2,7 +2,7 @@ module Clusters class CreateService < BaseService attr_reader :access_token - def execute(access_token) + def execute(access_token = nil) @access_token = access_token create_cluster.tap do |cluster| diff --git a/app/services/merge_requests/build_service.rb b/app/services/merge_requests/build_service.rb index c2fb01466df..9622a5c5462 100644 --- a/app/services/merge_requests/build_service.rb +++ b/app/services/merge_requests/build_service.rb @@ -10,8 +10,12 @@ module MergeRequests merge_request.target_branch = find_target_branch merge_request.can_be_created = branches_valid? - compare_branches if branches_present? - assign_title_and_description if merge_request.can_be_created + # compare branches only if branches are valid, otherwise + # compare_branches may raise an error + if merge_request.can_be_created + compare_branches + assign_title_and_description + end merge_request end diff --git a/app/services/merge_requests/create_service.rb b/app/services/merge_requests/create_service.rb index 820709583fa..49cf534dc0d 100644 --- a/app/services/merge_requests/create_service.rb +++ b/app/services/merge_requests/create_service.rb @@ -35,6 +35,12 @@ module MergeRequests super end + # expose issuable create method so it can be called from email + # handler CreateMergeRequestHandler + def create(merge_request) + super + end + private def update_merge_requests_head_pipeline(merge_request) diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index d34903c9989..a773222bf17 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -18,7 +18,7 @@ module Projects @status.enqueue! @status.run! - raise 'missing pages artifacts' unless build.artifacts_file? + raise 'missing pages artifacts' unless build.artifacts? raise 'pages are outdated' unless latest? # Create temporary directory in which we will extract the artifacts diff --git a/app/services/system_hooks_service.rb b/app/services/system_hooks_service.rb index 911cc919bb8..690918b4a00 100644 --- a/app/services/system_hooks_service.rb +++ b/app/services/system_hooks_service.rb @@ -1,6 +1,10 @@ class SystemHooksService def execute_hooks_for(model, event) - execute_hooks(build_event_data(model, event)) + data = build_event_data(model, event) + + model.run_after_commit_or_now do + SystemHooksService.new.execute_hooks(data) + end end def execute_hooks(data, hooks_scope = :all) diff --git a/app/services/web_hook_service.rb b/app/services/web_hook_service.rb index cd99e0b90f9..6ebc7c89500 100644 --- a/app/services/web_hook_service.rb +++ b/app/services/web_hook_service.rb @@ -63,7 +63,7 @@ class WebHookService end def async_execute - Sidekiq::Client.enqueue(WebHookWorker, hook.id, data, hook_name) + WebHookWorker.perform_async(hook.id, data, hook_name) end private diff --git a/app/uploaders/artifact_uploader.rb b/app/uploaders/artifact_uploader.rb deleted file mode 100644 index 14addb6cf14..00000000000 --- a/app/uploaders/artifact_uploader.rb +++ /dev/null @@ -1,39 +0,0 @@ -class ArtifactUploader < GitlabUploader - storage :file - - attr_reader :job, :field - - def self.local_artifacts_store - Gitlab.config.artifacts.path - end - - def self.artifacts_upload_path - File.join(self.local_artifacts_store, 'tmp/uploads/') - end - - def initialize(job, field) - @job, @field = job, field - end - - def store_dir - default_local_path - end - - def cache_dir - File.join(self.class.local_artifacts_store, 'tmp/cache') - end - - def work_dir - File.join(self.class.local_artifacts_store, 'tmp/work') - end - - private - - def default_local_path - File.join(self.class.local_artifacts_store, default_path) - end - - def default_path - File.join(job.created_at.utc.strftime('%Y_%m'), job.project_id.to_s, job.id.to_s) - end -end diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb new file mode 100644 index 00000000000..15dfb5a5763 --- /dev/null +++ b/app/uploaders/job_artifact_uploader.rb @@ -0,0 +1,46 @@ +class JobArtifactUploader < GitlabUploader + storage :file + + def self.local_store_path + Gitlab.config.artifacts.path + end + + def self.artifacts_upload_path + File.join(self.local_store_path, 'tmp/uploads/') + end + + def size + return super if model.size.nil? + + model.size + end + + def store_dir + default_local_path + end + + def cache_dir + File.join(self.class.local_store_path, 'tmp/cache') + end + + def work_dir + File.join(self.class.local_store_path, 'tmp/work') + end + + private + + def default_local_path + File.join(self.class.local_store_path, default_path) + end + + def default_path + creation_date = model.created_at.utc.strftime('%Y_%m_%d') + + File.join(disk_hash[0..1], disk_hash[2..3], disk_hash, + creation_date, model.job_id.to_s, model.id.to_s) + end + + def disk_hash + @disk_hash ||= Digest::SHA2.hexdigest(model.project_id.to_s) + end +end diff --git a/app/uploaders/legacy_artifact_uploader.rb b/app/uploaders/legacy_artifact_uploader.rb new file mode 100644 index 00000000000..4f7f8a63108 --- /dev/null +++ b/app/uploaders/legacy_artifact_uploader.rb @@ -0,0 +1,33 @@ +class LegacyArtifactUploader < GitlabUploader + storage :file + + def self.local_store_path + Gitlab.config.artifacts.path + end + + def self.artifacts_upload_path + File.join(self.local_store_path, 'tmp/uploads/') + end + + def store_dir + default_local_path + end + + def cache_dir + File.join(self.class.local_store_path, 'tmp/cache') + end + + def work_dir + File.join(self.class.local_store_path, 'tmp/work') + end + + private + + def default_local_path + File.join(self.class.local_store_path, default_path) + end + + def default_path + File.join(model.created_at.utc.strftime('%Y_%m'), model.project_id.to_s, model.id.to_s) + end +end diff --git a/app/validators/cluster_name_validator.rb b/app/validators/cluster_name_validator.rb index 13ec342f399..e7d32550176 100644 --- a/app/validators/cluster_name_validator.rb +++ b/app/validators/cluster_name_validator.rb @@ -3,11 +3,7 @@ # Custom validator for ClusterName. class ClusterNameValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) - if record.user? - unless value.present? - record.errors.add(attribute, " has to be present") - end - elsif record.gcp? + if record.managed? if record.persisted? && record.name_changed? record.errors.add(attribute, " can not be changed because it's synchronized with provider") end @@ -19,6 +15,10 @@ class ClusterNameValidator < ActiveModel::EachValidator unless value =~ Gitlab::Regex.kubernetes_namespace_regex record.errors.add(attribute, Gitlab::Regex.kubernetes_namespace_regex_message) end + else + unless value.present? + record.errors.add(attribute, " has to be present") + end end end end diff --git a/app/views/admin/appearances/_form.html.haml b/app/views/admin/appearances/_form.html.haml index 4a2238fe277..15bda97c3b5 100644 --- a/app/views/admin/appearances/_form.html.haml +++ b/app/views/admin/appearances/_form.html.haml @@ -1,6 +1,23 @@ = form_for @appearance, url: admin_appearances_path, html: { class: 'form-horizontal'} do |f| = form_errors(@appearance) + %fieldset.app_logo + %legend + Navigation bar: + .form-group + = f.label :header_logo, 'Header logo', class: 'control-label' + .col-sm-10 + - if @appearance.header_logo? + = image_tag @appearance.header_logo_url, class: 'appearance-light-logo-preview' + - if @appearance.persisted? + %br + = link_to 'Remove header logo', header_logos_admin_appearances_path, data: { confirm: "Header logo will be removed. Are you sure?"}, method: :delete, class: "btn btn-remove btn-sm remove-logo" + %hr + = f.hidden_field :header_logo_cache + = f.file_field :header_logo, class: "" + .hint + Maximum file size is 1MB. Pages are optimized for a 28px tall header logo + %fieldset.sign-in %legend Sign in/Sign up pages: @@ -28,27 +45,22 @@ .hint Maximum file size is 1MB. Pages are optimized for a 640x360 px logo. - %fieldset.app_logo + %fieldset %legend - Navigation bar: + New project pages: .form-group - = f.label :header_logo, 'Header logo', class: 'control-label' + = f.label :new_project_guidelines, class: 'control-label' .col-sm-10 - - if @appearance.header_logo? - = image_tag @appearance.header_logo_url, class: 'appearance-light-logo-preview' - - if @appearance.persisted? - %br - = link_to 'Remove header logo', header_logos_admin_appearances_path, data: { confirm: "Header logo will be removed. Are you sure?"}, method: :delete, class: "btn btn-remove btn-sm remove-logo" - %hr - = f.hidden_field :header_logo_cache - = f.file_field :header_logo, class: "" + = f.text_area :new_project_guidelines, class: "form-control", rows: 10 .hint - Maximum file size is 1MB. Pages are optimized for a 28px tall header logo + Guidelines parsed with #{link_to "GitLab Flavored Markdown", help_page_path('user/markdown'), target: '_blank'}. .form-actions = f.submit 'Save', class: 'btn btn-save append-right-10' - if @appearance.persisted? - = link_to 'Preview last save', preview_admin_appearances_path, class: 'btn', target: '_blank', rel: 'noopener noreferrer' + Preview last save: + = link_to 'Sign-in page', preview_sign_in_admin_appearances_path, class: 'btn', target: '_blank', rel: 'noopener noreferrer' + = link_to 'New project page', new_project_path, class: 'btn', target: '_blank', rel: 'noopener noreferrer' - if @appearance.updated_at %span.pull-right diff --git a/app/views/admin/appearances/preview.html.haml b/app/views/admin/appearances/preview_sign_in.html.haml index 1af7dd5bb67..1af7dd5bb67 100644 --- a/app/views/admin/appearances/preview.html.haml +++ b/app/views/admin/appearances/preview_sign_in.html.haml diff --git a/app/views/layouts/devise.html.haml b/app/views/layouts/devise.html.haml index 97016d28535..691d2528022 100644 --- a/app/views/layouts/devise.html.haml +++ b/app/views/layouts/devise.html.haml @@ -15,8 +15,8 @@ .col-sm-7.brand-holder.pull-left %h1 = brand_title - - if brand_item = brand_image + - if brand_item&.description? = brand_text - else %h3 Open source software to collaborate on code diff --git a/app/views/layouts/nav/sidebar/_project.html.haml b/app/views/layouts/nav/sidebar/_project.html.haml index 2ce960df13c..8b2d2a5c74d 100644 --- a/app/views/layouts/nav/sidebar/_project.html.haml +++ b/app/views/layouts/nav/sidebar/_project.html.haml @@ -146,7 +146,7 @@ = number_with_delimiter(@project.open_merge_requests_count) - if project_nav_tab? :pipelines - = nav_link(controller: [:pipelines, :builds, :jobs, :pipeline_schedules, :environments, :artifacts, :clusters]) do + = nav_link(controller: [:pipelines, :builds, :jobs, :pipeline_schedules, :environments, :artifacts, :clusters, :user, :gcp]) do = link_to project_pipelines_path(@project), class: 'shortcuts-pipelines' do .nav-icon-container = sprite_icon('pipeline') @@ -154,7 +154,7 @@ CI / CD %ul.sidebar-sub-level-items - = nav_link(controller: [:pipelines, :builds, :jobs, :pipeline_schedules, :environments, :artifacts], html_options: { class: "fly-out-top-item" } ) do + = nav_link(controller: [:pipelines, :builds, :jobs, :pipeline_schedules, :environments, :artifacts, :clusters, :user, :gcp], html_options: { class: "fly-out-top-item" } ) do = link_to project_pipelines_path(@project) do %strong.fly-out-top-item-name #{ _('CI / CD') } @@ -183,18 +183,18 @@ %span Environments + - if project_nav_tab? :clusters + = nav_link(controller: [:clusters, :user, :gcp]) do + = link_to project_clusters_path(@project), title: 'Cluster', class: 'shortcuts-cluster' do + %span + Cluster + - if @project.feature_available?(:builds, current_user) && !@project.empty_repo? = nav_link(path: 'pipelines#charts') do = link_to charts_project_pipelines_path(@project), title: 'Charts', class: 'shortcuts-pipelines-charts' do %span Charts - - if project_nav_tab? :clusters - = nav_link(controller: :clusters) do - = link_to project_clusters_path(@project), title: 'Cluster', class: 'shortcuts-cluster' do - %span - Cluster - - if project_nav_tab? :wiki = nav_link(controller: :wikis) do = link_to get_project_wiki_path(@project), class: 'shortcuts-wiki' do diff --git a/app/views/projects/_issuable_by_email.html.haml b/app/views/projects/_issuable_by_email.html.haml new file mode 100644 index 00000000000..749e273b2e2 --- /dev/null +++ b/app/views/projects/_issuable_by_email.html.haml @@ -0,0 +1,30 @@ +- name = issuable_type == 'issue' ? 'issue' : 'merge request' + +.issuable-footer.text-center + %button.issuable-email-modal-btn{ type: "button", data: { toggle: "modal", target: "#issuable-email-modal" } } + Email a new #{name} to this project + +#issuable-email-modal.modal.fade{ tabindex: "-1", role: "dialog" } + .modal-dialog{ role: "document" } + .modal-content + .modal-header + %button.close{ type: "button", data: { dismiss: "modal" }, aria: { label: "close" } } + %span{ aria: { hidden: "true" } }= icon("times") + %h4.modal-title + Create new #{name} by email + .modal-body + %p + You can create a new #{name} inside this project by sending an email to the following email address: + .email-modal-input-group.input-group + = text_field_tag :issuable_email, email, class: "monospace js-select-on-focus form-control", readonly: true + .input-group-btn + = clipboard_button(target: '#issuable_email') + %p + = render 'by_email_description' + %p + This is a private email address, generated just for you. + + Anyone who gets ahold of it can create issues or merge requests as if they were you. + You should + = link_to 'reset it', new_issuable_address_project_path(@project, issuable_type: issuable_type), class: 'incoming-email-token-reset' + if that ever happens. diff --git a/app/views/projects/clusters/_advanced_settings.html.haml b/app/views/projects/clusters/_advanced_settings.html.haml index 97532f1e2bd..2b3095eb94b 100644 --- a/app/views/projects/clusters/_advanced_settings.html.haml +++ b/app/views/projects/clusters/_advanced_settings.html.haml @@ -1,10 +1,11 @@ - if can?(current_user, :admin_cluster, @cluster) - .append-bottom-20 - %label.append-bottom-10 - = s_('ClusterIntegration|Google Container Engine') - %p - - link_gke = link_to(s_('ClusterIntegration|Google Container Engine'), @cluster.gke_cluster_url, target: '_blank', rel: 'noopener noreferrer') - = s_('ClusterIntegration|Manage your cluster by visiting %{link_gke}').html_safe % { link_gke: link_gke } + - if @cluster.managed? + .append-bottom-20 + %label.append-bottom-10 + = s_('ClusterIntegration|Google Container Engine') + %p + - link_gke = link_to(s_('ClusterIntegration|Google Container Engine'), @cluster.gke_cluster_url, target: '_blank', rel: 'noopener noreferrer') + = s_('ClusterIntegration|Manage your cluster by visiting %{link_gke}').html_safe % { link_gke: link_gke } .well.form-group %label.text-danger diff --git a/app/views/projects/clusters/_banner.html.haml b/app/views/projects/clusters/_banner.html.haml new file mode 100644 index 00000000000..a1cc66eac92 --- /dev/null +++ b/app/views/projects/clusters/_banner.html.haml @@ -0,0 +1,21 @@ +%h4= s_('ClusterIntegration|Enable cluster integration') +.settings-content + + .hidden.js-cluster-error.alert.alert-danger.alert-block.append-bottom-10{ role: 'alert' } + = s_('ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine') + %p.js-error-reason + + .hidden.js-cluster-creating.alert.alert-info.alert-block.append-bottom-10{ role: 'alert' } + = s_('ClusterIntegration|Cluster is being created on Google Container Engine...') + + .hidden.js-cluster-success.alert.alert-success.alert-block.append-bottom-10{ role: 'alert' } + = s_('ClusterIntegration|Cluster was successfully created on Google Container Engine. Refresh the page to see cluster\'s details') + + %p + - if @cluster.enabled? + - if can?(current_user, :update_cluster, @cluster) + = s_('ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab\'s connection to it.') + - else + = s_('ClusterIntegration|Cluster integration is enabled for this project.') + - else + = s_('ClusterIntegration|Cluster integration is disabled for this project.') diff --git a/app/views/projects/clusters/_dropdown.html.haml b/app/views/projects/clusters/_dropdown.html.haml new file mode 100644 index 00000000000..39188c7ca27 --- /dev/null +++ b/app/views/projects/clusters/_dropdown.html.haml @@ -0,0 +1,12 @@ +%h4.prepend-top-0= s_('ClusterIntegration|Choose how to set up cluster integration') + +.dropdown.clusters-dropdown + %button.dropdown-menu-toggle.dropdown-menu-full-width{ type: 'button', data: { toggle: 'dropdown' }, 'aria-haspopup': true, 'aria-expanded': false } + %span.dropdown-toggle-text + = dropdown_text + = icon('chevron-down') + %ul.dropdown-menu.clusters-dropdown-menu.dropdown-menu-full-width + %li + = link_to(s_('ClusterIntegration|Create cluster on Google Container Engine'), gcp_new_namespace_project_clusters_path(@project.namespace, @project)) + %li + = link_to(s_('ClusterIntegration|Add an existing cluster'), user_new_namespace_project_clusters_path(@project.namespace, @project)) diff --git a/app/views/projects/clusters/_enabled.html.haml b/app/views/projects/clusters/_enabled.html.haml new file mode 100644 index 00000000000..f4d261df8f5 --- /dev/null +++ b/app/views/projects/clusters/_enabled.html.haml @@ -0,0 +1,16 @@ += form_for @cluster, url: namespace_project_cluster_path(@project.namespace, @project, @cluster), as: :cluster do |field| + = form_errors(@cluster) + .form-group.append-bottom-20 + %label.append-bottom-10 + = field.hidden_field :enabled, { class: 'js-toggle-input'} + + %button{ type: 'button', + class: "js-toggle-cluster project-feature-toggle #{'checked' unless !@cluster.enabled?} #{'disabled' unless can?(current_user, :update_cluster, @cluster)}", + 'aria-label': s_('ClusterIntegration|Toggle Cluster'), + disabled: !can?(current_user, :update_cluster, @cluster), + data: { 'enabled-text': 'Enabled', 'disabled-text': 'Disabled' } } + + - if can?(current_user, :update_cluster, @cluster) + .form-group + = field.submit _('Save'), class: 'btn btn-success' + diff --git a/app/views/projects/clusters/_form.html.haml b/app/views/projects/clusters/_form.html.haml deleted file mode 100644 index 1f8ae463d0f..00000000000 --- a/app/views/projects/clusters/_form.html.haml +++ /dev/null @@ -1,35 +0,0 @@ -.row - .col-sm-8.col-sm-offset-4 - %p - - link_to_help_page = link_to(s_('ClusterIntegration|help page'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer') - = s_('ClusterIntegration|Read our %{link_to_help_page} on cluster integration.').html_safe % { link_to_help_page: link_to_help_page} - - = form_for @cluster, url: namespace_project_clusters_path(@project.namespace, @project, @cluster), as: :cluster do |field| - = field.hidden_field :provider_type, value: :gcp - = form_errors(@cluster) - .form-group - = field.label :name, s_('ClusterIntegration|Cluster name') - = field.text_field :name, class: 'form-control' - - = field.fields_for :provider_gcp, @cluster.provider_gcp do |provider_gcp_field| - .form-group - = provider_gcp_field.label :gcp_project_id, s_('ClusterIntegration|Google Cloud Platform project ID') - = link_to(s_('ClusterIntegration|See your projects'), 'https://console.cloud.google.com/home/dashboard', target: '_blank', rel: 'noopener noreferrer') - = provider_gcp_field.text_field :gcp_project_id, class: 'form-control' - - .form-group - = provider_gcp_field.label :zone, s_('ClusterIntegration|Zone') - = link_to(s_('ClusterIntegration|See zones'), 'https://cloud.google.com/compute/docs/regions-zones/regions-zones', target: '_blank', rel: 'noopener noreferrer') - = provider_gcp_field.text_field :zone, class: 'form-control', placeholder: 'us-central1-a' - - .form-group - = provider_gcp_field.label :num_nodes, s_('ClusterIntegration|Number of nodes') - = provider_gcp_field.text_field :num_nodes, class: 'form-control', placeholder: '3' - - .form-group - = provider_gcp_field.label :machine_type, s_('ClusterIntegration|Machine type') - = link_to(s_('ClusterIntegration|See machine types'), 'https://cloud.google.com/compute/docs/machine-types', target: '_blank', rel: 'noopener noreferrer') - = provider_gcp_field.text_field :machine_type, class: 'form-control', placeholder: 'n1-standard-2' - - .form-group - = field.submit s_('ClusterIntegration|Create cluster'), class: 'btn btn-save' diff --git a/app/views/projects/clusters/gcp/_form.html.haml b/app/views/projects/clusters/gcp/_form.html.haml new file mode 100644 index 00000000000..0f6bae97571 --- /dev/null +++ b/app/views/projects/clusters/gcp/_form.html.haml @@ -0,0 +1,32 @@ +%p + - link_to_help_page = link_to(s_('ClusterIntegration|help page'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer') + = s_('ClusterIntegration|Read our %{link_to_help_page} on cluster integration.').html_safe % { link_to_help_page: link_to_help_page} + += form_for @cluster, html: { class: 'prepend-top-20' }, url: gcp_namespace_project_clusters_path(@project.namespace, @project), as: :cluster do |field| + = form_errors(@cluster) + .form-group + = field.label :name, s_('ClusterIntegration|Cluster name') + = field.text_field :name, class: 'form-control', placeholder: s_('ClusterIntegration|Cluster name') + + = field.fields_for :provider_gcp, @cluster.provider_gcp do |provider_gcp_field| + .form-group + = provider_gcp_field.label :gcp_project_id, s_('ClusterIntegration|Google Cloud Platform project ID') + = link_to(s_('ClusterIntegration|See your projects'), 'https://console.cloud.google.com/home/dashboard', target: '_blank', rel: 'noopener noreferrer') + = provider_gcp_field.text_field :gcp_project_id, class: 'form-control', placeholder: s_('ClusterIntegration|Project ID') + + .form-group + = provider_gcp_field.label :zone, s_('ClusterIntegration|Zone') + = link_to(s_('ClusterIntegration|See zones'), 'https://cloud.google.com/compute/docs/regions-zones/regions-zones', target: '_blank', rel: 'noopener noreferrer') + = provider_gcp_field.text_field :zone, class: 'form-control', placeholder: 'us-central1-a' + + .form-group + = provider_gcp_field.label :num_nodes, s_('ClusterIntegration|Number of nodes') + = provider_gcp_field.text_field :num_nodes, class: 'form-control', placeholder: '3' + + .form-group + = provider_gcp_field.label :machine_type, s_('ClusterIntegration|Machine type') + = link_to(s_('ClusterIntegration|See machine types'), 'https://cloud.google.com/compute/docs/machine-types', target: '_blank', rel: 'noopener noreferrer') + = provider_gcp_field.text_field :machine_type, class: 'form-control', placeholder: 'n1-standard-4' + + .form-group + = field.submit s_('ClusterIntegration|Create cluster'), class: 'btn btn-success' diff --git a/app/views/projects/clusters/_header.html.haml b/app/views/projects/clusters/gcp/_header.html.haml index beb798e7154..cddb53c2688 100644 --- a/app/views/projects/clusters/_header.html.haml +++ b/app/views/projects/clusters/gcp/_header.html.haml @@ -1,5 +1,5 @@ -%h4.prepend-top-0 - = s_('ClusterIntegration|Create new cluster on Google Container Engine') +%h4.prepend-top-20 + = s_('ClusterIntegration|Enter the details for your cluster') %p = s_('ClusterIntegration|Please make sure that your Google account meets the following requirements:') %ul diff --git a/app/views/projects/clusters/gcp/_show.html.haml b/app/views/projects/clusters/gcp/_show.html.haml new file mode 100644 index 00000000000..3fa9f69708a --- /dev/null +++ b/app/views/projects/clusters/gcp/_show.html.haml @@ -0,0 +1,40 @@ +.form-group + %label.append-bottom-10{ for: 'cluster-name' } + = s_('ClusterIntegration|Cluster name') + .input-group + %input.form-control.cluster-name.js-select-on-focus{ value: @cluster.name, readonly: true } + %span.input-group-btn + = clipboard_button(text: @cluster.name, title: s_('ClusterIntegration|Copy cluster name'), class: 'btn-default') + += form_for @cluster, url: namespace_project_cluster_path(@project.namespace, @project, @cluster), as: :cluster do |field| + = form_errors(@cluster) + = field.fields_for :platform_kubernetes, @cluster.platform_kubernetes do |platform_kubernetes_field| + .form-group + = platform_kubernetes_field.label :api_url, s_('ClusterIntegration|API URL') + .input-group + = platform_kubernetes_field.text_field :api_url, class: 'form-control js-select-on-focus', placeholder: s_('ClusterIntegration|API URL'), readonly: true + %span.input-group-btn + = clipboard_button(text: @cluster.platform_kubernetes.api_url, title: s_('ClusterIntegration|Copy API URL'), class: 'btn-default') + + .form-group + = platform_kubernetes_field.label :ca_cert, s_('ClusterIntegration|CA Certificate') + .input-group + = platform_kubernetes_field.text_area :ca_cert, class: 'form-control js-select-on-focus', placeholder: s_('ClusterIntegration|Certificate Authority bundle (PEM format)'), readonly: true + %span.input-group-addon.clipboard-addon + = clipboard_button(text: @cluster.platform_kubernetes.ca_cert, title: s_('ClusterIntegration|Copy CA Certificate'), class: 'btn-blank') + + .form-group + = platform_kubernetes_field.label :token, s_('ClusterIntegration|Token') + .input-group + = platform_kubernetes_field.text_field :token, class: 'form-control js-cluster-token js-select-on-focus', type: 'password', placeholder: s_('ClusterIntegration|Token'), readonly: true + %span.input-group-btn + %button.btn.btn-default.js-show-cluster-token{ type: 'button' } + = s_('ClusterIntegration|Show') + = clipboard_button(text: @cluster.platform_kubernetes.token, title: s_('ClusterIntegration|Copy Token'), class: 'btn-default') + + .form-group + = platform_kubernetes_field.label :namespace, s_('ClusterIntegration|Project namespace (optional, unique)') + = platform_kubernetes_field.text_field :namespace, class: 'form-control', placeholder: s_('ClusterIntegration|Project namespace') + + .form-group + = field.submit s_('ClusterIntegration|Save changes'), class: 'btn btn-success' diff --git a/app/views/projects/clusters/login.html.haml b/app/views/projects/clusters/gcp/login.html.haml index fde030b500b..790ba61fd86 100644 --- a/app/views/projects/clusters/login.html.haml +++ b/app/views/projects/clusters/gcp/login.html.haml @@ -3,8 +3,9 @@ .row.prepend-top-default .col-sm-4 - = render 'sidebar' + = render 'projects/clusters/sidebar' .col-sm-8 + = render 'projects/clusters/dropdown', dropdown_text: s_('ClusterIntegration|Create cluster on Google Container Engine') = render 'header' .row .col-sm-8.col-sm-offset-4.signin-with-google diff --git a/app/views/projects/clusters/gcp/new.html.haml b/app/views/projects/clusters/gcp/new.html.haml new file mode 100644 index 00000000000..9a79480c82f --- /dev/null +++ b/app/views/projects/clusters/gcp/new.html.haml @@ -0,0 +1,10 @@ +- breadcrumb_title "Cluster" +- page_title _("New Cluster") + +.row.prepend-top-default + .col-sm-4 + = render 'projects/clusters/sidebar' + .col-sm-8 + = render 'projects/clusters/dropdown', dropdown_text: s_('ClusterIntegration|Create cluster on Google Container Engine') + = render 'header' + = render 'form' diff --git a/app/views/projects/clusters/new.html.haml b/app/views/projects/clusters/new.html.haml index 665120c7e49..2e5bc34f64a 100644 --- a/app/views/projects/clusters/new.html.haml +++ b/app/views/projects/clusters/new.html.haml @@ -5,16 +5,9 @@ .col-sm-4 = render 'sidebar' .col-sm-8 - - if @project.deployment_platform&.active? - %h4.prepend-top-0= s_('ClusterIntegration|Cluster management') + %h4.prepend-top-0= s_('ClusterIntegration|Choose how to set up cluster integration') - %p= s_('ClusterIntegration|A cluster has been set up on this project through the Kubernetes integration page') - = link_to s_('ClusterIntegration|Manage Kubernetes integration'), edit_project_service_path(@project, :kubernetes), class: 'btn append-bottom-20' - - - else - %h4.prepend-top-0= s_('ClusterIntegration|Choose how to set up cluster integration') - - %p= s_('ClusterIntegration|Create a new cluster on Google Container Engine right from GitLab') - = link_to s_('ClusterIntegration|Create on GKE'), providers_gcp_new_namespace_project_clusters_path(@project.namespace, @project), class: 'btn append-bottom-20' - %p= s_('ClusterIntegration|Enter the details for an existing Kubernetes cluster') - = link_to s_('ClusterIntegration|Add an existing cluster'), edit_project_service_path(@project, :kubernetes), class: 'btn append-bottom-20' + %p= s_('ClusterIntegration|Create a new cluster on Google Engine right from GitLab') + = link_to s_('ClusterIntegration|Create on GKE'), gcp_new_namespace_project_clusters_path(@project.namespace, @project), class: 'btn append-bottom-20' + %p= s_('ClusterIntegration|Enter the details for an existing Kubernetes cluster') + = link_to s_('ClusterIntegration|Add an existing cluster'), user_new_namespace_project_clusters_path(@project.namespace, @project), class: 'btn append-bottom-20' diff --git a/app/views/projects/clusters/new_gcp.html.haml b/app/views/projects/clusters/new_gcp.html.haml deleted file mode 100644 index 48e6b6ae8e8..00000000000 --- a/app/views/projects/clusters/new_gcp.html.haml +++ /dev/null @@ -1,10 +0,0 @@ -- breadcrumb_title "Cluster" -- page_title _("New Cluster") - -.row.prepend-top-default - .col-sm-4 - = render 'sidebar' - .col-sm-8 - = render 'header' - -= render 'form' diff --git a/app/views/projects/clusters/show.html.haml b/app/views/projects/clusters/show.html.haml index b7671f5e3c4..d23efe4d9aa 100644 --- a/app/views/projects/clusters/show.html.haml +++ b/app/views/projects/clusters/show.html.haml @@ -13,52 +13,16 @@ cluster_status_reason: @cluster.status_reason, help_path: help_page_path('user/project/clusters/index.md', anchor: 'installing-applications') } } - .js-cluster-application-notice .flash-container %section.settings.no-animate.expanded - %h4= s_('ClusterIntegration|Enable cluster integration') - .settings-content - - .hidden.js-cluster-error.alert.alert-danger.alert-block.append-bottom-10{ role: 'alert' } - = s_('ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine') - %p.js-error-reason - - .hidden.js-cluster-creating.alert.alert-info.alert-block.append-bottom-10{ role: 'alert' } - = s_('ClusterIntegration|Cluster is being created on Google Container Engine...') - - .hidden.js-cluster-success.alert.alert-success.alert-block.append-bottom-10{ role: 'alert' } - = s_('ClusterIntegration|Cluster was successfully created on Google Container Engine') - - %p - - if @cluster.enabled? - - if can?(current_user, :update_cluster, @cluster) - = s_('ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab\'s connection to it.') - - else - = s_('ClusterIntegration|Cluster integration is enabled for this project.') - - else - = s_('ClusterIntegration|Cluster integration is disabled for this project.') - - = form_for @cluster, url: namespace_project_cluster_path(@project.namespace, @project, @cluster), as: :cluster do |field| - = form_errors(@cluster) - .form-group.append-bottom-20 - %label.append-bottom-10 - = field.hidden_field :enabled, { class: 'js-toggle-input'} - - %button{ type: 'button', - class: "js-toggle-cluster project-feature-toggle #{'checked' unless !@cluster.enabled?} #{'disabled' unless can?(current_user, :update_cluster, @cluster)}", - 'aria-label': s_('ClusterIntegration|Toggle Cluster'), - disabled: !can?(current_user, :update_cluster, @cluster), - data: { 'enabled-text': 'Enabled', 'disabled-text': 'Disabled' } } - - - if can?(current_user, :update_cluster, @cluster) - .form-group - = field.submit _('Save'), class: 'btn btn-success' + = render 'banner' + = render 'enabled' .cluster-applications-table#js-cluster-applications - %section.settings#js-cluster-details + %section.settings#js-cluster-details{ class: ('expanded' if expanded) } .settings-header %h4= s_('ClusterIntegration|Cluster details') %button.btn.js-settings-toggle @@ -66,20 +30,16 @@ %p= s_('ClusterIntegration|See and edit the details for your cluster') .settings-content - - .form_group.append-bottom-20 - %label.append-bottom-10{ for: 'cluster-name' } - = s_('ClusterIntegration|Cluster name') - .input-group - %input.form-control.cluster-name{ value: @cluster.name, disabled: true } - %span.input-group-addon.clipboard-addon - = clipboard_button(text: @cluster.name, title: s_('ClusterIntegration|Copy cluster name')) + - if @cluster.managed? + = render 'projects/clusters/gcp/show' + - else + = render 'projects/clusters/user/show' %section.settings.no-animate#js-cluster-advanced-settings{ class: ('expanded' if expanded) } .settings-header %h4= _('Advanced settings') %button.btn.js-settings-toggle = expanded ? 'Collapse' : 'Expand' - %p= s_('ClusterIntegration|Manage Cluster integration on your GitLab project') + %p= s_('ClusterIntegration|Manage cluster integration on your GitLab project') .settings-content = render 'advanced_settings' diff --git a/app/views/projects/clusters/user/_form.html.haml b/app/views/projects/clusters/user/_form.html.haml new file mode 100644 index 00000000000..4a9bd5186c6 --- /dev/null +++ b/app/views/projects/clusters/user/_form.html.haml @@ -0,0 +1,25 @@ += form_for @cluster, url: user_namespace_project_clusters_path(@project.namespace, @project), as: :cluster do |field| + = form_errors(@cluster) + .form-group + = field.label :name, s_('ClusterIntegration|Cluster name') + = field.text_field :name, class: 'form-control', placeholder: s_('ClusterIntegration|Cluster name') + + = field.fields_for :platform_kubernetes, @cluster.platform_kubernetes do |platform_kubernetes_field| + .form-group + = platform_kubernetes_field.label :api_url, s_('ClusterIntegration|API URL') + = platform_kubernetes_field.text_field :api_url, class: 'form-control', placeholder: s_('ClusterIntegration|API URL') + + .form-group + = platform_kubernetes_field.label :ca_cert, s_('ClusterIntegration|CA Certificate') + = platform_kubernetes_field.text_area :ca_cert, class: 'form-control', placeholder: s_('ClusterIntegration|Certificate Authority bundle (PEM format)') + + .form-group + = platform_kubernetes_field.label :token, s_('ClusterIntegration|Token') + = platform_kubernetes_field.text_field :token, class: 'form-control', placeholder: s_('ClusterIntegration|Service token'), autocomplete: 'off' + + .form-group + = platform_kubernetes_field.label :namespace, s_('ClusterIntegration|Project namespace (optional, unique)') + = platform_kubernetes_field.text_field :namespace, class: 'form-control', placeholder: s_('ClusterIntegration|Project namespace') + + .form-group + = field.submit s_('ClusterIntegration|Add cluster'), class: 'btn btn-success' diff --git a/app/views/projects/clusters/user/_header.html.haml b/app/views/projects/clusters/user/_header.html.haml new file mode 100644 index 00000000000..06ac210a06d --- /dev/null +++ b/app/views/projects/clusters/user/_header.html.haml @@ -0,0 +1,5 @@ +%h4.prepend-top-20 + = s_('ClusterIntegration|Enter the details for your cluster') +%p + - link_to_help_page = link_to(s_('ClusterIntegration|documentation'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer') + = s_('ClusterIntegration|Please enter access information for your cluster. If you need help, you can read our %{link_to_help_page} on clusters').html_safe % { link_to_help_page: link_to_help_page } diff --git a/app/views/projects/clusters/user/_show.html.haml b/app/views/projects/clusters/user/_show.html.haml new file mode 100644 index 00000000000..5931e0b7f17 --- /dev/null +++ b/app/views/projects/clusters/user/_show.html.haml @@ -0,0 +1,29 @@ += form_for @cluster, url: namespace_project_cluster_path(@project.namespace, @project, @cluster), as: :cluster do |field| + = form_errors(@cluster) + .form-group + = field.label :name, s_('ClusterIntegration|Cluster name') + = field.text_field :name, class: 'form-control', placeholder: s_('ClusterIntegration|Cluster name') + + = field.fields_for :platform_kubernetes, @cluster.platform_kubernetes do |platform_kubernetes_field| + .form-group + = platform_kubernetes_field.label :api_url, s_('ClusterIntegration|API URL') + = platform_kubernetes_field.text_field :api_url, class: 'form-control', placeholder: s_('ClusterIntegration|API URL') + + .form-group + = platform_kubernetes_field.label :ca_cert, s_('ClusterIntegration|CA Certificate') + = platform_kubernetes_field.text_area :ca_cert, class: 'form-control', placeholder: s_('ClusterIntegration|Certificate Authority bundle (PEM format)') + + .form-group + = platform_kubernetes_field.label :token, s_('ClusterIntegration|Token') + .input-group + = platform_kubernetes_field.text_field :token, class: 'form-control js-cluster-token', type: 'password', placeholder: s_('ClusterIntegration|Token'), autocomplete: 'off' + %span.input-group-addon.clipboard-addon + %button.js-show-cluster-token.btn-blank{ type: 'button' } + = s_('ClusterIntegration|Show') + + .form-group + = platform_kubernetes_field.label :namespace, s_('ClusterIntegration|Project namespace (optional, unique)') + = platform_kubernetes_field.text_field :namespace, class: 'form-control', placeholder: s_('ClusterIntegration|Project namespace') + + .form-group + = field.submit s_('ClusterIntegration|Save changes'), class: 'btn btn-success' diff --git a/app/views/projects/clusters/user/new.html.haml b/app/views/projects/clusters/user/new.html.haml new file mode 100644 index 00000000000..68f38f83453 --- /dev/null +++ b/app/views/projects/clusters/user/new.html.haml @@ -0,0 +1,11 @@ +- breadcrumb_title "Cluster" +- page_title _("New Cluster") + +.row.prepend-top-default + .col-sm-4 + = render 'projects/clusters/sidebar' + .col-sm-8 + = render 'projects/clusters/dropdown', dropdown_text: s_('ClusterIntegration|Add an existing cluster') + = render 'header' + .prepend-top-20 + = render 'form' diff --git a/app/views/projects/issues/_by_email_description.html.haml b/app/views/projects/issues/_by_email_description.html.haml new file mode 100644 index 00000000000..f2d58534903 --- /dev/null +++ b/app/views/projects/issues/_by_email_description.html.haml @@ -0,0 +1,6 @@ +The subject will be used as the title of the new issue, and the message will be the description. + += link_to 'Quick actions', help_page_path('user/project/quick_actions'), target: '_blank', tabindex: -1 +and styling with += link_to 'Markdown', help_page_path('user/markdown'), target: '_blank', tabindex: -1 +are supported. diff --git a/app/views/projects/issues/_issue_by_email.html.haml b/app/views/projects/issues/_issue_by_email.html.haml deleted file mode 100644 index 264032a3a31..00000000000 --- a/app/views/projects/issues/_issue_by_email.html.haml +++ /dev/null @@ -1,34 +0,0 @@ -.issues-footer.text-center - %button.issue-email-modal-btn{ type: "button", data: { toggle: "modal", target: "#issue-email-modal" } } - Email a new issue to this project - -#issue-email-modal.modal.fade{ tabindex: "-1", role: "dialog" } - .modal-dialog{ role: "document" } - .modal-content - .modal-header - %button.close{ type: "button", data: { dismiss: "modal" }, aria: { label: "close" } } - %span{ aria: { hidden: "true" } }= icon("times") - %h4.modal-title - Create new issue by email - .modal-body - %p - You can create a new issue inside this project by sending an email to the following email address: - .email-modal-input-group.input-group - = text_field_tag :issue_email, email, class: "monospace js-select-on-focus form-control", readonly: true - .input-group-btn - = clipboard_button(target: '#issue_email') - %p - The subject will be used as the title of the new issue, and the message will be the description. - - = link_to 'Quick actions', help_page_path('user/project/quick_actions'), target: '_blank', tabindex: -1 - and styling with - = link_to 'Markdown', help_page_path('user/markdown'), target: '_blank', tabindex: -1 - are supported. - - %p - This is a private email address, generated just for you. - - Anyone who gets ahold of it can create issues as if they were you. - You should - = link_to 'reset it', new_issue_address_project_path(@project), class: 'incoming-email-token-reset' - if that ever happens. diff --git a/app/views/projects/issues/index.html.haml b/app/views/projects/issues/index.html.haml index bfaf024428d..193111b4cee 100644 --- a/app/views/projects/issues/index.html.haml +++ b/app/views/projects/issues/index.html.haml @@ -2,7 +2,7 @@ - @can_bulk_update = can?(current_user, :admin_issue, @project) - page_title "Issues" -- new_issue_email = @project.new_issue_address(current_user) +- new_issue_email = @project.new_issuable_address(current_user, 'issue') - content_for :page_specific_javascripts do = webpack_bundle_tag 'common_vue' @@ -25,6 +25,6 @@ .issues-holder = render 'issues' - if new_issue_email - = render 'issue_by_email', email: new_issue_email + = render 'projects/issuable_by_email', email: new_issue_email, issuable_type: 'issue' - else = render 'shared/empty_states/issues', button_path: new_project_issue_path(@project) diff --git a/app/views/projects/merge_requests/_by_email_description.html.haml b/app/views/projects/merge_requests/_by_email_description.html.haml new file mode 100644 index 00000000000..8ba251749b8 --- /dev/null +++ b/app/views/projects/merge_requests/_by_email_description.html.haml @@ -0,0 +1 @@ +The subject will be used as the source branch name for the new merge request and the target branch will be the default branch for the project. diff --git a/app/views/projects/merge_requests/index.html.haml b/app/views/projects/merge_requests/index.html.haml index 8da2243adef..2ded7484151 100644 --- a/app/views/projects/merge_requests/index.html.haml +++ b/app/views/projects/merge_requests/index.html.haml @@ -4,6 +4,7 @@ - new_merge_request_path = project_new_merge_request_path(merge_project) if merge_project - page_title "Merge Requests" +- new_merge_request_email = @project.new_issuable_address(current_user, 'merge_request') - content_for :page_specific_javascripts do = webpack_bundle_tag 'common_vue' @@ -25,5 +26,7 @@ .merge-requests-holder = render 'merge_requests' + - if new_merge_request_email + = render 'projects/issuable_by_email', email: new_merge_request_email, issuable_type: 'merge_request' - else = render 'shared/empty_states/merge_requests', button_path: new_merge_request_path diff --git a/app/views/projects/new.html.haml b/app/views/projects/new.html.haml index 0a7880ce4cd..cad7c2e83db 100644 --- a/app/views/projects/new.html.haml +++ b/app/views/projects/new.html.haml @@ -18,6 +18,7 @@ A project is where you house your files (repository), plan your work (issues), and publish your documentation (wiki), #{link_to 'among other things', help_page_path("user/project/index.md", anchor: "projects-features"), target: '_blank'}. %p All features are enabled when you create a project, but you can disable the ones you don’t need in the project settings. + = brand_new_project_guidelines .col-lg-9.js-toggle-container %ul.nav-links.gitlab-tabs{ role: 'tablist' } %li.active{ role: 'presentation' } diff --git a/app/views/shared/issuable/_nav.html.haml b/app/views/shared/issuable/_nav.html.haml index 3f03cc7a275..6d8a4668cec 100644 --- a/app/views/shared/issuable/_nav.html.haml +++ b/app/views/shared/issuable/_nav.html.haml @@ -1,6 +1,5 @@ - type = local_assigns.fetch(:type, :issues) - page_context_word = type.to_s.humanize(capitalize: false) -- issuables = @issues || @merge_requests %ul.nav-links.issues-state-filters %li{ class: active_when(params[:state] == 'opened') }> @@ -20,6 +19,4 @@ = link_to page_filter_path(state: 'closed', label: true), id: 'state-closed', title: 'Filter by issues that are currently closed.', data: { state: 'closed' } do #{issuables_state_counter_text(type, :closed)} - %li{ class: active_when(params[:state] == 'all') }> - = link_to page_filter_path(state: 'all', label: true), id: 'state-all', title: "Show all #{page_context_word}.", data: { state: 'all' } do - #{issuables_state_counter_text(type, :all)} + = render 'shared/issuable/nav_links/all', page_context_word: page_context_word, counter: issuables_state_counter_text(type, :all) diff --git a/app/views/shared/issuable/nav_links/_all.html.haml b/app/views/shared/issuable/nav_links/_all.html.haml new file mode 100644 index 00000000000..d7ad7090a45 --- /dev/null +++ b/app/views/shared/issuable/nav_links/_all.html.haml @@ -0,0 +1,6 @@ +- page_context_word = local_assigns.fetch(:page_context_word) +- counter = local_assigns.fetch(:counter) + +%li{ class: active_when(params[:state] == 'all') }> + = link_to page_filter_path(state: 'all', label: true), id: 'state-all', title: "Show all #{page_context_word}.", data: { state: 'all' } do + #{counter} diff --git a/app/workers/admin_email_worker.rb b/app/workers/admin_email_worker.rb index c2dc955b27c..bec0a003a1c 100644 --- a/app/workers/admin_email_worker.rb +++ b/app/workers/admin_email_worker.rb @@ -1,5 +1,5 @@ class AdminEmailWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform diff --git a/app/workers/authorized_projects_worker.rb b/app/workers/authorized_projects_worker.rb index 55d8d0c69d1..09559e3b696 100644 --- a/app/workers/authorized_projects_worker.rb +++ b/app/workers/authorized_projects_worker.rb @@ -1,6 +1,5 @@ class AuthorizedProjectsWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker # Schedules multiple jobs and waits for them to be completed. def self.bulk_perform_and_wait(args_list) @@ -17,11 +16,6 @@ class AuthorizedProjectsWorker waiter.wait end - # Schedules multiple jobs to run in sidekiq without waiting for completion - def self.bulk_perform_async(args_list) - Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], 'args' => args_list) - end - # Performs multiple jobs directly. Failed jobs will be put into sidekiq so # they can benefit from retries def self.bulk_perform_inline(args_list) diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index 45ce49bb5c0..aeb3bc019b9 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -1,34 +1,5 @@ class BackgroundMigrationWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue - - # Enqueues a number of jobs in bulk. - # - # The `jobs` argument should be an Array of Arrays, each sub-array must be in - # the form: - # - # [migration-class, [arg1, arg2, ...]] - def self.perform_bulk(jobs) - Sidekiq::Client.push_bulk('class' => self, - 'queue' => sidekiq_options['queue'], - 'args' => jobs) - end - - # Schedules multiple jobs in bulk, with a delay. - # - def self.perform_bulk_in(delay, jobs) - now = Time.now.to_i - schedule = now + delay.to_i - - if schedule <= now - raise ArgumentError, 'The schedule time must be in the future!' - end - - Sidekiq::Client.push_bulk('class' => self, - 'queue' => sidekiq_options['queue'], - 'args' => jobs, - 'at' => schedule) - end + include ApplicationWorker # Performs the background migration. # diff --git a/app/workers/build_coverage_worker.rb b/app/workers/build_coverage_worker.rb index cd4af85d047..62b212c79be 100644 --- a/app/workers/build_coverage_worker.rb +++ b/app/workers/build_coverage_worker.rb @@ -1,5 +1,5 @@ class BuildCoverageWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue def perform(build_id) diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index 52e7d346e74..5efa9180f5e 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -1,5 +1,5 @@ class BuildFinishedWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue enqueue_in group: :processing diff --git a/app/workers/build_hooks_worker.rb b/app/workers/build_hooks_worker.rb index dedaf2835e6..6705a1c2709 100644 --- a/app/workers/build_hooks_worker.rb +++ b/app/workers/build_hooks_worker.rb @@ -1,5 +1,5 @@ class BuildHooksWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue enqueue_in group: :hooks diff --git a/app/workers/build_queue_worker.rb b/app/workers/build_queue_worker.rb index e5ceb9ef715..fc775a84dc0 100644 --- a/app/workers/build_queue_worker.rb +++ b/app/workers/build_queue_worker.rb @@ -1,5 +1,5 @@ class BuildQueueWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue enqueue_in group: :processing diff --git a/app/workers/build_success_worker.rb b/app/workers/build_success_worker.rb index 20ec24bd18a..ec049821ad7 100644 --- a/app/workers/build_success_worker.rb +++ b/app/workers/build_success_worker.rb @@ -1,5 +1,5 @@ class BuildSuccessWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue enqueue_in group: :processing diff --git a/app/workers/build_trace_sections_worker.rb b/app/workers/build_trace_sections_worker.rb index 8c57e8f767b..c0f5c144e10 100644 --- a/app/workers/build_trace_sections_worker.rb +++ b/app/workers/build_trace_sections_worker.rb @@ -1,5 +1,5 @@ class BuildTraceSectionsWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue def perform(build_id) diff --git a/app/workers/cluster_install_app_worker.rb b/app/workers/cluster_install_app_worker.rb index 899aed904e4..f771cb4939f 100644 --- a/app/workers/cluster_install_app_worker.rb +++ b/app/workers/cluster_install_app_worker.rb @@ -1,5 +1,5 @@ class ClusterInstallAppWorker - include Sidekiq::Worker + include ApplicationWorker include ClusterQueue include ClusterApplications diff --git a/app/workers/cluster_provision_worker.rb b/app/workers/cluster_provision_worker.rb index b01f9708424..1ab4de3b647 100644 --- a/app/workers/cluster_provision_worker.rb +++ b/app/workers/cluster_provision_worker.rb @@ -1,5 +1,5 @@ class ClusterProvisionWorker - include Sidekiq::Worker + include ApplicationWorker include ClusterQueue def perform(cluster_id) diff --git a/app/workers/cluster_wait_for_app_installation_worker.rb b/app/workers/cluster_wait_for_app_installation_worker.rb index 4bb8c293e5d..d564d5e48bf 100644 --- a/app/workers/cluster_wait_for_app_installation_worker.rb +++ b/app/workers/cluster_wait_for_app_installation_worker.rb @@ -1,5 +1,5 @@ class ClusterWaitForAppInstallationWorker - include Sidekiq::Worker + include ApplicationWorker include ClusterQueue include ClusterApplications diff --git a/app/workers/concerns/application_worker.rb b/app/workers/concerns/application_worker.rb new file mode 100644 index 00000000000..9c3bdabc49e --- /dev/null +++ b/app/workers/concerns/application_worker.rb @@ -0,0 +1,40 @@ +Sidekiq::Worker.extend ActiveSupport::Concern + +module ApplicationWorker + extend ActiveSupport::Concern + + include Sidekiq::Worker + + included do + sidekiq_options queue: base_queue_name + end + + module ClassMethods + def base_queue_name + name + .sub(/\AGitlab::/, '') + .sub(/Worker\z/, '') + .underscore + .tr('/', '_') + end + + def queue + get_sidekiq_options['queue'].to_s + end + + def bulk_perform_async(args_list) + Sidekiq::Client.push_bulk('class' => self, 'args' => args_list) + end + + def bulk_perform_in(delay, args_list) + now = Time.now.to_i + schedule = now + delay.to_i + + if schedule <= now + raise ArgumentError, 'The schedule time must be in the future!' + end + + Sidekiq::Client.push_bulk('class' => self, 'args' => args_list, 'at' => schedule) + end + end +end diff --git a/app/workers/concerns/dedicated_sidekiq_queue.rb b/app/workers/concerns/dedicated_sidekiq_queue.rb deleted file mode 100644 index 132bae6022b..00000000000 --- a/app/workers/concerns/dedicated_sidekiq_queue.rb +++ /dev/null @@ -1,9 +0,0 @@ -# Concern that sets the queue of a Sidekiq worker based on the worker's class -# name/namespace. -module DedicatedSidekiqQueue - extend ActiveSupport::Concern - - included do - sidekiq_options queue: name.sub(/Worker\z/, '').underscore.tr('/', '_') - end -end diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb index 67e36c811de..9a9fbaad653 100644 --- a/app/workers/concerns/gitlab/github_import/object_importer.rb +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -8,7 +8,7 @@ module Gitlab extend ActiveSupport::Concern included do - include Sidekiq::Worker + include ApplicationWorker include GithubImport::Queue include ReschedulingMethods include NotifyUponDeath diff --git a/app/workers/create_gpg_signature_worker.rb b/app/workers/create_gpg_signature_worker.rb index 9b5ff17aafa..f371731f68c 100644 --- a/app/workers/create_gpg_signature_worker.rb +++ b/app/workers/create_gpg_signature_worker.rb @@ -1,6 +1,5 @@ class CreateGpgSignatureWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker def perform(commit_sha, project_id) project = Project.find_by(id: project_id) diff --git a/app/workers/create_pipeline_worker.rb b/app/workers/create_pipeline_worker.rb index 865ad1ba420..00cd7b85b9f 100644 --- a/app/workers/create_pipeline_worker.rb +++ b/app/workers/create_pipeline_worker.rb @@ -1,5 +1,5 @@ class CreatePipelineWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue enqueue_in group: :creation diff --git a/app/workers/delete_merged_branches_worker.rb b/app/workers/delete_merged_branches_worker.rb index f870da4ecfd..07cd1f02fb5 100644 --- a/app/workers/delete_merged_branches_worker.rb +++ b/app/workers/delete_merged_branches_worker.rb @@ -1,6 +1,5 @@ class DeleteMergedBranchesWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker def perform(project_id, user_id) begin diff --git a/app/workers/delete_user_worker.rb b/app/workers/delete_user_worker.rb index 3340a7be4fe..6c431b02979 100644 --- a/app/workers/delete_user_worker.rb +++ b/app/workers/delete_user_worker.rb @@ -1,6 +1,5 @@ class DeleteUserWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker def perform(current_user_id, delete_user_id, options = {}) delete_user = User.find(delete_user_id) diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb index 1afa24c8e2a..dd8a6cbbef1 100644 --- a/app/workers/email_receiver_worker.rb +++ b/app/workers/email_receiver_worker.rb @@ -1,6 +1,5 @@ class EmailReceiverWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker def perform(raw) return unless Gitlab::IncomingEmail.enabled? @@ -39,8 +38,7 @@ class EmailReceiverWorker "You are not allowed to perform this action. If you believe this is in error, contact a staff member." when Gitlab::Email::NoteableNotFoundError "The thread you are replying to no longer exists, perhaps it was deleted? If you believe this is in error, contact a staff member." - when Gitlab::Email::InvalidNoteError, - Gitlab::Email::InvalidIssueError + when Gitlab::Email::InvalidRecordError can_retry = true e.message end diff --git a/app/workers/emails_on_push_worker.rb b/app/workers/emails_on_push_worker.rb index f5ccc84c160..21da27973fe 100644 --- a/app/workers/emails_on_push_worker.rb +++ b/app/workers/emails_on_push_worker.rb @@ -1,6 +1,5 @@ class EmailsOnPushWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker attr_reader :email, :skip_premailer diff --git a/app/workers/expire_build_artifacts_worker.rb b/app/workers/expire_build_artifacts_worker.rb index a27585fd389..87e5dca01fd 100644 --- a/app/workers/expire_build_artifacts_worker.rb +++ b/app/workers/expire_build_artifacts_worker.rb @@ -1,5 +1,5 @@ class ExpireBuildArtifactsWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform @@ -8,6 +8,6 @@ class ExpireBuildArtifactsWorker build_ids = Ci::Build.with_expired_artifacts.pluck(:id) build_ids = build_ids.map { |build_id| [build_id] } - Sidekiq::Client.push_bulk('class' => ExpireBuildInstanceArtifactsWorker, 'args' => build_ids ) + ExpireBuildInstanceArtifactsWorker.bulk_perform_async(build_ids) end end diff --git a/app/workers/expire_build_instance_artifacts_worker.rb b/app/workers/expire_build_instance_artifacts_worker.rb index 7b59e976492..234b4357cf7 100644 --- a/app/workers/expire_build_instance_artifacts_worker.rb +++ b/app/workers/expire_build_instance_artifacts_worker.rb @@ -1,6 +1,5 @@ class ExpireBuildInstanceArtifactsWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker def perform(build_id) build = Ci::Build diff --git a/app/workers/expire_job_cache_worker.rb b/app/workers/expire_job_cache_worker.rb index 98a7500bffe..a591e2da519 100644 --- a/app/workers/expire_job_cache_worker.rb +++ b/app/workers/expire_job_cache_worker.rb @@ -1,5 +1,5 @@ class ExpireJobCacheWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue enqueue_in group: :cache diff --git a/app/workers/expire_pipeline_cache_worker.rb b/app/workers/expire_pipeline_cache_worker.rb index 1a0e7f92875..a3ac32b437d 100644 --- a/app/workers/expire_pipeline_cache_worker.rb +++ b/app/workers/expire_pipeline_cache_worker.rb @@ -1,5 +1,5 @@ class ExpirePipelineCacheWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue enqueue_in group: :cache diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb index ec65d3ff65e..8e26275669e 100644 --- a/app/workers/git_garbage_collect_worker.rb +++ b/app/workers/git_garbage_collect_worker.rb @@ -1,6 +1,5 @@ class GitGarbageCollectWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker include Gitlab::CurrentSettings sidekiq_options retry: false diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb index 877f88c043f..400396d5755 100644 --- a/app/workers/gitlab/github_import/advance_stage_worker.rb +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -7,7 +7,7 @@ module Gitlab # been completed this worker will advance the import process to the next # stage. class AdvanceStageWorker - include Sidekiq::Worker + include ApplicationWorker sidekiq_options queue: 'github_importer_advance_stage', dead: false diff --git a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb index 45a38927225..7108b531bc2 100644 --- a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb +++ b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb @@ -3,7 +3,7 @@ module Gitlab module GithubImport class RefreshImportJidWorker - include Sidekiq::Worker + include ApplicationWorker include GithubImport::Queue # The interval to schedule new instances of this job at. diff --git a/app/workers/gitlab/github_import/stage/finish_import_worker.rb b/app/workers/gitlab/github_import/stage/finish_import_worker.rb index 1a09497780a..073d6608082 100644 --- a/app/workers/gitlab/github_import/stage/finish_import_worker.rb +++ b/app/workers/gitlab/github_import/stage/finish_import_worker.rb @@ -4,7 +4,7 @@ module Gitlab module GithubImport module Stage class FinishImportWorker - include Sidekiq::Worker + include ApplicationWorker include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb index f8a3684c6ba..5726fbb573d 100644 --- a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb @@ -4,7 +4,7 @@ module Gitlab module GithubImport module Stage class ImportBaseDataWorker - include Sidekiq::Worker + include ApplicationWorker include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb index e110b7c1c36..7007754ff2e 100644 --- a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb @@ -4,7 +4,7 @@ module Gitlab module GithubImport module Stage class ImportIssuesAndDiffNotesWorker - include Sidekiq::Worker + include ApplicationWorker include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_notes_worker.rb index 9810ed25cf9..5f4678a595f 100644 --- a/app/workers/gitlab/github_import/stage/import_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_notes_worker.rb @@ -4,7 +4,7 @@ module Gitlab module GithubImport module Stage class ImportNotesWorker - include Sidekiq::Worker + include ApplicationWorker include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb index c531f26e897..1c5a7139802 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb @@ -4,7 +4,7 @@ module Gitlab module GithubImport module Stage class ImportPullRequestsWorker - include Sidekiq::Worker + include ApplicationWorker include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_repository_worker.rb b/app/workers/gitlab/github_import/stage/import_repository_worker.rb index aa5762e773d..4d16cef1130 100644 --- a/app/workers/gitlab/github_import/stage/import_repository_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_repository_worker.rb @@ -4,7 +4,7 @@ module Gitlab module GithubImport module Stage class ImportRepositoryWorker - include Sidekiq::Worker + include ApplicationWorker include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab_shell_worker.rb b/app/workers/gitlab_shell_worker.rb index 0ec871e00e1..a0028e41332 100644 --- a/app/workers/gitlab_shell_worker.rb +++ b/app/workers/gitlab_shell_worker.rb @@ -1,7 +1,6 @@ class GitlabShellWorker - include Sidekiq::Worker + include ApplicationWorker include Gitlab::ShellAdapter - include DedicatedSidekiqQueue def perform(action, *arg) gitlab_shell.__send__(action, *arg) # rubocop:disable GitlabSecurity/PublicSend diff --git a/app/workers/gitlab_usage_ping_worker.rb b/app/workers/gitlab_usage_ping_worker.rb index 0a55aab63fd..6dd281b1147 100644 --- a/app/workers/gitlab_usage_ping_worker.rb +++ b/app/workers/gitlab_usage_ping_worker.rb @@ -1,7 +1,7 @@ class GitlabUsagePingWorker LEASE_TIMEOUT = 86400 - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform diff --git a/app/workers/group_destroy_worker.rb b/app/workers/group_destroy_worker.rb index bd8e212e928..f577b310b20 100644 --- a/app/workers/group_destroy_worker.rb +++ b/app/workers/group_destroy_worker.rb @@ -1,6 +1,5 @@ class GroupDestroyWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker include ExceptionBacktrace def perform(group_id, user_id) diff --git a/app/workers/import_export_project_cleanup_worker.rb b/app/workers/import_export_project_cleanup_worker.rb index 7957ed807ab..9788c8df3a3 100644 --- a/app/workers/import_export_project_cleanup_worker.rb +++ b/app/workers/import_export_project_cleanup_worker.rb @@ -1,5 +1,5 @@ class ImportExportProjectCleanupWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform diff --git a/app/workers/invalid_gpg_signature_update_worker.rb b/app/workers/invalid_gpg_signature_update_worker.rb index db6b1ea8e8d..6774ab307c6 100644 --- a/app/workers/invalid_gpg_signature_update_worker.rb +++ b/app/workers/invalid_gpg_signature_update_worker.rb @@ -1,6 +1,5 @@ class InvalidGpgSignatureUpdateWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker def perform(gpg_key_id) gpg_key = GpgKey.find_by(id: gpg_key_id) diff --git a/app/workers/irker_worker.rb b/app/workers/irker_worker.rb index 311fc187e49..9ae5456be4c 100644 --- a/app/workers/irker_worker.rb +++ b/app/workers/irker_worker.rb @@ -2,8 +2,7 @@ require 'json' require 'socket' class IrkerWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker def perform(project_id, chans, colors, push_data, settings) project = Project.find(project_id) diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb index 48e2da338f6..ba832fe30c6 100644 --- a/app/workers/merge_worker.rb +++ b/app/workers/merge_worker.rb @@ -1,6 +1,5 @@ class MergeWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker def perform(merge_request_id, current_user_id, params) params = params.with_indifferent_access diff --git a/app/workers/namespaceless_project_destroy_worker.rb b/app/workers/namespaceless_project_destroy_worker.rb index f1cd1769421..adb25c2a170 100644 --- a/app/workers/namespaceless_project_destroy_worker.rb +++ b/app/workers/namespaceless_project_destroy_worker.rb @@ -5,14 +5,9 @@ # The worker will reject doing anything for projects that *do* have a # namespace. For those use ProjectDestroyWorker instead. class NamespacelessProjectDestroyWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker include ExceptionBacktrace - def self.bulk_perform_async(args_list) - Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], 'args' => args_list) - end - def perform(project_id) begin project = Project.unscoped.find(project_id) diff --git a/app/workers/new_issue_worker.rb b/app/workers/new_issue_worker.rb index d9a8e892e90..3bc030f9c62 100644 --- a/app/workers/new_issue_worker.rb +++ b/app/workers/new_issue_worker.rb @@ -1,6 +1,5 @@ class NewIssueWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker include NewIssuable def perform(issue_id, user_id) diff --git a/app/workers/new_merge_request_worker.rb b/app/workers/new_merge_request_worker.rb index 1910c490159..bda2a0ab59d 100644 --- a/app/workers/new_merge_request_worker.rb +++ b/app/workers/new_merge_request_worker.rb @@ -1,6 +1,5 @@ class NewMergeRequestWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker include NewIssuable def perform(merge_request_id, user_id) diff --git a/app/workers/new_note_worker.rb b/app/workers/new_note_worker.rb index 926162b8c53..67c54fbf10e 100644 --- a/app/workers/new_note_worker.rb +++ b/app/workers/new_note_worker.rb @@ -1,6 +1,5 @@ class NewNoteWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker # Keep extra parameter to preserve backwards compatibility with # old `NewNoteWorker` jobs (can remove later) diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb index 64788da7299..62f733c02fc 100644 --- a/app/workers/pages_worker.rb +++ b/app/workers/pages_worker.rb @@ -1,5 +1,5 @@ class PagesWorker - include Sidekiq::Worker + include ApplicationWorker sidekiq_options queue: :pages, retry: false diff --git a/app/workers/pipeline_hooks_worker.rb b/app/workers/pipeline_hooks_worker.rb index 30a75ec8435..661c29efe88 100644 --- a/app/workers/pipeline_hooks_worker.rb +++ b/app/workers/pipeline_hooks_worker.rb @@ -1,5 +1,5 @@ class PipelineHooksWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue enqueue_in group: :hooks diff --git a/app/workers/pipeline_metrics_worker.rb b/app/workers/pipeline_metrics_worker.rb index 070943f1ecc..d46d1f122fc 100644 --- a/app/workers/pipeline_metrics_worker.rb +++ b/app/workers/pipeline_metrics_worker.rb @@ -1,5 +1,5 @@ class PipelineMetricsWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue def perform(pipeline_id) diff --git a/app/workers/pipeline_notification_worker.rb b/app/workers/pipeline_notification_worker.rb index cdb860b6675..a9a1168a6e3 100644 --- a/app/workers/pipeline_notification_worker.rb +++ b/app/workers/pipeline_notification_worker.rb @@ -1,5 +1,5 @@ class PipelineNotificationWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue def perform(pipeline_id, recipients = nil) diff --git a/app/workers/pipeline_process_worker.rb b/app/workers/pipeline_process_worker.rb index 8c067d05081..07dbf6a971e 100644 --- a/app/workers/pipeline_process_worker.rb +++ b/app/workers/pipeline_process_worker.rb @@ -1,5 +1,5 @@ class PipelineProcessWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue enqueue_in group: :processing diff --git a/app/workers/pipeline_schedule_worker.rb b/app/workers/pipeline_schedule_worker.rb index 7320db1065e..c49758878a4 100644 --- a/app/workers/pipeline_schedule_worker.rb +++ b/app/workers/pipeline_schedule_worker.rb @@ -1,5 +1,5 @@ class PipelineScheduleWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform diff --git a/app/workers/pipeline_success_worker.rb b/app/workers/pipeline_success_worker.rb index cb8bb2ffe75..68c40a259e1 100644 --- a/app/workers/pipeline_success_worker.rb +++ b/app/workers/pipeline_success_worker.rb @@ -1,5 +1,5 @@ class PipelineSuccessWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue enqueue_in group: :processing diff --git a/app/workers/pipeline_update_worker.rb b/app/workers/pipeline_update_worker.rb index 5fa399dff4c..24a8a9fbed5 100644 --- a/app/workers/pipeline_update_worker.rb +++ b/app/workers/pipeline_update_worker.rb @@ -1,5 +1,5 @@ class PipelineUpdateWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue enqueue_in group: :processing diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index b8f8d3750d9..f2b2c4428d3 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -1,6 +1,5 @@ class PostReceive - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker def perform(gl_repository, identifier, changes) project, is_wiki = Gitlab::GlRepository.parse(gl_repository) diff --git a/app/workers/process_commit_worker.rb b/app/workers/process_commit_worker.rb index c0c03848a40..52eebe475ec 100644 --- a/app/workers/process_commit_worker.rb +++ b/app/workers/process_commit_worker.rb @@ -5,8 +5,7 @@ # Consider using an extra worker if you need to add any extra (and potentially # slow) processing of commits. class ProcessCommitWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker # project_id - The ID of the project this commit belongs to. # user_id - The ID of the user that pushed the commit. diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb index 505ff9e086e..f19bcbf946a 100644 --- a/app/workers/project_cache_worker.rb +++ b/app/workers/project_cache_worker.rb @@ -1,7 +1,6 @@ # Worker for updating any project specific caches. class ProjectCacheWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker LEASE_TIMEOUT = 15.minutes.to_i diff --git a/app/workers/project_destroy_worker.rb b/app/workers/project_destroy_worker.rb index 3be7e686609..1ba854ca4cb 100644 --- a/app/workers/project_destroy_worker.rb +++ b/app/workers/project_destroy_worker.rb @@ -1,6 +1,5 @@ class ProjectDestroyWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker include ExceptionBacktrace def perform(project_id, user_id, params) diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index f13ac9e5db2..c100852374a 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -1,6 +1,5 @@ class ProjectExportWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker include ExceptionBacktrace sidekiq_options retry: 3 diff --git a/app/workers/project_migrate_hashed_storage_worker.rb b/app/workers/project_migrate_hashed_storage_worker.rb index 127aa6b9d7d..d01eb744e5d 100644 --- a/app/workers/project_migrate_hashed_storage_worker.rb +++ b/app/workers/project_migrate_hashed_storage_worker.rb @@ -1,6 +1,5 @@ class ProjectMigrateHashedStorageWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker LEASE_TIMEOUT = 30.seconds.to_i diff --git a/app/workers/project_service_worker.rb b/app/workers/project_service_worker.rb index 4883d848c53..75c4b8b3663 100644 --- a/app/workers/project_service_worker.rb +++ b/app/workers/project_service_worker.rb @@ -1,6 +1,5 @@ class ProjectServiceWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker sidekiq_options dead: false diff --git a/app/workers/propagate_service_template_worker.rb b/app/workers/propagate_service_template_worker.rb index 6b607451c7a..635a97c99af 100644 --- a/app/workers/propagate_service_template_worker.rb +++ b/app/workers/propagate_service_template_worker.rb @@ -1,7 +1,6 @@ # Worker for updating any project specific caches. class PropagateServiceTemplateWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker LEASE_TIMEOUT = 4.hours.to_i diff --git a/app/workers/prune_old_events_worker.rb b/app/workers/prune_old_events_worker.rb index 2b43bb19ad1..5ff62ab1369 100644 --- a/app/workers/prune_old_events_worker.rb +++ b/app/workers/prune_old_events_worker.rb @@ -1,5 +1,5 @@ class PruneOldEventsWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform diff --git a/app/workers/reactive_caching_worker.rb b/app/workers/reactive_caching_worker.rb index 18b8daf4e1e..ef3ddb9024b 100644 --- a/app/workers/reactive_caching_worker.rb +++ b/app/workers/reactive_caching_worker.rb @@ -1,6 +1,5 @@ class ReactiveCachingWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker def perform(class_name, id, *args) klass = begin diff --git a/app/workers/remove_expired_group_links_worker.rb b/app/workers/remove_expired_group_links_worker.rb index 2a619f83410..7e64c3070a8 100644 --- a/app/workers/remove_expired_group_links_worker.rb +++ b/app/workers/remove_expired_group_links_worker.rb @@ -1,5 +1,5 @@ class RemoveExpiredGroupLinksWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform diff --git a/app/workers/remove_expired_members_worker.rb b/app/workers/remove_expired_members_worker.rb index 31f652e5f9b..d80b3b15840 100644 --- a/app/workers/remove_expired_members_worker.rb +++ b/app/workers/remove_expired_members_worker.rb @@ -1,5 +1,5 @@ class RemoveExpiredMembersWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform diff --git a/app/workers/remove_old_web_hook_logs_worker.rb b/app/workers/remove_old_web_hook_logs_worker.rb index 555e1bb8691..87fed42d7ce 100644 --- a/app/workers/remove_old_web_hook_logs_worker.rb +++ b/app/workers/remove_old_web_hook_logs_worker.rb @@ -1,5 +1,5 @@ class RemoveOldWebHookLogsWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue WEB_HOOK_LOG_LIFETIME = 2.days diff --git a/app/workers/remove_unreferenced_lfs_objects_worker.rb b/app/workers/remove_unreferenced_lfs_objects_worker.rb index b80f131d5f7..8daf079fc31 100644 --- a/app/workers/remove_unreferenced_lfs_objects_worker.rb +++ b/app/workers/remove_unreferenced_lfs_objects_worker.rb @@ -1,5 +1,5 @@ class RemoveUnreferencedLfsObjectsWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform diff --git a/app/workers/repository_archive_cache_worker.rb b/app/workers/repository_archive_cache_worker.rb index e47069df189..86a258cf94f 100644 --- a/app/workers/repository_archive_cache_worker.rb +++ b/app/workers/repository_archive_cache_worker.rb @@ -1,5 +1,5 @@ class RepositoryArchiveCacheWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform diff --git a/app/workers/repository_check/batch_worker.rb b/app/workers/repository_check/batch_worker.rb index b94d83bd709..76688cf51c1 100644 --- a/app/workers/repository_check/batch_worker.rb +++ b/app/workers/repository_check/batch_worker.rb @@ -1,6 +1,6 @@ module RepositoryCheck class BatchWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue RUN_TIME = 3600 diff --git a/app/workers/repository_check/clear_worker.rb b/app/workers/repository_check/clear_worker.rb index 85bc9103538..97b89dc3db5 100644 --- a/app/workers/repository_check/clear_worker.rb +++ b/app/workers/repository_check/clear_worker.rb @@ -1,6 +1,6 @@ module RepositoryCheck class ClearWorker - include Sidekiq::Worker + include ApplicationWorker include RepositoryCheckQueue def perform diff --git a/app/workers/repository_check/single_repository_worker.rb b/app/workers/repository_check/single_repository_worker.rb index 3d4bee15f1c..4e3c691e8da 100644 --- a/app/workers/repository_check/single_repository_worker.rb +++ b/app/workers/repository_check/single_repository_worker.rb @@ -1,6 +1,6 @@ module RepositoryCheck class SingleRepositoryWorker - include Sidekiq::Worker + include ApplicationWorker include RepositoryCheckQueue def perform(project_id) diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index 001c11b73e1..a07ef1705a1 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -1,9 +1,8 @@ class RepositoryForkWorker ForkError = Class.new(StandardError) - include Sidekiq::Worker + include ApplicationWorker include Gitlab::ShellAdapter - include DedicatedSidekiqQueue include ProjectStartImport sidekiq_options status_expiration: StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 4e90b137b26..55715c83cb1 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -1,8 +1,7 @@ class RepositoryImportWorker ImportError = Class.new(StandardError) - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker include ExceptionBacktrace include ProjectStartImport diff --git a/app/workers/requests_profiles_worker.rb b/app/workers/requests_profiles_worker.rb index 703b025d76e..55c236e9e9d 100644 --- a/app/workers/requests_profiles_worker.rb +++ b/app/workers/requests_profiles_worker.rb @@ -1,5 +1,5 @@ class RequestsProfilesWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform diff --git a/app/workers/schedule_update_user_activity_worker.rb b/app/workers/schedule_update_user_activity_worker.rb index 6c2c3e437f3..d9376577597 100644 --- a/app/workers/schedule_update_user_activity_worker.rb +++ b/app/workers/schedule_update_user_activity_worker.rb @@ -1,5 +1,5 @@ class ScheduleUpdateUserActivityWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform(batch_size = 500) diff --git a/app/workers/stage_update_worker.rb b/app/workers/stage_update_worker.rb index c301cea5ad6..69f2318d83b 100644 --- a/app/workers/stage_update_worker.rb +++ b/app/workers/stage_update_worker.rb @@ -1,5 +1,5 @@ class StageUpdateWorker - include Sidekiq::Worker + include ApplicationWorker include PipelineQueue enqueue_in group: :processing diff --git a/app/workers/storage_migrator_worker.rb b/app/workers/storage_migrator_worker.rb index b48ead799b9..f92421a667d 100644 --- a/app/workers/storage_migrator_worker.rb +++ b/app/workers/storage_migrator_worker.rb @@ -1,6 +1,5 @@ class StorageMigratorWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker BATCH_SIZE = 100 diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index 367e227f680..fb26fa4c515 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -1,5 +1,5 @@ class StuckCiJobsWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue EXCLUSIVE_LEASE_KEY = 'stuck_ci_builds_worker_lease'.freeze diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb index f850e459cd9..e0e6d1418de 100644 --- a/app/workers/stuck_import_jobs_worker.rb +++ b/app/workers/stuck_import_jobs_worker.rb @@ -1,5 +1,5 @@ class StuckImportJobsWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue IMPORT_JOBS_EXPIRATION = 15.hours.to_i diff --git a/app/workers/stuck_merge_jobs_worker.rb b/app/workers/stuck_merge_jobs_worker.rb index a396c0f27b2..36d2a2e6466 100644 --- a/app/workers/stuck_merge_jobs_worker.rb +++ b/app/workers/stuck_merge_jobs_worker.rb @@ -1,5 +1,5 @@ class StuckMergeJobsWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform diff --git a/app/workers/system_hook_push_worker.rb b/app/workers/system_hook_push_worker.rb index e43bbe35de9..ceeaaf8d189 100644 --- a/app/workers/system_hook_push_worker.rb +++ b/app/workers/system_hook_push_worker.rb @@ -1,6 +1,5 @@ class SystemHookPushWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker def perform(push_data, hook_id) SystemHooksService.new.execute_hooks(push_data, hook_id) diff --git a/app/workers/trending_projects_worker.rb b/app/workers/trending_projects_worker.rb index 0531630d13a..7eb65452a7d 100644 --- a/app/workers/trending_projects_worker.rb +++ b/app/workers/trending_projects_worker.rb @@ -1,5 +1,5 @@ class TrendingProjectsWorker - include Sidekiq::Worker + include ApplicationWorker include CronjobQueue def perform diff --git a/app/workers/update_merge_requests_worker.rb b/app/workers/update_merge_requests_worker.rb index afc47fc63d6..74bb9993275 100644 --- a/app/workers/update_merge_requests_worker.rb +++ b/app/workers/update_merge_requests_worker.rb @@ -1,6 +1,5 @@ class UpdateMergeRequestsWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker LOG_TIME_THRESHOLD = 90 # seconds diff --git a/app/workers/update_user_activity_worker.rb b/app/workers/update_user_activity_worker.rb index 31bbdb69edb..27ec5cd33fb 100644 --- a/app/workers/update_user_activity_worker.rb +++ b/app/workers/update_user_activity_worker.rb @@ -1,6 +1,5 @@ class UpdateUserActivityWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker def perform(pairs) pairs = cast_data(pairs) diff --git a/app/workers/upload_checksum_worker.rb b/app/workers/upload_checksum_worker.rb index 78931f1258f..9222760c031 100644 --- a/app/workers/upload_checksum_worker.rb +++ b/app/workers/upload_checksum_worker.rb @@ -1,6 +1,5 @@ class UploadChecksumWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker def perform(upload_id) upload = Upload.find(upload_id) diff --git a/app/workers/wait_for_cluster_creation_worker.rb b/app/workers/wait_for_cluster_creation_worker.rb index 241ed3901dc..19cdb279aaa 100644 --- a/app/workers/wait_for_cluster_creation_worker.rb +++ b/app/workers/wait_for_cluster_creation_worker.rb @@ -1,5 +1,5 @@ class WaitForClusterCreationWorker - include Sidekiq::Worker + include ApplicationWorker include ClusterQueue def perform(cluster_id) diff --git a/app/workers/web_hook_worker.rb b/app/workers/web_hook_worker.rb index 713c0228040..dfc3f33ad9d 100644 --- a/app/workers/web_hook_worker.rb +++ b/app/workers/web_hook_worker.rb @@ -1,6 +1,5 @@ class WebHookWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue + include ApplicationWorker sidekiq_options retry: 4, dead: false diff --git a/changelogs/unreleased/32878-merge-request-from-email.yml b/changelogs/unreleased/32878-merge-request-from-email.yml new file mode 100644 index 00000000000..2df148d5a81 --- /dev/null +++ b/changelogs/unreleased/32878-merge-request-from-email.yml @@ -0,0 +1,5 @@ +--- +title: Allow creation of merge request from email +merge_request: 13817 +author: janp +type: added diff --git a/changelogs/unreleased/35616-move-k8-to-cluster-page.yml b/changelogs/unreleased/35616-move-k8-to-cluster-page.yml new file mode 100644 index 00000000000..032a39608ce --- /dev/null +++ b/changelogs/unreleased/35616-move-k8-to-cluster-page.yml @@ -0,0 +1,5 @@ +--- +title: Create a new form to add Existing Kubernetes Cluster +merge_request: 14805 +author: +type: added diff --git a/changelogs/unreleased/bvl-limit-fork-queries-on-project-show.yml b/changelogs/unreleased/bvl-limit-fork-queries-on-project-show.yml new file mode 100644 index 00000000000..299d9bf6b9c --- /dev/null +++ b/changelogs/unreleased/bvl-limit-fork-queries-on-project-show.yml @@ -0,0 +1,5 @@ +--- +title: Reduce requests for project forks on show page of projects that have forks +merge_request: 15663 +author: +type: performance diff --git a/changelogs/unreleased/feature-custom-text-for-new-projects.yml b/changelogs/unreleased/feature-custom-text-for-new-projects.yml new file mode 100644 index 00000000000..905d76dab33 --- /dev/null +++ b/changelogs/unreleased/feature-custom-text-for-new-projects.yml @@ -0,0 +1,5 @@ +--- +title: Add custom brand text on new project pages +merge_request: 15541 +author: Markus Koller +type: changed diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 7f6e68ceed6..c8b6018bc1b 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -649,6 +649,8 @@ test: # user: YOUR_USERNAME pages: path: tmp/tests/pages + artifacts: + path: tmp/tests/artifacts repositories: storages: default: diff --git a/config/initializers/forbid_sidekiq_in_transactions.rb b/config/initializers/forbid_sidekiq_in_transactions.rb index a78711fe599..bedd57ede04 100644 --- a/config/initializers/forbid_sidekiq_in_transactions.rb +++ b/config/initializers/forbid_sidekiq_in_transactions.rb @@ -13,20 +13,19 @@ module Sidekiq module ClassMethods module NoSchedulingFromTransactions - NESTING = ::Rails.env.test? ? 1 : 0 - %i(perform_async perform_at perform_in).each do |name| define_method(name) do |*args| - return super(*args) if Sidekiq::Worker.skip_transaction_check - return super(*args) unless ActiveRecord::Base.connection.open_transactions > NESTING + if !Sidekiq::Worker.skip_transaction_check && AfterCommitQueue.inside_transaction? + raise <<-MSG.strip_heredoc + `#{self}.#{name}` cannot be called inside a transaction as this can lead to + race conditions when the worker runs before the transaction is committed and + tries to access a model that has not been saved yet. - raise <<-MSG.strip_heredoc - `#{self}.#{name}` cannot be called inside a transaction as this can lead to - race conditions when the worker runs before the transaction is committed and - tries to access a model that has not been saved yet. + Use an `after_commit` hook, or include `AfterCommitQueue` and use a `run_after_commit` block instead. + MSG + end - Use an `after_commit` hook, or include `AfterCommitQueue` and use a `run_after_commit` block instead. - MSG + super(*args) end end end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index a1cc9655319..ba4481ae602 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -64,13 +64,13 @@ end # The Sidekiq client API always adds the queue to the Sidekiq queue # list, but mail_room and gitlab-shell do not. This is only necessary # for monitoring. -config = YAML.load_file(Rails.root.join('config', 'sidekiq_queues.yml').to_s) - begin + queues = Gitlab::SidekiqConfig.worker_queues + Sidekiq.redis do |conn| conn.pipelined do - config[:queues].each do |queue| - conn.sadd('queues', queue[0]) + queues.each do |queue| + conn.sadd('queues', queue) end end end diff --git a/config/routes/admin.rb b/config/routes/admin.rb index c0748231813..e22fb440abc 100644 --- a/config/routes/admin.rb +++ b/config/routes/admin.rb @@ -97,7 +97,7 @@ namespace :admin do resource :appearances, only: [:show, :create, :update], path: 'appearance' do member do - get :preview + get :preview_sign_in delete :logo delete :header_logos end diff --git a/config/routes/project.rb b/config/routes/project.rb index bdafaba3ab3..093da10f57f 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -183,10 +183,16 @@ constraints(ProjectUrlConstrainer.new) do end end - resources :clusters, except: [:edit] do + resources :clusters, except: [:edit, :create] do collection do - get :login - get '/providers/gcp/new', action: :new_gcp + scope :providers do + get '/user/new', to: 'clusters/user#new' + post '/user', to: 'clusters/user#create' + + get '/gcp/new', to: 'clusters/gcp#new' + get '/gcp/login', to: 'clusters/gcp#login' + post '/gcp', to: 'clusters/gcp#create' + end end member do @@ -429,7 +435,7 @@ constraints(ProjectUrlConstrainer.new) do get :download_export get :activity get :refs - put :new_issue_address + put :new_issuable_address end end end diff --git a/db/fixtures/development/14_pipelines.rb b/db/fixtures/development/14_pipelines.rb index 5de5339b70e..d3a63aa2a78 100644 --- a/db/fixtures/development/14_pipelines.rb +++ b/db/fixtures/development/14_pipelines.rb @@ -124,11 +124,11 @@ class Gitlab::Seeder::Pipelines return unless %w[build test].include?(build.stage) artifacts_cache_file(artifacts_archive_path) do |file| - build.artifacts_file = file + build.job_artifacts.build(project: build.project, file_type: :archive, file: file) end artifacts_cache_file(artifacts_metadata_path) do |file| - build.artifacts_metadata = file + build.job_artifacts.build(project: build.project, file_type: :metadata, file: file) end end diff --git a/db/migrate/20170918072948_create_job_artifacts.rb b/db/migrate/20170918072948_create_job_artifacts.rb new file mode 100644 index 00000000000..95f2c6c8ce8 --- /dev/null +++ b/db/migrate/20170918072948_create_job_artifacts.rb @@ -0,0 +1,23 @@ +class CreateJobArtifacts < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + create_table :ci_job_artifacts do |t| + t.belongs_to :project, null: false, index: true, foreign_key: { on_delete: :cascade } + t.integer :job_id, null: false + t.integer :file_type, null: false + t.integer :size, limit: 8 + + t.datetime_with_timezone :created_at, null: false + t.datetime_with_timezone :updated_at, null: false + t.datetime_with_timezone :expire_at + + t.string :file + + t.foreign_key :ci_builds, column: :job_id, on_delete: :cascade + t.index [:job_id, :file_type], unique: true + end + end +end diff --git a/db/migrate/20171122131600_add_new_project_guidelines_to_appearances.rb b/db/migrate/20171122131600_add_new_project_guidelines_to_appearances.rb new file mode 100644 index 00000000000..f141c442d97 --- /dev/null +++ b/db/migrate/20171122131600_add_new_project_guidelines_to_appearances.rb @@ -0,0 +1,12 @@ +class AddNewProjectGuidelinesToAppearances < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + change_table :appearances do |t| + t.text :new_project_guidelines + t.text :new_project_guidelines_html + end + end +end diff --git a/db/post_migrate/20170627101016_schedule_event_migrations.rb b/db/post_migrate/20170627101016_schedule_event_migrations.rb index 1f34375ff0d..1e020d05f78 100644 --- a/db/post_migrate/20170627101016_schedule_event_migrations.rb +++ b/db/post_migrate/20170627101016_schedule_event_migrations.rb @@ -25,14 +25,14 @@ class ScheduleEventMigrations < ActiveRecord::Migration # We push multiple jobs at a time to reduce the time spent in # Sidekiq/Redis operations. We're using this buffer based approach so we # don't need to run additional queries for every range. - BackgroundMigrationWorker.perform_bulk(jobs) + BackgroundMigrationWorker.bulk_perform_async(jobs) jobs.clear end jobs << ['MigrateEventsToPushEventPayloads', [min, max]] end - BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty? + BackgroundMigrationWorker.bulk_perform_async(jobs) unless jobs.empty? end def down diff --git a/db/post_migrate/20171005130944_schedule_create_gpg_key_subkeys_from_gpg_keys.rb b/db/post_migrate/20171005130944_schedule_create_gpg_key_subkeys_from_gpg_keys.rb index 01d56fbd490..467c584c2e0 100644 --- a/db/post_migrate/20171005130944_schedule_create_gpg_key_subkeys_from_gpg_keys.rb +++ b/db/post_migrate/20171005130944_schedule_create_gpg_key_subkeys_from_gpg_keys.rb @@ -19,7 +19,7 @@ class ScheduleCreateGpgKeySubkeysFromGpgKeys < ActiveRecord::Migration [MIGRATION, [id]] end - BackgroundMigrationWorker.perform_bulk(jobs) + BackgroundMigrationWorker.bulk_perform_async(jobs) end end diff --git a/db/schema.rb b/db/schema.rb index effb2604af2..0984ca6487f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -36,6 +36,8 @@ ActiveRecord::Schema.define(version: 20171124150326) do t.datetime_with_timezone "updated_at", null: false t.text "description_html" t.integer "cached_markdown_version" + t.text "new_project_guidelines" + t.text "new_project_guidelines_html" end create_table "application_settings", force: :cascade do |t| @@ -319,6 +321,20 @@ ActiveRecord::Schema.define(version: 20171124150326) do add_index "ci_group_variables", ["group_id", "key"], name: "index_ci_group_variables_on_group_id_and_key", unique: true, using: :btree + create_table "ci_job_artifacts", force: :cascade do |t| + t.integer "project_id", null: false + t.integer "job_id", null: false + t.integer "file_type", null: false + t.integer "size", limit: 8 + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + t.datetime_with_timezone "expire_at" + t.string "file" + end + + add_index "ci_job_artifacts", ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true, using: :btree + add_index "ci_job_artifacts", ["project_id"], name: "index_ci_job_artifacts_on_project_id", using: :btree + create_table "ci_pipeline_schedule_variables", force: :cascade do |t| t.string "key", null: false t.text "value" @@ -1909,6 +1925,8 @@ ActiveRecord::Schema.define(version: 20171124150326) do add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade add_foreign_key "ci_builds", "projects", name: "fk_befce0568a", on_delete: :cascade add_foreign_key "ci_group_variables", "namespaces", column: "group_id", name: "fk_33ae4d58d8", on_delete: :cascade + add_foreign_key "ci_job_artifacts", "ci_builds", column: "job_id", on_delete: :cascade + add_foreign_key "ci_job_artifacts", "projects", on_delete: :cascade add_foreign_key "ci_pipeline_schedule_variables", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_41c35fda51", on_delete: :cascade add_foreign_key "ci_pipeline_schedules", "projects", name: "fk_8ead60fcc4", on_delete: :cascade add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify diff --git a/doc/README.md b/doc/README.md index d4119d35162..95cb9683a15 100644 --- a/doc/README.md +++ b/doc/README.md @@ -189,6 +189,7 @@ have access to GitLab administration tools and settings. - [Issue closing pattern](administration/issue_closing_pattern.md): Customize how to close an issue from commit messages. - [Libravatar](customization/libravatar.md): Use Libravatar instead of Gravatar for user avatars. - [Welcome message](customization/welcome_message.md): Add a custom welcome message to the sign-in page. +- [New project page](customization/new_project_page.md): Customize the new project page. ### Admin tools diff --git a/doc/customization/new_project_page.md b/doc/customization/new_project_page.md new file mode 100644 index 00000000000..148bf9512c6 --- /dev/null +++ b/doc/customization/new_project_page.md @@ -0,0 +1,20 @@ +# Customizing the new project page + +It is possible to add a markdown-formatted message to your GitLab +new project page. + +By default, the new project page shows a sidebar with general information: + +![](new_project_page/default_new_project_page.png) + +## Changing the appearance of the new project page + +Navigate to the **Admin** area and go to the **Appearance** page. + +Fill in your project guidelines: + +![](new_project_page/appearance_settings.png) + +After saving the page, your new project page will show the guidelines in the sidebar, below the general information: + +![](new_project_page/custom_new_project_page.png) diff --git a/doc/customization/new_project_page/appearance_settings.png b/doc/customization/new_project_page/appearance_settings.png Binary files differnew file mode 100644 index 00000000000..08eea684e14 --- /dev/null +++ b/doc/customization/new_project_page/appearance_settings.png diff --git a/doc/customization/new_project_page/custom_new_project_page.png b/doc/customization/new_project_page/custom_new_project_page.png Binary files differnew file mode 100644 index 00000000000..662c715f193 --- /dev/null +++ b/doc/customization/new_project_page/custom_new_project_page.png diff --git a/doc/customization/new_project_page/default_new_project_page.png b/doc/customization/new_project_page/default_new_project_page.png Binary files differnew file mode 100644 index 00000000000..4a0bcf09903 --- /dev/null +++ b/doc/customization/new_project_page/default_new_project_page.png diff --git a/doc/development/background_migrations.md b/doc/development/background_migrations.md index 5452b0e7a2f..fd2b9d0e908 100644 --- a/doc/development/background_migrations.md +++ b/doc/development/background_migrations.md @@ -68,10 +68,10 @@ BackgroundMigrationWorker.perform_async('BackgroundMigrationClassName', [arg1, a ``` Usually it's better to enqueue jobs in bulk, for this you can use -`BackgroundMigrationWorker.perform_bulk`: +`BackgroundMigrationWorker.bulk_perform_async`: ```ruby -BackgroundMigrationWorker.perform_bulk( +BackgroundMigrationWorker.bulk_perform_async( [['BackgroundMigrationClassName', [1]], ['BackgroundMigrationClassName', [2]]] ) @@ -85,13 +85,13 @@ updates. Removals in turn can be handled by simply defining foreign keys with cascading deletes. If you would like to schedule jobs in bulk with a delay, you can use -`BackgroundMigrationWorker.perform_bulk_in`: +`BackgroundMigrationWorker.bulk_perform_in`: ```ruby jobs = [['BackgroundMigrationClassName', [1]], ['BackgroundMigrationClassName', [2]]] -BackgroundMigrationWorker.perform_bulk_in(5.minutes, jobs) +BackgroundMigrationWorker.bulk_perform_in(5.minutes, jobs) ``` ## Cleaning Up @@ -201,7 +201,7 @@ class ScheduleExtractServicesUrl < ActiveRecord::Migration ['ExtractServicesUrl', [id]] end - BackgroundMigrationWorker.perform_bulk(jobs) + BackgroundMigrationWorker.bulk_perform_async(jobs) end end diff --git a/doc/development/sidekiq_style_guide.md b/doc/development/sidekiq_style_guide.md index 1e9fdbc65e2..085fb8f902c 100644 --- a/doc/development/sidekiq_style_guide.md +++ b/doc/development/sidekiq_style_guide.md @@ -3,6 +3,12 @@ This document outlines various guidelines that should be followed when adding or modifying Sidekiq workers. +## ApplicationWorker + +All workers should include `ApplicationWorker` instead of `Sidekiq::Worker`, +which adds some convenience methods and automatically sets the queue based on +the worker's name. + ## Default Queue Use of the "default" queue is not allowed. Every worker should use a queue that @@ -13,19 +19,10 @@ A list of all available queues can be found in `config/sidekiq_queues.yml`. ## Dedicated Queues -Most workers should use their own queue. To ease this process a worker can -include the `DedicatedSidekiqQueue` concern as follows: - -```ruby -class ProcessSomethingWorker - include Sidekiq::Worker - include DedicatedSidekiqQueue -end -``` - -This will set the queue name based on the class' name, minus the `Worker` -suffix. In the above example this would lead to the queue being -`process_something`. +Most workers should use their own queue, which is automatically set based on the +worker class name. For a worker named `ProcessSomethingWorker`, the queue name +would be `process_something`. If you're not sure what a worker's queue name is, +you can find it using `SomeWorker.queue`. In some cases multiple workers do use the same queue. For example, the various workers for updating CI pipelines all use the `pipeline` queue. Adding workers @@ -39,7 +36,7 @@ tests should be placed in `spec/workers`. ## Removing or renaming queues -Try to avoid renaming or removing queues in minor and patch releases. -During online update instance can have pending jobs and removing the queue can -lead to those jobs being stuck forever. If you can't write migration for those -Sidekiq jobs, please consider doing rename or remove queue in major release only.
\ No newline at end of file +Try to avoid renaming or removing queues in minor and patch releases. +During online update instance can have pending jobs and removing the queue can +lead to those jobs being stuck forever. If you can't write migration for those +Sidekiq jobs, please consider doing rename or remove queue in major release only. diff --git a/doc/development/writing_documentation.md b/doc/development/writing_documentation.md index 68ba3dd2da3..b6def7ef541 100644 --- a/doc/development/writing_documentation.md +++ b/doc/development/writing_documentation.md @@ -152,12 +152,23 @@ CE and EE. ## Previewing the changes live If you want to preview the doc changes of your merge request live, you can use -the manual `review-docs-deploy` job in your merge request. +the manual `review-docs-deploy` job in your merge request. You will need at +least Master permissions to be able to run it and is currently enabled for the +following projects: + +- https://gitlab.com/gitlab-org/gitlab-ce +- https://gitlab.com/gitlab-org/gitlab-ee + +NOTE: **Note:** +You will need to push a branch to those repositories, it doesn't work for forks. TIP: **Tip:** If your branch contains only documentation changes, you can use [special branch names](#testing) to avoid long running pipelines. +In the mini pipeline graph, you should see an `>>` icon. Clicking on it will +reveal the `review-docs-deploy` job. Hit the play button for the job to start. + ![Manual trigger a docs build](img/manual_build_docs.png) This job will: diff --git a/doc/integration/slash_commands.md b/doc/integration/slash_commands.md index aa52b5415cf..36a8844e953 100644 --- a/doc/integration/slash_commands.md +++ b/doc/integration/slash_commands.md @@ -17,6 +17,9 @@ Taking the trigger term as `project-name`, the commands are: | `/project-name issue search <query>` | Shows up to 5 issues matching `<query>` | | `/project-name deploy <from> to <to>` | Deploy from the `<from>` environment to the `<to>` environment | +Note that if you are using the [GitLab Slack application](https://docs.gitlab.com/ee/user/project/integrations/gitlab_slack_application.html) for +your GitLab.com projects, you need to [add the `gitlab` keyword at the beginning of the command](https://docs.gitlab.com/ee/user/project/integrations/gitlab_slack_application.html#usage). + ## Issue commands It is possible to create new issue, display issue details and search up to 5 issues. diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index 4b2e042251b..d76ea259301 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -27,7 +27,7 @@ With GitLab merge requests, you can: - [Resolve merge conflicts from the UI](#resolve-conflicts) - Enable [fast-forward merge requests](#fast-forward-merge-requests) - Enable [semi-linear history merge requests](#semi-linear-history-merge-requests) as another security layer to guarantee the pipeline is passing in the target branch - +- [Create new merge requests by email](#create_by_email) With **[GitLab Enterprise Edition][ee]**, you can also: @@ -132,6 +132,14 @@ those conflicts in the GitLab UI. [Learn more about resolving merge conflicts in the UI.](resolve_conflicts.md) +## Create new merge requests by email + +You can create a new merge request by sending an email to a user-specific email +address. The address can be obtained on the merge requests page by clicking on +a **Email a new merge request to this project** button. The subject will be +used as the source branch name for the new merge request and the target branch +will be the default branch for the project. + ## Revert changes GitLab implements Git's powerful feature to revert any commit with introducing diff --git a/features/steps/project/pages.rb b/features/steps/project/pages.rb index 124a132d688..f03630e5a91 100644 --- a/features/steps/project/pages.rb +++ b/features/steps/project/pages.rb @@ -44,8 +44,8 @@ class Spinach::Features::ProjectPages < Spinach::FeatureSteps project: @project, pipeline: pipeline, ref: 'HEAD', - artifacts_file: fixture_file_upload(Rails.root + 'spec/fixtures/pages.zip'), - artifacts_metadata: fixture_file_upload(Rails.root + 'spec/fixtures/pages.zip.meta') + legacy_artifacts_file: fixture_file_upload(Rails.root + 'spec/fixtures/pages.zip'), + legacy_artifacts_metadata: fixture_file_upload(Rails.root + 'spec/fixtures/pages.zip.meta') ) result = ::Projects::UpdatePagesService.new(@project, build).execute diff --git a/features/steps/shared/builds.rb b/features/steps/shared/builds.rb index 3b4c98ec00d..c267195f0e8 100644 --- a/features/steps/shared/builds.rb +++ b/features/steps/shared/builds.rb @@ -37,13 +37,13 @@ module SharedBuilds step 'recent build has artifacts available' do artifacts = Rails.root + 'spec/fixtures/ci_build_artifacts.zip' archive = fixture_file_upload(artifacts, 'application/zip') - @build.update_attributes(artifacts_file: archive) + @build.update_attributes(legacy_artifacts_file: archive) end step 'recent build has artifacts metadata available' do metadata = Rails.root + 'spec/fixtures/ci_build_artifacts_metadata.gz' gzip = fixture_file_upload(metadata, 'application/x-gzip') - @build.update_attributes(artifacts_metadata: gzip) + @build.update_attributes(legacy_artifacts_metadata: gzip) end step 'recent build has a build trace' do diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb index 4750a2c373a..db63c5038ae 100644 --- a/lib/after_commit_queue.rb +++ b/lib/after_commit_queue.rb @@ -6,12 +6,34 @@ module AfterCommitQueue after_rollback :_clear_after_commit_queue end - def run_after_commit(method = nil, &block) - _after_commit_queue << proc { self.send(method) } if method # rubocop:disable GitlabSecurity/PublicSend + def run_after_commit(&block) _after_commit_queue << block if block + + true + end + + def run_after_commit_or_now(&block) + if AfterCommitQueue.inside_transaction? + run_after_commit(&block) + else + instance_eval(&block) + end + true end + def self.open_transactions_baseline + if ::Rails.env.test? + return DatabaseCleaner.connections.count { |conn| conn.strategy.is_a?(DatabaseCleaner::ActiveRecord::Transaction) } + end + + 0 + end + + def self.inside_transaction? + ActiveRecord::Base.connection.open_transactions > open_transactions_baseline + end + protected def _run_after_commit_queue diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 7cec8da013d..62ee20bf7de 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1050,13 +1050,9 @@ module API expose :type, :url, :username, :password end - class ArtifactFile < Grape::Entity - expose :filename, :size - end - class Dependency < Grape::Entity expose :id, :name, :token - expose :artifacts_file, using: ArtifactFile, if: ->(job, _) { job.artifacts? } + expose :artifacts_file, using: JobArtifactFile, if: ->(job, _) { job.artifacts? } end class Response < Grape::Entity diff --git a/lib/api/runner.rb b/lib/api/runner.rb index a3987c560dd..80feb629d54 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -215,18 +215,20 @@ module API job = authenticate_job! forbidden!('Job is not running!') unless job.running? - artifacts_upload_path = ArtifactUploader.artifacts_upload_path + artifacts_upload_path = JobArtifactUploader.artifacts_upload_path artifacts = uploaded_file(:file, artifacts_upload_path) metadata = uploaded_file(:metadata, artifacts_upload_path) bad_request!('Missing artifacts file!') unless artifacts file_to_large! unless artifacts.size < max_artifacts_size - job.artifacts_file = artifacts - job.artifacts_metadata = metadata - job.artifacts_expire_in = params['expire_in'] || + expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in + job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, expire_in: expire_in) + job.build_job_artifacts_metadata(project: job.project, file_type: :metadata, file: metadata, expire_in: expire_in) if metadata + job.artifacts_expire_in = expire_in + if job.save present job, with: Entities::JobRequest::Response else diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 1f4bda6f588..7a582a20056 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -3,7 +3,7 @@ require 'backup/files' module Backup class Artifacts < Files def initialize - super('artifacts', ArtifactUploader.local_artifacts_store) + super('artifacts', LegacyArtifactUploader.local_store_path) end def create_files_dir diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index c276c3566b4..3f65bc912de 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -703,14 +703,14 @@ into similar problems in the future (e.g. when new tables are created). # We push multiple jobs at a time to reduce the time spent in # Sidekiq/Redis operations. We're using this buffer based approach so we # don't need to run additional queries for every range. - BackgroundMigrationWorker.perform_bulk(jobs) + BackgroundMigrationWorker.bulk_perform_async(jobs) jobs.clear end jobs << [job_class_name, [start_id, end_id]] end - BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty? + BackgroundMigrationWorker.bulk_perform_async(jobs) unless jobs.empty? end # Queues background migration jobs for an entire table, batched by ID range. diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index b07c68d1498..e08b5be8984 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -1,3 +1,4 @@ +require 'gitlab/email/handler/create_merge_request_handler' require 'gitlab/email/handler/create_note_handler' require 'gitlab/email/handler/create_issue_handler' require 'gitlab/email/handler/unsubscribe_handler' @@ -8,6 +9,7 @@ module Gitlab HANDLERS = [ UnsubscribeHandler, CreateNoteHandler, + CreateMergeRequestHandler, CreateIssueHandler ].freeze diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb new file mode 100644 index 00000000000..c63666b98c1 --- /dev/null +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -0,0 +1,67 @@ +require 'gitlab/email/handler/base_handler' +require 'gitlab/email/handler/reply_processing' + +module Gitlab + module Email + module Handler + class CreateMergeRequestHandler < BaseHandler + include ReplyProcessing + attr_reader :project_path, :incoming_email_token + + def initialize(mail, mail_key) + super(mail, mail_key) + if m = /\A([^\+]*)\+merge-request\+(.*)/.match(mail_key.to_s) + @project_path, @incoming_email_token = m.captures + end + end + + def can_handle? + @project_path && @incoming_email_token + end + + def execute + raise ProjectNotFound unless project + + validate_permission!(:create_merge_request) + + verify_record!( + record: create_merge_request, + invalid_exception: InvalidMergeRequestError, + record_name: 'merge_request') + end + + def author + @author ||= User.find_by(incoming_email_token: incoming_email_token) + end + + def project + @project ||= Project.find_by_full_path(project_path) + end + + def metrics_params + super.merge(project: project&.full_path) + end + + private + + def create_merge_request + merge_request = MergeRequests::BuildService.new(project, author, merge_request_params).execute + + if merge_request.errors.any? + merge_request + else + MergeRequests::CreateService.new(project, author).create(merge_request) + end + end + + def merge_request_params + { + source_project_id: project.id, + source_branch: mail.subject, + target_project_id: project.id + } + end + end + end + end +end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index c8f4591d060..d8c594ad0e7 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -13,8 +13,10 @@ module Gitlab UserBlockedError = Class.new(ProcessingError) UserNotAuthorizedError = Class.new(ProcessingError) NoteableNotFoundError = Class.new(ProcessingError) - InvalidNoteError = Class.new(ProcessingError) - InvalidIssueError = Class.new(ProcessingError) + InvalidRecordError = Class.new(ProcessingError) + InvalidNoteError = Class.new(InvalidRecordError) + InvalidIssueError = Class.new(InvalidRecordError) + InvalidMergeRequestError = Class.new(InvalidRecordError) UnknownIncomingEmail = Class.new(ProcessingError) class Receiver diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb index fc1595f1faf..b2a625e08fa 100644 --- a/lib/gitlab/git/conflict/file.rb +++ b/lib/gitlab/git/conflict/file.rb @@ -2,7 +2,7 @@ module Gitlab module Git module Conflict class File - attr_reader :content, :their_path, :our_path, :our_mode, :repository + attr_reader :content, :their_path, :our_path, :our_mode, :repository, :commit_oid def initialize(repository, commit_oid, conflict, content) @repository = repository diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index df509c5f4ce..de8cce41b6d 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -75,7 +75,7 @@ module Gitlab resolved_lines = file.resolve_lines(params[:sections]) new_file = resolved_lines.map { |line| line[:full_line] }.join("\n") - new_file << "\n" if file.our_blob.data.ends_with?("\n") + new_file << "\n" if file.our_blob.data.end_with?("\n") elsif params[:content] new_file = file.resolve_content(params[:content]) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index f2bbaed64a7..eab04bcac65 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -18,6 +18,8 @@ module Gitlab GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE ].freeze SEARCH_CONTEXT_LINES = 3 + REBASE_WORKTREE_PREFIX = 'rebase'.freeze + SQUASH_WORKTREE_PREFIX = 'squash'.freeze NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) @@ -1070,13 +1072,8 @@ module Gitlab raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") - command = [Gitlab.config.git.bin_path] + %w[update-ref --stdin -z] input = "update #{ref_path}\x00#{ref}\x00\x00" - output, status = circuit_breaker.perform do - popen(command, path) { |stdin| stdin.write(input) } - end - - raise GitError, output unless status.zero? + run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } end def fetch_ref(source_repository, source_ref:, target_ref:) @@ -1098,14 +1095,22 @@ module Gitlab end # Refactoring aid; allows us to copy code from app/models/repository.rb - def run_git(args, env: {}, nice: false) + def run_git(args, chdir: path, env: {}, nice: false, &block) cmd = [Gitlab.config.git.bin_path, *args] cmd.unshift("nice") if nice circuit_breaker.perform do - popen(cmd, path, env) + popen(cmd, chdir, env, &block) end end + def run_git!(args, chdir: path, env: {}, nice: false, &block) + output, status = run_git(args, chdir: chdir, env: env, nice: nice, &block) + + raise GitError, output unless status.zero? + + output + end + # Refactoring aid; allows us to copy code from app/models/repository.rb def run_git_with_timeout(args, timeout, env: {}) circuit_breaker.perform do @@ -1175,6 +1180,64 @@ module Gitlab raise GitError.new("Could not fsck repository:\n#{output}") unless status.zero? end + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) + env = git_env_for_user(user) + + with_worktree(rebase_path, branch, env: env) do + run_git!( + %W(pull --rebase #{remote_repository.path} #{remote_branch}), + chdir: rebase_path, env: env + ) + + rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip + + Gitlab::Git::OperationService.new(user, self) + .update_branch(branch, rebase_sha, branch_sha) + + rebase_sha + end + end + + def rebase_in_progress?(rebase_id) + fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + end + + def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) + squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) + env = git_env_for_user(user).merge( + 'GIT_AUTHOR_NAME' => author.name, + 'GIT_AUTHOR_EMAIL' => author.email + ) + diff_range = "#{start_sha}...#{end_sha}" + diff_files = run_git!( + %W(diff --name-only --diff-filter=a --binary #{diff_range}) + ).chomp + + with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do + # Apply diff of the `diff_range` to the worktree + diff = run_git!(%W(diff --binary #{diff_range})) + run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| + stdin.write(diff) + end + + # Commit the `diff_range` diff + run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) + + # Return the squash sha. May print a warning for ambiguous refs, but + # we can ignore that with `--quiet` and just take the SHA, if present. + # HEAD here always refers to the current HEAD commit, even if there is + # another ref called HEAD. + run_git!( + %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env + ).chomp + end + end + + def squash_in_progress?(squash_id) + fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end @@ -1211,6 +1274,57 @@ module Gitlab private + def fresh_worktree?(path) + File.exist?(path) && !clean_stuck_worktree(path) + end + + def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) + base_args = %w(worktree add --detach) + + # Note that we _don't_ want to test for `.present?` here: If the caller + # passes an non nil empty value it means it still wants sparse checkout + # but just isn't interested in any file, perhaps because it wants to + # checkout files in by a changeset but that changeset only adds files. + if sparse_checkout_files + # Create worktree without checking out + run_git!(base_args + ['--no-checkout', worktree_path], env: env) + worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path) + + configure_sparse_checkout(worktree_git_path, sparse_checkout_files) + + # After sparse checkout configuration, checkout `branch` in worktree + run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env) + else + # Create worktree and checkout `branch` in it + run_git!(base_args + [worktree_path, branch], env: env) + end + + yield + ensure + FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path) + FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) + end + + def clean_stuck_worktree(path) + return false unless File.mtime(path) < 15.minutes.ago + + FileUtils.rm_rf(path) + true + end + + # Adding a worktree means checking out the repository. For large repos, + # this can be very expensive, so set up sparse checkout for the worktree + # to only check out the files we're interested in. + def configure_sparse_checkout(worktree_git_path, files) + run_git!(%w(config core.sparseCheckout true)) + + return if files.empty? + + worktree_info_path = File.join(worktree_git_path, 'info') + FileUtils.mkdir_p(worktree_info_path) + File.write(File.join(worktree_info_path, 'sparse-checkout'), files) + end + def rugged_fetch_source_branch(source_repository, source_branch, local_ref) with_repo_branch_commit(source_repository, source_branch) do |commit| if commit @@ -1222,6 +1336,24 @@ module Gitlab end end + def worktree_path(prefix, id) + id = id.to_s + raise ArgumentError, "worktree id can't be empty" unless id.present? + raise ArgumentError, "worktree id can't contain slashes " if id.include?("/") + + File.join(path, 'gitlab-worktree', "#{prefix}-#{id}") + end + + def git_env_for_user(user) + { + 'GIT_COMMITTER_NAME' => user.name, + 'GIT_COMMITTER_EMAIL' => user.email, + 'GL_ID' => Gitlab::GlId.gl_id(user), + 'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL, + 'GL_REPOSITORY' => gl_repository + } + end + # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. def branches_filter(filter: nil, sort_by: nil) # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464 diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb new file mode 100644 index 00000000000..dc9886732b5 --- /dev/null +++ b/lib/gitlab/sidekiq_config.rb @@ -0,0 +1,50 @@ +require 'yaml' + +module Gitlab + module SidekiqConfig + def self.redis_queues + @redis_queues ||= Sidekiq::Queue.all.map(&:name) + end + + # This method is called by `bin/sidekiq-cluster` in EE, which runs outside + # of bundler/Rails context, so we cannot use any gem or Rails methods. + def self.config_queues(rails_path = Rails.root.to_s) + @config_queues ||= begin + config = YAML.load_file(File.join(rails_path, 'config', 'sidekiq_queues.yml')) + config[:queues].map(&:first) + end + end + + def self.cron_workers + @cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize } + end + + def self.workers + @workers ||= find_workers(Rails.root.join('app', 'workers')) + end + + def self.default_queues + [ActionMailer::DeliveryJob.queue_name, 'default'] + end + + def self.worker_queues + @worker_queues ||= (workers.map(&:queue) + default_queues).uniq + end + + def self.find_workers(root) + concerns = root.join('concerns').to_s + + workers = Dir[root.join('**', '*.rb')] + .reject { |path| path.start_with?(concerns) } + + workers.map! do |path| + ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '') + + ns.camelize.constantize + end + + # Skip concerns + workers.select { |w| w < Sidekiq::Worker } + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 864a9e04888..5ab6cd5a4ef 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -58,7 +58,7 @@ module Gitlab end def artifact_upload_ok - { TempPath: ArtifactUploader.artifacts_upload_path } + { TempPath: JobArtifactUploader.artifacts_upload_path } end def send_git_blob(repository, blob) diff --git a/spec/controllers/projects/clusters/gcp_controller_spec.rb b/spec/controllers/projects/clusters/gcp_controller_spec.rb new file mode 100644 index 00000000000..bb5ef7bb365 --- /dev/null +++ b/spec/controllers/projects/clusters/gcp_controller_spec.rb @@ -0,0 +1,185 @@ +require 'spec_helper' + +describe Projects::Clusters::GcpController do + include AccessMatchersForController + include GoogleApi::CloudPlatformHelpers + + set(:project) { create(:project) } + + describe 'GET login' do + describe 'functionality' do + let(:user) { create(:user) } + + before do + project.add_master(user) + sign_in(user) + end + + context 'when omniauth has been configured' do + let(:key) { 'secret-key' } + + let(:session_key_for_redirect_uri) do + GoogleApi::CloudPlatform::Client.session_key_for_redirect_uri(key) + end + + before do + allow(SecureRandom).to receive(:hex).and_return(key) + end + + it 'has authorize_url' do + go + + expect(assigns(:authorize_url)).to include(key) + expect(session[session_key_for_redirect_uri]).to eq(gcp_new_project_clusters_path(project)) + end + end + + context 'when omniauth has not configured' do + before do + stub_omniauth_setting(providers: []) + end + + it 'does not have authorize_url' do + go + + expect(assigns(:authorize_url)).to be_nil + end + end + end + + describe 'security' do + it { expect { go }.to be_allowed_for(:admin) } + it { expect { go }.to be_allowed_for(:owner).of(project) } + it { expect { go }.to be_allowed_for(:master).of(project) } + it { expect { go }.to be_denied_for(:developer).of(project) } + it { expect { go }.to be_denied_for(:reporter).of(project) } + it { expect { go }.to be_denied_for(:guest).of(project) } + it { expect { go }.to be_denied_for(:user) } + it { expect { go }.to be_denied_for(:external) } + end + + def go + get :login, namespace_id: project.namespace, project_id: project + end + end + + describe 'GET new' do + describe 'functionality' do + let(:user) { create(:user) } + + before do + project.add_master(user) + sign_in(user) + end + + context 'when access token is valid' do + before do + stub_google_api_validate_token + end + + it 'has new object' do + go + + expect(assigns(:cluster)).to be_an_instance_of(Clusters::Cluster) + end + end + + context 'when access token is expired' do + before do + stub_google_api_expired_token + end + + it { expect(go).to redirect_to(gcp_login_project_clusters_path(project)) } + end + + context 'when access token is not stored in session' do + it { expect(go).to redirect_to(gcp_login_project_clusters_path(project)) } + end + end + + describe 'security' do + it { expect { go }.to be_allowed_for(:admin) } + it { expect { go }.to be_allowed_for(:owner).of(project) } + it { expect { go }.to be_allowed_for(:master).of(project) } + it { expect { go }.to be_denied_for(:developer).of(project) } + it { expect { go }.to be_denied_for(:reporter).of(project) } + it { expect { go }.to be_denied_for(:guest).of(project) } + it { expect { go }.to be_denied_for(:user) } + it { expect { go }.to be_denied_for(:external) } + end + + def go + get :new, namespace_id: project.namespace, project_id: project + end + end + + describe 'POST create' do + let(:params) do + { + cluster: { + name: 'new-cluster', + provider_gcp_attributes: { + gcp_project_id: '111' + } + } + } + end + + describe 'functionality' do + let(:user) { create(:user) } + + before do + project.add_master(user) + sign_in(user) + end + + context 'when access token is valid' do + before do + stub_google_api_validate_token + end + + context 'when creates a cluster on gke' do + it 'creates a new cluster' do + expect(ClusterProvisionWorker).to receive(:perform_async) + expect { go }.to change { Clusters::Cluster.count } + .and change { Clusters::Providers::Gcp.count } + expect(response).to redirect_to(project_cluster_path(project, project.cluster)) + expect(project.cluster).to be_gcp + expect(project.cluster).to be_kubernetes + end + end + end + + context 'when access token is expired' do + before do + stub_google_api_expired_token + end + + it 'redirects to login page' do + expect(go).to redirect_to(gcp_login_project_clusters_path(project)) + end + end + + context 'when access token is not stored in session' do + it 'redirects to login page' do + expect(go).to redirect_to(gcp_login_project_clusters_path(project)) + end + end + end + + describe 'security' do + it { expect { go }.to be_allowed_for(:admin) } + it { expect { go }.to be_allowed_for(:owner).of(project) } + it { expect { go }.to be_allowed_for(:master).of(project) } + it { expect { go }.to be_denied_for(:developer).of(project) } + it { expect { go }.to be_denied_for(:reporter).of(project) } + it { expect { go }.to be_denied_for(:guest).of(project) } + it { expect { go }.to be_denied_for(:user) } + it { expect { go }.to be_denied_for(:external) } + end + + def go + post :create, params.merge(namespace_id: project.namespace, project_id: project) + end + end +end diff --git a/spec/controllers/projects/clusters/user_controller_spec.rb b/spec/controllers/projects/clusters/user_controller_spec.rb new file mode 100644 index 00000000000..22005e0dc66 --- /dev/null +++ b/spec/controllers/projects/clusters/user_controller_spec.rb @@ -0,0 +1,87 @@ +require 'spec_helper' + +describe Projects::Clusters::UserController do + include AccessMatchersForController + + set(:project) { create(:project) } + + describe 'GET new' do + describe 'functionality' do + let(:user) { create(:user) } + + before do + project.add_master(user) + sign_in(user) + end + + it 'has new object' do + go + + expect(assigns(:cluster)).to be_an_instance_of(Clusters::Cluster) + end + end + + describe 'security' do + it { expect { go }.to be_allowed_for(:admin) } + it { expect { go }.to be_allowed_for(:owner).of(project) } + it { expect { go }.to be_allowed_for(:master).of(project) } + it { expect { go }.to be_denied_for(:developer).of(project) } + it { expect { go }.to be_denied_for(:reporter).of(project) } + it { expect { go }.to be_denied_for(:guest).of(project) } + it { expect { go }.to be_denied_for(:user) } + it { expect { go }.to be_denied_for(:external) } + end + + def go + get :new, namespace_id: project.namespace, project_id: project + end + end + + describe 'POST create' do + let(:params) do + { + cluster: { + name: 'new-cluster', + platform_kubernetes_attributes: { + api_url: 'http://my-url', + token: 'test', + namespace: 'aaa' + } + } + } + end + + describe 'functionality' do + let(:user) { create(:user) } + + before do + project.add_master(user) + sign_in(user) + end + + context 'when creates a cluster' do + it 'creates a new cluster' do + expect(ClusterProvisionWorker).to receive(:perform_async) + expect { go }.to change { Clusters::Cluster.count } + .and change { Clusters::Platforms::Kubernetes.count } + expect(response).to redirect_to(project_cluster_path(project, project.cluster)) + end + end + end + + describe 'security' do + it { expect { go }.to be_allowed_for(:admin) } + it { expect { go }.to be_allowed_for(:owner).of(project) } + it { expect { go }.to be_allowed_for(:master).of(project) } + it { expect { go }.to be_denied_for(:developer).of(project) } + it { expect { go }.to be_denied_for(:reporter).of(project) } + it { expect { go }.to be_denied_for(:guest).of(project) } + it { expect { go }.to be_denied_for(:user) } + it { expect { go }.to be_denied_for(:external) } + end + + def go + post :create, params.merge(namespace_id: project.namespace, project_id: project) + end + end +end diff --git a/spec/controllers/projects/clusters_controller_spec.rb b/spec/controllers/projects/clusters_controller_spec.rb index ca2bcb2b5ae..66e67652dad 100644 --- a/spec/controllers/projects/clusters_controller_spec.rb +++ b/spec/controllers/projects/clusters_controller_spec.rb @@ -4,6 +4,8 @@ describe Projects::ClustersController do include AccessMatchersForController include GoogleApi::CloudPlatformHelpers + set(:project) { create(:project) } + describe 'GET index' do describe 'functionality' do let(:user) { create(:user) } @@ -14,22 +16,18 @@ describe Projects::ClustersController do end context 'when project has a cluster' do - let(:cluster) { create(:cluster, :project, :provided_by_gcp) } - let(:project) { cluster.project } + let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } it { expect(go).to redirect_to(project_cluster_path(project, project.cluster)) } end context 'when project does not have a cluster' do - let(:project) { create(:project) } - it { expect(go).to redirect_to(new_project_cluster_path(project)) } end end describe 'security' do - let(:cluster) { create(:cluster, :project, :provided_by_gcp) } - let(:project) { cluster.project } + let(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } it { expect { go }.to be_allowed_for(:admin) } it { expect { go }.to be_allowed_for(:owner).of(project) } @@ -46,198 +44,8 @@ describe Projects::ClustersController do end end - describe 'GET login' do - let(:project) { create(:project) } - - describe 'functionality' do - let(:user) { create(:user) } - - before do - project.add_master(user) - sign_in(user) - end - - context 'when omniauth has been configured' do - let(:key) { 'secere-key' } - - let(:session_key_for_redirect_uri) do - GoogleApi::CloudPlatform::Client.session_key_for_redirect_uri(key) - end - - before do - allow(SecureRandom).to receive(:hex).and_return(key) - end - - it 'has authorize_url' do - go - - expect(assigns(:authorize_url)).to include(key) - expect(session[session_key_for_redirect_uri]).to eq(providers_gcp_new_project_clusters_url(project)) - end - end - - context 'when omniauth has not configured' do - before do - stub_omniauth_setting(providers: []) - end - - it 'does not have authorize_url' do - go - - expect(assigns(:authorize_url)).to be_nil - end - end - end - - describe 'security' do - it { expect { go }.to be_allowed_for(:admin) } - it { expect { go }.to be_allowed_for(:owner).of(project) } - it { expect { go }.to be_allowed_for(:master).of(project) } - it { expect { go }.to be_denied_for(:developer).of(project) } - it { expect { go }.to be_denied_for(:reporter).of(project) } - it { expect { go }.to be_denied_for(:guest).of(project) } - it { expect { go }.to be_denied_for(:user) } - it { expect { go }.to be_denied_for(:external) } - end - - def go - get :login, namespace_id: project.namespace, project_id: project - end - end - - shared_examples 'requires to login' do - it 'redirects to create a cluster' do - subject - - expect(response).to redirect_to(login_project_clusters_path(project)) - end - end - - describe 'GET new_gcp' do - let(:project) { create(:project) } - - describe 'functionality' do - let(:user) { create(:user) } - - before do - project.add_master(user) - sign_in(user) - end - - context 'when access token is valid' do - before do - stub_google_api_validate_token - end - - it 'has new object' do - go - - expect(assigns(:cluster)).to be_an_instance_of(Clusters::Cluster) - end - end - - context 'when access token is expired' do - before do - stub_google_api_expired_token - end - - it { expect(go).to redirect_to(login_project_clusters_path(project)) } - end - - context 'when access token is not stored in session' do - it { expect(go).to redirect_to(login_project_clusters_path(project)) } - end - end - - describe 'security' do - it { expect { go }.to be_allowed_for(:admin) } - it { expect { go }.to be_allowed_for(:owner).of(project) } - it { expect { go }.to be_allowed_for(:master).of(project) } - it { expect { go }.to be_denied_for(:developer).of(project) } - it { expect { go }.to be_denied_for(:reporter).of(project) } - it { expect { go }.to be_denied_for(:guest).of(project) } - it { expect { go }.to be_denied_for(:user) } - it { expect { go }.to be_denied_for(:external) } - end - - def go - get :new_gcp, namespace_id: project.namespace, project_id: project - end - end - - describe 'POST create' do - let(:project) { create(:project) } - - let(:params) do - { - cluster: { - name: 'new-cluster', - provider_type: :gcp, - provider_gcp_attributes: { - gcp_project_id: '111' - } - } - } - end - - describe 'functionality' do - let(:user) { create(:user) } - - before do - project.add_master(user) - sign_in(user) - end - - context 'when access token is valid' do - before do - stub_google_api_validate_token - end - - context 'when creates a cluster on gke' do - it 'creates a new cluster' do - expect(ClusterProvisionWorker).to receive(:perform_async) - expect { go }.to change { Clusters::Cluster.count } - expect(response).to redirect_to(project_cluster_path(project, project.cluster)) - end - end - end - - context 'when access token is expired' do - before do - stub_google_api_expired_token - end - - it 'redirects to login page' do - expect(go).to redirect_to(login_project_clusters_path(project)) - end - end - - context 'when access token is not stored in session' do - it 'redirects to login page' do - expect(go).to redirect_to(login_project_clusters_path(project)) - end - end - end - - describe 'security' do - it { expect { go }.to be_allowed_for(:admin) } - it { expect { go }.to be_allowed_for(:owner).of(project) } - it { expect { go }.to be_allowed_for(:master).of(project) } - it { expect { go }.to be_denied_for(:developer).of(project) } - it { expect { go }.to be_denied_for(:reporter).of(project) } - it { expect { go }.to be_denied_for(:guest).of(project) } - it { expect { go }.to be_denied_for(:user) } - it { expect { go }.to be_denied_for(:external) } - end - - def go - post :create, params.merge(namespace_id: project.namespace, project_id: project) - end - end - describe 'GET status' do - let(:cluster) { create(:cluster, :project, :providing_by_gcp) } - let(:project) { cluster.project } + let(:cluster) { create(:cluster, :providing_by_gcp, projects: [project]) } describe 'functionality' do let(:user) { create(:user) } @@ -275,8 +83,7 @@ describe Projects::ClustersController do end describe 'GET show' do - let(:cluster) { create(:cluster, :project, :provided_by_gcp) } - let(:project) { cluster.project } + let(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } describe 'functionality' do let(:user) { create(:user) } @@ -313,10 +120,8 @@ describe Projects::ClustersController do end describe 'PUT update' do - let(:cluster) { create(:cluster, :project, :provided_by_gcp) } - let(:project) { cluster.project } - - describe 'functionality' do + context 'when cluster is provided by GCP' do + let(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } let(:user) { create(:user) } before do @@ -324,10 +129,16 @@ describe Projects::ClustersController do sign_in(user) end - context 'when update enabled' do + context 'when changing parameters' do let(:params) do { - cluster: { enabled: false } + cluster: { + enabled: false, + name: 'my-new-cluster-name', + platform_kubernetes_attributes: { + namespace: 'my-namespace' + } + } } end @@ -340,8 +151,14 @@ describe Projects::ClustersController do expect(cluster.enabled).to be_falsey end + it "does not change cluster name" do + go + + expect(cluster.name).to eq('test-cluster') + end + context 'when cluster is being created' do - let(:cluster) { create(:cluster, :project, :providing_by_gcp) } + let(:cluster) { create(:cluster, :providing_by_gcp, projects: [project]) } it "rejects changes" do go @@ -354,11 +171,46 @@ describe Projects::ClustersController do end end + context 'when cluster is provided by user' do + let(:cluster) { create(:cluster, :provided_by_user, projects: [project]) } + let(:user) { create(:user) } + + before do + project.add_master(user) + sign_in(user) + end + + context 'when changing parameters' do + let(:params) do + { + cluster: { + enabled: false, + name: 'my-new-cluster-name', + platform_kubernetes_attributes: { + namespace: 'my-namespace' + } + } + } + end + + it "updates and redirects back to show page" do + go + + cluster.reload + expect(response).to redirect_to(project_cluster_path(project, project.cluster)) + expect(flash[:notice]).to eq('Cluster was successfully updated.') + expect(cluster.enabled).to be_falsey + expect(cluster.name).to eq('my-new-cluster-name') + expect(cluster.platform_kubernetes.namespace).to eq('my-namespace') + end + end + end + describe 'security' do + set(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } + let(:params) do - { - cluster: { enabled: false } - } + { cluster: { enabled: false } } end it { expect { go }.to be_allowed_for(:admin) } @@ -378,10 +230,7 @@ describe Projects::ClustersController do end end - describe 'delete update' do - let(:cluster) { create(:cluster, :project, :provided_by_gcp) } - let(:project) { cluster.project } - + describe 'DELETE destroy' do describe 'functionality' do let(:user) { create(:user) } @@ -390,31 +239,37 @@ describe Projects::ClustersController do sign_in(user) end - it "destroys and redirects back to clusters list" do - expect { go } - .to change { Clusters::Cluster.count }.by(-1) - .and change { Clusters::Platforms::Kubernetes.count }.by(-1) - .and change { Clusters::Providers::Gcp.count }.by(-1) + context 'when cluster is provided by GCP' do + context 'when cluster is created' do + let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } - expect(response).to redirect_to(project_clusters_path(project)) - expect(flash[:notice]).to eq('Cluster integration was successfully removed.') - end + it "destroys and redirects back to clusters list" do + expect { go } + .to change { Clusters::Cluster.count }.by(-1) + .and change { Clusters::Platforms::Kubernetes.count }.by(-1) + .and change { Clusters::Providers::Gcp.count }.by(-1) - context 'when cluster is being created' do - let(:cluster) { create(:cluster, :project, :providing_by_gcp) } + expect(response).to redirect_to(project_clusters_path(project)) + expect(flash[:notice]).to eq('Cluster integration was successfully removed.') + end + end - it "destroys and redirects back to clusters list" do - expect { go } - .to change { Clusters::Cluster.count }.by(-1) - .and change { Clusters::Providers::Gcp.count }.by(-1) + context 'when cluster is being created' do + let!(:cluster) { create(:cluster, :providing_by_gcp, projects: [project]) } - expect(response).to redirect_to(project_clusters_path(project)) - expect(flash[:notice]).to eq('Cluster integration was successfully removed.') + it "destroys and redirects back to clusters list" do + expect { go } + .to change { Clusters::Cluster.count }.by(-1) + .and change { Clusters::Providers::Gcp.count }.by(-1) + + expect(response).to redirect_to(project_clusters_path(project)) + expect(flash[:notice]).to eq('Cluster integration was successfully removed.') + end end end - context 'when provider is user' do - let(:cluster) { create(:cluster, :project, :provided_by_user) } + context 'when cluster is provided by user' do + let!(:cluster) { create(:cluster, :provided_by_user, projects: [project]) } it "destroys and redirects back to clusters list" do expect { go } @@ -429,6 +284,8 @@ describe Projects::ClustersController do end describe 'security' do + set(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } + it { expect { go }.to be_allowed_for(:admin) } it { expect { go }.to be_allowed_for(:owner).of(project) } it { expect { go }.to be_allowed_for(:master).of(project) } diff --git a/spec/controllers/projects_controller_spec.rb b/spec/controllers/projects_controller_spec.rb index e7ab714c550..e61187fb518 100644 --- a/spec/controllers/projects_controller_spec.rb +++ b/spec/controllers/projects_controller_spec.rb @@ -261,6 +261,27 @@ describe ProjectsController do expect(response).to redirect_to(namespace_project_path) end end + + context 'when the project is forked and has a repository', :request_store do + let(:public_project) { create(:project, :public, :repository) } + let(:other_user) { create(:user) } + + render_views + + before do + # View the project as a user that does not have any rights + sign_in(other_user) + + fork_project(public_project) + end + + it 'does not increase the number of queries when the project is forked' do + expected_query = /#{public_project.fork_network.find_forks_in(other_user.namespace).to_sql}/ + + expect { get(:show, namespace_id: public_project.namespace, id: public_project) } + .not_to exceed_query_limit(1).for_query(expected_query) + end + end end describe "#update" do @@ -405,11 +426,12 @@ describe ProjectsController do end end - describe 'PUT #new_issue_address' do + describe 'PUT #new_issuable_address for issue' do subject do - put :new_issue_address, + put :new_issuable_address, namespace_id: project.namespace, - id: project + id: project, + issuable_type: 'issue' user.reload end @@ -428,7 +450,35 @@ describe ProjectsController do end it 'changes projects new issue address' do - expect { subject }.to change { project.new_issue_address(user) } + expect { subject }.to change { project.new_issuable_address(user, 'issue') } + end + end + + describe 'PUT #new_issuable_address for merge request' do + subject do + put :new_issuable_address, + namespace_id: project.namespace, + id: project, + issuable_type: 'merge_request' + user.reload + end + + before do + sign_in(user) + project.team << [user, :developer] + allow(Gitlab.config.incoming_email).to receive(:enabled).and_return(true) + end + + it 'has http status 200' do + expect(response).to have_http_status(200) + end + + it 'changes the user incoming email token' do + expect { subject }.to change { user.incoming_email_token } + end + + it 'changes projects new merge request address' do + expect { subject }.to change { project.new_issuable_address(user, 'merge_request') } end end diff --git a/spec/factories/appearances.rb b/spec/factories/appearances.rb index cf2a2b76bcb..860973024c9 100644 --- a/spec/factories/appearances.rb +++ b/spec/factories/appearances.rb @@ -4,5 +4,6 @@ FactoryGirl.define do factory :appearance do title "MepMep" description "This is my Community Edition instance" + new_project_guidelines "Custom project guidelines" end end diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb index cf38066dedc..c868525cbc0 100644 --- a/spec/factories/ci/builds.rb +++ b/spec/factories/ci/builds.rb @@ -154,36 +154,29 @@ FactoryGirl.define do runner factory: :ci_runner end - trait :artifacts do + trait :legacy_artifacts do after(:create) do |build, _| - build.artifacts_file = - fixture_file_upload(Rails.root.join('spec/fixtures/ci_build_artifacts.zip'), - 'application/zip') - - build.artifacts_metadata = - fixture_file_upload(Rails.root.join('spec/fixtures/ci_build_artifacts_metadata.gz'), - 'application/x-gzip') - - build.save! + build.update!( + legacy_artifacts_file: fixture_file_upload( + Rails.root.join('spec/fixtures/ci_build_artifacts.zip'), 'application/zip'), + legacy_artifacts_metadata: fixture_file_upload( + Rails.root.join('spec/fixtures/ci_build_artifacts_metadata.gz'), 'application/x-gzip') + ) end end - trait :artifacts_expired do - after(:create) do |build, _| - build.artifacts_file = - fixture_file_upload(Rails.root.join('spec/fixtures/ci_build_artifacts.zip'), - 'application/zip') - - build.artifacts_metadata = - fixture_file_upload(Rails.root.join('spec/fixtures/ci_build_artifacts_metadata.gz'), - 'application/x-gzip') - - build.artifacts_expire_at = 1.minute.ago - - build.save! + trait :artifacts do + after(:create) do |build| + create(:ci_job_artifact, :archive, job: build) + create(:ci_job_artifact, :metadata, job: build) + build.reload end end + trait :expired do + artifacts_expire_at 1.minute.ago + end + trait :with_commit do after(:build) do |build| allow(build).to receive(:commit).and_return build(:commit, :without_author) diff --git a/spec/factories/ci/job_artifacts.rb b/spec/factories/ci/job_artifacts.rb new file mode 100644 index 00000000000..538dc422832 --- /dev/null +++ b/spec/factories/ci/job_artifacts.rb @@ -0,0 +1,30 @@ +include ActionDispatch::TestProcess + +FactoryGirl.define do + factory :ci_job_artifact, class: Ci::JobArtifact do + job factory: :ci_build + file_type :archive + + after :build do |artifact| + artifact.project ||= artifact.job.project + end + + trait :archive do + file_type :archive + + after(:build) do |artifact, _| + artifact.file = fixture_file_upload( + Rails.root.join('spec/fixtures/ci_build_artifacts.zip'), 'application/zip') + end + end + + trait :metadata do + file_type :metadata + + after(:build) do |artifact, _| + artifact.file = fixture_file_upload( + Rails.root.join('spec/fixtures/ci_build_artifacts_metadata.gz'), 'application/x-gzip') + end + end + end +end diff --git a/spec/factories/clusters/cluster.rb b/spec/factories/clusters/clusters.rb index c4261178f2d..81866845a20 100644 --- a/spec/factories/clusters/cluster.rb +++ b/spec/factories/clusters/clusters.rb @@ -13,27 +13,20 @@ FactoryGirl.define do provider_type :user platform_type :kubernetes - platform_kubernetes do - create(:cluster_platform_kubernetes, :configured) - end + platform_kubernetes factory: [:cluster_platform_kubernetes, :configured] end trait :provided_by_gcp do provider_type :gcp platform_type :kubernetes - before(:create) do |cluster, evaluator| - cluster.platform_kubernetes = build(:cluster_platform_kubernetes, :configured) - cluster.provider_gcp = build(:cluster_provider_gcp, :created) - end + provider_gcp factory: [:cluster_provider_gcp, :created] + platform_kubernetes factory: [:cluster_platform_kubernetes, :configured] end trait :providing_by_gcp do provider_type :gcp - - provider_gcp do - create(:cluster_provider_gcp, :creating) - end + provider_gcp factory: [:cluster_provider_gcp, :creating] end end end diff --git a/spec/features/admin/admin_appearance_spec.rb b/spec/features/admin/admin_appearance_spec.rb index 5f3a37c1dcc..d91dcf76191 100644 --- a/spec/features/admin/admin_appearance_spec.rb +++ b/spec/features/admin/admin_appearance_spec.rb @@ -9,6 +9,7 @@ feature 'Admin Appearance' do fill_in 'appearance_title', with: 'MyCompany' fill_in 'appearance_description', with: 'dev server' + fill_in 'appearance_new_project_guidelines', with: 'Custom project guidelines' click_button 'Save' expect(current_path).to eq admin_appearances_path @@ -16,21 +17,39 @@ feature 'Admin Appearance' do expect(page).to have_field('appearance_title', with: 'MyCompany') expect(page).to have_field('appearance_description', with: 'dev server') + expect(page).to have_field('appearance_new_project_guidelines', with: 'Custom project guidelines') expect(page).to have_content 'Last edit' end - scenario 'Preview appearance' do + scenario 'Preview sign-in page appearance' do sign_in(create(:admin)) visit admin_appearances_path - click_link "Preview" + click_link "Sign-in page" - expect_page_has_custom_appearance(appearance) + expect_custom_sign_in_appearance(appearance) + end + + scenario 'Preview new project page appearance' do + sign_in(create(:admin)) + + visit admin_appearances_path + click_link "New project page" + + expect_custom_new_project_appearance(appearance) end scenario 'Custom sign-in page' do visit new_user_session_path - expect_page_has_custom_appearance(appearance) + + expect_custom_sign_in_appearance(appearance) + end + + scenario 'Custom new project page' do + sign_in create(:user) + visit new_project_path + + expect_custom_new_project_appearance(appearance) end scenario 'Appearance logo' do @@ -57,11 +76,15 @@ feature 'Admin Appearance' do expect(page).not_to have_css(header_logo_selector) end - def expect_page_has_custom_appearance(appearance) + def expect_custom_sign_in_appearance(appearance) expect(page).to have_content appearance.title expect(page).to have_content appearance.description end + def expect_custom_new_project_appearance(appearance) + expect(page).to have_content appearance.new_project_guidelines + end + def logo_selector '//img[data-src^="/uploads/-/system/appearance/logo"]' end diff --git a/spec/features/commits_spec.rb b/spec/features/commits_spec.rb index 98586ddbd81..c870910c8ea 100644 --- a/spec/features/commits_spec.rb +++ b/spec/features/commits_spec.rb @@ -89,7 +89,7 @@ describe 'Commits' do context 'Download artifacts' do before do - build.update_attributes(artifacts_file: artifacts_file) + build.update_attributes(legacy_artifacts_file: artifacts_file) end it do @@ -146,7 +146,7 @@ describe 'Commits' do context "when logged as reporter" do before do project.team << [user, :reporter] - build.update_attributes(artifacts_file: artifacts_file) + build.update_attributes(legacy_artifacts_file: artifacts_file) visit pipeline_path(pipeline) end @@ -168,7 +168,7 @@ describe 'Commits' do project.update( visibility_level: Gitlab::VisibilityLevel::INTERNAL, public_builds: false) - build.update_attributes(artifacts_file: artifacts_file) + build.update_attributes(legacy_artifacts_file: artifacts_file) visit pipeline_path(pipeline) end diff --git a/spec/features/issues_spec.rb b/spec/features/issues_spec.rb index b9af77f918a..852d9e368aa 100644 --- a/spec/features/issues_spec.rb +++ b/spec/features/issues_spec.rb @@ -365,16 +365,16 @@ describe 'Issues' do end it 'changes incoming email address token', :js do - find('.issue-email-modal-btn').click - previous_token = find('input#issue_email').value + find('.issuable-email-modal-btn').click + previous_token = find('input#issuable_email').value find('.incoming-email-token-reset').click wait_for_requests - expect(page).to have_no_field('issue_email', with: previous_token) - new_token = project1.new_issue_address(user.reload) + expect(page).to have_no_field('issuable_email', with: previous_token) + new_token = project1.new_issuable_address(user.reload, 'issue') expect(page).to have_field( - 'issue_email', + 'issuable_email', with: new_token ) end @@ -630,8 +630,8 @@ describe 'Issues' do end it 'click the button to show modal for the new email' do - page.within '#issue-email-modal' do - email = project.new_issue_address(user) + page.within '#issuable-email-modal' do + email = project.new_issuable_address(user, 'issue') expect(page).to have_selector("input[value='#{email}']") end diff --git a/spec/features/merge_requests/mini_pipeline_graph_spec.rb b/spec/features/merge_requests/mini_pipeline_graph_spec.rb index bac56270362..93c5e945453 100644 --- a/spec/features/merge_requests/mini_pipeline_graph_spec.rb +++ b/spec/features/merge_requests/mini_pipeline_graph_spec.rb @@ -28,14 +28,14 @@ feature 'Mini Pipeline Graph', :js do let(:artifacts_file2) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/png') } before do - create(:ci_build, pipeline: pipeline, artifacts_file: artifacts_file1) + create(:ci_build, pipeline: pipeline, legacy_artifacts_file: artifacts_file1) create(:ci_build, pipeline: pipeline, when: 'manual') end it 'avoids repeated database queries' do before = ActiveRecord::QueryRecorder.new { visit_merge_request(:json) } - create(:ci_build, pipeline: pipeline, artifacts_file: artifacts_file2) + create(:ci_build, pipeline: pipeline, legacy_artifacts_file: artifacts_file2) create(:ci_build, pipeline: pipeline, when: 'manual') after = ActiveRecord::QueryRecorder.new { visit_merge_request(:json) } diff --git a/spec/features/projects/clusters/applications_spec.rb b/spec/features/projects/clusters/applications_spec.rb new file mode 100644 index 00000000000..b34cd061ec6 --- /dev/null +++ b/spec/features/projects/clusters/applications_spec.rb @@ -0,0 +1,107 @@ +require 'spec_helper' + +feature 'Clusters Applications', :js do + include GoogleApi::CloudPlatformHelpers + + let(:project) { create(:project) } + let(:user) { create(:user) } + + before do + project.add_master(user) + sign_in(user) + end + + describe 'Installing applications' do + before do + visit project_cluster_path(project, cluster) + end + + context 'when cluster is being created' do + let(:cluster) { create(:cluster, :providing_by_gcp, projects: [project])} + + scenario 'user is unable to install applications' do + page.within('.js-cluster-application-row-helm') do + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button').text).to eq('Install') + end + end + end + + context 'when cluster is created' do + let(:cluster) { create(:cluster, :provided_by_gcp, projects: [project])} + + scenario 'user can install applications' do + page.within('.js-cluster-application-row-helm') do + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to be_nil + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Install') + end + end + + context 'when user installs Helm' do + before do + allow(ClusterInstallAppWorker).to receive(:perform_async).and_return(nil) + + page.within('.js-cluster-application-row-helm') do + page.find(:css, '.js-cluster-application-install-button').click + end + end + + it 'he sees status transition' do + page.within('.js-cluster-application-row-helm') do + # FE sends request and gets the response, then the buttons is "Install" + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Install') + + Clusters::Cluster.last.application_helm.make_installing! + + # FE starts polling and update the buttons to "Installing" + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Installing') + + Clusters::Cluster.last.application_helm.make_installed! + + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Installed') + end + + expect(page).to have_content('Helm Tiller was successfully installed on your cluster') + end + end + + context 'when user installs Ingress' do + context 'when user installs application: Ingress' do + before do + allow(ClusterInstallAppWorker).to receive(:perform_async).and_return(nil) + + create(:cluster_applications_helm, :installed, cluster: cluster) + + page.within('.js-cluster-application-row-ingress') do + page.find(:css, '.js-cluster-application-install-button').click + end + end + + it 'he sees status transition' do + page.within('.js-cluster-application-row-ingress') do + # FE sends request and gets the response, then the buttons is "Install" + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Install') + + Clusters::Cluster.last.application_ingress.make_installing! + + # FE starts polling and update the buttons to "Installing" + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Installing') + + Clusters::Cluster.last.application_ingress.make_installed! + + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Installed') + end + + expect(page).to have_content('Ingress was successfully installed on your cluster') + end + end + end + end + end +end diff --git a/spec/features/projects/clusters/gcp_spec.rb b/spec/features/projects/clusters/gcp_spec.rb new file mode 100644 index 00000000000..8a0da669147 --- /dev/null +++ b/spec/features/projects/clusters/gcp_spec.rb @@ -0,0 +1,136 @@ +require 'spec_helper' + +feature 'Gcp Cluster', :js do + include GoogleApi::CloudPlatformHelpers + + let(:project) { create(:project) } + let(:user) { create(:user) } + + before do + project.add_master(user) + gitlab_sign_in(user) + allow(Projects::ClustersController).to receive(:STATUS_POLLING_INTERVAL) { 100 } + end + + context 'when user has signed with Google' do + before do + allow_any_instance_of(Projects::Clusters::GcpController) + .to receive(:token_in_session).and_return('token') + allow_any_instance_of(Projects::Clusters::GcpController) + .to receive(:expires_at_in_session).and_return(1.hour.since.to_i.to_s) + end + + context 'when user does not have a cluster and visits cluster index page' do + before do + visit project_clusters_path(project) + + click_link 'Create on GKE' + end + + context 'when user filled form with valid parameters' do + before do + allow_any_instance_of(GoogleApi::CloudPlatform::Client) + .to receive(:projects_zones_clusters_create) do + OpenStruct.new( + self_link: 'projects/gcp-project-12345/zones/us-central1-a/operations/ope-123', + status: 'RUNNING' + ) + end + + allow(WaitForClusterCreationWorker).to receive(:perform_in).and_return(nil) + + fill_in 'cluster_provider_gcp_attributes_gcp_project_id', with: 'gcp-project-123' + fill_in 'cluster_name', with: 'dev-cluster' + click_button 'Create cluster' + end + + it 'user sees a cluster details page and creation status' do + expect(page).to have_content('Cluster is being created on Google Container Engine...') + + Clusters::Cluster.last.provider.make_created! + + expect(page).to have_content('Cluster was successfully created on Google Container Engine') + end + + it 'user sees a error if something worng during creation' do + expect(page).to have_content('Cluster is being created on Google Container Engine...') + + Clusters::Cluster.last.provider.make_errored!('Something wrong!') + + expect(page).to have_content('Something wrong!') + end + end + + context 'when user filled form with invalid parameters' do + before do + click_button 'Create cluster' + end + + it 'user sees a validation error' do + expect(page).to have_css('#error_explanation') + end + end + end + + context 'when user does have a cluster and visits cluster page' do + let(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } + + before do + visit project_cluster_path(project, cluster) + end + + it 'user sees a cluster details page' do + expect(page).to have_button('Save') + expect(page.find(:css, '.cluster-name').value).to eq(cluster.name) + end + + context 'when user disables the cluster' do + before do + page.find(:css, '.js-toggle-cluster').click + click_button 'Save' + end + + it 'user sees the successful message' do + expect(page).to have_content('Cluster was successfully updated.') + end + end + + context 'when user changes cluster parameters' do + before do + fill_in 'cluster_platform_kubernetes_attributes_namespace', with: 'my-namespace' + click_button 'Save changes' + end + + it 'user sees the successful message' do + expect(page).to have_content('Cluster was successfully updated.') + expect(cluster.reload.platform_kubernetes.namespace).to eq('my-namespace') + end + end + + context 'when user destroy the cluster' do + before do + page.accept_confirm do + click_link 'Remove integration' + end + end + + it 'user sees creation form with the successful message' do + expect(page).to have_content('Cluster integration was successfully removed.') + expect(page).to have_link('Create on GKE') + end + end + end + end + + context 'when user has not signed with Google' do + before do + visit project_clusters_path(project) + + click_link 'Create on GKE' + end + + it 'user sees a login page' do + expect(page).to have_css('.signin-with-google') + end + end +end diff --git a/spec/features/projects/clusters/user_spec.rb b/spec/features/projects/clusters/user_spec.rb new file mode 100644 index 00000000000..e97ba88f2f4 --- /dev/null +++ b/spec/features/projects/clusters/user_spec.rb @@ -0,0 +1,101 @@ +require 'spec_helper' + +feature 'User Cluster', :js do + include GoogleApi::CloudPlatformHelpers + + let(:project) { create(:project) } + let(:user) { create(:user) } + + before do + project.add_master(user) + gitlab_sign_in(user) + allow(Projects::ClustersController).to receive(:STATUS_POLLING_INTERVAL) { 100 } + end + + context 'when user does not have a cluster and visits cluster index page' do + before do + visit project_clusters_path(project) + + click_link 'Add an existing cluster' + end + + context 'when user filled form with valid parameters' do + before do + fill_in 'cluster_name', with: 'dev-cluster' + fill_in 'cluster_platform_kubernetes_attributes_api_url', with: 'http://example.com' + fill_in 'cluster_platform_kubernetes_attributes_token', with: 'my-token' + click_button 'Add cluster' + end + + it 'user sees a cluster details page' do + expect(page).to have_content('Enable cluster integration') + expect(page.find_field('cluster[name]').value).to eq('dev-cluster') + expect(page.find_field('cluster[platform_kubernetes_attributes][api_url]').value) + .to have_content('http://example.com') + expect(page.find_field('cluster[platform_kubernetes_attributes][token]').value) + .to have_content('my-token') + end + end + + context 'when user filled form with invalid parameters' do + before do + click_button 'Add cluster' + end + + it 'user sees a validation error' do + expect(page).to have_css('#error_explanation') + end + end + end + + context 'when user does have a cluster and visits cluster page' do + let(:cluster) { create(:cluster, :provided_by_user, projects: [project]) } + + before do + visit project_cluster_path(project, cluster) + end + + it 'user sees a cluster details page' do + expect(page).to have_button('Save') + end + + context 'when user disables the cluster' do + before do + page.find(:css, '.js-toggle-cluster').click + fill_in 'cluster_name', with: 'dev-cluster' + click_button 'Save' + end + + it 'user sees the successful message' do + expect(page).to have_content('Cluster was successfully updated.') + end + end + + context 'when user changes cluster parameters' do + before do + fill_in 'cluster_name', with: 'my-dev-cluster' + fill_in 'cluster_platform_kubernetes_attributes_namespace', with: 'my-namespace' + click_button 'Save changes' + end + + it 'user sees the successful message' do + expect(page).to have_content('Cluster was successfully updated.') + expect(cluster.reload.name).to eq('my-dev-cluster') + expect(cluster.reload.platform_kubernetes.namespace).to eq('my-namespace') + end + end + + context 'when user destroy the cluster' do + before do + page.accept_confirm do + click_link 'Remove integration' + end + end + + it 'user sees creation form with the successful message' do + expect(page).to have_content('Cluster integration was successfully removed.') + expect(page).to have_link('Add an existing cluster') + end + end + end +end diff --git a/spec/features/projects/clusters_spec.rb b/spec/features/projects/clusters_spec.rb index 197e6df4997..4243c4fd266 100644 --- a/spec/features/projects/clusters_spec.rb +++ b/spec/features/projects/clusters_spec.rb @@ -3,204 +3,23 @@ require 'spec_helper' feature 'Clusters', :js do include GoogleApi::CloudPlatformHelpers - let!(:project) { create(:project, :repository) } - let!(:user) { create(:user) } + let(:project) { create(:project) } + let(:user) { create(:user) } before do project.add_master(user) gitlab_sign_in(user) end - context 'when user has signed in Google' do - before do - allow_any_instance_of(Projects::ClustersController) - .to receive(:token_in_session).and_return('token') - allow_any_instance_of(Projects::ClustersController) - .to receive(:expires_at_in_session).and_return(1.hour.since.to_i.to_s) - end - - context 'when user does not have a cluster and visits cluster index page' do - before do - visit project_clusters_path(project) - - click_link 'Create on GKE' - end - - it 'user sees a new page' do - expect(page).to have_button('Create cluster') - end - - context 'when user filled form with valid parameters' do - before do - double.tap do |dbl| - allow(dbl).to receive(:status).and_return('RUNNING') - allow(dbl).to receive(:self_link) - .and_return('projects/gcp-project-12345/zones/us-central1-a/operations/ope-123') - allow_any_instance_of(GoogleApi::CloudPlatform::Client) - .to receive(:projects_zones_clusters_create).and_return(dbl) - end - - allow(WaitForClusterCreationWorker).to receive(:perform_in).and_return(nil) - - fill_in 'cluster_provider_gcp_attributes_gcp_project_id', with: 'gcp-project-123' - fill_in 'cluster_name', with: 'dev-cluster' - click_button 'Create cluster' - end - - it 'user sees a cluster details page and creation status' do - expect(page).to have_content('Cluster is being created on Google Container Engine...') - - # Application Installation buttons - page.within('.js-cluster-application-row-helm') do - expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') - expect(page.find(:css, '.js-cluster-application-install-button').text).to eq('Install') - end - - Clusters::Cluster.last.provider.make_created! - - expect(page).to have_content('Cluster was successfully created on Google Container Engine') - end - - it 'user sees a error if something worng during creation' do - expect(page).to have_content('Cluster is being created on Google Container Engine...') - - Clusters::Cluster.last.provider.make_errored!('Something wrong!') - - expect(page).to have_content('Something wrong!') - end - end - - context 'when user filled form with invalid parameters' do - before do - click_button 'Create cluster' - end - - it 'user sees a validation error' do - expect(page).to have_css('#error_explanation') - end - end - end - - context 'when user has a cluster and visits cluster index page' do - let!(:cluster) { create(:cluster, :project, :provided_by_gcp) } - let(:project) { cluster.project } - - before do - visit project_clusters_path(project) - end - - it 'user sees an cluster details page' do - expect(page).to have_button('Save') - expect(page.find(:css, '.cluster-name').value).to eq(cluster.name) - - # Application Installation buttons - page.within('.js-cluster-application-row-helm') do - expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to be_nil - expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Install') - end - end - - context 'when user installs application: Helm Tiller' do - before do - allow(ClusterInstallAppWorker).to receive(:perform_async).and_return(nil) - - page.within('.js-cluster-application-row-helm') do - page.find(:css, '.js-cluster-application-install-button').click - end - end - - it 'user sees status transition' do - page.within('.js-cluster-application-row-helm') do - # FE sends request and gets the response, then the buttons is "Install" - expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') - expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Install') - - Clusters::Cluster.last.application_helm.make_installing! - - # FE starts polling and update the buttons to "Installing" - expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') - expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Installing') - - Clusters::Cluster.last.application_helm.make_installed! - - expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') - expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Installed') - end - - expect(page).to have_content('Helm Tiller was successfully installed on your cluster') - end - end - - context 'when user installs application: Ingress' do - before do - allow(ClusterInstallAppWorker).to receive(:perform_async).and_return(nil) - # Helm Tiller needs to be installed before you can install Ingress - create(:cluster_applications_helm, :installed, cluster: cluster) - - visit project_clusters_path(project) - - page.within('.js-cluster-application-row-ingress') do - page.find(:css, '.js-cluster-application-install-button').click - end - end - - it 'user sees status transition' do - page.within('.js-cluster-application-row-ingress') do - # FE sends request and gets the response, then the buttons is "Install" - expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') - expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Install') - - Clusters::Cluster.last.application_ingress.make_installing! - - # FE starts polling and update the buttons to "Installing" - expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') - expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Installing') - - Clusters::Cluster.last.application_ingress.make_installed! - - expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') - expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Installed') - end - - expect(page).to have_content('Ingress was successfully installed on your cluster') - end - end - - context 'when user disables the cluster' do - before do - page.find(:css, '.js-toggle-cluster').click - click_button 'Save' - end - - it 'user sees the succeccful message' do - expect(page).to have_content('Cluster was successfully updated.') - end - end - - context 'when user destory the cluster' do - before do - page.accept_confirm do - click_link 'Remove integration' - end - end - - it 'user sees creation form with the succeccful message' do - expect(page).to have_content('Cluster integration was successfully removed.') - expect(page).to have_link('Create on GKE') - end - end - end - end - - context 'when user has not signed in Google' do + context 'when user does not have a cluster and visits cluster index page' do before do visit project_clusters_path(project) click_link 'Create on GKE' end - it 'user sees a login page' do - expect(page).to have_css('.signin-with-google') + it 'user sees a new page' do + expect(page).to have_button('Create cluster') end end end diff --git a/spec/features/projects/jobs_spec.rb b/spec/features/projects/jobs_spec.rb index c2a0d2395a9..0b0d5a2dce8 100644 --- a/spec/features/projects/jobs_spec.rb +++ b/spec/features/projects/jobs_spec.rb @@ -187,7 +187,7 @@ feature 'Jobs' do context "Download artifacts" do before do - job.update_attributes(artifacts_file: artifacts_file) + job.update_attributes(legacy_artifacts_file: artifacts_file) visit project_job_path(project, job) end @@ -198,7 +198,7 @@ feature 'Jobs' do context 'Artifacts expire date' do before do - job.update_attributes(artifacts_file: artifacts_file, + job.update_attributes(legacy_artifacts_file: artifacts_file, artifacts_expire_at: expire_at) visit project_job_path(project, job) @@ -422,14 +422,14 @@ feature 'Jobs' do describe "GET /:project/jobs/:id/download" do before do - job.update_attributes(artifacts_file: artifacts_file) + job.update_attributes(legacy_artifacts_file: artifacts_file) visit project_job_path(project, job) click_link 'Download' end context "Build from other project" do before do - job2.update_attributes(artifacts_file: artifacts_file) + job2.update_attributes(legacy_artifacts_file: artifacts_file) visit download_project_job_artifacts_path(project, job2) end diff --git a/spec/features/projects/pipelines/pipelines_spec.rb b/spec/features/projects/pipelines/pipelines_spec.rb index a1b1d94ae06..b87b47d0e1a 100644 --- a/spec/features/projects/pipelines/pipelines_spec.rb +++ b/spec/features/projects/pipelines/pipelines_spec.rb @@ -304,7 +304,7 @@ describe 'Pipelines', :js do context 'with artifacts expired' do let!(:with_artifacts_expired) do - create(:ci_build, :artifacts_expired, :success, + create(:ci_build, :expired, :success, pipeline: pipeline, name: 'rspec', stage: 'test') diff --git a/spec/fixtures/emails/valid_new_merge_request.eml b/spec/fixtures/emails/valid_new_merge_request.eml new file mode 100644 index 00000000000..480675a6d7e --- /dev/null +++ b/spec/fixtures/emails/valid_new_merge_request.eml @@ -0,0 +1,18 @@ +Return-Path: <jake@adventuretime.ooo> +Received: from iceking.adventuretime.ooo ([unix socket]) by iceking (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400 +Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <incoming+gitlabhq/gitlabhq@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 17:03:50 -0400 +Received: by mail-ie0-f180.google.com with SMTP id f4so21977375iea.25 for <incoming+gitlabhq/gitlabhq@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 14:03:48 -0700 +Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700 +Date: Thu, 13 Jun 2013 17:03:48 -0400 +From: Jake the Dog <jake@adventuretime.ooo> +To: incoming+gitlabhq/gitlabhq+merge-request+auth_token@appmail.adventuretime.ooo +Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com> +Subject: feature +Mime-Version: 1.0 +Content-Type: text/plain; + charset=ISO-8859-1 +Content-Transfer-Encoding: 7bit +X-Sieve: CMU Sieve 2.2 +X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu, + 13 Jun 2013 14:03:48 -0700 (PDT) +X-Scanned-By: MIMEDefang 2.69 on IPv6:2001:470:1d:165::1 diff --git a/spec/fixtures/emails/valid_new_merge_request_no_subject.eml b/spec/fixtures/emails/valid_new_merge_request_no_subject.eml new file mode 100644 index 00000000000..27eb1b7d922 --- /dev/null +++ b/spec/fixtures/emails/valid_new_merge_request_no_subject.eml @@ -0,0 +1,18 @@ +Return-Path: <jake@adventuretime.ooo> +Received: from iceking.adventuretime.ooo ([unix socket]) by iceking (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400 +Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <incoming+gitlabhq/gitlabhq@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 17:03:50 -0400 +Received: by mail-ie0-f180.google.com with SMTP id f4so21977375iea.25 for <incoming+gitlabhq/gitlabhq@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 14:03:48 -0700 +Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700 +Date: Thu, 13 Jun 2013 17:03:48 -0400 +From: Jake the Dog <jake@adventuretime.ooo> +To: incoming+gitlabhq/gitlabhq+merge-request+auth_token@appmail.adventuretime.ooo +Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com> +Subject: +Mime-Version: 1.0 +Content-Type: text/plain; + charset=ISO-8859-1 +Content-Transfer-Encoding: 7bit +X-Sieve: CMU Sieve 2.2 +X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu, + 13 Jun 2013 14:03:48 -0700 (PDT) +X-Scanned-By: MIMEDefang 2.69 on IPv6:2001:470:1d:165::1 diff --git a/spec/javascripts/clusters/clusters_bundle_spec.js b/spec/javascripts/clusters/clusters_bundle_spec.js index 027e8001053..6d6e71cc215 100644 --- a/spec/javascripts/clusters/clusters_bundle_spec.js +++ b/spec/javascripts/clusters/clusters_bundle_spec.js @@ -36,6 +36,20 @@ describe('Clusters', () => { }); }); + describe('showToken', () => { + it('should update tye field type', () => { + cluster.showTokenButton.click(); + expect( + cluster.tokenField.getAttribute('type'), + ).toEqual('text'); + + cluster.showTokenButton.click(); + expect( + cluster.tokenField.getAttribute('type'), + ).toEqual('password'); + }); + }); + describe('checkForNewInstalls', () => { const INITIAL_APP_MAP = { helm: { status: null, title: 'Helm Tiller' }, @@ -113,7 +127,7 @@ describe('Clusters', () => { }); describe('when cluster is created', () => { - it('should show the success container', () => { + it('should show the success container and fresh the page', () => { cluster.updateContainer(null, 'created'); expect( diff --git a/spec/javascripts/issuable_spec.js b/spec/javascripts/issuable_spec.js index ceee08d47c5..5a9112716f4 100644 --- a/spec/javascripts/issuable_spec.js +++ b/spec/javascripts/issuable_spec.js @@ -26,7 +26,7 @@ describe('Issuable', () => { document.body.appendChild(element); const input = document.createElement('input'); - input.setAttribute('id', 'issue_email'); + input.setAttribute('id', 'issuable_email'); document.body.appendChild(input); Issuable = new IssuableIndex('issue_'); diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 3c8350b3aad..664ba0f7234 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -942,8 +942,8 @@ describe Gitlab::Database::MigrationHelpers do end it 'queues jobs in groups of buffer size 1' do - expect(BackgroundMigrationWorker).to receive(:perform_bulk).with([['FooJob', [id1, id2]]]) - expect(BackgroundMigrationWorker).to receive(:perform_bulk).with([['FooJob', [id3, id3]]]) + expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]]]) + expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id3, id3]]]) model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) end @@ -960,8 +960,8 @@ describe Gitlab::Database::MigrationHelpers do end it 'queues jobs in bulk all at once (big buffer size)' do - expect(BackgroundMigrationWorker).to receive(:perform_bulk).with([['FooJob', [id1, id2]], - ['FooJob', [id3, id3]]]) + expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]], + ['FooJob', [id3, id3]]]) model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) end diff --git a/spec/lib/gitlab/email/handler/create_merge_request_handler_spec.rb b/spec/lib/gitlab/email/handler/create_merge_request_handler_spec.rb new file mode 100644 index 00000000000..e361d1a7393 --- /dev/null +++ b/spec/lib/gitlab/email/handler/create_merge_request_handler_spec.rb @@ -0,0 +1,84 @@ +require 'spec_helper' +require_relative '../email_shared_blocks' + +describe Gitlab::Email::Handler::CreateMergeRequestHandler do + include_context :email_shared_context + it_behaves_like :reply_processing_shared_examples + + before do + stub_incoming_email_setting(enabled: true, address: "incoming+%{key}@appmail.adventuretime.ooo") + stub_config_setting(host: 'localhost') + end + + let(:email_raw) { fixture_file('emails/valid_new_merge_request.eml') } + let(:namespace) { create(:namespace, path: 'gitlabhq') } + + # project's git repository is not deleted when project is deleted + # between tests. Then tests fail because re-creation of the project with + # the same name fails on existing git repository -> skip_disk_validation + # ignores repository existence on disk + let!(:project) { create(:project, :public, :repository, skip_disk_validation: true, namespace: namespace, path: 'gitlabhq') } + let!(:user) do + create( + :user, + email: 'jake@adventuretime.ooo', + incoming_email_token: 'auth_token' + ) + end + + context "as a non-developer" do + before do + project.add_guest(user) + end + + it "raises UserNotAuthorizedError if the user is not a member" do + expect { receiver.execute }.to raise_error(Gitlab::Email::UserNotAuthorizedError) + end + end + + context "as a developer" do + before do + project.add_developer(user) + end + + context "when everything is fine" do + it "creates a new merge request" do + expect { receiver.execute }.to change { project.merge_requests.count }.by(1) + merge_request = project.merge_requests.last + + expect(merge_request.author).to eq(user) + expect(merge_request.source_branch).to eq('feature') + expect(merge_request.title).to eq('Feature added') + expect(merge_request.target_branch).to eq(project.default_branch) + end + end + + context "something is wrong" do + context "when the merge request could not be saved" do + before do + allow_any_instance_of(MergeRequest).to receive(:save).and_return(false) + end + + it "raises an InvalidMergeRequestError" do + expect { receiver.execute }.to raise_error(Gitlab::Email::InvalidMergeRequestError) + end + end + + context "when we can't find the incoming_email_token" do + let(:email_raw) { fixture_file("emails/wrong_incoming_email_token.eml") } + + it "raises an UserNotFoundError" do + expect { receiver.execute }.to raise_error(Gitlab::Email::UserNotFoundError) + end + end + + context "when the subject is blank" do + let(:email_raw) { fixture_file("emails/valid_new_merge_request_no_subject.eml") } + + it "raises an InvalidMergeRequestError" do + expect { receiver.execute }.to raise_error(Gitlab::Email::InvalidMergeRequestError) + end + end + end + end +end diff --git a/spec/lib/gitlab/email/handler_spec.rb b/spec/lib/gitlab/email/handler_spec.rb new file mode 100644 index 00000000000..650b01c4df4 --- /dev/null +++ b/spec/lib/gitlab/email/handler_spec.rb @@ -0,0 +1,17 @@ +require 'spec_helper' + +describe Gitlab::Email::Handler do + describe '.for' do + it 'picks issue handler if there is not merge request prefix' do + expect(described_class.for('email', 'project+key')).to be_an_instance_of(Gitlab::Email::Handler::CreateIssueHandler) + end + + it 'picks merge request handler if there is merge request key' do + expect(described_class.for('email', 'project+merge-request+key')).to be_an_instance_of(Gitlab::Email::Handler::CreateMergeRequestHandler) + end + + it 'returns nil if no handler is found' do + expect(described_class.for('email', '')).to be_nil + end + end +end diff --git a/spec/lib/gitlab/sidekiq_config_spec.rb b/spec/lib/gitlab/sidekiq_config_spec.rb new file mode 100644 index 00000000000..09f95be2213 --- /dev/null +++ b/spec/lib/gitlab/sidekiq_config_spec.rb @@ -0,0 +1,24 @@ +require 'rails_helper' + +describe Gitlab::SidekiqConfig do + describe '.workers' do + it 'includes all workers' do + workers = described_class.workers + + expect(workers).to include(PostReceive) + expect(workers).to include(MergeWorker) + end + end + + describe '.worker_queues' do + it 'includes all queues' do + queues = described_class.worker_queues + + expect(queues).to include('post_receive') + expect(queues).to include('merge') + expect(queues).to include('cronjob') + expect(queues).to include('mailers') + expect(queues).to include('default') + end + end +end diff --git a/spec/migrations/migrate_old_artifacts_spec.rb b/spec/migrations/migrate_old_artifacts_spec.rb index 81366d15b34..92eb1d9ce86 100644 --- a/spec/migrations/migrate_old_artifacts_spec.rb +++ b/spec/migrations/migrate_old_artifacts_spec.rb @@ -16,20 +16,22 @@ describe MigrateOldArtifacts do end context 'with migratable data' do - let(:project1) { create(:project, ci_id: 2) } - let(:project2) { create(:project, ci_id: 3) } - let(:project3) { create(:project) } + set(:project1) { create(:project, ci_id: 2) } + set(:project2) { create(:project, ci_id: 3) } + set(:project3) { create(:project) } - let(:pipeline1) { create(:ci_empty_pipeline, project: project1) } - let(:pipeline2) { create(:ci_empty_pipeline, project: project2) } - let(:pipeline3) { create(:ci_empty_pipeline, project: project3) } + set(:pipeline1) { create(:ci_empty_pipeline, project: project1) } + set(:pipeline2) { create(:ci_empty_pipeline, project: project2) } + set(:pipeline3) { create(:ci_empty_pipeline, project: project3) } let!(:build_with_legacy_artifacts) { create(:ci_build, pipeline: pipeline1) } let!(:build_without_artifacts) { create(:ci_build, pipeline: pipeline1) } - let!(:build2) { create(:ci_build, :artifacts, pipeline: pipeline2) } - let!(:build3) { create(:ci_build, :artifacts, pipeline: pipeline3) } + let!(:build2) { create(:ci_build, pipeline: pipeline2) } + let!(:build3) { create(:ci_build, pipeline: pipeline3) } before do + setup_builds(build2, build3) + store_artifacts_in_legacy_path(build_with_legacy_artifacts) end @@ -38,7 +40,7 @@ describe MigrateOldArtifacts do end it "legacy artifacts are set" do - expect(build_with_legacy_artifacts.artifacts_file_identifier).not_to be_nil + expect(build_with_legacy_artifacts.legacy_artifacts_file_identifier).not_to be_nil end describe '#min_id' do @@ -113,5 +115,24 @@ describe MigrateOldArtifacts do build.project.ci_id.to_s, build.id.to_s) end + + def new_legacy_path(build) + File.join(directory, + build.created_at.utc.strftime('%Y_%m'), + build.project_id.to_s, + build.id.to_s) + end + + def setup_builds(*builds) + builds.each do |build| + FileUtils.mkdir_p(new_legacy_path(build)) + + build.update_columns( + artifacts_file: 'ci_build_artifacts.zip', + artifacts_metadata: 'ci_build_artifacts_metadata.gz') + + build.reload + end + end end end diff --git a/spec/models/appearance_spec.rb b/spec/models/appearance_spec.rb index 49f44525b29..56b5d616284 100644 --- a/spec/models/appearance_spec.rb +++ b/spec/models/appearance_spec.rb @@ -5,9 +5,6 @@ describe Appearance do it { is_expected.to be_valid } - it { is_expected.to validate_presence_of(:title) } - it { is_expected.to validate_presence_of(:description) } - it { is_expected.to have_many(:uploads).dependent(:destroy) } describe '.current', :use_clean_rails_memory_store_caching do diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 1795ee8e9a4..9070692abfe 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -23,6 +23,8 @@ describe Ci::Build do it { is_expected.to respond_to(:has_trace?) } it { is_expected.to respond_to(:trace) } + it { is_expected.to be_a(ArtifactMigratable) } + describe 'callbacks' do context 'when running after_create callback' do it 'triggers asynchronous build hooks worker' do @@ -130,34 +132,55 @@ describe Ci::Build do end describe '#artifacts?' do - subject { build.artifacts? } + context 'when new artifacts are used' do + let(:build) { create(:ci_build, :artifacts) } - context 'artifacts archive does not exist' do - before do - build.update_attributes(artifacts_file: nil) + subject { build.artifacts? } + + context 'artifacts archive does not exist' do + let(:build) { create(:ci_build) } + + it { is_expected.to be_falsy } end - it { is_expected.to be_falsy } - end + context 'artifacts archive exists' do + it { is_expected.to be_truthy } - context 'artifacts archive exists' do - let(:build) { create(:ci_build, :artifacts) } - it { is_expected.to be_truthy } + context 'is expired' do + let!(:build) { create(:ci_build, :artifacts, :expired) } - context 'is expired' do - before do - build.update(artifacts_expire_at: Time.now - 7.days) + it { is_expected.to be_falsy } end + context 'is not expired' do + it { is_expected.to be_truthy } + end + end + end + + context 'when legacy artifacts are used' do + let(:build) { create(:ci_build, :legacy_artifacts) } + + subject { build.artifacts? } + + context 'artifacts archive does not exist' do + let(:build) { create(:ci_build) } + it { is_expected.to be_falsy } end - context 'is not expired' do - before do - build.update(artifacts_expire_at: Time.now + 7.days) + context 'artifacts archive exists' do + it { is_expected.to be_truthy } + + context 'is expired' do + let!(:build) { create(:ci_build, :legacy_artifacts, :expired) } + + it { is_expected.to be_falsy } end - it { is_expected.to be_truthy } + context 'is not expired' do + it { is_expected.to be_truthy } + end end end end @@ -612,71 +635,144 @@ describe Ci::Build do describe '#erasable?' do subject { build.erasable? } + it { is_expected.to eq false } end end context 'build is erasable' do - let!(:build) { create(:ci_build, :trace, :success, :artifacts) } + context 'new artifacts' do + let!(:build) { create(:ci_build, :trace, :success, :artifacts) } - describe '#erase' do - before do - build.erase(erased_by: user) - end + describe '#erase' do + before do + build.erase(erased_by: user) + end - context 'erased by user' do - let!(:user) { create(:user, username: 'eraser') } + context 'erased by user' do + let!(:user) { create(:user, username: 'eraser') } - include_examples 'erasable' + include_examples 'erasable' - it 'records user who erased a build' do - expect(build.erased_by).to eq user + it 'records user who erased a build' do + expect(build.erased_by).to eq user + end end - end - context 'erased by system' do - let(:user) { nil } + context 'erased by system' do + let(:user) { nil } - include_examples 'erasable' + include_examples 'erasable' - it 'does not set user who erased a build' do - expect(build.erased_by).to be_nil + it 'does not set user who erased a build' do + expect(build.erased_by).to be_nil + end end end - end - describe '#erasable?' do - subject { build.erasable? } - it { is_expected.to be_truthy } - end + describe '#erasable?' do + subject { build.erasable? } + it { is_expected.to be_truthy } + end - describe '#erased?' do - let!(:build) { create(:ci_build, :trace, :success, :artifacts) } - subject { build.erased? } + describe '#erased?' do + let!(:build) { create(:ci_build, :trace, :success, :artifacts) } + subject { build.erased? } - context 'job has not been erased' do - it { is_expected.to be_falsey } + context 'job has not been erased' do + it { is_expected.to be_falsey } + end + + context 'job has been erased' do + before do + build.erase + end + + it { is_expected.to be_truthy } + end end - context 'job has been erased' do + context 'metadata and build trace are not available' do + let!(:build) { create(:ci_build, :success, :artifacts) } + before do - build.erase + build.remove_artifacts_metadata! end - it { is_expected.to be_truthy } + describe '#erase' do + it 'does not raise error' do + expect { build.erase }.not_to raise_error + end + end end end + end - context 'metadata and build trace are not available' do - let!(:build) { create(:ci_build, :success, :artifacts) } + context 'old artifacts' do + context 'build is erasable' do + context 'new artifacts' do + let!(:build) { create(:ci_build, :trace, :success, :legacy_artifacts) } - before do - build.remove_artifacts_metadata! - end + describe '#erase' do + before do + build.erase(erased_by: user) + end - describe '#erase' do - it 'does not raise error' do - expect { build.erase }.not_to raise_error + context 'erased by user' do + let!(:user) { create(:user, username: 'eraser') } + + include_examples 'erasable' + + it 'records user who erased a build' do + expect(build.erased_by).to eq user + end + end + + context 'erased by system' do + let(:user) { nil } + + include_examples 'erasable' + + it 'does not set user who erased a build' do + expect(build.erased_by).to be_nil + end + end + end + + describe '#erasable?' do + subject { build.erasable? } + it { is_expected.to be_truthy } + end + + describe '#erased?' do + let!(:build) { create(:ci_build, :trace, :success, :legacy_artifacts) } + subject { build.erased? } + + context 'job has not been erased' do + it { is_expected.to be_falsey } + end + + context 'job has been erased' do + before do + build.erase + end + + it { is_expected.to be_truthy } + end + end + + context 'metadata and build trace are not available' do + let!(:build) { create(:ci_build, :success, :legacy_artifacts) } + + before do + build.remove_artifacts_metadata! + end + + describe '#erase' do + it 'does not raise error' do + expect { build.erase }.not_to raise_error + end + end end end end @@ -912,11 +1008,23 @@ describe Ci::Build do describe '#keep_artifacts!' do let(:build) { create(:ci_build, artifacts_expire_at: Time.now + 7.days) } + subject { build.keep_artifacts! } + it 'to reset expire_at' do - build.keep_artifacts! + subject expect(build.artifacts_expire_at).to be_nil end + + context 'when having artifacts files' do + let!(:artifact) { create(:ci_job_artifact, job: build, expire_in: '7 days') } + + it 'to reset dependent objects' do + subject + + expect(artifact.reload.expire_at).to be_nil + end + end end describe '#merge_request' do @@ -1241,10 +1349,10 @@ describe Ci::Build do context 'when config does not have a questioned job' do let(:config) do YAML.dump({ - test_other: { - script: 'Hello World' - } - }) + test_other: { + script: 'Hello World' + } + }) end it { is_expected.to eq('on_success') } @@ -1253,11 +1361,11 @@ describe Ci::Build do context 'when config has `when`' do let(:config) do YAML.dump({ - test: { - script: 'Hello World', - when: 'always' - } - }) + test: { + script: 'Hello World', + when: 'always' + } + }) end it { is_expected.to eq('always') } @@ -1338,10 +1446,10 @@ describe Ci::Build do let!(:environment) do create(:environment, - project: build.project, - name: 'production', - slug: 'prod-slug', - external_url: '') + project: build.project, + name: 'production', + slug: 'prod-slug', + external_url: '') end before do @@ -1565,8 +1673,8 @@ describe Ci::Build do let!(:pipeline_schedule_variable) do create(:ci_pipeline_schedule_variable, - key: 'SCHEDULE_VARIABLE_KEY', - pipeline_schedule: pipeline_schedule) + key: 'SCHEDULE_VARIABLE_KEY', + pipeline_schedule: pipeline_schedule) end before do @@ -1708,8 +1816,8 @@ describe Ci::Build do allow_any_instance_of(Project) .to receive(:secret_variables_for) .with(ref: 'master', environment: nil) do - [create(:ci_variable, key: 'secret', value: 'value')] - end + [create(:ci_variable, key: 'secret', value: 'value')] + end allow_any_instance_of(Ci::Pipeline) .to receive(:predefined_variables) { [pipeline_pre_var] } diff --git a/spec/models/ci/job_artifact_spec.rb b/spec/models/ci/job_artifact_spec.rb new file mode 100644 index 00000000000..0e18a326c68 --- /dev/null +++ b/spec/models/ci/job_artifact_spec.rb @@ -0,0 +1,74 @@ +require 'spec_helper' + +describe Ci::JobArtifact do + set(:artifact) { create(:ci_job_artifact, :archive) } + + describe "Associations" do + it { is_expected.to belong_to(:project) } + it { is_expected.to belong_to(:job) } + end + + it { is_expected.to respond_to(:file) } + it { is_expected.to respond_to(:created_at) } + it { is_expected.to respond_to(:updated_at) } + + describe '#set_size' do + it 'sets the size' do + expect(artifact.size).to eq(106365) + end + end + + describe '#file' do + subject { artifact.file } + + context 'the uploader api' do + it { is_expected.to respond_to(:store_dir) } + it { is_expected.to respond_to(:cache_dir) } + it { is_expected.to respond_to(:work_dir) } + end + end + + describe '#expire_in' do + subject { artifact.expire_in } + + it { is_expected.to be_nil } + + context 'when expire_at is specified' do + let(:expire_at) { Time.now + 7.days } + + before do + artifact.expire_at = expire_at + end + + it { is_expected.to be_within(5).of(expire_at - Time.now) } + end + end + + describe '#expire_in=' do + subject { artifact.expire_in } + + it 'when assigning valid duration' do + artifact.expire_in = '7 days' + + is_expected.to be_within(10).of(7.days.to_i) + end + + it 'when assigning invalid duration' do + expect { artifact.expire_in = '7 elephants' }.to raise_error(ChronicDuration::DurationParseError) + + is_expected.to be_nil + end + + it 'when resetting value' do + artifact.expire_in = nil + + is_expected.to be_nil + end + + it 'when setting to 0' do + artifact.expire_in = '0' + + is_expected.to be_nil + end + end +end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 90b768f595e..3817f20bfe7 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -531,7 +531,7 @@ describe Namespace do end end - describe '#has_forks_of?' do + describe '#find_fork_of?' do let(:project) { create(:project, :public) } let!(:forked_project) { fork_project(project, namespace.owner, namespace: namespace) } @@ -550,5 +550,13 @@ describe Namespace do expect(other_namespace.find_fork_of(project)).to eq(other_fork) end + + context 'with request store enabled', :request_store do + it 'only queries once' do + expect(project.fork_network).to receive(:find_forks_in).once.and_call_original + + 2.times { namespace.find_fork_of(project) } + end + end end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 000d5b7126d..6bda1eb15a8 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -451,7 +451,7 @@ describe Project do end end - describe "#new_issue_address" do + describe "#new_issuable_address" do let(:project) { create(:project, path: "somewhere") } let(:user) { create(:user) } @@ -463,7 +463,13 @@ describe Project do it 'returns the address to create a new issue' do address = "p+#{project.full_path}+#{user.incoming_email_token}@gl.ab" - expect(project.new_issue_address(user)).to eq(address) + expect(project.new_issuable_address(user, 'issue')).to eq(address) + end + + it 'returns the address to create a new merge request' do + address = "p+#{project.full_path}+merge-request+#{user.incoming_email_token}@gl.ab" + + expect(project.new_issuable_address(user, 'merge_request')).to eq(address) end end @@ -473,7 +479,11 @@ describe Project do end it 'returns nil' do - expect(project.new_issue_address(user)).to be_nil + expect(project.new_issuable_address(user, 'issue')).to be_nil + end + + it 'returns nil' do + expect(project.new_issuable_address(user, 'merge_request')).to be_nil end end end diff --git a/spec/models/project_statistics_spec.rb b/spec/models/project_statistics_spec.rb index 59e20e84c2f..e78ed1df821 100644 --- a/spec/models/project_statistics_spec.rb +++ b/spec/models/project_statistics_spec.rb @@ -133,15 +133,29 @@ describe ProjectStatistics do describe '#update_build_artifacts_size' do let!(:pipeline) { create(:ci_pipeline, project: project) } - let!(:build1) { create(:ci_build, pipeline: pipeline, artifacts_size: 45.megabytes) } - let!(:build2) { create(:ci_build, pipeline: pipeline, artifacts_size: 56.megabytes) } - before do - statistics.update_build_artifacts_size + context 'when new job artifacts are calculated' do + let(:ci_build) { create(:ci_build, pipeline: pipeline) } + + before do + create(:ci_job_artifact, :archive, project: pipeline.project, job: ci_build) + end + + it "stores the size of related build artifacts" do + statistics.update_build_artifacts_size + + expect(statistics.build_artifacts_size).to be(106365) + end end - it "stores the size of related build artifacts" do - expect(statistics.build_artifacts_size).to eq 101.megabytes + context 'when legacy artifacts are used' do + let!(:ci_build) { create(:ci_build, pipeline: pipeline, artifacts_size: 10.megabytes) } + + it "stores the size of related build artifacts" do + statistics.update_build_artifacts_size + + expect(statistics.build_artifacts_size).to eq(10.megabytes) + end end end diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb index 47f4ccd4887..679d391caa5 100644 --- a/spec/requests/api/runner_spec.rb +++ b/spec/requests/api/runner_spec.rb @@ -945,7 +945,7 @@ describe API::Runner do context 'when artifacts are being stored inside of tmp path' do before do # by configuring this path we allow to pass temp file from any path - allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return('/') + allow(JobArtifactUploader).to receive(:artifacts_upload_path).and_return('/') end context 'when job has been erased' do @@ -985,15 +985,6 @@ describe API::Runner do it_behaves_like 'successful artifacts upload' end - context 'when updates artifact' do - before do - upload_artifacts(file_upload2, headers_with_token) - upload_artifacts(file_upload, headers_with_token) - end - - it_behaves_like 'successful artifacts upload' - end - context 'when using runners token' do it 'responds with forbidden' do upload_artifacts(file_upload, headers.merge(API::Helpers::Runner::JOB_TOKEN_HEADER => job.project.runners_token)) @@ -1106,7 +1097,7 @@ describe API::Runner do expect(response).to have_gitlab_http_status(201) expect(stored_artifacts_file.original_filename).to eq(artifacts.original_filename) expect(stored_metadata_file.original_filename).to eq(metadata.original_filename) - expect(stored_artifacts_size).to eq(71759) + expect(stored_artifacts_size).to eq(72821) end end @@ -1131,7 +1122,7 @@ describe API::Runner do # by configuring this path we allow to pass file from @tmpdir only # but all temporary files are stored in system tmp directory @tmpdir = Dir.mktmpdir - allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return(@tmpdir) + allow(JobArtifactUploader).to receive(:artifacts_upload_path).and_return(@tmpdir) end after do diff --git a/spec/serializers/pipeline_serializer_spec.rb b/spec/serializers/pipeline_serializer_spec.rb index 8fc1ceedc34..88d347322a6 100644 --- a/spec/serializers/pipeline_serializer_spec.rb +++ b/spec/serializers/pipeline_serializer_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe PipelineSerializer do - let(:user) { create(:user) } + set(:user) { create(:user) } let(:serializer) do described_class.new(current_user: user) @@ -117,7 +117,7 @@ describe PipelineSerializer do shared_examples 'no N+1 queries' do it 'verifies number of queries', :request_store do recorded = ActiveRecord::QueryRecorder.new { subject } - expect(recorded.count).to be_within(1).of(57) + expect(recorded.count).to be_within(1).of(36) expect(recorded.cached_count).to eq(0) end end diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index b61d1cb765e..d48a44fa57f 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -17,7 +17,7 @@ describe Ci::RetryBuildService do %i[id status user token coverage trace runner artifacts_expire_at artifacts_file artifacts_metadata artifacts_size created_at updated_at started_at finished_at queued_at erased_by - erased_at auto_canceled_by].freeze + erased_at auto_canceled_by job_artifacts job_artifacts_archive job_artifacts_metadata].freeze IGNORE_ACCESSORS = %i[type lock_version target_url base_tags trace_sections @@ -34,7 +34,7 @@ describe Ci::RetryBuildService do end let(:build) do - create(:ci_build, :failed, :artifacts_expired, :erased, + create(:ci_build, :failed, :artifacts, :expired, :erased, :queued, :coverage, :tags, :allowed_to_fail, :on_tag, :triggered, :trace, :teardown_environment, description: 'my-job', stage: 'test', pipeline: pipeline, diff --git a/spec/services/merge_requests/build_service_spec.rb b/spec/services/merge_requests/build_service_spec.rb index fee293760f5..b5c92e681fb 100644 --- a/spec/services/merge_requests/build_service_spec.rb +++ b/spec/services/merge_requests/build_service_spec.rb @@ -39,6 +39,7 @@ describe MergeRequests::BuildService do describe '#execute' do it 'calls the compare service with the correct arguments' do + allow_any_instance_of(described_class).to receive(:branches_valid?).and_return(true) expect(CompareService).to receive(:new) .with(project, Gitlab::Git::BRANCH_REF_PREFIX + source_branch) .and_call_original diff --git a/spec/services/projects/update_pages_service_spec.rb b/spec/services/projects/update_pages_service_spec.rb index d4ac1f6ad81..bfb86284d86 100644 --- a/spec/services/projects/update_pages_service_spec.rb +++ b/spec/services/projects/update_pages_service_spec.rb @@ -1,10 +1,18 @@ require "spec_helper" describe Projects::UpdatePagesService do - let(:project) { create(:project, :repository) } - let(:pipeline) { create(:ci_pipeline, project: project, sha: project.commit('HEAD').sha) } - let(:build) { create(:ci_build, pipeline: pipeline, ref: 'HEAD') } + set(:project) { create(:project, :repository) } + set(:pipeline) { create(:ci_pipeline, project: project, sha: project.commit('HEAD').sha) } + set(:build) { create(:ci_build, pipeline: pipeline, ref: 'HEAD') } let(:invalid_file) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png') } + let(:extension) { 'zip' } + + let(:file) { fixture_file_upload(Rails.root + "spec/fixtures/pages.#{extension}") } + let(:empty_file) { fixture_file_upload(Rails.root + "spec/fixtures/pages_empty.#{extension}") } + let(:metadata) do + filename = Rails.root + "spec/fixtures/pages.#{extension}.meta" + fixture_file_upload(filename) if File.exist?(filename) + end subject { described_class.new(project, build) } @@ -12,18 +20,85 @@ describe Projects::UpdatePagesService do project.remove_pages end - %w(tar.gz zip).each do |format| - context "for valid #{format}" do - let(:file) { fixture_file_upload(Rails.root + "spec/fixtures/pages.#{format}") } - let(:empty_file) { fixture_file_upload(Rails.root + "spec/fixtures/pages_empty.#{format}") } - let(:metadata) do - filename = Rails.root + "spec/fixtures/pages.#{format}.meta" - fixture_file_upload(filename) if File.exist?(filename) + context 'legacy artifacts' do + %w(tar.gz zip).each do |format| + let(:extension) { format } + + context "for valid #{format}" do + before do + build.update_attributes(legacy_artifacts_file: file) + build.update_attributes(legacy_artifacts_metadata: metadata) + end + + describe 'pages artifacts' do + context 'with expiry date' do + before do + build.artifacts_expire_in = "2 days" + end + + it "doesn't delete artifacts" do + expect(execute).to eq(:success) + + expect(build.reload.artifacts?).to eq(true) + end + end + + context 'without expiry date' do + it "does delete artifacts" do + expect(execute).to eq(:success) + + expect(build.reload.artifacts?).to eq(false) + end + end + end + + it 'succeeds' do + expect(project.pages_deployed?).to be_falsey + expect(execute).to eq(:success) + expect(project.pages_deployed?).to be_truthy + + # Check that all expected files are extracted + %w[index.html zero .hidden/file].each do |filename| + expect(File.exist?(File.join(project.public_pages_path, filename))).to be_truthy + end + end + + it 'limits pages size' do + stub_application_setting(max_pages_size: 1) + expect(execute).not_to eq(:success) + end + + it 'removes pages after destroy' do + expect(PagesWorker).to receive(:perform_in) + expect(project.pages_deployed?).to be_falsey + expect(execute).to eq(:success) + expect(project.pages_deployed?).to be_truthy + project.destroy + expect(project.pages_deployed?).to be_falsey + end + + it 'fails if sha on branch is not latest' do + build.update_attributes(ref: 'feature') + + expect(execute).not_to eq(:success) + end + + it 'fails for empty file fails' do + build.update_attributes(legacy_artifacts_file: empty_file) + + expect(execute).not_to eq(:success) + end end + end + end + context 'for new artifacts' do + context "for a valid job" do before do - build.update_attributes(artifacts_file: file) - build.update_attributes(artifacts_metadata: metadata) + create(:ci_job_artifact, file: file, job: build) + create(:ci_job_artifact, file_type: :metadata, file: metadata, job: build) + + build.reload end describe 'pages artifacts' do @@ -35,7 +110,7 @@ describe Projects::UpdatePagesService do it "doesn't delete artifacts" do expect(execute).to eq(:success) - expect(build.reload.artifacts_file?).to eq(true) + expect(build.artifacts?).to eq(true) end end @@ -43,7 +118,7 @@ describe Projects::UpdatePagesService do it "does delete artifacts" do expect(execute).to eq(:success) - expect(build.reload.artifacts_file?).to eq(false) + expect(build.reload.artifacts?).to eq(false) end end end @@ -74,13 +149,14 @@ describe Projects::UpdatePagesService do end it 'fails if sha on branch is not latest' do - pipeline.update_attributes(sha: 'old_sha') - build.update_attributes(artifacts_file: file) + build.update_attributes(ref: 'feature') + expect(execute).not_to eq(:success) end it 'fails for empty file fails' do - build.update_attributes(artifacts_file: empty_file) + build.job_artifacts_archive.update_attributes(file: empty_file) + expect(execute).not_to eq(:success) end end @@ -97,7 +173,7 @@ describe Projects::UpdatePagesService do end it 'fails for invalid archive' do - build.update_attributes(artifacts_file: invalid_file) + build.update_attributes(legacy_artifacts_file: invalid_file) expect(execute).not_to eq(:success) end @@ -108,8 +184,8 @@ describe Projects::UpdatePagesService do file = fixture_file_upload(Rails.root + 'spec/fixtures/pages.zip') metafile = fixture_file_upload(Rails.root + 'spec/fixtures/pages.zip.meta') - build.update_attributes(artifacts_file: file) - build.update_attributes(artifacts_metadata: metafile) + build.update_attributes(legacy_artifacts_file: file) + build.update_attributes(legacy_artifacts_metadata: metafile) allow(build).to receive(:artifacts_metadata_entry) .and_return(metadata) diff --git a/spec/services/web_hook_service_spec.rb b/spec/services/web_hook_service_spec.rb index a669429ce3e..21910e69d2e 100644 --- a/spec/services/web_hook_service_spec.rb +++ b/spec/services/web_hook_service_spec.rb @@ -146,7 +146,7 @@ describe WebHookService do let(:system_hook) { create(:system_hook) } it 'enqueue WebHookWorker' do - expect(Sidekiq::Client).to receive(:enqueue).with(WebHookWorker, project_hook.id, data, 'push_hooks') + expect(WebHookWorker).to receive(:perform_async).with(project_hook.id, data, 'push_hooks') described_class.new(project_hook, data, 'push_hooks').async_execute end diff --git a/spec/support/query_recorder.rb b/spec/support/query_recorder.rb index 369775db462..8cf8f45a8b2 100644 --- a/spec/support/query_recorder.rb +++ b/spec/support/query_recorder.rb @@ -41,7 +41,8 @@ RSpec::Matchers.define :exceed_query_limit do |expected| supports_block_expectations match do |block| - query_count(&block) > expected_count + threshold + @subject_block = block + actual_count > expected_count + threshold end failure_message_when_negated do |actual| @@ -55,6 +56,11 @@ RSpec::Matchers.define :exceed_query_limit do |expected| self end + def for_query(query) + @query = query + self + end + def threshold @threshold.to_i end @@ -68,12 +74,15 @@ RSpec::Matchers.define :exceed_query_limit do |expected| end def actual_count - @recorder.count + @actual_count ||= if @query + recorder.log.select { |recorded| recorded =~ @query }.size + else + recorder.count + end end - def query_count(&block) - @recorder = ActiveRecord::QueryRecorder.new(&block) - @recorder.count + def recorder + @recorder ||= ActiveRecord::QueryRecorder.new(&@subject_block) end def count_queries(queries) diff --git a/spec/support/test_env.rb b/spec/support/test_env.rb index fff120fcb88..b300b493f86 100644 --- a/spec/support/test_env.rb +++ b/spec/support/test_env.rb @@ -120,6 +120,7 @@ module TestEnv FileUtils.mkdir_p(repos_path) FileUtils.mkdir_p(backup_path) FileUtils.mkdir_p(pages_path) + FileUtils.mkdir_p(artifacts_path) end def clean_gitlab_test_path @@ -233,6 +234,10 @@ module TestEnv Gitlab.config.pages.path end + def artifacts_path + Gitlab.config.artifacts.path + end + # When no cached assets exist, manually hit the root path to create them # # Otherwise they'd be created by the first test, often timing out and diff --git a/spec/uploaders/artifact_uploader_spec.rb b/spec/uploaders/artifact_uploader_spec.rb deleted file mode 100644 index 2a3bd0e3bb2..00000000000 --- a/spec/uploaders/artifact_uploader_spec.rb +++ /dev/null @@ -1,61 +0,0 @@ -require 'rails_helper' - -describe ArtifactUploader do - let(:job) { create(:ci_build) } - let(:uploader) { described_class.new(job, :artifacts_file) } - let(:path) { Gitlab.config.artifacts.path } - - describe '.local_artifacts_store' do - subject { described_class.local_artifacts_store } - - it "delegate to artifacts path" do - expect(Gitlab.config.artifacts).to receive(:path) - - subject - end - end - - describe '.artifacts_upload_path' do - subject { described_class.artifacts_upload_path } - - it { is_expected.to start_with(path) } - it { is_expected.to end_with('tmp/uploads/') } - end - - describe '#store_dir' do - subject { uploader.store_dir } - - it { is_expected.to start_with(path) } - it { is_expected.to end_with("#{job.project_id}/#{job.id}") } - end - - describe '#cache_dir' do - subject { uploader.cache_dir } - - it { is_expected.to start_with(path) } - it { is_expected.to end_with('/tmp/cache') } - end - - describe '#work_dir' do - subject { uploader.work_dir } - - it { is_expected.to start_with(path) } - it { is_expected.to end_with('/tmp/work') } - end - - describe '#filename' do - # we need to use uploader, as this makes to use mounter - # which initialises uploader.file object - let(:uploader) { job.artifacts_file } - - subject { uploader.filename } - - it { is_expected.to be_nil } - - context 'with artifacts' do - let(:job) { create(:ci_build, :artifacts) } - - it { is_expected.not_to be_nil } - end - end -end diff --git a/spec/uploaders/job_artifact_uploader_spec.rb b/spec/uploaders/job_artifact_uploader_spec.rb new file mode 100644 index 00000000000..14fd5f3600f --- /dev/null +++ b/spec/uploaders/job_artifact_uploader_spec.rb @@ -0,0 +1,51 @@ +require 'spec_helper' + +describe JobArtifactUploader do + let(:job_artifact) { create(:ci_job_artifact) } + let(:uploader) { described_class.new(job_artifact, :file) } + let(:local_path) { Gitlab.config.artifacts.path } + + describe '#store_dir' do + subject { uploader.store_dir } + + let(:path) { "#{job_artifact.created_at.utc.strftime('%Y_%m_%d')}/#{job_artifact.project_id}/#{job_artifact.id}" } + + context 'when using local storage' do + it { is_expected.to start_with(local_path) } + it { is_expected.to match(/\h{2}\/\h{2}\/\h{64}\/\d{4}_\d{1,2}_\d{1,2}\/\d+\/\d+\z/) } + it { is_expected.to end_with(path) } + end + end + + describe '#cache_dir' do + subject { uploader.cache_dir } + + it { is_expected.to start_with(local_path) } + it { is_expected.to end_with('/tmp/cache') } + end + + describe '#work_dir' do + subject { uploader.work_dir } + + it { is_expected.to start_with(local_path) } + it { is_expected.to end_with('/tmp/work') } + end + + context 'file is stored in valid local_path' do + let(:file) do + fixture_file_upload( + Rails.root.join('spec/fixtures/ci_build_artifacts.zip'), 'application/zip') + end + + before do + uploader.store!(file) + end + + subject { uploader.file.path } + + it { is_expected.to start_with(local_path) } + it { is_expected.to include("/#{job_artifact.created_at.utc.strftime('%Y_%m_%d')}/") } + it { is_expected.to include("/#{job_artifact.project_id}/") } + it { is_expected.to end_with("ci_build_artifacts.zip") } + end +end diff --git a/spec/uploaders/legacy_artifact_uploader_spec.rb b/spec/uploaders/legacy_artifact_uploader_spec.rb new file mode 100644 index 00000000000..efeffb78772 --- /dev/null +++ b/spec/uploaders/legacy_artifact_uploader_spec.rb @@ -0,0 +1,77 @@ +require 'rails_helper' + +describe LegacyArtifactUploader do + let(:job) { create(:ci_build) } + let(:uploader) { described_class.new(job, :legacy_artifacts_file) } + let(:local_path) { Gitlab.config.artifacts.path } + + describe '.local_store_path' do + subject { described_class.local_store_path } + + it "delegate to artifacts path" do + expect(Gitlab.config.artifacts).to receive(:path) + + subject + end + end + + describe '.artifacts_upload_path' do + subject { described_class.artifacts_upload_path } + + it { is_expected.to start_with(local_path) } + it { is_expected.to end_with('tmp/uploads/') } + end + + describe '#store_dir' do + subject { uploader.store_dir } + + let(:path) { "#{job.created_at.utc.strftime('%Y_%m')}/#{job.project_id}/#{job.id}" } + + context 'when using local storage' do + it { is_expected.to start_with(local_path) } + it { is_expected.to end_with(path) } + end + end + + describe '#cache_dir' do + subject { uploader.cache_dir } + + it { is_expected.to start_with(local_path) } + it { is_expected.to end_with('/tmp/cache') } + end + + describe '#work_dir' do + subject { uploader.work_dir } + + it { is_expected.to start_with(local_path) } + it { is_expected.to end_with('/tmp/work') } + end + + describe '#filename' do + # we need to use uploader, as this makes to use mounter + # which initialises uploader.file object + let(:uploader) { job.artifacts_file } + + subject { uploader.filename } + + it { is_expected.to be_nil } + end + + context 'file is stored in valid path' do + let(:file) do + fixture_file_upload( + Rails.root.join('spec/fixtures/ci_build_artifacts.zip'), 'application/zip') + end + + before do + uploader.store!(file) + end + + subject { uploader.file.path } + + it { is_expected.to start_with(local_path) } + it { is_expected.to include("/#{job.created_at.utc.strftime('%Y_%m')}/") } + it { is_expected.to include("/#{job.project_id}/") } + it { is_expected.to end_with("ci_build_artifacts.zip") } + end +end diff --git a/spec/workers/authorized_projects_worker_spec.rb b/spec/workers/authorized_projects_worker_spec.rb index 90ed1309d4a..0d6eb536c33 100644 --- a/spec/workers/authorized_projects_worker_spec.rb +++ b/spec/workers/authorized_projects_worker_spec.rb @@ -65,7 +65,6 @@ describe AuthorizedProjectsWorker do args_list = build_args_list(project.owner.id) push_bulk_args = { 'class' => described_class, - 'queue' => described_class.sidekiq_options['queue'], 'args' => args_list } diff --git a/spec/workers/background_migration_worker_spec.rb b/spec/workers/background_migration_worker_spec.rb index 4f6e3474634..1c54cf55fa0 100644 --- a/spec/workers/background_migration_worker_spec.rb +++ b/spec/workers/background_migration_worker_spec.rb @@ -10,35 +10,4 @@ describe BackgroundMigrationWorker, :sidekiq do described_class.new.perform('Foo', [10, 20]) end end - - describe '.perform_bulk' do - it 'enqueues background migrations in bulk' do - Sidekiq::Testing.fake! do - described_class.perform_bulk([['Foo', [1]], ['Foo', [2]]]) - - expect(described_class.jobs.count).to eq 2 - expect(described_class.jobs).to all(include('enqueued_at')) - end - end - end - - describe '.perform_bulk_in' do - context 'when delay is valid' do - it 'correctly schedules background migrations' do - Sidekiq::Testing.fake! do - described_class.perform_bulk_in(1.minute, [['Foo', [1]], ['Foo', [2]]]) - - expect(described_class.jobs.count).to eq 2 - expect(described_class.jobs).to all(include('at')) - end - end - end - - context 'when delay is invalid' do - it 'raises an ArgumentError exception' do - expect { described_class.perform_bulk_in(-60, [['Foo']]) } - .to raise_error(ArgumentError) - end - end - end end diff --git a/spec/workers/concerns/application_worker_spec.rb b/spec/workers/concerns/application_worker_spec.rb new file mode 100644 index 00000000000..0145563e0ed --- /dev/null +++ b/spec/workers/concerns/application_worker_spec.rb @@ -0,0 +1,58 @@ +require 'spec_helper' + +describe ApplicationWorker do + let(:worker) do + Class.new do + def self.name + 'Gitlab::Foo::Bar::DummyWorker' + end + + include ApplicationWorker + end + end + + describe 'Sidekiq options' do + it 'sets the queue name based on the class name' do + expect(worker.sidekiq_options['queue']).to eq('foo_bar_dummy') + end + end + + describe '.queue' do + it 'returns the queue name' do + worker.sidekiq_options queue: :some_queue + + expect(worker.queue).to eq('some_queue') + end + end + + describe '.bulk_perform_async' do + it 'enqueues jobs in bulk' do + Sidekiq::Testing.fake! do + worker.bulk_perform_async([['Foo', [1]], ['Foo', [2]]]) + + expect(worker.jobs.count).to eq 2 + expect(worker.jobs).to all(include('enqueued_at')) + end + end + end + + describe '.bulk_perform_in' do + context 'when delay is valid' do + it 'correctly schedules jobs' do + Sidekiq::Testing.fake! do + worker.bulk_perform_in(1.minute, [['Foo', [1]], ['Foo', [2]]]) + + expect(worker.jobs.count).to eq 2 + expect(worker.jobs).to all(include('at')) + end + end + end + + context 'when delay is invalid' do + it 'raises an ArgumentError exception' do + expect { worker.bulk_perform_in(-60, [['Foo']]) } + .to raise_error(ArgumentError) + end + end + end +end diff --git a/spec/workers/concerns/cluster_queue_spec.rb b/spec/workers/concerns/cluster_queue_spec.rb index 1050651fa51..5049886b55c 100644 --- a/spec/workers/concerns/cluster_queue_spec.rb +++ b/spec/workers/concerns/cluster_queue_spec.rb @@ -3,7 +3,11 @@ require 'spec_helper' describe ClusterQueue do let(:worker) do Class.new do - include Sidekiq::Worker + def self.name + 'DummyWorker' + end + + include ApplicationWorker include ClusterQueue end end diff --git a/spec/workers/concerns/cronjob_queue_spec.rb b/spec/workers/concerns/cronjob_queue_spec.rb index 5d1336c21a6..3ae1c5f54d8 100644 --- a/spec/workers/concerns/cronjob_queue_spec.rb +++ b/spec/workers/concerns/cronjob_queue_spec.rb @@ -3,7 +3,11 @@ require 'spec_helper' describe CronjobQueue do let(:worker) do Class.new do - include Sidekiq::Worker + def self.name + 'DummyWorker' + end + + include ApplicationWorker include CronjobQueue end end diff --git a/spec/workers/concerns/dedicated_sidekiq_queue_spec.rb b/spec/workers/concerns/dedicated_sidekiq_queue_spec.rb deleted file mode 100644 index 512baec8b7e..00000000000 --- a/spec/workers/concerns/dedicated_sidekiq_queue_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -require 'spec_helper' - -describe DedicatedSidekiqQueue do - let(:worker) do - Class.new do - def self.name - 'Foo::Bar::DummyWorker' - end - - include Sidekiq::Worker - include DedicatedSidekiqQueue - end - end - - describe 'queue names' do - it 'sets the queue name based on the class name' do - expect(worker.sidekiq_options['queue']).to eq('foo_bar_dummy') - end - end -end diff --git a/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb b/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb index 3ccf06f2d7d..68cfe9d5545 100644 --- a/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb +++ b/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb @@ -3,6 +3,10 @@ require 'spec_helper' describe Gitlab::GithubImport::ObjectImporter do let(:worker) do Class.new do + def self.name + 'DummyWorker' + end + include(Gitlab::GithubImport::ObjectImporter) def counter_name diff --git a/spec/workers/concerns/gitlab/github_import/queue_spec.rb b/spec/workers/concerns/gitlab/github_import/queue_spec.rb index 321ae3fe978..9c69ee32da1 100644 --- a/spec/workers/concerns/gitlab/github_import/queue_spec.rb +++ b/spec/workers/concerns/gitlab/github_import/queue_spec.rb @@ -3,7 +3,11 @@ require 'spec_helper' describe Gitlab::GithubImport::Queue do it 'sets the Sidekiq options for the worker' do worker = Class.new do - include Sidekiq::Worker + def self.name + 'DummyWorker' + end + + include ApplicationWorker include Gitlab::GithubImport::Queue end diff --git a/spec/workers/concerns/pipeline_queue_spec.rb b/spec/workers/concerns/pipeline_queue_spec.rb index eac5a770e5f..dd911760948 100644 --- a/spec/workers/concerns/pipeline_queue_spec.rb +++ b/spec/workers/concerns/pipeline_queue_spec.rb @@ -3,7 +3,11 @@ require 'spec_helper' describe PipelineQueue do let(:worker) do Class.new do - include Sidekiq::Worker + def self.name + 'DummyWorker' + end + + include ApplicationWorker include PipelineQueue end end diff --git a/spec/workers/concerns/repository_check_queue_spec.rb b/spec/workers/concerns/repository_check_queue_spec.rb index 8868e969829..fdbbfcc90a5 100644 --- a/spec/workers/concerns/repository_check_queue_spec.rb +++ b/spec/workers/concerns/repository_check_queue_spec.rb @@ -3,7 +3,11 @@ require 'spec_helper' describe RepositoryCheckQueue do let(:worker) do Class.new do - include Sidekiq::Worker + def self.name + 'DummyWorker' + end + + include ApplicationWorker include RepositoryCheckQueue end end diff --git a/spec/workers/every_sidekiq_worker_spec.rb b/spec/workers/every_sidekiq_worker_spec.rb index 30908534eb3..7ee0a51a263 100644 --- a/spec/workers/every_sidekiq_worker_spec.rb +++ b/spec/workers/every_sidekiq_worker_spec.rb @@ -1,44 +1,21 @@ require 'spec_helper' describe 'Every Sidekiq worker' do - let(:workers) do - root = Rails.root.join('app', 'workers') - concerns = root.join('concerns').to_s - - workers = Dir[root.join('**', '*.rb')] - .reject { |path| path.start_with?(concerns) } - - workers.map do |path| - ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '') - - ns.camelize.constantize - end + it 'includes ApplicationWorker' do + expect(Gitlab::SidekiqConfig.workers).to all(include(ApplicationWorker)) end it 'does not use the default queue' do - workers.each do |worker| - expect(worker.sidekiq_options['queue'].to_s).not_to eq('default') - end + expect(Gitlab::SidekiqConfig.workers.map(&:queue)).not_to include('default') end it 'uses the cronjob queue when the worker runs as a cronjob' do - cron_workers = Settings.cron_jobs - .map { |job_name, options| options['job_class'].constantize } - .to_set - - workers.each do |worker| - next unless cron_workers.include?(worker) - - expect(worker.sidekiq_options['queue'].to_s).to eq('cronjob') - end + expect(Gitlab::SidekiqConfig.cron_workers.map(&:queue)).to all(eq('cronjob')) end it 'defines the queue in the Sidekiq configuration file' do - config = YAML.load_file(Rails.root.join('config', 'sidekiq_queues.yml').to_s) - queue_names = config[:queues].map { |(queue, _)| queue }.to_set + config_queue_names = Gitlab::SidekiqConfig.config_queues.to_set - workers.each do |worker| - expect(queue_names).to include(worker.sidekiq_options['queue'].to_s) - end + expect(Gitlab::SidekiqConfig.worker_queues).to all(be_in(config_queue_names)) end end diff --git a/spec/workers/expire_build_instance_artifacts_worker_spec.rb b/spec/workers/expire_build_instance_artifacts_worker_spec.rb index bed5c5e2ecb..e1a56c72162 100644 --- a/spec/workers/expire_build_instance_artifacts_worker_spec.rb +++ b/spec/workers/expire_build_instance_artifacts_worker_spec.rb @@ -11,12 +11,8 @@ describe ExpireBuildInstanceArtifactsWorker do end context 'with expired artifacts' do - let(:artifacts_expiry) { { artifacts_expire_at: Time.now - 7.days } } - context 'when associated project is valid' do - let(:build) do - create(:ci_build, :artifacts, artifacts_expiry) - end + let(:build) { create(:ci_build, :artifacts, :expired) } it 'does expire' do expect(build.reload.artifacts_expired?).to be_truthy @@ -26,14 +22,14 @@ describe ExpireBuildInstanceArtifactsWorker do expect(build.reload.artifacts_file.exists?).to be_falsey end - it 'does nullify artifacts_file column' do - expect(build.reload.artifacts_file_identifier).to be_nil + it 'does remove the job artifact record' do + expect(build.reload.job_artifacts_archive).to be_nil end end end context 'with not yet expired artifacts' do - let(:build) do + set(:build) do create(:ci_build, :artifacts, artifacts_expire_at: Time.now + 7.days) end @@ -45,8 +41,8 @@ describe ExpireBuildInstanceArtifactsWorker do expect(build.reload.artifacts_file.exists?).to be_truthy end - it 'does not nullify artifacts_file column' do - expect(build.reload.artifacts_file_identifier).not_to be_nil + it 'does not remove the job artifact record' do + expect(build.reload.job_artifacts_archive).not_to be_nil end end @@ -61,13 +57,13 @@ describe ExpireBuildInstanceArtifactsWorker do expect(build.reload.artifacts_file.exists?).to be_truthy end - it 'does not nullify artifacts_file column' do - expect(build.reload.artifacts_file_identifier).not_to be_nil + it 'does not remove the job artifact record' do + expect(build.reload.job_artifacts_archive).not_to be_nil end end context 'for expired artifacts' do - let(:build) { create(:ci_build, artifacts_expire_at: Time.now - 7.days) } + let(:build) { create(:ci_build, :expired) } it 'is still expired' do expect(build.reload.artifacts_expired?).to be_truthy |