diff options
132 files changed, 3002 insertions, 1061 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79ec1b881d4..d738a13531b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,6 +48,7 @@ after_script: stages: - build - prepare + - merge - test - post-test - pages @@ -666,7 +667,7 @@ gitlab:assets:compile: only: - //@gitlab-org/gitlab-ce - //@gitlab-org/gitlab-ee - - //@gitlab/gitabhq + - //@gitlab/gitlabhq - //@gitlab/gitlab-ee tags: - gitlab-org-delivery @@ -1025,3 +1026,27 @@ schedule:review-cleanup: - gem install gitlab --no-document script: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb + +merge:master: + image: registry.gitlab.com/gitlab-org/merge-train + stage: merge + # The global before_script/after_script blocks break this job, or aren't + # necessary. These two lines result in them being ignored. + before_script: [] + after_script: [] + only: + refs: + - master + - schedules + variables: + - $CI_PROJECT_PATH == "gitlab-org/gitlab-ce" + - $MERGE_TRAIN_SSH_PUBLIC_KEY + - $MERGE_TRAIN_SSH_PRIVATE_KEY + - $MERGE_TRAIN_API_TOKEN + - $MERGE_FORCE_ENABLE + script: + - scripts/merge-train + cache: + paths: + - gitlab-ee + key: "merge:master" @@ -298,7 +298,7 @@ gem 'gettext_i18n_rails', '~> 1.8.0' gem 'gettext_i18n_rails_js', '~> 1.3' gem 'gettext', '~> 3.2.2', require: false, group: :development -gem 'batch-loader', '~> 1.2.1' +gem 'batch-loader', '~> 1.2.2' # Perf bar gem 'peek', '~> 1.0.1' diff --git a/Gemfile.lock b/Gemfile.lock index e7873932dad..699d77615aa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -73,7 +73,7 @@ GEM thread_safe (~> 0.3, >= 0.3.1) babosa (1.0.2) base32 (0.3.2) - batch-loader (1.2.1) + batch-loader (1.2.2) bcrypt (3.1.12) bcrypt_pbkdf (1.0.0) benchmark-ips (2.3.0) @@ -950,7 +950,7 @@ DEPENDENCIES awesome_print babosa (~> 1.0.2) base32 (~> 0.3.0) - batch-loader (~> 1.2.1) + batch-loader (~> 1.2.2) bcrypt_pbkdf (~> 1.0) benchmark-ips (~> 2.3.0) better_errors (~> 2.5.0) diff --git a/Gemfile.rails4.lock b/Gemfile.rails4.lock index 7478e2173bd..15e0b782d5b 100644 --- a/Gemfile.rails4.lock +++ b/Gemfile.rails4.lock @@ -70,7 +70,7 @@ GEM thread_safe (~> 0.3, >= 0.3.1) babosa (1.0.2) base32 (0.3.2) - batch-loader (1.2.1) + batch-loader (1.2.2) bcrypt (3.1.12) bcrypt_pbkdf (1.0.0) benchmark-ips (2.3.0) @@ -941,7 +941,7 @@ DEPENDENCIES awesome_print babosa (~> 1.0.2) base32 (~> 0.3.0) - batch-loader (~> 1.2.1) + batch-loader (~> 1.2.2) bcrypt_pbkdf (~> 1.0) benchmark-ips (~> 2.3.0) better_errors (~> 2.5.0) diff --git a/app/assets/javascripts/diffs/components/diff_gutter_avatars.vue b/app/assets/javascripts/diffs/components/diff_gutter_avatars.vue index b969017a2bb..0c0a0faa59d 100644 --- a/app/assets/javascripts/diffs/components/diff_gutter_avatars.vue +++ b/app/assets/javascripts/diffs/components/diff_gutter_avatars.vue @@ -56,9 +56,12 @@ export default { return `${noteData.author.name}: ${note}`; }, toggleDiscussions() { + const forceExpanded = this.discussions.some(discussion => !discussion.expanded); + this.discussions.forEach(discussion => { this.toggleDiscussion({ discussionId: discussion.id, + forceExpanded, }); }); }, diff --git a/app/assets/javascripts/notes/components/noteable_discussion.vue b/app/assets/javascripts/notes/components/noteable_discussion.vue index 4eb3b49392c..f4991a41325 100644 --- a/app/assets/javascripts/notes/components/noteable_discussion.vue +++ b/app/assets/javascripts/notes/components/noteable_discussion.vue @@ -66,11 +66,13 @@ export default { }, }, data() { + const { diff_discussion: isDiffDiscussion, resolved } = this.discussion; + return { isReplying: false, isResolving: false, resolveAsThread: true, - isRepliesToggledByUser: false, + isRepliesCollapsed: Boolean(!isDiffDiscussion && resolved), }; }, computed: { @@ -150,15 +152,6 @@ export default { return expanded || this.alwaysExpanded || isResolvedNonDiffDiscussion; }, - isRepliesCollapsed() { - const { discussion, isRepliesToggledByUser } = this; - const { resolved, notes } = discussion; - const hasReplies = notes.length > 1; - - return ( - (!discussion.diff_discussion && resolved && hasReplies && !isRepliesToggledByUser) || false - ); - }, actionText() { const commitId = this.discussion.commit_id ? truncateSha(this.discussion.commit_id) : ''; const linkStart = `<a href="${_.escape(this.discussion.discussion_path)}">`; @@ -234,7 +227,7 @@ export default { this.toggleDiscussion({ discussionId: this.discussion.id }); }, toggleReplies() { - this.isRepliesToggledByUser = !this.isRepliesToggledByUser; + this.isRepliesCollapsed = !this.isRepliesCollapsed; }, showReplyForm() { this.isReplying = true; diff --git a/app/assets/javascripts/notes/stores/mutations.js b/app/assets/javascripts/notes/stores/mutations.js index 667c8a97cf3..bea396e5bb6 100644 --- a/app/assets/javascripts/notes/stores/mutations.js +++ b/app/assets/javascripts/notes/stores/mutations.js @@ -178,9 +178,11 @@ export default { } }, - [types.TOGGLE_DISCUSSION](state, { discussionId }) { + [types.TOGGLE_DISCUSSION](state, { discussionId, forceExpanded = null }) { const discussion = utils.findNoteObjectById(state.discussions, discussionId); - Object.assign(discussion, { expanded: !discussion.expanded }); + Object.assign(discussion, { + expanded: forceExpanded === null ? !discussion.expanded : forceExpanded, + }); }, [types.UPDATE_NOTE](state, note) { diff --git a/app/assets/stylesheets/framework.scss b/app/assets/stylesheets/framework.scss index d320fa5f595..4041f2b4479 100644 --- a/app/assets/stylesheets/framework.scss +++ b/app/assets/stylesheets/framework.scss @@ -6,7 +6,6 @@ @import 'bootstrap_migration'; @import 'framework/layout'; -@import 'framework/alerts'; @import 'framework/animations'; @import 'framework/vue_transitions'; @import 'framework/avatar'; diff --git a/app/assets/stylesheets/framework/alerts.scss b/app/assets/stylesheets/framework/alerts.scss deleted file mode 100644 index 866792a6a1b..00000000000 --- a/app/assets/stylesheets/framework/alerts.scss +++ /dev/null @@ -1,4 +0,0 @@ -.alert-tip { - background-color: $theme-gray-100; - color: $theme-gray-900; -} diff --git a/app/assets/stylesheets/framework/highlight.scss b/app/assets/stylesheets/framework/highlight.scss index a0bf6907b5f..73533571a2f 100644 --- a/app/assets/stylesheets/framework/highlight.scss +++ b/app/assets/stylesheets/framework/highlight.scss @@ -47,7 +47,7 @@ a { font-family: $monospace-font; display: block; - font-size: $code_font_size !important; + font-size: $code-font-size !important; min-height: 19px; white-space: nowrap; diff --git a/app/assets/stylesheets/pages/notes.scss b/app/assets/stylesheets/pages/notes.scss index 0d1d0b4d2d6..39d01c49fd7 100644 --- a/app/assets/stylesheets/pages/notes.scss +++ b/app/assets/stylesheets/pages/notes.scss @@ -371,10 +371,10 @@ $note-form-margin-left: 72px; &::after { content: ''; - width: 100%; height: 70px; position: absolute; - left: 0; + left: $gl-padding-24; + right: 0; bottom: 0; background: linear-gradient(rgba($white-light, 0.1) -100px, $white-light 100%); } diff --git a/app/controllers/concerns/issuable_collections.rb b/app/controllers/concerns/issuable_collections.rb index 34a8c50fcbd..0837599977f 100644 --- a/app/controllers/concerns/issuable_collections.rb +++ b/app/controllers/concerns/issuable_collections.rb @@ -91,7 +91,7 @@ module IssuableCollections options = { scope: params[:scope], state: params[:state], - sort: set_sort_order_from_cookie || default_sort_order + sort: set_sort_order } # Used by view to highlight active option @@ -113,6 +113,32 @@ module IssuableCollections 'opened' end + def set_sort_order + set_sort_order_from_user_preference || set_sort_order_from_cookie || default_sort_order + end + + def set_sort_order_from_user_preference + return unless current_user + return unless issuable_sorting_field + + user_preference = current_user.user_preference + + sort_param = params[:sort] + sort_param ||= user_preference[issuable_sorting_field] + + if user_preference[issuable_sorting_field] != sort_param + user_preference.update_attribute(issuable_sorting_field, sort_param) + end + + sort_param + end + + # Implement default_sorting_field method on controllers + # to choose which column to store the sorting parameter. + def issuable_sorting_field + nil + end + def set_sort_order_from_cookie sort_param = params[:sort] if params[:sort].present? # fallback to legacy cookie value for backward compatibility diff --git a/app/controllers/concerns/uploads_actions.rb b/app/controllers/concerns/uploads_actions.rb index 7a1c7abfb8f..5912fffc058 100644 --- a/app/controllers/concerns/uploads_actions.rb +++ b/app/controllers/concerns/uploads_actions.rb @@ -1,17 +1,11 @@ # frozen_string_literal: true module UploadsActions - extend ActiveSupport::Concern - include Gitlab::Utils::StrongMemoize include SendFileUpload UPLOAD_MOUNTS = %w(avatar attachment file logo header_logo favicon).freeze - included do - prepend_before_action :set_html_format, only: :show - end - def create link_to_file = UploadService.new(model, params[:file], uploader_class).execute @@ -61,13 +55,6 @@ module UploadsActions private - # Explicitly set the format. - # Otherwise rails 5 will set it from a file extension. - # See https://github.com/rails/rails/commit/84e8accd6fb83031e4c27e44925d7596655285f7#diff-2b8f2fbb113b55ca8e16001c393da8f1 - def set_html_format - request.format = :html - end - def uploader_class raise NotImplementedError end diff --git a/app/controllers/projects/artifacts_controller.rb b/app/controllers/projects/artifacts_controller.rb index ae9c17802b9..1a91e07b97f 100644 --- a/app/controllers/projects/artifacts_controller.rb +++ b/app/controllers/projects/artifacts_controller.rb @@ -9,7 +9,6 @@ class Projects::ArtifactsController < Projects::ApplicationController before_action :authorize_read_build! before_action :authorize_update_build!, only: [:keep] before_action :extract_ref_name_and_path - before_action :set_request_format, only: [:file] before_action :validate_artifacts!, except: [:download] before_action :entry, only: [:file] @@ -110,12 +109,4 @@ class Projects::ArtifactsController < Projects::ApplicationController render_404 unless @entry.exists? end - - def set_request_format - request.format = :html if set_request_format? - end - - def set_request_format? - request.format != :json - end end diff --git a/app/controllers/projects/blob_controller.rb b/app/controllers/projects/blob_controller.rb index 873c96a5523..60fabd15333 100644 --- a/app/controllers/projects/blob_controller.rb +++ b/app/controllers/projects/blob_controller.rb @@ -9,7 +9,6 @@ class Projects::BlobController < Projects::ApplicationController include ActionView::Helpers::SanitizeHelper prepend_before_action :authenticate_user!, only: [:edit] - before_action :set_request_format, only: [:edit, :show, :update, :destroy] before_action :require_non_empty_project, except: [:new, :create] before_action :authorize_download_code! @@ -242,18 +241,6 @@ class Projects::BlobController < Projects::ApplicationController .last_for_path(@repository, @ref, @path).sha end - # In Rails 4.2 if params[:format] is empty, Rails set it to :html - # But since Rails 5.0 the framework now looks for an extension. - # E.g. for `blob/master/CHANGELOG.md` in Rails 4 the format would be `:html`, but in Rails 5 on it'd be `:md` - # This before_action explicitly sets the `:html` format for all requests unless `:format` is set by a client e.g. by JS for XHR requests. - def set_request_format - request.format = :html if set_request_format? - end - - def set_request_format? - params[:id].present? && params[:format].blank? && request.format != "json" - end - def show_html environment_params = @repository.branch_exists?(@ref) ? { ref: @ref } : { commit: @commit } @environment = EnvironmentsFinder.new(@project, current_user, environment_params).execute.last diff --git a/app/controllers/projects/commits_controller.rb b/app/controllers/projects/commits_controller.rb index 8ba18aacc58..e40a1a1d744 100644 --- a/app/controllers/projects/commits_controller.rb +++ b/app/controllers/projects/commits_controller.rb @@ -12,7 +12,6 @@ class Projects::CommitsController < Projects::ApplicationController before_action :assign_ref_vars, except: :commits_root before_action :authorize_download_code! before_action :set_commits, except: :commits_root - before_action :set_request_format, only: :show def commits_root redirect_to project_commits_path(@project, @project.default_branch) @@ -71,19 +70,6 @@ class Projects::CommitsController < Projects::ApplicationController @commits = set_commits_for_rendering(@commits) end - # Rails 5 sets request.format from the extension. - # Explicitly set to :html. - def set_request_format - request.format = :html if set_request_format? - end - - # Rails 5 sets request.format from extension. - # In this case if the ref ends with `.atom`, it's expected to be the html response, - # not the atom one. So explicitly set request.format as :html to act like rails4. - def set_request_format? - request.format.to_s == "text/html" || @commits.ref.ends_with?("atom") - end - def whitelist_query_limiting Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42330') end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 207ffae873a..4319db42019 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -7,7 +7,7 @@ class ApplicationSetting < ActiveRecord::Base include IgnorableColumn include ChronicDurationAttribute - add_authentication_token_field :runners_registration_token + add_authentication_token_field :runners_registration_token, encrypted: true, fallback: true add_authentication_token_field :health_check_access_token DOMAIN_LIST_SEPARATOR = %r{\s*[,;]\s* # comma or semicolon, optionally surrounded by whitespace diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 1487b9d3bca..a0b2acd502b 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -171,6 +171,8 @@ module Ci scope :internal, -> { where(source: internal_sources) } + scope :for_user, -> (user) { where(user: user) } + # Returns the pipelines in descending order (= newest first), optionally # limited to a number of references. # @@ -496,6 +498,8 @@ module Ci end def ci_yaml_file_path + return unless repository_source? || unknown_source? + if project.ci_config_path.blank? '.gitlab-ci.yml' else @@ -664,6 +668,7 @@ module Ci def ci_yaml_from_repo return unless project return unless sha + return unless ci_yaml_file_path project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path) rescue GRPC::NotFound, GRPC::Internal diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 31330d0682e..260348c97b2 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -8,6 +8,9 @@ module Ci include RedisCacheable include ChronicDurationAttribute include FromUnion + include TokenAuthenticatable + + add_authentication_token_field :token, encrypted: true, migrating: true enum access_level: { not_protected: 0, @@ -39,7 +42,7 @@ module Ci has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build' - before_validation :set_default_values + before_save :ensure_token scope :active, -> { where(active: true) } scope :paused, -> { where(active: false) } @@ -145,10 +148,6 @@ module Ci end end - def set_default_values - self.token = SecureRandom.hex(15) if self.token.blank? - end - def assign_to(project, current_user = nil) if instance_type? self.runner_type = :project_type diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 3c5d7756eec..dc8b52105cc 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -85,7 +85,7 @@ module Clusters if kubernetes_namespace = cluster.kubernetes_namespaces.has_service_account_token.find_by(project: project) variables.concat(kubernetes_namespace.predefined_variables) - else + elsif cluster.project_type? # From 11.5, every Clusters::Project should have at least one # Clusters::KubernetesNamespace, so once migration has been completed, # this 'else' branch will be removed. For more information, please see diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb index 23a43aec677..f5bb559ceda 100644 --- a/app/models/concerns/token_authenticatable.rb +++ b/app/models/concerns/token_authenticatable.rb @@ -9,24 +9,18 @@ module TokenAuthenticatable private # rubocop:disable Lint/UselessAccessModifier def add_authentication_token_field(token_field, options = {}) - @token_fields = [] unless @token_fields - unique = options.fetch(:unique, true) - - if @token_fields.include?(token_field) + if token_authenticatable_fields.include?(token_field) raise ArgumentError.new("#{token_field} already configured via add_authentication_token_field") end - @token_fields << token_field + token_authenticatable_fields.push(token_field) attr_accessor :cleartext_tokens - strategy = if options[:digest] - TokenAuthenticatableStrategies::Digest.new(self, token_field, options) - else - TokenAuthenticatableStrategies::Insecure.new(self, token_field, options) - end + strategy = TokenAuthenticatableStrategies::Base + .fabricate(self, token_field, options) - if unique + if options.fetch(:unique, true) define_singleton_method("find_by_#{token_field}") do |token| strategy.find_token_authenticatable(token) end @@ -53,6 +47,15 @@ module TokenAuthenticatable define_method("reset_#{token_field}!") do strategy.reset_token!(self) end + + define_method("#{token_field}_matches?") do |other_token| + token = read_attribute(token_field) + token.present? && ActiveSupport::SecurityUtils.variable_size_secure_compare(other_token, token) + end + end + + def token_authenticatable_fields + @token_authenticatable_fields ||= [] end end end diff --git a/app/models/concerns/token_authenticatable_strategies/base.rb b/app/models/concerns/token_authenticatable_strategies/base.rb index 413721d3e6c..01fb194281a 100644 --- a/app/models/concerns/token_authenticatable_strategies/base.rb +++ b/app/models/concerns/token_authenticatable_strategies/base.rb @@ -2,6 +2,8 @@ module TokenAuthenticatableStrategies class Base + attr_reader :klass, :token_field, :options + def initialize(klass, token_field, options) @klass = klass @token_field = token_field @@ -22,6 +24,7 @@ module TokenAuthenticatableStrategies def ensure_token(instance) write_new_token(instance) unless token_set?(instance) + get_token(instance) end # Returns a token, but only saves when the database is in read & write mode @@ -36,6 +39,36 @@ module TokenAuthenticatableStrategies instance.save! if Gitlab::Database.read_write? end + def fallback? + unless options[:fallback].in?([true, false, nil]) + raise ArgumentError, 'fallback: needs to be a boolean value!' + end + + options[:fallback] == true + end + + def migrating? + unless options[:migrating].in?([true, false, nil]) + raise ArgumentError, 'migrating: needs to be a boolean value!' + end + + options[:migrating] == true + end + + def self.fabricate(model, field, options) + if options[:digest] && options[:encrypted] + raise ArgumentError, 'Incompatible options set!' + end + + if options[:digest] + TokenAuthenticatableStrategies::Digest.new(model, field, options) + elsif options[:encrypted] + TokenAuthenticatableStrategies::Encrypted.new(model, field, options) + else + TokenAuthenticatableStrategies::Insecure.new(model, field, options) + end + end + protected def write_new_token(instance) @@ -65,9 +98,5 @@ module TokenAuthenticatableStrategies def token_set?(instance) raise NotImplementedError end - - def token_field_name - @token_field - end end end diff --git a/app/models/concerns/token_authenticatable_strategies/encrypted.rb b/app/models/concerns/token_authenticatable_strategies/encrypted.rb new file mode 100644 index 00000000000..152491aa6e9 --- /dev/null +++ b/app/models/concerns/token_authenticatable_strategies/encrypted.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module TokenAuthenticatableStrategies + class Encrypted < Base + def initialize(*) + super + + if migrating? && fallback? + raise ArgumentError, '`fallback` and `migrating` options are not compatible!' + end + end + + def find_token_authenticatable(token, unscoped = false) + return if token.blank? + + if fully_encrypted? + return find_by_encrypted_token(token, unscoped) + end + + if fallback? + find_by_encrypted_token(token, unscoped) || + find_by_plaintext_token(token, unscoped) + elsif migrating? + find_by_plaintext_token(token, unscoped) + else + raise ArgumentError, 'Unknown encryption phase!' + end + end + + def ensure_token(instance) + # TODO, tech debt, because some specs are testing migrations, but are still + # using factory bot to create resources, it might happen that a database + # schema does not have "#{token_name}_encrypted" field yet, however a bunch + # of models call `ensure_#{token_name}` in `before_save`. + # + # In that case we are using insecure strategy, but this should only happen + # in tests, because otherwise `encrypted_field` is going to exist. + # + # Another use case is when we are caching resources / columns, like we do + # in case of ApplicationSetting. + + return super if instance.has_attribute?(encrypted_field) + + if fully_encrypted? + raise ArgumentError, 'Using encrypted strategy when encrypted field is missing!' + else + insecure_strategy.ensure_token(instance) + end + end + + def get_token(instance) + return insecure_strategy.get_token(instance) if migrating? + + encrypted_token = instance.read_attribute(encrypted_field) + token = Gitlab::CryptoHelper.aes256_gcm_decrypt(encrypted_token) + + token || (insecure_strategy.get_token(instance) if fallback?) + end + + def set_token(instance, token) + raise ArgumentError unless token.present? + + instance[encrypted_field] = Gitlab::CryptoHelper.aes256_gcm_encrypt(token) + instance[token_field] = token if migrating? + instance[token_field] = nil if fallback? + token + end + + def fully_encrypted? + !migrating? && !fallback? + end + + protected + + def find_by_plaintext_token(token, unscoped) + insecure_strategy.find_token_authenticatable(token, unscoped) + end + + def find_by_encrypted_token(token, unscoped) + encrypted_value = Gitlab::CryptoHelper.aes256_gcm_encrypt(token) + relation(unscoped).find_by(encrypted_field => encrypted_value) + end + + def insecure_strategy + @insecure_strategy ||= TokenAuthenticatableStrategies::Insecure + .new(klass, token_field, options) + end + + def token_set?(instance) + raw_token = instance.read_attribute(encrypted_field) + + unless fully_encrypted? + raw_token ||= insecure_strategy.get_token(instance) + end + + raw_token.present? + end + + def encrypted_field + @encrypted_field ||= "#{@token_field}_encrypted" + end + end +end diff --git a/app/models/group.rb b/app/models/group.rb index adb9169cfcd..02ddc8762af 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -55,7 +55,7 @@ class Group < Namespace validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } - add_authentication_token_field :runners_token + add_authentication_token_field :runners_token, encrypted: true, migrating: true after_create :post_create_hook after_destroy :post_destroy_hook diff --git a/app/models/project.rb b/app/models/project.rb index ade20cc8948..1c5c34111b0 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -85,7 +85,7 @@ class Project < ActiveRecord::Base default_value_for :snippets_enabled, gitlab_config_features.snippets default_value_for :only_allow_merge_if_all_discussions_are_resolved, false - add_authentication_token_field :runners_token + add_authentication_token_field :runners_token, encrypted: true, migrating: true before_validation :mark_remote_mirrors_for_removal, if: -> { RemoteMirror.table_exists? } diff --git a/changelogs/unreleased/ab-approximate-counts.yml b/changelogs/unreleased/ab-approximate-counts.yml new file mode 100644 index 00000000000..8a67239d031 --- /dev/null +++ b/changelogs/unreleased/ab-approximate-counts.yml @@ -0,0 +1,5 @@ +--- +title: Approximate counting strategy with TABLESAMPLE. +merge_request: 22650 +author: +type: performance diff --git a/changelogs/unreleased/auto-devops-support-for-group-security-dashboard.yml b/changelogs/unreleased/auto-devops-support-for-group-security-dashboard.yml deleted file mode 100644 index 7fb11f24902..00000000000 --- a/changelogs/unreleased/auto-devops-support-for-group-security-dashboard.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Auto DevOps support for Group Security Dashboard -merge_request: 23165 -author: -type: fixed diff --git a/changelogs/unreleased/diff-fix-expanding.yml b/changelogs/unreleased/diff-fix-expanding.yml new file mode 100644 index 00000000000..8ba7f87addc --- /dev/null +++ b/changelogs/unreleased/diff-fix-expanding.yml @@ -0,0 +1,5 @@ +--- +title: Fixed multiple diff line discussions not expanding +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/document-raw-snippet-api.yml b/changelogs/unreleased/document-raw-snippet-api.yml new file mode 100644 index 00000000000..3b8818cea5c --- /dev/null +++ b/changelogs/unreleased/document-raw-snippet-api.yml @@ -0,0 +1,5 @@ +--- +title: Fix lack of documentation on how to fetch a snippet's content using API +merge_request: 23448 +author: Colin Leroy +type: other diff --git a/changelogs/unreleased/fix-gb-encrypt-runners-tokens.yml b/changelogs/unreleased/fix-gb-encrypt-runners-tokens.yml new file mode 100644 index 00000000000..4ce4f96c1dd --- /dev/null +++ b/changelogs/unreleased/fix-gb-encrypt-runners-tokens.yml @@ -0,0 +1,5 @@ +--- +title: Encrypt runners tokens +merge_request: 23412 +author: +type: security diff --git a/changelogs/unreleased/fix-multiple-comments-shade-overlap.yml b/changelogs/unreleased/fix-multiple-comments-shade-overlap.yml new file mode 100644 index 00000000000..20005ba355e --- /dev/null +++ b/changelogs/unreleased/fix-multiple-comments-shade-overlap.yml @@ -0,0 +1,5 @@ +--- +title: Fix multiple commits shade overlapping vertical discussion line +merge_request: 23515 +author: +type: fixed diff --git a/changelogs/unreleased/legacy_fallback_for_project_clusters_only.yml b/changelogs/unreleased/legacy_fallback_for_project_clusters_only.yml new file mode 100644 index 00000000000..c8e959176d0 --- /dev/null +++ b/changelogs/unreleased/legacy_fallback_for_project_clusters_only.yml @@ -0,0 +1,5 @@ +--- +title: Fallback to admin KUBE_TOKEN for project clusters only +merge_request: 23527 +author: +type: other diff --git a/changelogs/unreleased/winh-collapse-discussions.yml b/changelogs/unreleased/winh-collapse-discussions.yml new file mode 100644 index 00000000000..19d04506318 --- /dev/null +++ b/changelogs/unreleased/winh-collapse-discussions.yml @@ -0,0 +1,5 @@ +--- +title: Fix collapsing discussion replies +merge_request: 23462 +author: +type: fixed diff --git a/config/initializers/action_dispatch_http_mime_negotiation.rb b/config/initializers/action_dispatch_http_mime_negotiation.rb new file mode 100644 index 00000000000..bdf5b0babfb --- /dev/null +++ b/config/initializers/action_dispatch_http_mime_negotiation.rb @@ -0,0 +1,19 @@ +# Starting with Rails 5, Rails tries to determine the request format based on +# the extension of the full URL path if no explicit `format` param or `Accept` +# header is provided, like when simply browsing to a page in your browser. +# +# This is undesireable in GitLab, because many of our paths will end in a ref or +# blob name that can end with any extension, while these pages should still be +# presented as HTML unless otherwise specified. + +# We override `format_from_path_extension` to disable this behavior. + +module ActionDispatch + module Http + module MimeNegotiation + def format_from_path_extension + nil + end + end + end +end diff --git a/config/routes/wiki.rb b/config/routes/wiki.rb index 1a07b1c206b..2ca52e55fca 100644 --- a/config/routes/wiki.rb +++ b/config/routes/wiki.rb @@ -6,7 +6,7 @@ scope(controller: :wikis) do post '/', to: 'wikis#create' end - scope(path: 'wikis/*id', as: :wiki, format: false, defaults: { format: :html }) do + scope(path: 'wikis/*id', as: :wiki, format: false) do get :edit get :history post :preview_markdown diff --git a/config/settings.rb b/config/settings.rb index 3f3481bb65d..1b94df785a7 100644 --- a/config/settings.rb +++ b/config/settings.rb @@ -95,6 +95,14 @@ class Settings < Settingslogic Gitlab::Application.secrets.db_key_base[0..31] end + def attr_encrypted_db_key_base_32 + Gitlab::Utils.ensure_utf8_size(attr_encrypted_db_key_base, bytes: 32.bytes) + end + + def attr_encrypted_db_key_base_12 + Gitlab::Utils.ensure_utf8_size(attr_encrypted_db_key_base, bytes: 12.bytes) + end + # This should be used for :per_attribute_salt_and_iv mode. There is no # need to truncate the key because the encryptor will use the salt to # generate a hash of the password: diff --git a/db/migrate/20181115140140_add_encrypted_runners_token_to_settings.rb b/db/migrate/20181115140140_add_encrypted_runners_token_to_settings.rb new file mode 100644 index 00000000000..36d9ad45b19 --- /dev/null +++ b/db/migrate/20181115140140_add_encrypted_runners_token_to_settings.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddEncryptedRunnersTokenToSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :application_settings, :runners_registration_token_encrypted, :string + end +end diff --git a/db/migrate/20181116141415_add_encrypted_runners_token_to_namespaces.rb b/db/migrate/20181116141415_add_encrypted_runners_token_to_namespaces.rb new file mode 100644 index 00000000000..b92b1b50218 --- /dev/null +++ b/db/migrate/20181116141415_add_encrypted_runners_token_to_namespaces.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddEncryptedRunnersTokenToNamespaces < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :namespaces, :runners_token_encrypted, :string + end +end diff --git a/db/migrate/20181116141504_add_encrypted_runners_token_to_projects.rb b/db/migrate/20181116141504_add_encrypted_runners_token_to_projects.rb new file mode 100644 index 00000000000..53e475bd180 --- /dev/null +++ b/db/migrate/20181116141504_add_encrypted_runners_token_to_projects.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddEncryptedRunnersTokenToProjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :projects, :runners_token_encrypted, :string + end +end diff --git a/db/migrate/20181120151656_add_token_encrypted_to_ci_runners.rb b/db/migrate/20181120151656_add_token_encrypted_to_ci_runners.rb new file mode 100644 index 00000000000..40db6b399ab --- /dev/null +++ b/db/migrate/20181120151656_add_token_encrypted_to_ci_runners.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddTokenEncryptedToCiRunners < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :ci_runners, :token_encrypted, :string + end +end diff --git a/db/post_migrate/20181121111200_schedule_runners_token_encryption.rb b/db/post_migrate/20181121111200_schedule_runners_token_encryption.rb new file mode 100644 index 00000000000..753e052f7a7 --- /dev/null +++ b/db/post_migrate/20181121111200_schedule_runners_token_encryption.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +class ScheduleRunnersTokenEncryption < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 10000 + RANGE_SIZE = 2000 + MIGRATION = 'EncryptRunnersTokens' + + MODELS = [ + ::Gitlab::BackgroundMigration::Models::EncryptColumns::Settings, + ::Gitlab::BackgroundMigration::Models::EncryptColumns::Namespace, + ::Gitlab::BackgroundMigration::Models::EncryptColumns::Project, + ::Gitlab::BackgroundMigration::Models::EncryptColumns::Runner + ].freeze + + disable_ddl_transaction! + + def up + MODELS.each do |model| + model.each_batch(of: BATCH_SIZE) do |relation, index| + delay = index * 4.minutes + + relation.each_batch(of: RANGE_SIZE) do |relation| + range = relation.pluck('MIN(id)', 'MAX(id)').first + args = [model.name.demodulize.downcase, *range] + + BackgroundMigrationWorker.perform_in(delay, MIGRATION, args) + end + end + end + end + + def down + # no-op + end +end diff --git a/db/schema.rb b/db/schema.rb index 995619bdc69..d048be08d77 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -166,6 +166,7 @@ ActiveRecord::Schema.define(version: 20181126153547) do t.integer "diff_max_patch_bytes", default: 102400, null: false t.integer "archive_builds_in_seconds" t.string "commit_email_hostname" + t.string "runners_registration_token_encrypted" t.index ["usage_stats_set_by_user_id"], name: "index_application_settings_on_usage_stats_set_by_user_id", using: :btree end @@ -520,6 +521,7 @@ ActiveRecord::Schema.define(version: 20181126153547) do t.string "ip_address" t.integer "maximum_timeout" t.integer "runner_type", limit: 2, null: false + t.string "token_encrypted" t.index ["contacted_at"], name: "index_ci_runners_on_contacted_at", using: :btree t.index ["is_shared"], name: "index_ci_runners_on_is_shared", using: :btree t.index ["locked"], name: "index_ci_runners_on_locked", using: :btree @@ -1335,6 +1337,7 @@ ActiveRecord::Schema.define(version: 20181126153547) do t.integer "two_factor_grace_period", default: 48, null: false t.integer "cached_markdown_version" t.string "runners_token" + t.string "runners_token_encrypted" t.index ["created_at"], name: "index_namespaces_on_created_at", using: :btree t.index ["name", "parent_id"], name: "index_namespaces_on_name_and_parent_id", unique: true, using: :btree t.index ["name"], name: "index_namespaces_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"} @@ -1675,6 +1678,7 @@ ActiveRecord::Schema.define(version: 20181126153547) do t.boolean "pages_https_only", default: true t.boolean "remote_mirror_available_overridden" t.bigint "pool_repository_id" + t.string "runners_token_encrypted" t.index ["ci_id"], name: "index_projects_on_ci_id", using: :btree t.index ["created_at"], name: "index_projects_on_created_at", using: :btree t.index ["creator_id"], name: "index_projects_on_creator_id", using: :btree diff --git a/doc/api/snippets.md b/doc/api/snippets.md index 7892866cd8e..e840e640377 100644 --- a/doc/api/snippets.md +++ b/doc/api/snippets.md @@ -37,13 +37,13 @@ Parameters: | --------- | ---- | -------- | ----------- | | `id` | Integer | yes | The ID of a snippet | -``` bash +```bash curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/snippets/1 ``` Example response: -``` json +```json { "id": 1, "title": "test", @@ -65,6 +65,30 @@ Example response: } ``` +## Single snippet contents + +Get a single snippet's raw contents. + +``` +GET /snippets/:id/raw +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | Integer | yes | The ID of a snippet | + +```bash +curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/snippets/1/raw +``` + +Example response: + +``` +Hello World snippet +``` + ## Create new snippet Creates a new snippet. The user must have permission to create new snippets. @@ -84,7 +108,7 @@ Parameters: | `visibility` | String | no | The snippet's visibility | -``` bash +```bash curl --request POST \ --data '{"title": "This is a snippet", "content": "Hello world", "description": "Hello World snippet", "file_name": "test.txt", "visibility": "internal" }' \ --header 'Content-Type: application/json' \ @@ -94,7 +118,7 @@ curl --request POST \ Example response: -``` json +```json { "id": 1, "title": "This is a snippet", @@ -136,7 +160,7 @@ Parameters: | `visibility` | String | no | The snippet's visibility | -``` bash +```bash curl --request PUT \ --data '{"title": "foo", "content": "bar"}' \ --header 'Content-Type: application/json' \ @@ -146,7 +170,7 @@ curl --request PUT \ Example response: -``` json +```json { "id": 1, "title": "test", @@ -201,13 +225,13 @@ GET /snippets/public | `per_page` | Integer | no | number of snippets to return per page | | `page` | Integer | no | the page to retrieve | -``` bash +```bash curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/snippets/public?per_page=2&page=1 ``` Example response: -``` json +```json [ { "author": { diff --git a/doc/development/automatic_ce_ee_merge.md b/doc/development/automatic_ce_ee_merge.md index 58e08d432cc..e4eb26b3aca 100644 --- a/doc/development/automatic_ce_ee_merge.md +++ b/doc/development/automatic_ce_ee_merge.md @@ -1,57 +1,33 @@ # Automatic CE->EE merge -GitLab Community Edition is merged automatically every 3 hours into the -Enterprise Edition (look for the [`CE Upstream` merge requests]). - -This merge is done automatically in a -[scheduled pipeline](https://gitlab.com/gitlab-org/release-tools/-/jobs/43201679). - -## What to do if you are pinged in a `CE Upstream` merge request to resolve a conflict? - -1. Please resolve the conflict as soon as possible or ask someone else to do it - - It's ok to resolve more conflicts than the one that you are asked to resolve. - In that case, it's a good habit to ask for a double-check on your resolution - by someone who is familiar with the code you touched. -1. Once you have resolved your conflicts, push to the branch (no force-push) -1. Assign the merge request to the next person that has to resolve a conflict -1. If all conflicts are resolved after your resolution is pushed, keep the merge - request assigned to you: **you are now responsible for the merge request to be - green** -1. If you are the last person to resolve the conflicts, the pipeline is green, - and you have merge rights, merge the MR, but **do not** choose to squash. - Otherwise, assign the MR to someone that can merge. -1. If you need any help, you can ping the current [release managers], or ask in - the `#ce-to-ee` Slack channel - -A few notes about the automatic CE->EE merge job: - -- If a merge is already in progress, the job - [doesn't create a new one](https://gitlab.com/gitlab-org/release-tools/-/jobs/43157687). -- If there is nothing to merge (i.e. EE is up-to-date with CE), the job doesn't - create a new one -- The job posts messages to the `#ce-to-ee` Slack channel to inform what's the - current CE->EE merge status (e.g. "A new MR has been created", "A MR is still pending") - -[`CE Upstream` merge requests]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests?label_name%5B%5D=CE+upstream -[release managers]: https://about.gitlab.com/release-managers/ +Whenever a commit is pushed to the CE `master` branch, it is automatically +merged into the EE `master` branch. If the commit produces any conflicts, it is +instead reverted from CE `master`. When this happens, a merge request will be +set up automatically that can be used to reinstate the changes. This merge +request will be assigned to the author of the conflicting commit, or the merge +request author if the commit author could not be associated with a GitLab user. +If no author could be found, the merge request is assigned to a random member of +the Delivery team. It is then up to this team member to figure out who to assign +the merge request to. + +Because some commits can not be reverted if new commits depend on them, we also +run a job periodically that processes a range of commits and tries to merge or +revert them. This should ensure that all commits are either merged into EE +`master`, or reverted, instead of just being left behind in CE. ## Always merge EE merge requests before their CE counterparts **In order to avoid conflicts in the CE->EE merge, you should always merge the EE version of your CE merge request first, if present.** -The rationale for this is that as CE->EE merges are done automatically every few -hours, it can happen that: +The rationale for this is that as CE->EE merges are done automatically, it can +happen that: -1. A CE merge request that needs EE-specific changes is merged -1. The automatic CE->EE merge happens +1. A CE merge request that needs EE-specific changes is merged. +1. The automatic CE->EE merge happens. 1. Conflicts due to the CE merge request occur since its EE merge request isn't - merged yet -1. The automatic merge bot will ping someone to resolve the conflict **that are - already resolved in the EE merge request that isn't merged yet** - -That's a waste of time, and that's why you should merge EE merge request before -their CE counterpart. + merged yet. +1. The CE changes are reverted. ## Avoiding CE->EE merge conflicts beforehand @@ -69,136 +45,89 @@ detect if the current branch's changes will conflict during the CE->EE merge. The job reports what files are conflicting and how to set up a merge request against EE. -#### How the job works - -1. Generates the diff between your branch and current CE `master` -1. Tries to apply it to current EE `master` -1. If it applies cleanly, the job succeeds, otherwise... -1. Detects a branch with the `ee-` prefix or `-ee` suffix in EE -1. If it exists, generate the diff between this branch and current EE `master` -1. Tries to apply it to current EE `master` -1. If it applies cleanly, the job succeeds - -In the case where the job fails, it means you should create an `ee-<ce_branch>` -or `<ce_branch>-ee` branch, push it to EE and open a merge request against EE -`master`. -At this point if you retry the failing job in your CE merge request, it should -now pass. - -Notes: - -- This task is not a silver-bullet, its current goal is to bring awareness to - developers that their work needs to be ported to EE. -- Community contributors shouldn't be required to submit merge requests against - EE, but reviewers should take actions by either creating such EE merge request - or asking a GitLab developer to do it **before the merge request is merged**. -- If you branch is too far behind `master`, the job will fail. In that case you - should rebase your branch upon latest `master`. -- Code reviews for merge requests often consist of multiple iterations of - feedback and fixes. There is no need to update your EE MR after each - iteration. Instead, create an EE MR as soon as you see the - `ee_compat_check` job failing. After you receive the final approval - from a Maintainer (but **before the CE MR is merged**) update the EE MR. - This helps to identify significant conflicts sooner, but also reduces the - number of times you have to resolve conflicts. -- Please remember to - [always have your EE merge request merged before the CE version](#always-merge-ee-merge-requests-before-their-ce-counterparts). -- You can use [`git rerere`](https://git-scm.com/docs/git-rerere) - to avoid resolving the same conflicts multiple times. - -### Cherry-picking from CE to EE - -For avoiding merge conflicts, we use a method of creating equivalent branches -for CE and EE. If the `ee-compat-check` job fails, this process is required. - -This method only requires that you have cloned both CE and EE into your computer. -If you don't have them yet, please go ahead and clone them: - -- Clone CE repo: `git clone git@gitlab.com:gitlab-org/gitlab-ce.git` -- Clone EE repo: `git clone git@gitlab.com:gitlab-org/gitlab-ee.git` - -And the only additional setup we need is to add CE as remote of EE and vice-versa: - -- Open two terminal windows, one in CE, and another one in EE: - - In EE: `git remote add ce git@gitlab.com:gitlab-org/gitlab-ce.git` - - In CE: `git remote add ee git@gitlab.com:gitlab-org/gitlab-ee.git` - -That's all setup we need, so that we can cherry-pick a commit from CE to EE, and -from EE to CE. - -Now, every time you create an MR for CE and EE: - -1. Open two terminal windows, one in CE, and another one in EE -1. In the CE terminal: - 1. Create the CE branch, e.g., `branch-example` - 1. Make your changes and push a commit (commit A) - 1. Create the CE merge request in GitLab -1. In the EE terminal: - 1. Create the EE-equivalent branch ending with `-ee`, e.g., - `git checkout -b branch-example-ee` - 1. Fetch the CE branch: `git fetch ce branch-example` - 1. Cherry-pick the commit A: `git cherry-pick commit-A-SHA` - 1. If Git prompts you to fix the conflicts, do a `git status` - to check which files contain conflicts, fix them, save the files - 1. Add the changes with `git add .` but **DO NOT commit** them - 1. Continue cherry-picking: `git cherry-pick --continue` - 1. Push to EE: `git push origin branch-example-ee` -1. Create the EE-equivalent MR and link to the CE MR from the -description "Ports [CE-MR-LINK] to EE" -1. Once all the jobs are passing in both CE and EE, you've addressed the -feedback from your own team, and got them approved, the merge requests can be merged. -1. When both MRs are ready, the EE merge request will be merged first, and the -CE-equivalent will be merged next. - -**Important notes:** - -- The commit SHA can be easily found from the GitLab UI. From a merge request, -open the tab **Commits** and click the copy icon to copy the commit SHA. -- To cherry-pick a **commit range**, such as [A > B > C > D] use: - - ```shell - git cherry-pick "oldest-commit-SHA^..newest-commit-SHA" - ``` - - For example, suppose the commit A is the oldest, and its SHA is `4f5e4018c09ed797fdf446b3752f82e46f5af502`, - and the commit D is the newest, and its SHA is `80e1c9e56783bd57bd7129828ec20b252ebc0538`. - The cherry-pick command will be: - - ```shell - git cherry-pick "4f5e4018c09ed797fdf446b3752f82e46f5af502^..80e1c9e56783bd57bd7129828ec20b252ebc0538" - ``` - -- To cherry-pick a **merge commit**, use the flag `-m 1`. For example, suppose that the -merge commit SHA is `138f5e2f20289bb376caffa0303adb0cac859ce1`: - - ```shell - git cherry-pick -m 1 138f5e2f20289bb376caffa0303adb0cac859ce1 - ``` -- To cherry-pick multiple commits, such as B and D in a range [A > B > C > D], use: - - ```shell - git cherry-pick commmit-B-SHA commit-D-SHA - ``` - - For example, suppose commit B SHA = `4f5e4018c09ed797fdf446b3752f82e46f5af502`, - and the commit D SHA = `80e1c9e56783bd57bd7129828ec20b252ebc0538`. - The cherry-pick command will be: - - ```shell - git cherry-pick 4f5e4018c09ed797fdf446b3752f82e46f5af502 80e1c9e56783bd57bd7129828ec20b252ebc0538 - ``` - - This case is particularly useful when you have a merge commit in a sequence of - commits and you want to cherry-pick all but the merge commit. - -- If you push more commits to the CE branch, you can safely repeat the procedure -to cherry-pick them to the EE-equivalent branch. You can do that as many times as -necessary, using the same CE and EE branches. -- If you submitted the merge request to the CE repo and the `ee-compat-check` job passed, -you are not required to submit the EE-equivalent MR, but it's still recommended. If the -job failed, you are required to submit the EE MR so that you can fix the conflicts in EE -before merging your changes into CE. - ---- - -[Return to Development documentation](README.md) +## How to reinstate changes + +When a commit is reverted, the corresponding merge request to reinstate the +changes will include all the details necessary to ensure the changes make it +back into CE and EE. However, you still need to manually set up an EE merge +request that resolves the conflicts. + +Each merge request used to reinstate changes will have the "reverted" label +applied. Please do not remove this label, as it will be used to determine how +many times commits are reverted and how long it takes to reinstate the changes. + +An example merge request can be found in [CE merge request +23280](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/23280). + +## How it works + +The automatic merging is performed using a project called [Merge +Train](https://gitlab.com/gitlab-org/merge-train/). For every commit to merge or +revert, we generate patches using `git format-patch` which we then try to apply +using `git am --3way`. If this succeeds we push the changes to EE, if this fails +we decide what to do based on the failure reason: + +1. If the patch could not be applied because it was already applied, we just + skip it. +1. If the patch caused conflicts, we revert the source commits. + +Commits are reverted in reverse order, ensuring that if commit B depends on A, +and both conflict, we first revert B followed by reverting A. + +## FAQ + +### Why? + +We want to work towards being able to deploy continuously, but this requires +that `master` is always stable and has all the changes we need. If CE `master` +can not be merged into EE `master` due to merge conflicts, this prevents _any_ +change from CE making its way into EE. Since GitLab.com runs on EE, this +effectively prevents us from deploying changes. + +Past experiences and data have shown that periodic CE to EE merge requests do +not scale, and often take a very long time to complete. For example, [in this +comment](https://gitlab.com/gitlab-org/release/framework/issues/49#note_114614619) +we determined that the average time to close an upstream merge request is around +5 hours, with peaks up to several days. Periodic merge requests are also +frustrating to work with, because they often include many changes unrelated to +your own changes. + +Automatically merging or reverting commits allows us to keep merging changes +from CE into EE, as we never have to wait hours for somebody to resolve a set of +merge conflicts. + +### Does the CE to EE merge take into account merge commits? + +No. When merging CE changes into EE, merge commits are ignored. + +### My changes are reverted, but I set up an EE MR to resolve conflicts + +Most likely the automatic merge job ran before the EE merge request was merged. +If this keeps happening, consider reporting a bug in the [Merge Train issue +tracker](https://gitlab.com/gitlab-org/merge-train/issues). + +### My changes keep getting reverted, and this is really annoying! + +This is understandable, but the solution to this is fairly straightforward: +simply set up an EE merge request for every CE merge request, and resolve your +conflicts before the changes are reverted. + +### Will we allow certain people to still merge changes, even if they conflict? + +No. + +### Some files I work with often conflict, how can I best deal with this? + +If you find you keep running into merge conflicts, consider refactoring the file +so that the EE specific changes are not intertwined with CE code. For Ruby code +you can do this by moving the EE code to a separate module, which can then be +injected into the appropriate classes or modules. See [Guidelines for +implementing Enterprise Edition features](ee_features.md) for more information. + +### Will changelog entries be reverted automatically? + +Only if the changelog was added in the commit that was reverted. If a changelog +entry was added in a separate commit, it is possible for it to be left behind. +Since changelog entries are related to the changes in question, there is no real +reason to commit the changelog separately, and as such this should not be a big +problem. diff --git a/doc/development/gotchas.md b/doc/development/gotchas.md index 84dea7ce9aa..a5a34d82bec 100644 --- a/doc/development/gotchas.md +++ b/doc/development/gotchas.md @@ -92,7 +92,7 @@ describe API::Labels do end ``` -## Avoid using `allow_any_instance_of` in RSpec +## Avoid using `expect_any_instance_of` or `allow_any_instance_of` in RSpec ### Why @@ -102,7 +102,7 @@ end error like this: ``` - 1.1) Failure/Error: allow_any_instance_of(ApplicationSetting).to receive_messages(messages) + 1.1) Failure/Error: expect_any_instance_of(ApplicationSetting).to receive_messages(messages) Using `any_instance` to stub a method (elasticsearch_indexing) that has been defined on a prepended module (EE::ApplicationSetting) is not supported. ``` @@ -112,7 +112,7 @@ Instead of writing: ```ruby # Don't do this: -allow_any_instance_of(Project).to receive(:add_import_job) +expect_any_instance_of(Project).to receive(:add_import_job) ``` We could write: diff --git a/doc/development/i18n/proofreader.md b/doc/development/i18n/proofreader.md index f75f5882b56..fd4df64999b 100644 --- a/doc/development/i18n/proofreader.md +++ b/doc/development/i18n/proofreader.md @@ -43,7 +43,7 @@ are very appreciative of the work done by translators and proofreaders! - Greek - Proofreaders needed. - Hebrew - - Proofreaders needed. + - Yaron Shahrabani - [GitLab](https://gitlab.com/yarons), [Crowdin](https://crowdin.com/profile/YaronSh) - Hungarian - Proofreaders needed. - Indonesian diff --git a/doc/install/installation.md b/doc/install/installation.md index 06293b8caf5..d041bfa863a 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -12,7 +12,7 @@ Since installations from source don't have Runit, Sidekiq can't be terminated an ## Select Version to Install -Make sure you view [this installation guide](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/install/installation.md) from the branch (version) of GitLab you would like to install (e.g., `11-5-stable`). +Make sure you view [this installation guide](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/install/installation.md) from the branch (version) of GitLab you would like to install (e.g., `11-6-stable`). You can select the branch in the version dropdown in the top left corner of GitLab (below the menu bar). If the highest number stable branch is unclear please check the [GitLab Blog](https://about.gitlab.com/blog/) for installation guide links by version. @@ -300,9 +300,9 @@ sudo usermod -aG redis git ### Clone the Source # Clone GitLab repository - sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 11-5-stable gitlab + sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 11-6-stable gitlab -**Note:** You can change `11-5-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server! +**Note:** You can change `11-6-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server! ### Configure It diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index a8b088395aa..63e7497cbbc 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -666,8 +666,6 @@ also be customized, and you can easily use a [custom buildpack](#custom-buildpac | `REVIEW_DISABLED` | From GitLab 11.0, this variable can be used to disable the `review` and the manual `review:stop` job. If the variable is present, these jobs will not be created. | | `DAST_DISABLED` | From GitLab 11.0, this variable can be used to disable the `dast` job. If the variable is present, the job will not be created. | | `PERFORMANCE_DISABLED` | From GitLab 11.0, this variable can be used to disable the `performance` job. If the variable is present, the job will not be created. | -| `OLD_REPORTS_DISABLED` | From GitLab 11.5, this variable can be used to disable the `sast` job. If the variable is present, the job will not be created. | -| `NEW_REPORTS_DISABLED` | From GitLab 11.5, this variable can be used to disable the `sast_dashboard` job. If the variable is present, the job will not be created. | TIP: **Tip:** Set up the replica variables using a diff --git a/doc/update/11.5-to-11.6.md b/doc/update/11.5-to-11.6.md new file mode 100644 index 00000000000..031abc434ca --- /dev/null +++ b/doc/update/11.5-to-11.6.md @@ -0,0 +1,390 @@ +--- +comments: false +--- + +# From 11.5 to 11.6 + +Make sure you view this update guide from the branch (version) of GitLab you would +like to install (e.g., `11-6-stable`. You can select the branch in the version +dropdown at the top left corner of GitLab (below the menu bar). + +If the highest number stable branch is unclear please check the +[GitLab Blog](https://about.gitlab.com/blog/archives.html) for installation +guide links by version. + +### 1. Stop server + +```bash +sudo service gitlab stop +``` + +### 2. Backup + +NOTE: If you installed GitLab from source, make sure `rsync` is installed. + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production +``` + +### 3. Update Ruby + +NOTE: GitLab 11.0 and higher only support Ruby 2.4.x and dropped support for Ruby 2.3.x. Be +sure to upgrade your interpreter if necessary. + +You can check which version you are running with `ruby -v`. + +Download Ruby and compile it: + +```bash +mkdir /tmp/ruby && cd /tmp/ruby +curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.3.tar.gz +echo 'f919a9fbcdb7abecd887157b49833663c5c15fda ruby-2.5.3.tar.gz' | shasum -c - && tar xzf ruby-2.5.3.tar.gz +cd ruby-2.5.3 + +./configure --disable-install-rdoc +make +sudo make install +``` + +Install Bundler: + +```bash +sudo gem install bundler --no-ri --no-rdoc +``` + +### 4. Update Node + +GitLab utilizes [webpack](http://webpack.js.org) to compile frontend assets. +This requires a minimum version of node v6.0.0. + +You can check which version you are running with `node -v`. If you are running +a version older than `v6.0.0` you will need to update to a newer version. You +can find instructions to install from community maintained packages or compile +from source at the nodejs.org website. + +<https://nodejs.org/en/download/> + +GitLab also requires the use of yarn `>= v1.2.0` to manage JavaScript +dependencies. + +```bash +curl --silent --show-error https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - +echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list +sudo apt-get update +sudo apt-get install yarn +``` + +More information can be found on the [yarn website](https://yarnpkg.com/en/docs/install). + +### 5. Update Go + +NOTE: GitLab 11.4 and higher only supports Go 1.10.x and newer, and dropped support for Go +1.9.x. Be sure to upgrade your installation if necessary. + +You can check which version you are running with `go version`. + +Download and install Go: + +```bash +# Remove former Go installation folder +sudo rm -rf /usr/local/go + +curl --remote-name --progress https://dl.google.com/go/go1.10.5.linux-amd64.tar.gz +echo 'a035d9beda8341b645d3f45a1b620cf2d8fb0c5eb409be36b389c0fd384ecc3a go1.10.5.linux-amd64.tar.gz' | shasum -a256 -c - && \ + sudo tar -C /usr/local -xzf go1.10.5.linux-amd64.tar.gz +sudo ln -sf /usr/local/go/bin/{go,godoc,gofmt} /usr/local/bin/ +rm go1.10.5.linux-amd64.tar.gz +``` + +### 6. Get latest code + +```bash +cd /home/git/gitlab + +sudo -u git -H git fetch --all --prune +sudo -u git -H git checkout -- db/schema.rb # local changes will be restored automatically +sudo -u git -H git checkout -- locale +``` + +For GitLab Community Edition: + +```bash +cd /home/git/gitlab + +sudo -u git -H git checkout 11-6-stable +``` + +OR + +For GitLab Enterprise Edition: + +```bash +cd /home/git/gitlab + +sudo -u git -H git checkout 11-6-stable-ee +``` + +### 7. Update gitlab-shell + +```bash +cd /home/git/gitlab-shell + +sudo -u git -H git fetch --all --tags --prune +sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_SHELL_VERSION) +sudo -u git -H bin/compile +``` + +### 8. Update gitlab-workhorse + +Install and compile gitlab-workhorse. GitLab-Workhorse uses +[GNU Make](https://www.gnu.org/software/make/). +If you are not using Linux you may have to run `gmake` instead of +`make` below. + +```bash +cd /home/git/gitlab-workhorse + +sudo -u git -H git fetch --all --tags --prune +sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_WORKHORSE_VERSION) +sudo -u git -H make +``` + +### 9. Update Gitaly + +#### Check Gitaly configuration + +Due to a bug in the `rake gitlab:gitaly:install` script your Gitaly +configuration file may contain syntax errors. The block name +`[[storages]]`, which may occur more than once in your `config.toml` +file, should be `[[storage]]` instead. + +```shell +sudo -u git -H sed -i.pre-10.1 's/\[\[storages\]\]/[[storage]]/' /home/git/gitaly/config.toml +``` + +#### Compile Gitaly + +```shell +cd /home/git/gitaly +sudo -u git -H git fetch --all --tags --prune +sudo -u git -H git checkout v$(</home/git/gitlab/GITALY_SERVER_VERSION) +sudo -u git -H make +``` + +### 10. Update gitlab-pages + +#### Only needed if you use GitLab Pages + +Install and compile gitlab-pages. GitLab-Pages uses +[GNU Make](https://www.gnu.org/software/make/). +If you are not using Linux you may have to run `gmake` instead of +`make` below. + +```bash +cd /home/git/gitlab-pages + +sudo -u git -H git fetch --all --tags --prune +sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_PAGES_VERSION) +sudo -u git -H make +``` + +### 11. Update MySQL permissions + +If you are using MySQL you need to grant the GitLab user the necessary +permissions on the database: + +```bash +mysql -u root -p -e "GRANT TRIGGER ON \`gitlabhq_production\`.* TO 'git'@'localhost';" +``` + +If you use MySQL with replication, or just have MySQL configured with binary logging, +you will need to also run the following on all of your MySQL servers: + +```bash +mysql -u root -p -e "SET GLOBAL log_bin_trust_function_creators = 1;" +``` + +You can make this setting permanent by adding it to your `my.cnf`: + +``` +log_bin_trust_function_creators=1 +``` + +### 12. Update configuration files + +#### New `unicorn.rb` configuration + +We have made [changes](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22372) to `unicorn.rb` to allow GitLab run with both Unicorn and Puma in future. + +Make `/home/git/gitlab/config/unicorn.rb` the same as https://gitlab.com/gitlab-org/gitlab-ce/blob/11-6-stable/config/unicorn.rb.example but with your settings. +In particular, make sure that `require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"` line exists and the `before_exec`, `before_fork`, and `after_fork` handlers are configured as shown below: + +```ruby +require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events" + +before_exec do |server| + # Signal application hooks that we're about to restart + Gitlab::Cluster::LifecycleEvents.do_master_restart +end + +before_fork do |server, worker| + # Signal application hooks that we're about to fork + Gitlab::Cluster::LifecycleEvents.do_before_fork +end + +after_fork do |server, worker| + # Signal application hooks of worker start + Gitlab::Cluster::LifecycleEvents.do_worker_start +end +``` + +#### New configuration options for `gitlab.yml` + +There might be configuration options available for [`gitlab.yml`][yaml]. View them with the command below and apply them manually to your current `gitlab.yml`: + +```sh +cd /home/git/gitlab + +git diff origin/11-5-stable:config/gitlab.yml.example origin/11-6-stable:config/gitlab.yml.example +``` + +#### Nginx configuration + +Ensure you're still up-to-date with the latest NGINX configuration changes: + +```sh +cd /home/git/gitlab + +# For HTTPS configurations +git diff origin/11-5-stable:lib/support/nginx/gitlab-ssl origin/11-6-stable:lib/support/nginx/gitlab-ssl + +# For HTTP configurations +git diff origin/11-5-stable:lib/support/nginx/gitlab origin/11-6-stable:lib/support/nginx/gitlab +``` + +If you are using Strict-Transport-Security in your installation to continue using it you must enable it in your Nginx +configuration as GitLab application no longer handles setting it. + +If you are using Apache instead of NGINX please see the updated [Apache templates]. +Also note that because Apache does not support upstreams behind Unix sockets you +will need to let gitlab-workhorse listen on a TCP port. You can do this +via [/etc/default/gitlab]. + +[Apache templates]: https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache +[/etc/default/gitlab]: https://gitlab.com/gitlab-org/gitlab-ce/blob/11-6-stable/lib/support/init.d/gitlab.default.example#L38 + +#### SMTP configuration + +If you're installing from source and use SMTP to deliver mail, you will need to add the following line +to config/initializers/smtp_settings.rb: + +```ruby +ActionMailer::Base.delivery_method = :smtp +``` + +See [smtp_settings.rb.sample] as an example. + +[smtp_settings.rb.sample]: https://gitlab.com/gitlab-org/gitlab-ce/blob/11-6-stable/config/initializers/smtp_settings.rb.sample#L13 + +#### Init script + +There might be new configuration options available for [`gitlab.default.example`][gl-example]. View them with the command below and apply them manually to your current `/etc/default/gitlab`: + +```sh +cd /home/git/gitlab + +git diff origin/11-5-stable:lib/support/init.d/gitlab.default.example origin/11-6-stable:lib/support/init.d/gitlab.default.example +``` + +Ensure you're still up-to-date with the latest init script changes: + +```bash +cd /home/git/gitlab + +sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab +``` + +For Ubuntu 16.04.1 LTS: + +```bash +sudo systemctl daemon-reload +``` + +### 13. Install libs, migrations, etc. + +```bash +cd /home/git/gitlab + +# MySQL installations (note: the line below states '--without postgres') +sudo -u git -H bundle install --without postgres development test --deployment + +# PostgreSQL installations (note: the line below states '--without mysql') +sudo -u git -H bundle install --without mysql development test --deployment + +# Optional: clean up old gems +sudo -u git -H bundle clean + +# Run database migrations +sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production + +# Compile GetText PO files + +sudo -u git -H bundle exec rake gettext:compile RAILS_ENV=production + +# Update node dependencies and recompile assets +sudo -u git -H bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile RAILS_ENV=production NODE_ENV=production + +# Clean up cache +sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production +``` + +**MySQL installations**: Run through the `MySQL strings limits` and `Tables and data conversion to utf8mb4` [tasks](../install/database_mysql.md). + +### 14. Start application + +```bash +sudo service gitlab start +sudo service nginx restart +``` + +### 15. Check application status + +Check if GitLab and its environment are configured correctly: + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production +``` + +To make sure you didn't miss anything run a more thorough check: + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production +``` + +If all items are green, then congratulations, the upgrade is complete! + +## Things went south? Revert to previous version (11.5) + +### 1. Revert the code to the previous version + +Follow the [upgrade guide from 11.4 to 11.5](11.4-to-11.5.md), except for the +database migration (the backup is already migrated to the previous version). + +### 2. Restore from the backup + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:backup:restore RAILS_ENV=production +``` + +If you have more than one backup `*.tar` file(s) please add `BACKUP=timestamp_of_backup` to the command above. + +[yaml]: https://gitlab.com/gitlab-org/gitlab-ce/blob/11-6-stable/config/gitlab.yml.example +[gl-example]: https://gitlab.com/gitlab-org/gitlab-ce/blob/11-6-stable/lib/support/init.d/gitlab.default.example diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 2f15f3a7d76..c60d25b88cb 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -19,7 +19,6 @@ module API optional :tag_list, type: Array[String], desc: %q(List of Runner's tags) optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' end - # rubocop: disable CodeReuse/ActiveRecord post '/' do attributes = attributes_for_keys([:description, :active, :locked, :run_untagged, :tag_list, :maximum_timeout]) .merge(get_runner_details_from_request) @@ -28,10 +27,10 @@ module API if runner_registration_token_valid? # Create shared runner. Requires admin access attributes.merge(runner_type: :instance_type) - elsif project = Project.find_by(runners_token: params[:token]) + elsif project = Project.find_by_runners_token(params[:token]) # Create a specific runner for the project attributes.merge(runner_type: :project_type, projects: [project]) - elsif group = Group.find_by(runners_token: params[:token]) + elsif group = Group.find_by_runners_token(params[:token]) # Create a specific runner for the group attributes.merge(runner_type: :group_type, groups: [group]) else @@ -46,7 +45,6 @@ module API render_validation_error!(runner) end end - # rubocop: enable CodeReuse/ActiveRecord desc 'Deletes a registered Runner' do http_codes [[204, 'Runner was deleted'], [403, 'Forbidden']] diff --git a/lib/gitlab/background_migration/encrypt_columns.rb b/lib/gitlab/background_migration/encrypt_columns.rb index bd5f12276ab..b9ad8267e37 100644 --- a/lib/gitlab/background_migration/encrypt_columns.rb +++ b/lib/gitlab/background_migration/encrypt_columns.rb @@ -5,15 +5,17 @@ module Gitlab # EncryptColumn migrates data from an unencrypted column - `foo`, say - to # an encrypted column - `encrypted_foo`, say. # + # To avoid depending on a particular version of the model in app/, add a + # model to `lib/gitlab/background_migration/models/encrypt_columns` and use + # it in the migration that enqueues the jobs, so code can be shared. + # # For this background migration to work, the table that is migrated _has_ to # have an `id` column as the primary key. Additionally, the encrypted column # should be managed by attr_encrypted, and map to an attribute with the same # name as the unencrypted column (i.e., the unencrypted column should be - # shadowed). + # shadowed), unless you want to define specific methods / accessors in the + # temporary model in `/models/encrypt_columns/your_model.rb`. # - # To avoid depending on a particular version of the model in app/, add a - # model to `lib/gitlab/background_migration/models/encrypt_columns` and use - # it in the migration that enqueues the jobs, so code can be shared. class EncryptColumns def perform(model, attributes, from, to) model = model.constantize if model.is_a?(String) @@ -36,6 +38,10 @@ module Gitlab end end + def clear_migrated_values? + true + end + private # Build a hash of { attribute => encrypted column name } @@ -72,7 +78,10 @@ module Gitlab if instance.changed? instance.save! - instance.update_columns(to_clear) + + if clear_migrated_values? + instance.update_columns(to_clear) + end end end diff --git a/lib/gitlab/background_migration/encrypt_runners_tokens.rb b/lib/gitlab/background_migration/encrypt_runners_tokens.rb new file mode 100644 index 00000000000..91e559a8765 --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_runners_tokens.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # EncryptColumn migrates data from an unencrypted column - `foo`, say - to + # an encrypted column - `encrypted_foo`, say. + # + # We only create a subclass here because we want to isolate this migration + # (migrating unencrypted runner registration tokens to encrypted columns) + # from other `EncryptColumns` migration. This class name is going to be + # serialized and stored in Redis and later picked by Sidekiq, so we need to + # create a separate class name in order to isolate these migration tasks. + # + # We can solve this differently, see tech debt issue: + # + # https://gitlab.com/gitlab-org/gitlab-ce/issues/54328 + # + class EncryptRunnersTokens < EncryptColumns + def perform(model, from, to) + resource = "::Gitlab::BackgroundMigration::Models::EncryptColumns::#{model.to_s.capitalize}" + model = resource.constantize + attributes = model.encrypted_attributes.keys + + super(model, attributes, from, to) + end + + def clear_migrated_values? + false + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb b/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb new file mode 100644 index 00000000000..41f18979d76 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `runners_token` column in `namespaces` table. + # + class Namespace < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + def runners_token=(value) + self.runners_token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { runners_token: { attribute: :runners_token_encrypted } } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/project.rb b/lib/gitlab/background_migration/models/encrypt_columns/project.rb new file mode 100644 index 00000000000..bfeae14584d --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/project.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `runners_token` column in `projects` table. + # + class Project < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'projects' + self.inheritance_column = :_type_disabled + + def runners_token=(value) + self.runners_token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { runners_token: { attribute: :runners_token_encrypted } } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/runner.rb b/lib/gitlab/background_migration/models/encrypt_columns/runner.rb new file mode 100644 index 00000000000..14ddce4b147 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/runner.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `token` column in `ci_runners` table. + # + class Runner < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'ci_runners' + self.inheritance_column = :_type_disabled + + def token=(value) + self.token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { token: { attribute: :token_encrypted } } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/settings.rb b/lib/gitlab/background_migration/models/encrypt_columns/settings.rb new file mode 100644 index 00000000000..08ae35c0671 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/settings.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `runners_token` column in `application_settings` table. + # + class Settings < ActiveRecord::Base + include ::EachBatch + include ::CacheableAttributes + + self.table_name = 'application_settings' + self.inheritance_column = :_type_disabled + + after_commit do + ::ApplicationSetting.expire + end + + def runners_registration_token=(value) + self.runners_registration_token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { + runners_registration_token: { + attribute: :runners_registration_token_encrypted + } + } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb index bb76eb8ed48..ccd9d4c6d44 100644 --- a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb +++ b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb @@ -15,12 +15,12 @@ module Gitlab attr_encrypted :token, mode: :per_attribute_iv, algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_truncated + key: ::Settings.attr_encrypted_db_key_base_truncated attr_encrypted :url, mode: :per_attribute_iv, algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_truncated + key: ::Settings.attr_encrypted_db_key_base_truncated end end end diff --git a/lib/gitlab/checks/base_checker.rb b/lib/gitlab/checks/base_checker.rb new file mode 100644 index 00000000000..f8cda0382fe --- /dev/null +++ b/lib/gitlab/checks/base_checker.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class BaseChecker + include Gitlab::Utils::StrongMemoize + + attr_reader :change_access + delegate(*ChangeAccess::ATTRIBUTES, to: :change_access) + + def initialize(change_access) + @change_access = change_access + end + + def validate! + raise NotImplementedError + end + + private + + def deletion? + Gitlab::Git.blank_ref?(newrev) + end + + def update? + !Gitlab::Git.blank_ref?(oldrev) && !deletion? + end + + def updated_from_web? + protocol == 'web' + end + + def tag_exists? + project.repository.tag_exists?(tag_name) + end + end + end +end diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb new file mode 100644 index 00000000000..d06b2df36f2 --- /dev/null +++ b/lib/gitlab/checks/branch_check.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class BranchCheck < BaseChecker + ERROR_MESSAGES = { + delete_default_branch: 'The default branch of a project cannot be deleted.', + force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.', + non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project maintainer or owner can delete a protected branch.', + non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.', + merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.', + push_protected_branch: 'You are not allowed to push code to protected branches on this project.' + }.freeze + + LOG_MESSAGES = { + delete_default_branch_check: "Checking if default branch is being deleted...", + protected_branch_checks: "Checking if you are force pushing to a protected branch...", + protected_branch_push_checks: "Checking if you are allowed to push to the protected branch...", + protected_branch_deletion_checks: "Checking if you are allowed to delete the protected branch..." + }.freeze + + def validate! + return unless branch_name + + logger.log_timed(LOG_MESSAGES[:delete_default_branch_check]) do + if deletion? && branch_name == project.default_branch + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] + end + end + + protected_branch_checks + end + + private + + def protected_branch_checks + logger.log_timed(LOG_MESSAGES[:protected_branch_checks]) do + return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks + + if forced_push? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] + end + end + + if deletion? + protected_branch_deletion_checks + else + protected_branch_push_checks + end + end + + def protected_branch_deletion_checks + logger.log_timed(LOG_MESSAGES[:protected_branch_deletion_checks]) do + unless user_access.can_delete_branch?(branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] + end + + unless updated_from_web? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] + end + end + end + + def protected_branch_push_checks + logger.log_timed(LOG_MESSAGES[:protected_branch_push_checks]) do + if matching_merge_request? + unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] + end + else + unless user_access.can_push_to_branch?(branch_name) + raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message + end + end + end + end + + def push_to_protected_branch_rejected_message + if project.empty_repo? + empty_project_push_message + else + ERROR_MESSAGES[:push_protected_branch] + end + end + + def empty_project_push_message + <<~MESSAGE + + A default branch (e.g. master) does not yet exist for #{project.full_path} + Ask a project Owner or Maintainer to create a default branch: + + #{project_members_url} + + MESSAGE + end + + def project_members_url + Gitlab::Routing.url_helpers.project_project_members_url(project) + end + + def matching_merge_request? + Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? + end + + def forced_push? + Gitlab::Checks::ForcePush.force_push?(project, oldrev, newrev) + end + end + end +end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 074afe9c412..7778d3068cc 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -3,35 +3,11 @@ module Gitlab module Checks class ChangeAccess - ERROR_MESSAGES = { - push_code: 'You are not allowed to push code to this project.', - delete_default_branch: 'The default branch of a project cannot be deleted.', - force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.', - non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project maintainer or owner can delete a protected branch.', - non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.', - merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.', - push_protected_branch: 'You are not allowed to push code to protected branches on this project.', - change_existing_tags: 'You are not allowed to change existing tags on this project.', - update_protected_tag: 'Protected tags cannot be updated.', - delete_protected_tag: 'Protected tags cannot be deleted.', - create_protected_tag: 'You are not allowed to create this tag as it is protected.', - lfs_objects_missing: 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' - }.freeze + ATTRIBUTES = %i[user_access project skip_authorization + skip_lfs_integrity_check protocol oldrev newrev ref + branch_name tag_name logger commits].freeze - LOG_MESSAGES = { - push_checks: "Checking if you are allowed to push...", - delete_default_branch_check: "Checking if default branch is being deleted...", - protected_branch_checks: "Checking if you are force pushing to a protected branch...", - protected_branch_push_checks: "Checking if you are allowed to push to the protected branch...", - protected_branch_deletion_checks: "Checking if you are allowed to delete the protected branch...", - tag_checks: "Checking if you are allowed to change existing tags...", - protected_tag_checks: "Checking if you are creating, updating or deleting a protected tag...", - lfs_objects_exist_check: "Scanning repository for blobs stored in LFS and verifying their files have been uploaded to GitLab...", - commits_check_file_paths_validation: "Validating commits' file paths...", - commits_check: "Validating commit contents..." - }.freeze - - attr_reader :user_access, :project, :skip_authorization, :skip_lfs_integrity_check, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name, :logger + attr_reader(*ATTRIBUTES) def initialize( change, user_access:, project:, skip_authorization: false, @@ -50,206 +26,32 @@ module Gitlab @logger.append_message("Running checks for ref: #{@branch_name || @tag_name}") end - def exec(skip_commits_check: false) + def exec return true if skip_authorization - push_checks - branch_checks - tag_checks - lfs_objects_exist_check unless skip_lfs_integrity_check - commits_check unless skip_commits_check + ref_level_checks + # Check of commits should happen as the last step + # given they're expensive in terms of performance + commits_check true end - protected - - def push_checks - logger.log_timed(LOG_MESSAGES[__method__]) do - unless can_push? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] - end - end - end - - def branch_checks - return unless branch_name - - logger.log_timed(LOG_MESSAGES[:delete_default_branch_check]) do - if deletion? && branch_name == project.default_branch - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] - end - end - - protected_branch_checks - end - - def protected_branch_checks - logger.log_timed(LOG_MESSAGES[__method__]) do - return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks - - if forced_push? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] - end - end - - if deletion? - protected_branch_deletion_checks - else - protected_branch_push_checks - end - end - - def protected_branch_deletion_checks - logger.log_timed(LOG_MESSAGES[__method__]) do - unless user_access.can_delete_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] - end - - unless updated_from_web? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] - end - end - end - - def protected_branch_push_checks - logger.log_timed(LOG_MESSAGES[__method__]) do - if matching_merge_request? - unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] - end - else - unless user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message - end - end - end - end - - def tag_checks - return unless tag_name - - logger.log_timed(LOG_MESSAGES[__method__]) do - if tag_exists? && user_access.cannot_do_action?(:admin_project) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] - end - end - - protected_tag_checks + def commits + @commits ||= project.repository.new_commits(newrev) end - def protected_tag_checks - logger.log_timed(LOG_MESSAGES[__method__]) do - return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks - - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? + protected - unless user_access.can_create_tag?(tag_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] - end - end + def ref_level_checks + Gitlab::Checks::PushCheck.new(self).validate! + Gitlab::Checks::BranchCheck.new(self).validate! + Gitlab::Checks::TagCheck.new(self).validate! + Gitlab::Checks::LfsCheck.new(self).validate! end def commits_check - return if deletion? || newrev.nil? - return unless should_run_commit_validations? - - logger.log_timed(LOG_MESSAGES[__method__]) do - # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - commits.each do |commit| - logger.check_timeout_reached - - commit_check.validate(commit, validations_for_commit(commit)) - end - end - end - - logger.log_timed(LOG_MESSAGES[:commits_check_file_paths_validation]) do - commit_check.validate_file_paths - end - end - - # Method overwritten in EE to inject custom validations - def validations_for_commit(_) - [] - end - - private - - def push_to_protected_branch_rejected_message - if project.empty_repo? - empty_project_push_message - else - ERROR_MESSAGES[:push_protected_branch] - end - end - - def empty_project_push_message - <<~MESSAGE - - A default branch (e.g. master) does not yet exist for #{project.full_path} - Ask a project Owner or Maintainer to create a default branch: - - #{project_members_url} - - MESSAGE - end - - def project_members_url - Gitlab::Routing.url_helpers.project_project_members_url(project) - end - - def should_run_commit_validations? - commit_check.validate_lfs_file_locks? - end - - def updated_from_web? - protocol == 'web' - end - - def tag_exists? - project.repository.tag_exists?(tag_name) - end - - def forced_push? - Gitlab::Checks::ForcePush.force_push?(project, oldrev, newrev) - end - - def update? - !Gitlab::Git.blank_ref?(oldrev) && !deletion? - end - - def deletion? - Gitlab::Git.blank_ref?(newrev) - end - - def matching_merge_request? - Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? - end - - def lfs_objects_exist_check - logger.log_timed(LOG_MESSAGES[__method__]) do - lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) - - if lfs_check.objects_missing? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] - end - end - end - - def commit_check - @commit_check ||= Gitlab::Checks::CommitCheck.new(project, user_access.user, newrev, oldrev) - end - - def commits - @commits ||= project.repository.new_commits(newrev) - end - - def can_push? - user_access.can_do_action?(:push_code) || - user_access.can_push_to_branch?(branch_name) + Gitlab::Checks::DiffCheck.new(self).validate! end end end diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb deleted file mode 100644 index 58267b6752f..00000000000 --- a/lib/gitlab/checks/commit_check.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Checks - class CommitCheck - include Gitlab::Utils::StrongMemoize - - attr_reader :project, :user, :newrev, :oldrev - - def initialize(project, user, newrev, oldrev) - @project = project - @user = user - @newrev = newrev - @oldrev = oldrev - @file_paths = [] - end - - def validate(commit, validations) - return if validations.empty? && path_validations.empty? - - commit.raw_deltas.each do |diff| - @file_paths << (diff.new_path || diff.old_path) - - validations.each do |validation| - if error = validation.call(diff) - raise ::Gitlab::GitAccess::UnauthorizedError, error - end - end - end - end - - def validate_file_paths - path_validations.each do |validation| - if error = validation.call(@file_paths) - raise ::Gitlab::GitAccess::UnauthorizedError, error - end - end - end - - def validate_lfs_file_locks? - strong_memoize(:validate_lfs_file_locks) do - project.lfs_enabled? && newrev && oldrev && project.any_lfs_file_locks? - end - end - - private - - # rubocop: disable CodeReuse/ActiveRecord - def lfs_file_locks_validation - lambda do |paths| - lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user.id).first - - if lfs_lock - return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}" - end - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def path_validations - validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] - end - end - end -end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb new file mode 100644 index 00000000000..49d361fcef7 --- /dev/null +++ b/lib/gitlab/checks/diff_check.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class DiffCheck < BaseChecker + include Gitlab::Utils::StrongMemoize + + LOG_MESSAGES = { + validate_file_paths: "Validating diffs' file paths...", + diff_content_check: "Validating diff contents..." + }.freeze + + def validate! + return unless should_run_diff_validations? + return if commits.empty? + return unless uses_raw_delta_validations? + + file_paths = [] + process_raw_deltas do |diff| + file_paths << (diff.new_path || diff.old_path) + + validate_diff(diff) + end + + validate_file_paths(file_paths) + end + + private + + def should_run_diff_validations? + newrev && oldrev && !deletion? && validate_lfs_file_locks? + end + + def validate_lfs_file_locks? + strong_memoize(:validate_lfs_file_locks) do + project.lfs_enabled? && project.any_lfs_file_locks? + end + end + + def uses_raw_delta_validations? + validations_for_diff.present? || path_validations.present? + end + + def validate_diff(diff) + validations_for_diff.each do |validation| + if error = validation.call(diff) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + + # Method overwritten in EE to inject custom validations + def validations_for_diff + [] + end + + def path_validations + validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] + end + + def process_raw_deltas + logger.log_timed(LOG_MESSAGES[:diff_content_check]) do + # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + commits.each do |commit| + logger.check_timeout_reached + + commit.raw_deltas.each do |diff| + yield(diff) + end + end + end + end + end + + def validate_file_paths(file_paths) + logger.log_timed(LOG_MESSAGES[__method__]) do + path_validations.each do |validation| + if error = validation.call(file_paths) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def lfs_file_locks_validation + lambda do |paths| + lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user_access.user.id).take + + if lfs_lock + return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}" + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb new file mode 100644 index 00000000000..e42684e679a --- /dev/null +++ b/lib/gitlab/checks/lfs_check.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class LfsCheck < BaseChecker + LOG_MESSAGE = "Scanning repository for blobs stored in LFS and verifying their files have been uploaded to GitLab...".freeze + ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".'.freeze + + def validate! + return if skip_lfs_integrity_check + + logger.log_timed(LOG_MESSAGE) do + lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) + + if lfs_check.objects_missing? + raise GitAccess::UnauthorizedError, ERROR_MESSAGE + end + end + end + end + end +end diff --git a/lib/gitlab/checks/push_check.rb b/lib/gitlab/checks/push_check.rb new file mode 100644 index 00000000000..f3a52f09868 --- /dev/null +++ b/lib/gitlab/checks/push_check.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class PushCheck < BaseChecker + def validate! + logger.log_timed("Checking if you are allowed to push...") do + unless can_push? + raise GitAccess::UnauthorizedError, 'You are not allowed to push code to this project.' + end + end + end + + private + + def can_push? + user_access.can_do_action?(:push_code) || + user_access.can_push_to_branch?(branch_name) + end + end + end +end diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb new file mode 100644 index 00000000000..2a75c8059bd --- /dev/null +++ b/lib/gitlab/checks/tag_check.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class TagCheck < BaseChecker + ERROR_MESSAGES = { + change_existing_tags: 'You are not allowed to change existing tags on this project.', + update_protected_tag: 'Protected tags cannot be updated.', + delete_protected_tag: 'Protected tags cannot be deleted.', + create_protected_tag: 'You are not allowed to create this tag as it is protected.' + }.freeze + + LOG_MESSAGES = { + tag_checks: "Checking if you are allowed to change existing tags...", + protected_tag_checks: "Checking if you are creating, updating or deleting a protected tag..." + }.freeze + + def validate! + return unless tag_name + + logger.log_timed(LOG_MESSAGES[:tag_checks]) do + if tag_exists? && user_access.cannot_do_action?(:admin_project) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] + end + end + + protected_tag_checks + end + + private + + def protected_tag_checks + logger.log_timed(LOG_MESSAGES[__method__]) do + return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks + + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? + + unless user_access.can_create_tag?(tag_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 49f03d2ae19..3b2cae07c12 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -19,15 +19,6 @@ # * review: REVIEW_DISABLED # * stop_review: REVIEW_DISABLED # -# The sast and sast_dashboard jobs are executed to guarantee full compatibility -# with the group security dashboard and the security reports with old runners. -# If you use only runners with version 11.5 or above, you can disable the sast -# job by setting the OLD_REPORTS_DISABLED environment variable. If you use only -# runners with version below 11.5, you can disable the sast_dashboard job by -# setting the NEW_REPORTS_DISABLED environment variable. -# The sast_dashboard job will be removed in the future, when the sast job will -# use the new reports syntax. -# # In order to deploy, you must have a Kubernetes cluster configured either # via a project integration, or via group/project variables. # AUTO_DEVOPS_DOMAIN must also be set as a variable at the group or project @@ -182,29 +173,6 @@ sast: except: variables: - $SAST_DISABLED - - $OLD_REPORTS_DISABLED - -sast_dashboard: - stage: test - image: docker:stable - allow_failure: true - services: - - docker:stable-dind - script: - - setup_docker - - sast - artifacts: - reports: - sast: gl-sast-report.json - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ - except: - variables: - - $SAST_DISABLED - - $NEW_REPORTS_DISABLED dependency_scanning: stage: test diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb index 68d0b5d8f8a..87a03d9c58f 100644 --- a/lib/gitlab/crypto_helper.rb +++ b/lib/gitlab/crypto_helper.rb @@ -6,8 +6,8 @@ module Gitlab AES256_GCM_OPTIONS = { algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_truncated, - iv: Settings.attr_encrypted_db_key_base_truncated[0..11] + key: Settings.attr_encrypted_db_key_base_32, + iv: Settings.attr_encrypted_db_key_base_12 }.freeze def sha256(value) @@ -17,7 +17,7 @@ module Gitlab def aes256_gcm_encrypt(value) encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value)) - Base64.encode64(encrypted_token) + Base64.strict_encode64(encrypted_token) end def aes256_gcm_decrypt(value) diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index ea6529e2dc4..c996d786909 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -1,7 +1,12 @@ # frozen_string_literal: true # For large tables, PostgreSQL can take a long time to count rows due to MVCC. -# We can optimize this by using the reltuples count as described in https://wiki.postgresql.org/wiki/Slow_Counting. +# We can optimize this by using various strategies for approximate counting. +# +# For example, we can use the reltuples count as described in https://wiki.postgresql.org/wiki/Slow_Counting. +# +# However, since statistics are not always up to date, we also implement a table sampling strategy +# that performs an exact count but only on a sample of the table. See TablesampleCountStrategy. module Gitlab module Database module Count @@ -20,68 +25,30 @@ module Gitlab end # Takes in an array of models and returns a Hash for the approximate - # counts for them. If the model's table has not been vacuumed or - # analyzed recently, simply run the Model.count to get the data. + # counts for them. + # + # Various count strategies can be specified that are executed in + # sequence until all tables have an approximate count attached + # or we run out of strategies. + # + # Note that not all strategies are available on all supported RDBMS. # # @param [Array] # @return [Hash] of Model -> count mapping - def self.approximate_counts(models) - table_to_model_map = models.each_with_object({}) do |model, hash| - hash[model.table_name] = model - end - - table_names = table_to_model_map.keys - counts_by_table_name = Gitlab::Database.postgresql? ? reltuples_from_recently_updated(table_names) : {} + def self.approximate_counts(models, strategies: [TablesampleCountStrategy, ReltuplesCountStrategy, ExactCountStrategy]) + strategies.each_with_object({}) do |strategy, counts_by_model| + if strategy.enabled? + models_with_missing_counts = models - counts_by_model.keys - # Convert table -> count to Model -> count - counts_by_model = counts_by_table_name.each_with_object({}) do |pair, hash| - model = table_to_model_map[pair.first] - hash[model] = pair.second - end + break if models_with_missing_counts.empty? - missing_tables = table_names - counts_by_table_name.keys + counts = strategy.new(models_with_missing_counts).count - missing_tables.each do |table| - model = table_to_model_map[table] - counts_by_model[model] = model.count + counts.each do |model, count| + counts_by_model[model] = count + end + end end - - counts_by_model - end - - # Returns a hash of the table names that have recently updated tuples. - # - # @param [Array] table names - # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) - def self.reltuples_from_recently_updated(table_names) - query = postgresql_estimate_query(table_names) - rows = [] - - # Querying tuple stats only works on the primary. Due to load - # balancing, we need to ensure this query hits the load balancer. The - # easiest way to do this is to start a transaction. - ActiveRecord::Base.transaction do - rows = ActiveRecord::Base.connection.select_all(query) - end - - rows.each_with_object({}) { |row, data| data[row['table_name']] = row['estimate'].to_i } - rescue *CONNECTION_ERRORS - {} - end - - # Generates the PostgreSQL query to return the tuples for tables - # that have been vacuumed or analyzed in the last hour. - # - # @param [Array] table names - # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) - def self.postgresql_estimate_query(table_names) - time = "to_timestamp(#{1.hour.ago.to_i})" - <<~SQL - SELECT pg_class.relname AS table_name, reltuples::bigint AS estimate FROM pg_class - LEFT JOIN pg_stat_user_tables ON pg_class.relname = pg_stat_user_tables.relname - WHERE pg_class.relname IN (#{table_names.map { |table| "'#{table}'" }.join(',')}) - AND (last_vacuum > #{time} OR last_autovacuum > #{time} OR last_analyze > #{time} OR last_autoanalyze > #{time}) - SQL end end end diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb new file mode 100644 index 00000000000..0276fe2b54f --- /dev/null +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Count + # This strategy performs an exact count on the model. + # + # This is guaranteed to be accurate, however it also scans the + # whole table. Hence, there are no guarantees with respect + # to runtime. + # + # Note that for very large tables, this may even timeout. + class ExactCountStrategy + attr_reader :models + def initialize(models) + @models = models + end + + def count + models.each_with_object({}) do |model, data| + data[model] = model.count + end + end + + def self.enabled? + true + end + end + end + end +end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb new file mode 100644 index 00000000000..c3a674aeb7e --- /dev/null +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Count + class PgClass < ActiveRecord::Base + self.table_name = 'pg_class' + end + + # This strategy counts based on PostgreSQL's statistics in pg_stat_user_tables. + # + # Specifically, it relies on the column reltuples in said table. An additional + # check is performed to make sure statistics were updated within the last hour. + # + # Otherwise, this strategy skips tables with outdated statistics. + # + # There are no guarantees with respect to the accuracy of this strategy. Runtime + # however is guaranteed to be "fast", because it only looks up statistics. + class ReltuplesCountStrategy + attr_reader :models + def initialize(models) + @models = models + end + + # Returns a hash of the table names that have recently updated tuples. + # + # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) + def count + size_estimates + rescue *CONNECTION_ERRORS + {} + end + + def self.enabled? + Gitlab::Database.postgresql? + end + + private + + def table_names + models.map(&:table_name) + end + + def size_estimates(check_statistics: true) + table_to_model = models.each_with_object({}) { |model, h| h[model.table_name] = model } + + # Querying tuple stats only works on the primary. Due to load balancing, the + # easiest way to do this is to start a transaction. + ActiveRecord::Base.transaction do + get_statistics(table_names, check_statistics: check_statistics).each_with_object({}) do |row, data| + model = table_to_model[row.table_name] + data[model] = row.estimate + end + end + end + + # Generates the PostgreSQL query to return the tuples for tables + # that have been vacuumed or analyzed in the last hour. + # + # @param [Array] table names + # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) + def get_statistics(table_names, check_statistics: true) + time = 1.hour.ago + + query = PgClass.joins("LEFT JOIN pg_stat_user_tables USING (relname)") + .where(relname: table_names) + .select('pg_class.relname AS table_name, reltuples::bigint AS estimate') + + if check_statistics + query = query.where('last_vacuum > ? OR last_autovacuum > ? OR last_analyze > ? OR last_autoanalyze > ?', + time, time, time, time) + end + + query + end + end + end + end +end diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb new file mode 100644 index 00000000000..cf1cf054dbf --- /dev/null +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Count + # A tablesample count executes in two phases: + # * Estimate table sizes based on reltuples. + # * Based on the estimate: + # * If the table is considered 'small', execute an exact relation count. + # * Otherwise, count on a sample of the table using TABLESAMPLE. + # + # The size of the sample is chosen in a way that we always roughly scan + # the same amount of rows (see TABLESAMPLE_ROW_TARGET). + # + # There are no guarantees with respect to the accuracy of the result or runtime. + class TablesampleCountStrategy < ReltuplesCountStrategy + EXACT_COUNT_THRESHOLD = 10_000 + TABLESAMPLE_ROW_TARGET = 10_000 + + def count + estimates = size_estimates(check_statistics: false) + + models.each_with_object({}) do |model, count_by_model| + count = perform_count(model, estimates[model]) + count_by_model[model] = count if count + end + rescue *CONNECTION_ERRORS + {} + end + + def self.enabled? + Gitlab::Database.postgresql? && Feature.enabled?(:tablesample_counts) + end + + private + + def perform_count(model, estimate) + # If we estimate 0, we may not have statistics at all. Don't use them. + return nil unless estimate && estimate > 0 + + if estimate < EXACT_COUNT_THRESHOLD + # The table is considered small, the assumption here is that + # the exact count will be fast anyways. + model.count + else + # The table is considered large, let's only count on a sample. + tablesample_count(model, estimate) + end + end + + def tablesample_count(model, estimate) + portion = (TABLESAMPLE_ROW_TARGET.to_f / estimate).round(4) + inverse = 1 / portion + query = <<~SQL + SELECT (COUNT(*)*#{inverse})::integer AS count + FROM #{model.table_name} TABLESAMPLE SYSTEM (#{portion * 100}) + SQL + + rows = ActiveRecord::Base.connection.select_all(query) + + Integer(rows.first['count']) + end + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 8380e86c128..4f3298812a8 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -100,6 +100,7 @@ excluded_attributes: - :import_source - :mirror - :runners_token + - :runners_token_encrypted - :repository_storage - :repository_read_only - :lfs_enabled @@ -114,6 +115,9 @@ excluded_attributes: - :remote_mirror_available_overridden - :description_html - :repository_languages + namespaces: + - :runners_token + - :runners_token_encrypted project_import_state: - :last_error - :jid @@ -155,6 +159,9 @@ excluded_attributes: - :encrypted_token_iv - :encrypted_url - :encrypted_url_iv + runners: + - :token + - :token_encrypted services: - :template diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 097c7653754..9cc781ddf8d 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -10,6 +10,7 @@ module Gitlab triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', + runners: 'Ci::Runner', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', @@ -33,7 +34,7 @@ module Gitlab EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze - TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze + TOKEN_RESET_MODELS = %w[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze def self.create(*args) new(*args).create diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 9e59137a2c0..e0e8f598ba4 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -16,6 +16,21 @@ module Gitlab str.force_encoding(Encoding::UTF_8) end + def ensure_utf8_size(str, bytes:) + raise ArgumentError, 'Empty string provided!' if str.empty? + raise ArgumentError, 'Negative string size provided!' if bytes.negative? + + truncated = str.each_char.each_with_object(+'') do |char, object| + if object.bytesize + char.bytesize > bytes + break object + else + object.concat(char) + end + end + + truncated + ('0' * (bytes - truncated.bytesize)) + end + # Append path to host, making sure there's one single / in between def append_path(host, path) "#{host.to_s.sub(%r{\/+$}, '')}/#{path.to_s.sub(%r{^\/+}, '')}" diff --git a/scripts/merge-train b/scripts/merge-train new file mode 100755 index 00000000000..a934971b869 --- /dev/null +++ b/scripts/merge-train @@ -0,0 +1,73 @@ +#!/bin/sh + +set -e + +# The name (including namespace) of the EE repository to merge commits into. +EE_PROJECT='gitlab-org/gitlab-ee' + +# The directory to clone GitLab EE into. +EE_DIRECTORY="$CI_PROJECT_DIR/gitlab-ee" + +# The EE branch to merge the changes into. +EE_BRANCH='master' + +# Runs an incremental or periodic merge of CE to EE. This script should be run +# from a container built using https://gitlab.com/gitlab-org/merge-train. + +# Merges (or reverts) commits in a batch (based on CI_COMMIT_BEFORE_SHA and +# CI_COMMIT_SHA), or since a specific commit. +# +# The optional first argument of this function should be a SHA of a commit. When +# specified, all commits since this commit will be processed. +merge() { + # We need to source the configure-ssh script instead of running it with + # `sh`, since it uses `eval` for SSH agent and we want the result of that to + # persist. + # + # shellcheck disable=SC1091 + . /app/bin/configure-ssh + + # We can not perform a shallow clone, as this results in Git sometimes + # refusing to fetch from CE. To work around this, we perform a full clone + # but cache the repository in CI. If a cached repository exists, we simply + # just pull from `master`. + if [ -d "$EE_DIRECTORY" ] + then + echo "Updating existing clone of $EE_PROJECT" + + git -C "$EE_DIRECTORY" pull --quiet origin "$EE_BRANCH" + else + echo "Cloning $EE_PROJECT" + + git clone --quiet --single-branch --branch "$EE_BRANCH" \ + "git@gitlab.com:$EE_PROJECT.git" "$EE_DIRECTORY" + fi + + cd /app + + env GITLAB_TOKEN="$MERGE_TRAIN_API_TOKEN" \ + bundle exec /app/bin/merge-train "$CI_PROJECT_DIR" "$EE_DIRECTORY" \ + --source-name "$CI_PROJECT_PATH" \ + --target-branch "$EE_BRANCH" \ + ${1:+--before "$1"} +} + +# Merges (or reverts) all commits since a point in time as supported by `git log +# --since`, such as "12 hours ago". +merge_since() { + commit="$(git log --since="$MERGE_SINCE" --reverse --format=%H | head -n1)" + + if [ "$commit" = '' ] + then + echo "There are no commits to merge since $MERGE_SINCE" + else + merge "$commit" + fi +} + +if [ "$MERGE_SINCE" ] +then + merge_since +else + merge +fi diff --git a/spec/config/settings_spec.rb b/spec/config/settings_spec.rb index 83b2de47741..c89b5f48dc0 100644 --- a/spec/config/settings_spec.rb +++ b/spec/config/settings_spec.rb @@ -6,4 +6,102 @@ describe Settings do expect(described_class.omniauth.enabled).to be true end end + + describe '.attr_encrypted_db_key_base_truncated' do + it 'is a string with maximum 32 bytes size' do + expect(described_class.attr_encrypted_db_key_base_truncated.bytesize) + .to be <= 32 + end + end + + describe '.attr_encrypted_db_key_base_12' do + context 'when db key base secret is less than 12 bytes' do + before do + allow(described_class) + .to receive(:attr_encrypted_db_key_base) + .and_return('a' * 10) + end + + it 'expands db key base secret to 12 bytes' do + expect(described_class.attr_encrypted_db_key_base_12) + .to eq(('a' * 10) + ('0' * 2)) + end + end + + context 'when key has multiple multi-byte UTF chars exceeding 12 bytes' do + before do + allow(described_class) + .to receive(:attr_encrypted_db_key_base) + .and_return('❤' * 18) + end + + it 'does not use more than 32 bytes' do + db_key_base = described_class.attr_encrypted_db_key_base_12 + + expect(db_key_base).to eq('❤' * 4) + expect(db_key_base.bytesize).to eq 12 + end + end + end + + describe '.attr_encrypted_db_key_base_32' do + context 'when db key base secret is less than 32 bytes' do + before do + allow(described_class) + .to receive(:attr_encrypted_db_key_base) + .and_return('a' * 10) + end + + it 'expands db key base secret to 32 bytes' do + expanded_key_base = ('a' * 10) + ('0' * 22) + + expect(expanded_key_base.bytesize).to eq 32 + expect(described_class.attr_encrypted_db_key_base_32) + .to eq expanded_key_base + end + end + + context 'when db key base secret is 32 bytes' do + before do + allow(described_class) + .to receive(:attr_encrypted_db_key_base) + .and_return('a' * 32) + end + + it 'returns original value' do + expect(described_class.attr_encrypted_db_key_base_32) + .to eq 'a' * 32 + end + end + + context 'when db key base contains multi-byte UTF character' do + before do + allow(described_class) + .to receive(:attr_encrypted_db_key_base) + .and_return('❤' * 6) + end + + it 'does not use more than 32 bytes' do + db_key_base = described_class.attr_encrypted_db_key_base_32 + + expect(db_key_base).to eq '❤❤❤❤❤❤' + ('0' * 14) + expect(db_key_base.bytesize).to eq 32 + end + end + + context 'when db key base multi-byte UTF chars exceeding 32 bytes' do + before do + allow(described_class) + .to receive(:attr_encrypted_db_key_base) + .and_return('❤' * 18) + end + + it 'does not use more than 32 bytes' do + db_key_base = described_class.attr_encrypted_db_key_base_32 + + expect(db_key_base).to eq(('❤' * 10) + ('0' * 2)) + expect(db_key_base.bytesize).to eq 32 + end + end + end end diff --git a/spec/controllers/concerns/issuable_collections_spec.rb b/spec/controllers/concerns/issuable_collections_spec.rb index e93c923fd39..f87eed6ff9f 100644 --- a/spec/controllers/concerns/issuable_collections_spec.rb +++ b/spec/controllers/concerns/issuable_collections_spec.rb @@ -86,6 +86,7 @@ describe IssuableCollections do it 'only allows whitelisted params' do allow(controller).to receive(:cookies).and_return({}) + allow(controller).to receive(:current_user).and_return(nil) finder_options = controller.send(:finder_options) diff --git a/spec/controllers/projects/blob_controller_spec.rb b/spec/controllers/projects/blob_controller_spec.rb index 3c5a21c47fa..9fc6af6a045 100644 --- a/spec/controllers/projects/blob_controller_spec.rb +++ b/spec/controllers/projects/blob_controller_spec.rb @@ -35,6 +35,11 @@ describe Projects::BlobController do let(:id) { 'binary-encoding/encoding/binary-1.bin' } it { is_expected.to respond_with(:success) } end + + context "Markdown file" do + let(:id) { 'master/README.md' } + it { is_expected.to respond_with(:success) } + end end context 'with file path and JSON format' do diff --git a/spec/factories/clusters/kubernetes_namespaces.rb b/spec/factories/clusters/kubernetes_namespaces.rb index 6ad93fb0f45..3b50a57433f 100644 --- a/spec/factories/clusters/kubernetes_namespaces.rb +++ b/spec/factories/clusters/kubernetes_namespaces.rb @@ -5,10 +5,12 @@ FactoryBot.define do association :cluster, :project, :provided_by_gcp after(:build) do |kubernetes_namespace| - cluster_project = kubernetes_namespace.cluster.cluster_project + if kubernetes_namespace.cluster.project_type? + cluster_project = kubernetes_namespace.cluster.cluster_project - kubernetes_namespace.project = cluster_project.project - kubernetes_namespace.cluster_project = cluster_project + kubernetes_namespace.project = cluster_project.project + kubernetes_namespace.cluster_project = cluster_project + end end trait :with_token do diff --git a/spec/javascripts/diffs/components/diff_gutter_avatars_spec.js b/spec/javascripts/diffs/components/diff_gutter_avatars_spec.js index ad2605a5c5c..cdd30919b09 100644 --- a/spec/javascripts/diffs/components/diff_gutter_avatars_spec.js +++ b/spec/javascripts/diffs/components/diff_gutter_avatars_spec.js @@ -89,6 +89,35 @@ describe('DiffGutterAvatars', () => { expect(component.discussions[0].expanded).toEqual(false); component.$store.dispatch('setInitialNotes', []); }); + + it('forces expansion of all discussions', () => { + spyOn(component.$store, 'dispatch'); + + component.discussions[0].expanded = true; + component.discussions.push({ + ...component.discussions[0], + id: '123test', + expanded: false, + }); + + component.toggleDiscussions(); + + expect(component.$store.dispatch.calls.argsFor(0)).toEqual([ + 'toggleDiscussion', + { + discussionId: component.discussions[0].id, + forceExpanded: true, + }, + ]); + + expect(component.$store.dispatch.calls.argsFor(1)).toEqual([ + 'toggleDiscussion', + { + discussionId: component.discussions[1].id, + forceExpanded: true, + }, + ]); + }); }); }); diff --git a/spec/javascripts/notes/components/noteable_discussion_spec.js b/spec/javascripts/notes/components/noteable_discussion_spec.js index 76e9cd03d2d..ab9c52346d6 100644 --- a/spec/javascripts/notes/components/noteable_discussion_spec.js +++ b/spec/javascripts/notes/components/noteable_discussion_spec.js @@ -6,7 +6,6 @@ import { noteableDataMock, discussionMock, notesDataMock } from '../mock_data'; import mockDiffFile from '../../diffs/mock_data/diff_file'; const discussionWithTwoUnresolvedNotes = 'merge_requests/resolved_diff_discussion.json'; -const diffDiscussionFixture = 'merge_requests/diff_discussion.json'; describe('noteable_discussion component', () => { const Component = Vue.extend(noteableDiscussion); @@ -79,51 +78,6 @@ describe('noteable_discussion component', () => { }); }); - describe('computed', () => { - describe('isRepliesCollapsed', () => { - it('should return false for diff discussions', done => { - const diffDiscussion = getJSONFixture(diffDiscussionFixture)[0]; - vm.$store.dispatch('setInitialNotes', [diffDiscussion]); - - Vue.nextTick() - .then(() => { - expect(vm.isRepliesCollapsed).toEqual(false); - expect(vm.$el.querySelector('.js-toggle-replies')).not.toBeNull(); - expect(vm.$el.querySelector('.discussion-reply-holder')).not.toBeNull(); - }) - .then(done) - .catch(done.fail); - }); - - it('should return false if discussion does not have a reply', () => { - const discussion = { ...discussionMock, resolved: true }; - discussion.notes = discussion.notes.slice(0, 1); - const noRepliesVm = new Component({ - store, - propsData: { discussion }, - }).$mount(); - - expect(noRepliesVm.isRepliesCollapsed).toEqual(false); - expect(noRepliesVm.$el.querySelector('.js-toggle-replies')).toBeNull(); - expect(vm.$el.querySelector('.discussion-reply-holder')).not.toBeNull(); - noRepliesVm.$destroy(); - }); - - it('should return true for resolved non-diff discussion which has replies', () => { - const discussion = { ...discussionMock, resolved: true }; - const resolvedDiscussionVm = new Component({ - store, - propsData: { discussion }, - }).$mount(); - - expect(resolvedDiscussionVm.isRepliesCollapsed).toEqual(true); - expect(resolvedDiscussionVm.$el.querySelector('.js-toggle-replies')).not.toBeNull(); - expect(vm.$el.querySelector('.discussion-reply-holder')).not.toBeNull(); - resolvedDiscussionVm.$destroy(); - }); - }); - }); - describe('methods', () => { describe('jumpToNextDiscussion', () => { it('expands next unresolved discussion', done => { diff --git a/spec/javascripts/notes/stores/mutation_spec.js b/spec/javascripts/notes/stores/mutation_spec.js index 1c4449d1055..52cdc16353a 100644 --- a/spec/javascripts/notes/stores/mutation_spec.js +++ b/spec/javascripts/notes/stores/mutation_spec.js @@ -297,6 +297,16 @@ describe('Notes Store mutations', () => { expect(state.discussions[0].expanded).toEqual(false); }); + + it('forces a discussions expanded state', () => { + const state = { + discussions: [{ ...discussionMock, expanded: false }], + }; + + mutations.TOGGLE_DISCUSSION(state, { discussionId: discussionMock.id, forceExpanded: true }); + + expect(state.discussions[0].expanded).toEqual(true); + }); }); describe('UPDATE_NOTE', () => { diff --git a/spec/lib/gitlab/background_migration/encrypt_runners_tokens_spec.rb b/spec/lib/gitlab/background_migration/encrypt_runners_tokens_spec.rb new file mode 100644 index 00000000000..9d4921968b3 --- /dev/null +++ b/spec/lib/gitlab/background_migration/encrypt_runners_tokens_spec.rb @@ -0,0 +1,77 @@ +require 'spec_helper' + +describe Gitlab::BackgroundMigration::EncryptRunnersTokens, :migration, schema: 20181121111200 do + let(:settings) { table(:application_settings) } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:runners) { table(:ci_runners) } + + context 'when migrating application settings' do + before do + settings.create!(id: 1, runners_registration_token: 'plain-text-token1') + end + + it 'migrates runners registration tokens' do + migrate!(:settings, 1, 1) + + encrypted_token = settings.first.runners_registration_token_encrypted + decrypted_token = ::Gitlab::CryptoHelper.aes256_gcm_decrypt(encrypted_token) + + expect(decrypted_token).to eq 'plain-text-token1' + expect(settings.first.runners_registration_token).to eq 'plain-text-token1' + end + end + + context 'when migrating namespaces' do + before do + namespaces.create!(id: 11, name: 'gitlab', path: 'gitlab-org', runners_token: 'my-token1') + namespaces.create!(id: 12, name: 'gitlab', path: 'gitlab-org', runners_token: 'my-token2') + namespaces.create!(id: 22, name: 'gitlab', path: 'gitlab-org', runners_token: 'my-token3') + end + + it 'migrates runners registration tokens' do + migrate!(:namespace, 11, 22) + + expect(namespaces.all.reload).to all( + have_attributes(runners_token: be_a(String), runners_token_encrypted: be_a(String)) + ) + end + end + + context 'when migrating projects' do + before do + namespaces.create!(id: 11, name: 'gitlab', path: 'gitlab-org') + projects.create!(id: 111, namespace_id: 11, name: 'gitlab', path: 'gitlab-ce', runners_token: 'my-token1') + projects.create!(id: 114, namespace_id: 11, name: 'gitlab', path: 'gitlab-ce', runners_token: 'my-token2') + projects.create!(id: 116, namespace_id: 11, name: 'gitlab', path: 'gitlab-ce', runners_token: 'my-token3') + end + + it 'migrates runners registration tokens' do + migrate!(:project, 111, 116) + + expect(projects.all.reload).to all( + have_attributes(runners_token: be_a(String), runners_token_encrypted: be_a(String)) + ) + end + end + + context 'when migrating runners' do + before do + runners.create!(id: 201, runner_type: 1, token: 'plain-text-token1') + runners.create!(id: 202, runner_type: 1, token: 'plain-text-token2') + runners.create!(id: 203, runner_type: 1, token: 'plain-text-token3') + end + + it 'migrates runners communication tokens' do + migrate!(:runner, 201, 203) + + expect(runners.all.reload).to all( + have_attributes(token: be_a(String), token_encrypted: be_a(String)) + ) + end + end + + def migrate!(model, from, to) + subject.perform(model, from, to) + end +end diff --git a/spec/lib/gitlab/checks/branch_check_spec.rb b/spec/lib/gitlab/checks/branch_check_spec.rb new file mode 100644 index 00000000000..77366e91dca --- /dev/null +++ b/spec/lib/gitlab/checks/branch_check_spec.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Checks::BranchCheck do + include_context 'change access checks context' + + describe '#validate!' do + it 'does not raise any error' do + expect { subject.validate! }.not_to raise_error + end + + context 'trying to delete the default branch' do + let(:newrev) { '0000000000000000000000000000000000000000' } + let(:ref) { 'refs/heads/master' } + + it 'raises an error' do + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'The default branch of a project cannot be deleted.') + end + end + + context 'protected branches check' do + before do + allow(ProtectedBranch).to receive(:protected?).with(project, 'master').and_return(true) + allow(ProtectedBranch).to receive(:protected?).with(project, 'feature').and_return(true) + end + + it 'raises an error if the user is not allowed to do forced pushes to protected branches' do + expect(Gitlab::Checks::ForcePush).to receive(:force_push?).and_return(true) + + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to force push code to a protected branch on this project.') + end + + it 'raises an error if the user is not allowed to merge to protected branches' do + expect_any_instance_of(Gitlab::Checks::MatchingMergeRequest).to receive(:match?).and_return(true) + expect(user_access).to receive(:can_merge_to_branch?).and_return(false) + expect(user_access).to receive(:can_push_to_branch?).and_return(false) + + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to merge code into protected branches on this project.') + end + + it 'raises an error if the user is not allowed to push to protected branches' do + expect(user_access).to receive(:can_push_to_branch?).and_return(false) + + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to push code to protected branches on this project.') + end + + context 'when project repository is empty' do + let(:project) { create(:project) } + + it 'raises an error if the user is not allowed to push to protected branches' do + expect(user_access).to receive(:can_push_to_branch?).and_return(false) + + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /Ask a project Owner or Maintainer to create a default branch/) + end + end + + context 'branch deletion' do + let(:newrev) { '0000000000000000000000000000000000000000' } + let(:ref) { 'refs/heads/feature' } + + context 'if the user is not allowed to delete protected branches' do + it 'raises an error' do + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to delete protected branches from this project. Only a project maintainer or owner can delete a protected branch.') + end + end + + context 'if the user is allowed to delete protected branches' do + before do + project.add_maintainer(user) + end + + context 'through the web interface' do + let(:protocol) { 'web' } + + it 'allows branch deletion' do + expect { subject.validate! }.not_to raise_error + end + end + + context 'over SSH or HTTP' do + it 'raises an error' do + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You can only delete protected branches using the web interface.') + end + end + end + end + end + end +end diff --git a/spec/lib/gitlab/checks/change_access_spec.rb b/spec/lib/gitlab/checks/change_access_spec.rb index 81804ba5c76..45fb33e9e4a 100644 --- a/spec/lib/gitlab/checks/change_access_spec.rb +++ b/spec/lib/gitlab/checks/change_access_spec.rb @@ -2,245 +2,56 @@ require 'spec_helper' describe Gitlab::Checks::ChangeAccess do describe '#exec' do - let(:user) { create(:user) } - let(:project) { create(:project, :repository) } - let(:user_access) { Gitlab::UserAccess.new(user, project: project) } - let(:oldrev) { 'be93687618e4b132087f430a4d8fc3a609c9b77c' } - let(:newrev) { '54fcc214b94e78d7a41a9a8fe6d87a5e59500e51' } - let(:ref) { 'refs/heads/master' } - let(:changes) { { oldrev: oldrev, newrev: newrev, ref: ref } } - let(:protocol) { 'ssh' } - let(:timeout) { Gitlab::GitAccess::INTERNAL_TIMEOUT } - let(:logger) { Gitlab::Checks::TimedLogger.new(timeout: timeout) } + include_context 'change access checks context' - subject(:change_access) do - described_class.new( - changes, - project: project, - user_access: user_access, - protocol: protocol, - logger: logger - ) - end - - before do - project.add_developer(user) - end + subject { change_access } context 'without failed checks' do it "doesn't raise an error" do expect { subject.exec }.not_to raise_error end - end - context 'when time limit was reached' do - it 'raises a TimeoutError' do - logger = Gitlab::Checks::TimedLogger.new(start_time: timeout.ago, timeout: timeout) - access = described_class.new(changes, - project: project, - user_access: user_access, - protocol: protocol, - logger: logger) + it 'calls pushes checks' do + expect_any_instance_of(Gitlab::Checks::PushCheck).to receive(:validate!) - expect { access.exec }.to raise_error(Gitlab::Checks::TimedLogger::TimeoutError) + subject.exec end - end - context 'when the user is not allowed to push to the repo' do - it 'raises an error' do - expect(user_access).to receive(:can_do_action?).with(:push_code).and_return(false) - expect(user_access).to receive(:can_push_to_branch?).with('master').and_return(false) + it 'calls branches checks' do + expect_any_instance_of(Gitlab::Checks::BranchCheck).to receive(:validate!) - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to push code to this project.') + subject.exec end - end - context 'tags check' do - let(:ref) { 'refs/tags/v1.0.0' } + it 'calls tags checks' do + expect_any_instance_of(Gitlab::Checks::TagCheck).to receive(:validate!) - it 'raises an error if the user is not allowed to update tags' do - allow(user_access).to receive(:can_do_action?).with(:push_code).and_return(true) - expect(user_access).to receive(:can_do_action?).with(:admin_project).and_return(false) - - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to change existing tags on this project.') + subject.exec end - context 'with protected tag' do - let!(:protected_tag) { create(:protected_tag, project: project, name: 'v*') } - - context 'as maintainer' do - before do - project.add_maintainer(user) - end + it 'calls lfs checks' do + expect_any_instance_of(Gitlab::Checks::LfsCheck).to receive(:validate!) - context 'deletion' do - let(:oldrev) { 'be93687618e4b132087f430a4d8fc3a609c9b77c' } - let(:newrev) { '0000000000000000000000000000000000000000' } - - it 'is prevented' do - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /cannot be deleted/) - end - end - - context 'update' do - let(:oldrev) { 'be93687618e4b132087f430a4d8fc3a609c9b77c' } - let(:newrev) { '54fcc214b94e78d7a41a9a8fe6d87a5e59500e51' } - - it 'is prevented' do - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /cannot be updated/) - end - end - end - - context 'creation' do - let(:oldrev) { '0000000000000000000000000000000000000000' } - let(:newrev) { '54fcc214b94e78d7a41a9a8fe6d87a5e59500e51' } - let(:ref) { 'refs/tags/v9.1.0' } - - it 'prevents creation below access level' do - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /allowed to create this tag as it is protected/) - end - - context 'when user has access' do - let!(:protected_tag) { create(:protected_tag, :developers_can_create, project: project, name: 'v*') } - - it 'allows tag creation' do - expect { subject.exec }.not_to raise_error - end - end - end + subject.exec end - end - context 'branches check' do - context 'trying to delete the default branch' do - let(:newrev) { '0000000000000000000000000000000000000000' } - let(:ref) { 'refs/heads/master' } + it 'calls diff checks' do + expect_any_instance_of(Gitlab::Checks::DiffCheck).to receive(:validate!) - it 'raises an error' do - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'The default branch of a project cannot be deleted.') - end - end - - context 'protected branches check' do - before do - allow(ProtectedBranch).to receive(:protected?).with(project, 'master').and_return(true) - allow(ProtectedBranch).to receive(:protected?).with(project, 'feature').and_return(true) - end - - it 'raises an error if the user is not allowed to do forced pushes to protected branches' do - expect(Gitlab::Checks::ForcePush).to receive(:force_push?).and_return(true) - - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to force push code to a protected branch on this project.') - end - - it 'raises an error if the user is not allowed to merge to protected branches' do - expect_any_instance_of(Gitlab::Checks::MatchingMergeRequest).to receive(:match?).and_return(true) - expect(user_access).to receive(:can_merge_to_branch?).and_return(false) - expect(user_access).to receive(:can_push_to_branch?).and_return(false) - - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to merge code into protected branches on this project.') - end - - it 'raises an error if the user is not allowed to push to protected branches' do - expect(user_access).to receive(:can_push_to_branch?).and_return(false) - - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to push code to protected branches on this project.') - end - - context 'when project repository is empty' do - let(:project) { create(:project) } - - it 'raises an error if the user is not allowed to push to protected branches' do - expect(user_access).to receive(:can_push_to_branch?).and_return(false) - - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /Ask a project Owner or Maintainer to create a default branch/) - end - end - - context 'branch deletion' do - let(:newrev) { '0000000000000000000000000000000000000000' } - let(:ref) { 'refs/heads/feature' } - - context 'if the user is not allowed to delete protected branches' do - it 'raises an error' do - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to delete protected branches from this project. Only a project maintainer or owner can delete a protected branch.') - end - end - - context 'if the user is allowed to delete protected branches' do - before do - project.add_maintainer(user) - end - - context 'through the web interface' do - let(:protocol) { 'web' } - - it 'allows branch deletion' do - expect { subject.exec }.not_to raise_error - end - end - - context 'over SSH or HTTP' do - it 'raises an error' do - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You can only delete protected branches using the web interface.') - end - end - end - end + subject.exec end end - context 'LFS integrity check' do - it 'fails if any LFS blobs are missing' do - allow_any_instance_of(Gitlab::Checks::LfsIntegrity).to receive(:objects_missing?).and_return(true) - - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /LFS objects are missing/) - end - - it 'succeeds if LFS objects have already been uploaded' do - allow_any_instance_of(Gitlab::Checks::LfsIntegrity).to receive(:objects_missing?).and_return(false) - - expect { subject.exec }.not_to raise_error - end - end - - context 'LFS file lock check' do - let(:owner) { create(:user) } - let!(:lock) { create(:lfs_file_lock, user: owner, project: project, path: 'README') } - - before do - allow(project.repository).to receive(:new_commits).and_return( - project.repository.commits_between('be93687618e4b132087f430a4d8fc3a609c9b77c', '54fcc214b94e78d7a41a9a8fe6d87a5e59500e51') - ) - end - - context 'with LFS not enabled' do - it 'skips the validation' do - expect_any_instance_of(Gitlab::Checks::CommitCheck).not_to receive(:validate) - - subject.exec - end - end - - context 'with LFS enabled' do - before do - allow(project).to receive(:lfs_enabled?).and_return(true) - end - - context 'when change is sent by a different user' do - it 'raises an error if the user is not allowed to update the file' do - expect { subject.exec }.to raise_error(Gitlab::GitAccess::UnauthorizedError, "The path 'README' is locked in Git LFS by #{lock.user.name}") - end - end - - context 'when change is sent by the author of the lock' do - let(:user) { owner } + context 'when time limit was reached' do + it 'raises a TimeoutError' do + logger = Gitlab::Checks::TimedLogger.new(start_time: timeout.ago, timeout: timeout) + access = described_class.new(changes, + project: project, + user_access: user_access, + protocol: protocol, + logger: logger) - it "doesn't raise any error" do - expect { subject.exec }.not_to raise_error - end - end + expect { access.exec }.to raise_error(Gitlab::Checks::TimedLogger::TimeoutError) end end end diff --git a/spec/lib/gitlab/checks/diff_check_spec.rb b/spec/lib/gitlab/checks/diff_check_spec.rb new file mode 100644 index 00000000000..eeec1e83179 --- /dev/null +++ b/spec/lib/gitlab/checks/diff_check_spec.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Checks::DiffCheck do + include_context 'change access checks context' + + describe '#validate!' do + let(:owner) { create(:user) } + let!(:lock) { create(:lfs_file_lock, user: owner, project: project, path: 'README') } + + before do + allow(project.repository).to receive(:new_commits).and_return( + project.repository.commits_between('be93687618e4b132087f430a4d8fc3a609c9b77c', '54fcc214b94e78d7a41a9a8fe6d87a5e59500e51') + ) + end + + context 'with LFS not enabled' do + before do + allow(project).to receive(:lfs_enabled?).and_return(false) + end + + it 'skips the validation' do + expect(subject).not_to receive(:validate_diff) + expect(subject).not_to receive(:validate_file_paths) + + subject.validate! + end + end + + context 'with LFS enabled' do + before do + allow(project).to receive(:lfs_enabled?).and_return(true) + end + + context 'when change is sent by a different user' do + it 'raises an error if the user is not allowed to update the file' do + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, "The path 'README' is locked in Git LFS by #{lock.user.name}") + end + end + + context 'when change is sent by the author of the lock' do + let(:user) { owner } + + it "doesn't raise any error" do + expect { subject.validate! }.not_to raise_error + end + end + end + end +end diff --git a/spec/lib/gitlab/checks/lfs_check_spec.rb b/spec/lib/gitlab/checks/lfs_check_spec.rb new file mode 100644 index 00000000000..35f8069c8a4 --- /dev/null +++ b/spec/lib/gitlab/checks/lfs_check_spec.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Checks::LfsCheck do + include_context 'change access checks context' + + let(:blob_object) { project.repository.blob_at_branch('lfs', 'files/lfs/lfs_object.iso') } + + before do + allow_any_instance_of(Gitlab::Git::LfsChanges).to receive(:new_pointers) do + [blob_object] + end + end + + describe '#validate!' do + context 'with LFS not enabled' do + it 'skips integrity check' do + expect_any_instance_of(Gitlab::Git::LfsChanges).not_to receive(:new_pointers) + + subject.validate! + end + end + + context 'with LFS enabled' do + before do + allow(project).to receive(:lfs_enabled?).and_return(true) + end + + context 'deletion' do + let(:changes) { { oldrev: oldrev, ref: ref } } + + it 'skips integrity check' do + expect(project.repository).not_to receive(:new_objects) + + subject.validate! + end + end + + it 'fails if any LFS blobs are missing' do + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /LFS objects are missing/) + end + + it 'succeeds if LFS objects have already been uploaded' do + lfs_object = create(:lfs_object, oid: blob_object.lfs_oid) + create(:lfs_objects_project, project: project, lfs_object: lfs_object) + + expect { subject.validate! }.not_to raise_error + end + end + end +end diff --git a/spec/lib/gitlab/checks/push_check_spec.rb b/spec/lib/gitlab/checks/push_check_spec.rb new file mode 100644 index 00000000000..25f0d428cb9 --- /dev/null +++ b/spec/lib/gitlab/checks/push_check_spec.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Checks::PushCheck do + include_context 'change access checks context' + + describe '#validate!' do + it 'does not raise any error' do + expect { subject.validate! }.not_to raise_error + end + + context 'when the user is not allowed to push to the repo' do + it 'raises an error' do + expect(user_access).to receive(:can_do_action?).with(:push_code).and_return(false) + expect(user_access).to receive(:can_push_to_branch?).with('master').and_return(false) + + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to push code to this project.') + end + end + end +end diff --git a/spec/lib/gitlab/checks/tag_check_spec.rb b/spec/lib/gitlab/checks/tag_check_spec.rb new file mode 100644 index 00000000000..b1258270611 --- /dev/null +++ b/spec/lib/gitlab/checks/tag_check_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Checks::TagCheck do + include_context 'change access checks context' + + describe '#validate!' do + let(:ref) { 'refs/tags/v1.0.0' } + + it 'raises an error' do + allow(user_access).to receive(:can_do_action?).with(:push_code).and_return(true) + expect(user_access).to receive(:can_do_action?).with(:admin_project).and_return(false) + + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'You are not allowed to change existing tags on this project.') + end + + context 'with protected tag' do + let!(:protected_tag) { create(:protected_tag, project: project, name: 'v*') } + + context 'as maintainer' do + before do + project.add_maintainer(user) + end + + context 'deletion' do + let(:oldrev) { 'be93687618e4b132087f430a4d8fc3a609c9b77c' } + let(:newrev) { '0000000000000000000000000000000000000000' } + + it 'is prevented' do + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /cannot be deleted/) + end + end + + context 'update' do + let(:oldrev) { 'be93687618e4b132087f430a4d8fc3a609c9b77c' } + let(:newrev) { '54fcc214b94e78d7a41a9a8fe6d87a5e59500e51' } + + it 'is prevented' do + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /cannot be updated/) + end + end + end + + context 'creation' do + let(:oldrev) { '0000000000000000000000000000000000000000' } + let(:newrev) { '54fcc214b94e78d7a41a9a8fe6d87a5e59500e51' } + let(:ref) { 'refs/tags/v9.1.0' } + + it 'prevents creation below access level' do + expect { subject.validate! }.to raise_error(Gitlab::GitAccess::UnauthorizedError, /allowed to create this tag as it is protected/) + end + + context 'when user has access' do + let!(:protected_tag) { create(:protected_tag, :developers_can_create, project: project, name: 'v*') } + + it 'allows tag creation' do + expect { subject.validate! }.not_to raise_error + end + end + end + end + end +end diff --git a/spec/lib/gitlab/crypto_helper_spec.rb b/spec/lib/gitlab/crypto_helper_spec.rb new file mode 100644 index 00000000000..05cc6cf15de --- /dev/null +++ b/spec/lib/gitlab/crypto_helper_spec.rb @@ -0,0 +1,37 @@ +require 'spec_helper' + +describe Gitlab::CryptoHelper do + describe '.sha256' do + it 'generates SHA256 digest Base46 encoded' do + digest = described_class.sha256('some-value') + + expect(digest).to match %r{\A[A-Za-z0-9+/=]+\z} + expect(digest).to eq digest.strip + end + end + + describe '.aes256_gcm_encrypt' do + it 'is Base64 encoded string without new line character' do + encrypted = described_class.aes256_gcm_encrypt('some-value') + + expect(encrypted).to match %r{\A[A-Za-z0-9+/=]+\z} + expect(encrypted).not_to include "\n" + end + end + + describe '.aes256_gcm_decrypt' do + let(:encrypted) { described_class.aes256_gcm_encrypt('some-value') } + + it 'correctly decrypts encrypted string' do + decrypted = described_class.aes256_gcm_decrypt(encrypted) + + expect(decrypted).to eq 'some-value' + end + + it 'decrypts a value when it ends with a new line character' do + decrypted = described_class.aes256_gcm_decrypt(encrypted + "\n") + + expect(decrypted).to eq 'some-value' + end + end +end diff --git a/spec/lib/gitlab/database/count/exact_count_strategy_spec.rb b/spec/lib/gitlab/database/count/exact_count_strategy_spec.rb new file mode 100644 index 00000000000..f518bb3dc3e --- /dev/null +++ b/spec/lib/gitlab/database/count/exact_count_strategy_spec.rb @@ -0,0 +1,34 @@ +require 'spec_helper' + +describe Gitlab::Database::Count::ExactCountStrategy do + before do + create_list(:project, 3) + create(:identity) + end + + let(:models) { [Project, Identity] } + + subject { described_class.new(models).count } + + describe '#count' do + it 'counts all models' do + expect(models).to all(receive(:count).and_call_original) + + expect(subject).to eq({ Project => 3, Identity => 1 }) + end + end + + describe '.enabled?' do + it 'is enabled for PostgreSQL' do + allow(Gitlab::Database).to receive(:postgresql?).and_return(true) + + expect(described_class.enabled?).to be_truthy + end + + it 'is enabled for MySQL' do + allow(Gitlab::Database).to receive(:postgresql?).and_return(false) + + expect(described_class.enabled?).to be_truthy + end + end +end diff --git a/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb b/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb new file mode 100644 index 00000000000..b44e8c5a110 --- /dev/null +++ b/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb @@ -0,0 +1,48 @@ +require 'spec_helper' + +describe Gitlab::Database::Count::ReltuplesCountStrategy do + before do + create_list(:project, 3) + create(:identity) + end + + let(:models) { [Project, Identity] } + subject { described_class.new(models).count } + + describe '#count', :postgresql do + context 'when reltuples is up to date' do + before do + ActiveRecord::Base.connection.execute('ANALYZE projects') + ActiveRecord::Base.connection.execute('ANALYZE identities') + end + + it 'uses statistics to do the count' do + models.each { |model| expect(model).not_to receive(:count) } + + expect(subject).to eq({ Project => 3, Identity => 1 }) + end + end + + context 'insufficient permissions' do + it 'returns an empty hash' do + allow(ActiveRecord::Base).to receive(:transaction).and_raise(PG::InsufficientPrivilege) + + expect(subject).to eq({}) + end + end + end + + describe '.enabled?' do + it 'is enabled for PostgreSQL' do + allow(Gitlab::Database).to receive(:postgresql?).and_return(true) + + expect(described_class.enabled?).to be_truthy + end + + it 'is disabled for MySQL' do + allow(Gitlab::Database).to receive(:postgresql?).and_return(false) + + expect(described_class.enabled?).to be_falsey + end + end +end diff --git a/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb b/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb new file mode 100644 index 00000000000..203f9344a41 --- /dev/null +++ b/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb @@ -0,0 +1,65 @@ +require 'spec_helper' + +describe Gitlab::Database::Count::TablesampleCountStrategy do + before do + create_list(:project, 3) + create(:identity) + end + + let(:models) { [Project, Identity] } + let(:strategy) { described_class.new(models) } + + subject { strategy.count } + + describe '#count', :postgresql do + let(:estimates) { { Project => threshold + 1, Identity => threshold - 1 } } + let(:threshold) { Gitlab::Database::Count::TablesampleCountStrategy::EXACT_COUNT_THRESHOLD } + + before do + allow(strategy).to receive(:size_estimates).with(check_statistics: false).and_return(estimates) + end + + context 'for tables with an estimated small size' do + it 'performs an exact count' do + expect(Identity).to receive(:count).and_call_original + + expect(subject).to include({ Identity => 1 }) + end + end + + context 'for tables with an estimated large size' do + it 'performs a tablesample count' do + expect(Project).not_to receive(:count) + + result = subject + expect(result[Project]).to eq(3) + end + end + + context 'insufficient permissions' do + it 'returns an empty hash' do + allow(strategy).to receive(:size_estimates).and_raise(PG::InsufficientPrivilege) + + expect(subject).to eq({}) + end + end + end + + describe '.enabled?' do + before do + stub_feature_flags(tablesample_counts: true) + end + + it 'is enabled for PostgreSQL' do + allow(Gitlab::Database).to receive(:postgresql?).and_return(true) + + expect(described_class.enabled?).to be_truthy + end + + it 'is disabled for MySQL' do + allow(Gitlab::Database).to receive(:postgresql?).and_return(false) + + expect(described_class.enabled?).to be_falsey + end + end +end diff --git a/spec/lib/gitlab/database/count_spec.rb b/spec/lib/gitlab/database/count_spec.rb index 407d9470785..1d096b8fa7c 100644 --- a/spec/lib/gitlab/database/count_spec.rb +++ b/spec/lib/gitlab/database/count_spec.rb @@ -8,63 +8,51 @@ describe Gitlab::Database::Count do let(:models) { [Project, Identity] } - describe '.approximate_counts' do - context 'with MySQL' do - context 'when reltuples have not been updated' do - it 'counts all models the normal way' do - expect(Gitlab::Database).to receive(:postgresql?).and_return(false) + context '.approximate_counts' do + context 'selecting strategies' do + let(:strategies) { [double('s1', enabled?: true), double('s2', enabled?: false)] } - expect(Project).to receive(:count).and_call_original - expect(Identity).to receive(:count).and_call_original + it 'uses only enabled strategies' do + expect(strategies[0]).to receive(:new).and_return(double('strategy1', count: {})) + expect(strategies[1]).not_to receive(:new) - expect(described_class.approximate_counts(models)).to eq({ Project => 3, Identity => 1 }) - end + described_class.approximate_counts(models, strategies: strategies) end end - context 'with PostgreSQL', :postgresql do - describe 'when reltuples have not been updated' do - it 'counts all models the normal way' do - expect(described_class).to receive(:reltuples_from_recently_updated).with(%w(projects identities)).and_return({}) + context 'fallbacks' do + subject { described_class.approximate_counts(models, strategies: strategies) } - expect(Project).to receive(:count).and_call_original - expect(Identity).to receive(:count).and_call_original - expect(described_class.approximate_counts(models)).to eq({ Project => 3, Identity => 1 }) - end + let(:strategies) do + [ + double('s1', enabled?: true, new: first_strategy), + double('s2', enabled?: true, new: second_strategy) + ] end - describe 'no permission' do - it 'falls back to standard query' do - allow(described_class).to receive(:postgresql_estimate_query).and_raise(PG::InsufficientPrivilege) + let(:first_strategy) { double('first strategy', count: {}) } + let(:second_strategy) { double('second strategy', count: {}) } - expect(Project).to receive(:count).and_call_original - expect(Identity).to receive(:count).and_call_original - expect(described_class.approximate_counts(models)).to eq({ Project => 3, Identity => 1 }) - end + it 'gets results from first strategy' do + expect(strategies[0]).to receive(:new).with(models).and_return(first_strategy) + expect(first_strategy).to receive(:count) + + subject end - describe 'when some reltuples have been updated' do - it 'counts projects in the fast way' do - expect(described_class).to receive(:reltuples_from_recently_updated).with(%w(projects identities)).and_return({ 'projects' => 3 }) + it 'gets more results from second strategy if some counts are missing' do + expect(first_strategy).to receive(:count).and_return({ Project => 3 }) + expect(strategies[1]).to receive(:new).with([Identity]).and_return(second_strategy) + expect(second_strategy).to receive(:count).and_return({ Identity => 1 }) - expect(Project).not_to receive(:count).and_call_original - expect(Identity).to receive(:count).and_call_original - expect(described_class.approximate_counts(models)).to eq({ Project => 3, Identity => 1 }) - end + expect(subject).to eq({ Project => 3, Identity => 1 }) end - describe 'when all reltuples have been updated' do - before do - ActiveRecord::Base.connection.execute('ANALYZE projects') - ActiveRecord::Base.connection.execute('ANALYZE identities') - end - - it 'counts models with the standard way' do - expect(Project).not_to receive(:count) - expect(Identity).not_to receive(:count) + it 'does not get more results as soon as all counts are present' do + expect(first_strategy).to receive(:count).and_return({ Project => 3, Identity => 1 }) + expect(strategies[1]).not_to receive(:new) - expect(described_class.approximate_counts(models)).to eq({ Project => 3, Identity => 1 }) - end + subject end end end diff --git a/spec/lib/gitlab/utils_spec.rb b/spec/lib/gitlab/utils_spec.rb index ad2c9d7f2af..3579ed9a759 100644 --- a/spec/lib/gitlab/utils_spec.rb +++ b/spec/lib/gitlab/utils_spec.rb @@ -127,4 +127,42 @@ describe Gitlab::Utils do end end end + + describe '.ensure_utf8_size' do + context 'string is has less bytes than expected' do + it 'backfills string with null characters' do + transformed = described_class.ensure_utf8_size('a' * 10, bytes: 32) + + expect(transformed.bytesize).to eq 32 + expect(transformed).to eq(('a' * 10) + ('0' * 22)) + end + end + + context 'string size is exactly the one that is expected' do + it 'returns original value' do + transformed = described_class.ensure_utf8_size('a' * 32, bytes: 32) + + expect(transformed).to eq 'a' * 32 + expect(transformed.bytesize).to eq 32 + end + end + + context 'when string contains a few multi-byte UTF characters' do + it 'backfills string with null characters' do + transformed = described_class.ensure_utf8_size('❤' * 6, bytes: 32) + + expect(transformed).to eq '❤❤❤❤❤❤' + ('0' * 14) + expect(transformed.bytesize).to eq 32 + end + end + + context 'when string has multiple multi-byte UTF chars exceeding 32 bytes' do + it 'truncates string to 32 characters and backfills it if needed' do + transformed = described_class.ensure_utf8_size('❤' * 18, bytes: 32) + + expect(transformed).to eq(('❤' * 10) + ('0' * 2)) + expect(transformed.bytesize).to eq 32 + end + end + end end diff --git a/spec/migrations/schedule_runners_token_encryption_spec.rb b/spec/migrations/schedule_runners_token_encryption_spec.rb new file mode 100644 index 00000000000..376d2795277 --- /dev/null +++ b/spec/migrations/schedule_runners_token_encryption_spec.rb @@ -0,0 +1,38 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20181121111200_schedule_runners_token_encryption') + +describe ScheduleRunnersTokenEncryption, :migration do + let(:settings) { table(:application_settings) } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:runners) { table(:ci_runners) } + + before do + stub_const("#{described_class.name}::BATCH_SIZE", 1) + + settings.create!(id: 1, runners_registration_token: 'plain-text-token1') + namespaces.create!(id: 11, name: 'gitlab', path: 'gitlab-org', runners_token: 'my-token1') + namespaces.create!(id: 12, name: 'gitlab', path: 'gitlab-org', runners_token: 'my-token2') + projects.create!(id: 111, namespace_id: 11, name: 'gitlab', path: 'gitlab-ce', runners_token: 'my-token1') + projects.create!(id: 114, namespace_id: 11, name: 'gitlab', path: 'gitlab-ce', runners_token: 'my-token2') + runners.create!(id: 201, runner_type: 1, token: 'plain-text-token1') + runners.create!(id: 202, runner_type: 1, token: 'plain-text-token2') + end + + it 'schedules runners token encryption migration for multiple resources' do + Sidekiq::Testing.fake! do + Timecop.freeze do + migrate! + + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, 'settings', 1, 1) + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, 'namespace', 11, 11) + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(8.minutes, 'namespace', 12, 12) + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, 'project', 111, 111) + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(8.minutes, 'project', 114, 114) + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, 'runner', 201, 201) + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(8.minutes, 'runner', 202, 202) + expect(BackgroundMigrationWorker.jobs.size).to eq 7 + end + end + end +end diff --git a/spec/models/blob_spec.rb b/spec/models/blob_spec.rb index ed93f94d893..e8c03b587e2 100644 --- a/spec/models/blob_spec.rb +++ b/spec/models/blob_spec.rb @@ -18,6 +18,7 @@ describe Blob do describe '.lazy' do let(:project) { create(:project, :repository) } + let(:same_project) { Project.find(project.id) } let(:other_project) { create(:project, :repository) } let(:commit_id) { 'e63f41fe459e62e1228fcef60d7189127aeba95a' } @@ -32,7 +33,7 @@ describe Blob do expect(other_project.repository).not_to receive(:blobs_at) changelog = described_class.lazy(project, commit_id, 'CHANGELOG') - contributing = described_class.lazy(project, commit_id, 'CONTRIBUTING.md') + contributing = described_class.lazy(same_project, commit_id, 'CONTRIBUTING.md') described_class.lazy(other_project, commit_id, 'CHANGELOG') diff --git a/spec/models/ci/build_metadata_spec.rb b/spec/models/ci/build_metadata_spec.rb index 6dba132184c..519968b9e48 100644 --- a/spec/models/ci/build_metadata_spec.rb +++ b/spec/models/ci/build_metadata_spec.rb @@ -15,6 +15,8 @@ describe Ci::BuildMetadata do let(:build) { create(:ci_build, pipeline: pipeline) } let(:build_metadata) { build.metadata } + it_behaves_like 'having unique enum values' + describe '#update_timeout_state' do subject { build_metadata } diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index d02c3a5765f..4cdcae5f670 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -769,33 +769,15 @@ describe Ci::Build do let(:subject) { build.hide_secrets(data) } context 'hide runners token' do - let(:data) { 'new token data'} + let(:data) { "new #{project.runners_token} data"} - before do - build.project.update(runners_token: 'token') - end - - it { is_expected.to eq('new xxxxx data') } + it { is_expected.to match(/^new x+ data$/) } end context 'hide build token' do - let(:data) { 'new token data'} - - before do - build.update(token: 'token') - end - - it { is_expected.to eq('new xxxxx data') } - end - - context 'hide build token' do - let(:data) { 'new token data'} - - before do - build.update(token: 'token') - end + let(:data) { "new #{build.token} data"} - it { is_expected.to eq('new xxxxx data') } + it { is_expected.to match(/^new x+ data$/) } end end diff --git a/spec/models/ci/build_trace_chunk_spec.rb b/spec/models/ci/build_trace_chunk_spec.rb index 859287bb0c8..d214fdf369a 100644 --- a/spec/models/ci/build_trace_chunk_spec.rb +++ b/spec/models/ci/build_trace_chunk_spec.rb @@ -12,6 +12,8 @@ describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do described_class.new(build: build, chunk_index: chunk_index, data_store: data_store, raw_data: raw_data) end + it_behaves_like 'having unique enum values' + before do stub_feature_flags(ci_enable_live_trace: true) stub_artifacts_object_storage diff --git a/spec/models/ci/job_artifact_spec.rb b/spec/models/ci/job_artifact_spec.rb index fb5bec4108a..c68ba02b8de 100644 --- a/spec/models/ci/job_artifact_spec.rb +++ b/spec/models/ci/job_artifact_spec.rb @@ -15,6 +15,8 @@ describe Ci::JobArtifact do it { is_expected.to delegate_method(:open).to(:file) } it { is_expected.to delegate_method(:exists?).to(:file) } + it_behaves_like 'having unique enum values' + describe '.test_reports' do subject { described_class.test_reports } diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index 9e6146b8a44..3076a882445 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -8,6 +8,8 @@ describe Ci::Pipeline, :mailer do create(:ci_empty_pipeline, status: :created, project: project) end + it_behaves_like 'having unique enum values' + it { is_expected.to belong_to(:project) } it { is_expected.to belong_to(:user) } it { is_expected.to belong_to(:auto_canceled_by) } @@ -1247,22 +1249,40 @@ describe Ci::Pipeline, :mailer do describe '#ci_yaml_file_path' do subject { pipeline.ci_yaml_file_path } - it 'returns the path from project' do - allow(pipeline.project).to receive(:ci_config_path) { 'custom/path' } + %i[unknown_source repository_source].each do |source| + context source.to_s do + before do + pipeline.config_source = described_class.config_sources.fetch(source) + end - is_expected.to eq('custom/path') - end + it 'returns the path from project' do + allow(pipeline.project).to receive(:ci_config_path) { 'custom/path' } + + is_expected.to eq('custom/path') + end + + it 'returns default when custom path is nil' do + allow(pipeline.project).to receive(:ci_config_path) { nil } - it 'returns default when custom path is nil' do - allow(pipeline.project).to receive(:ci_config_path) { nil } + is_expected.to eq('.gitlab-ci.yml') + end + + it 'returns default when custom path is empty' do + allow(pipeline.project).to receive(:ci_config_path) { '' } - is_expected.to eq('.gitlab-ci.yml') + is_expected.to eq('.gitlab-ci.yml') + end + end end - it 'returns default when custom path is empty' do - allow(pipeline.project).to receive(:ci_config_path) { '' } + context 'when pipeline is for auto-devops' do + before do + pipeline.config_source = 'auto_devops_source' + end - is_expected.to eq('.gitlab-ci.yml') + it 'does not return config file' do + is_expected.to be_nil + end end end diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb index b545e036aa1..ad79f8d4ce0 100644 --- a/spec/models/ci/runner_spec.rb +++ b/spec/models/ci/runner_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' describe Ci::Runner do + it_behaves_like 'having unique enum values' + describe 'validation' do it { is_expected.to validate_presence_of(:access_level) } it { is_expected.to validate_presence_of(:runner_type) } diff --git a/spec/models/ci/stage_spec.rb b/spec/models/ci/stage_spec.rb index 5076f7faeac..3228c400155 100644 --- a/spec/models/ci/stage_spec.rb +++ b/spec/models/ci/stage_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' describe Ci::Stage, :models do let(:stage) { create(:ci_stage_entity) } + it_behaves_like 'having unique enum values' + describe 'associations' do before do create(:ci_build, stage_id: stage.id) diff --git a/spec/models/clusters/applications/ingress_spec.rb b/spec/models/clusters/applications/ingress_spec.rb index cfe0e216c78..cd28f1fe9c6 100644 --- a/spec/models/clusters/applications/ingress_spec.rb +++ b/spec/models/clusters/applications/ingress_spec.rb @@ -3,6 +3,8 @@ require 'rails_helper' describe Clusters::Applications::Ingress do let(:ingress) { create(:clusters_applications_ingress) } + it_behaves_like 'having unique enum values' + include_examples 'cluster application core specs', :clusters_applications_ingress include_examples 'cluster application status specs', :clusters_applications_ingress include_examples 'cluster application helm specs', :clusters_applications_ingress diff --git a/spec/models/clusters/cluster_spec.rb b/spec/models/clusters/cluster_spec.rb index eb68ebccdcb..7dcf97276b2 100644 --- a/spec/models/clusters/cluster_spec.rb +++ b/spec/models/clusters/cluster_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' describe Clusters::Cluster do + it_behaves_like 'having unique enum values' + it { is_expected.to belong_to(:user) } it { is_expected.to have_many(:cluster_projects) } it { is_expected.to have_many(:projects) } diff --git a/spec/models/clusters/platforms/kubernetes_spec.rb b/spec/models/clusters/platforms/kubernetes_spec.rb index 99fd6ccc4d8..062d2fd0768 100644 --- a/spec/models/clusters/platforms/kubernetes_spec.rb +++ b/spec/models/clusters/platforms/kubernetes_spec.rb @@ -18,6 +18,8 @@ describe Clusters::Platforms::Kubernetes, :use_clean_rails_memory_store_caching it { is_expected.to delegate_method(:managed?).to(:cluster) } it { is_expected.to delegate_method(:kubernetes_namespace).to(:cluster) } + it_behaves_like 'having unique enum values' + describe 'before_validation' do context 'when namespace includes upper case' do let(:kubernetes) { create(:cluster_platform_kubernetes, :configured, namespace: namespace) } @@ -273,6 +275,36 @@ describe Clusters::Platforms::Kubernetes, :use_clean_rails_memory_store_caching ) end end + + context 'group level cluster' do + let!(:cluster) { create(:cluster, :group, platform_kubernetes: kubernetes) } + + let(:project) { create(:project, group: cluster.group) } + + subject { kubernetes.predefined_variables(project: project) } + + context 'no kubernetes namespace for the project' do + it_behaves_like 'setting variables' + + it 'does not return KUBE_TOKEN' do + expect(subject).not_to include( + { key: 'KUBE_TOKEN', value: kubernetes.token, public: false } + ) + end + end + + context 'kubernetes namespace exists for the project' do + let!(:kubernetes_namespace) { create(:cluster_kubernetes_namespace, :with_token, cluster: cluster, project: project) } + + it_behaves_like 'setting variables' + + it 'sets KUBE_TOKEN' do + expect(subject).to include( + { key: 'KUBE_TOKEN', value: kubernetes_namespace.service_account_token, public: false } + ) + end + end + end end describe '#terminals' do diff --git a/spec/models/commit_status_spec.rb b/spec/models/commit_status_spec.rb index 917685399d4..8b7c88805c1 100644 --- a/spec/models/commit_status_spec.rb +++ b/spec/models/commit_status_spec.rb @@ -13,6 +13,8 @@ describe CommitStatus do create(:commit_status, pipeline: pipeline, **opts) end + it_behaves_like 'having unique enum values' + it { is_expected.to belong_to(:pipeline) } it { is_expected.to belong_to(:user) } it { is_expected.to belong_to(:project) } diff --git a/spec/models/concerns/token_authenticatable_spec.rb b/spec/models/concerns/token_authenticatable_spec.rb index 782687516ae..0cdf430e9ab 100644 --- a/spec/models/concerns/token_authenticatable_spec.rb +++ b/spec/models/concerns/token_authenticatable_spec.rb @@ -21,44 +21,59 @@ end describe ApplicationSetting, 'TokenAuthenticatable' do let(:token_field) { :runners_registration_token } + let(:settings) { described_class.new } + it_behaves_like 'TokenAuthenticatable' describe 'generating new token' do context 'token is not generated yet' do describe 'token field accessor' do - subject { described_class.new.send(token_field) } + subject { settings.send(token_field) } + it { is_expected.not_to be_blank } end - describe 'ensured token' do - subject { described_class.new.send("ensure_#{token_field}") } + describe "ensure_runners_registration_token" do + subject { settings.send("ensure_#{token_field}") } it { is_expected.to be_a String } it { is_expected.not_to be_blank } + + it 'does not persist token' do + expect(settings).not_to be_persisted + end end - describe 'ensured! token' do - subject { described_class.new.send("ensure_#{token_field}!") } + describe 'ensure_runners_registration_token!' do + subject { settings.send("ensure_#{token_field}!") } + + it 'persists new token as an encrypted string' do + expect(subject).to eq settings.reload.runners_registration_token + expect(settings.read_attribute('runners_registration_token_encrypted')) + .to eq Gitlab::CryptoHelper.aes256_gcm_encrypt(subject) + expect(settings).to be_persisted + end - it 'persists new token' do - expect(subject).to eq described_class.current[token_field] + it 'does not persist token in a clear text' do + expect(subject).not_to eq settings.reload + .read_attribute('runners_registration_token_encrypted') end end end context 'token is generated' do before do - subject.send("reset_#{token_field}!") + settings.send("reset_#{token_field}!") end it 'persists a new token' do - expect(subject.send(:read_attribute, token_field)).to be_a String + expect(settings.runners_registration_token).to be_a String end end end describe 'setting new token' do - subject { described_class.new.send("set_#{token_field}", '0123456789') } + subject { settings.send("set_#{token_field}", '0123456789') } it { is_expected.to eq '0123456789' } end diff --git a/spec/models/concerns/token_authenticatable_strategies/base_spec.rb b/spec/models/concerns/token_authenticatable_strategies/base_spec.rb new file mode 100644 index 00000000000..6605f1f5a5f --- /dev/null +++ b/spec/models/concerns/token_authenticatable_strategies/base_spec.rb @@ -0,0 +1,65 @@ +require 'spec_helper' + +describe TokenAuthenticatableStrategies::Base do + let(:instance) { double(:instance) } + let(:field) { double(:field) } + + describe '.fabricate' do + context 'when digest stragegy is specified' do + it 'fabricates digest strategy object' do + strategy = described_class.fabricate(instance, field, digest: true) + + expect(strategy).to be_a TokenAuthenticatableStrategies::Digest + end + end + + context 'when encrypted strategy is specified' do + it 'fabricates encrypted strategy object' do + strategy = described_class.fabricate(instance, field, encrypted: true) + + expect(strategy).to be_a TokenAuthenticatableStrategies::Encrypted + end + end + + context 'when no strategy is specified' do + it 'fabricates insecure strategy object' do + strategy = described_class.fabricate(instance, field, something: true) + + expect(strategy).to be_a TokenAuthenticatableStrategies::Insecure + end + end + + context 'when incompatible options are provided' do + it 'raises an error' do + expect { described_class.fabricate(instance, field, digest: true, encrypted: true) } + .to raise_error ArgumentError + end + end + end + + describe '#fallback?' do + context 'when fallback is set' do + it 'recognizes fallback setting' do + strategy = described_class.new(instance, field, fallback: true) + + expect(strategy.fallback?).to be true + end + end + + context 'when fallback is not a valid value' do + it 'raises an error' do + strategy = described_class.new(instance, field, fallback: 'something') + + expect { strategy.fallback? }.to raise_error ArgumentError + end + end + + context 'when fallback is not set' do + it 'raises an error' do + strategy = described_class.new(instance, field, {}) + + expect(strategy.fallback?).to eq false + end + end + end +end diff --git a/spec/models/concerns/token_authenticatable_strategies/encrypted_spec.rb b/spec/models/concerns/token_authenticatable_strategies/encrypted_spec.rb new file mode 100644 index 00000000000..93cab80cb1f --- /dev/null +++ b/spec/models/concerns/token_authenticatable_strategies/encrypted_spec.rb @@ -0,0 +1,156 @@ +require 'spec_helper' + +describe TokenAuthenticatableStrategies::Encrypted do + let(:model) { double(:model) } + let(:instance) { double(:instance) } + + let(:encrypted) do + Gitlab::CryptoHelper.aes256_gcm_encrypt('my-value') + end + + subject do + described_class.new(model, 'some_field', options) + end + + describe '.new' do + context 'when fallback and migration strategies are set' do + let(:options) { { fallback: true, migrating: true } } + + it 'raises an error' do + expect { subject }.to raise_error ArgumentError, /not compatible/ + end + end + end + + describe '#find_token_authenticatable' do + context 'when using fallback strategy' do + let(:options) { { fallback: true } } + + it 'finds the encrypted resource by cleartext' do + allow(model).to receive(:find_by) + .with('some_field_encrypted' => encrypted) + .and_return('encrypted resource') + + expect(subject.find_token_authenticatable('my-value')) + .to eq 'encrypted resource' + end + + it 'uses insecure strategy when encrypted token cannot be found' do + allow(subject.send(:insecure_strategy)) + .to receive(:find_token_authenticatable) + .and_return('plaintext resource') + + allow(model).to receive(:find_by) + .with('some_field_encrypted' => encrypted) + .and_return(nil) + + expect(subject.find_token_authenticatable('my-value')) + .to eq 'plaintext resource' + end + end + + context 'when using migration strategy' do + let(:options) { { migrating: true } } + + it 'finds the cleartext resource by cleartext' do + allow(model).to receive(:find_by) + .with('some_field' => 'my-value') + .and_return('cleartext resource') + + expect(subject.find_token_authenticatable('my-value')) + .to eq 'cleartext resource' + end + + it 'returns nil if resource cannot be found' do + allow(model).to receive(:find_by) + .with('some_field' => 'my-value') + .and_return(nil) + + expect(subject.find_token_authenticatable('my-value')) + .to be_nil + end + end + end + + describe '#get_token' do + context 'when using fallback strategy' do + let(:options) { { fallback: true } } + + it 'returns decrypted token when an encrypted token is present' do + allow(instance).to receive(:read_attribute) + .with('some_field_encrypted') + .and_return(encrypted) + + expect(subject.get_token(instance)).to eq 'my-value' + end + + it 'returns the plaintext token when encrypted token is not present' do + allow(instance).to receive(:read_attribute) + .with('some_field_encrypted') + .and_return(nil) + + allow(instance).to receive(:read_attribute) + .with('some_field') + .and_return('cleartext value') + + expect(subject.get_token(instance)).to eq 'cleartext value' + end + end + + context 'when using migration strategy' do + let(:options) { { migrating: true } } + + it 'returns cleartext token when an encrypted token is present' do + allow(instance).to receive(:read_attribute) + .with('some_field_encrypted') + .and_return(encrypted) + + allow(instance).to receive(:read_attribute) + .with('some_field') + .and_return('my-cleartext-value') + + expect(subject.get_token(instance)).to eq 'my-cleartext-value' + end + + it 'returns the cleartext token when encrypted token is not present' do + allow(instance).to receive(:read_attribute) + .with('some_field_encrypted') + .and_return(nil) + + allow(instance).to receive(:read_attribute) + .with('some_field') + .and_return('cleartext value') + + expect(subject.get_token(instance)).to eq 'cleartext value' + end + end + end + + describe '#set_token' do + context 'when using fallback strategy' do + let(:options) { { fallback: true } } + + it 'writes encrypted token and removes plaintext token and returns it' do + expect(instance).to receive(:[]=) + .with('some_field_encrypted', encrypted) + expect(instance).to receive(:[]=) + .with('some_field', nil) + + expect(subject.set_token(instance, 'my-value')).to eq 'my-value' + end + end + + context 'when using migration strategy' do + let(:options) { { migrating: true } } + + it 'writes encrypted token and writes plaintext token' do + expect(instance).to receive(:[]=) + .with('some_field_encrypted', encrypted) + expect(instance).to receive(:[]=) + .with('some_field', 'my-value') + + expect(subject.set_token(instance, 'my-value')).to eq 'my-value' + end + end + end +end diff --git a/spec/models/deployment_spec.rb b/spec/models/deployment_spec.rb index 270b2767c68..a8d53cfcd7d 100644 --- a/spec/models/deployment_spec.rb +++ b/spec/models/deployment_spec.rb @@ -16,6 +16,8 @@ describe Deployment do it { is_expected.to validate_presence_of(:ref) } it { is_expected.to validate_presence_of(:sha) } + it_behaves_like 'having unique enum values' + describe '#scheduled_actions' do subject { deployment.scheduled_actions } diff --git a/spec/models/gpg_signature_spec.rb b/spec/models/gpg_signature_spec.rb index 0136bb61c07..cdd7dea2064 100644 --- a/spec/models/gpg_signature_spec.rb +++ b/spec/models/gpg_signature_spec.rb @@ -8,6 +8,8 @@ RSpec.describe GpgSignature do let(:gpg_key) { create(:gpg_key) } let(:gpg_key_subkey) { create(:gpg_key_subkey) } + it_behaves_like 'having unique enum values' + describe 'associations' do it { is_expected.to belong_to(:project) } it { is_expected.to belong_to(:gpg_key) } diff --git a/spec/models/internal_id_spec.rb b/spec/models/internal_id_spec.rb index 52c00a74b4b..4696341c05f 100644 --- a/spec/models/internal_id_spec.rb +++ b/spec/models/internal_id_spec.rb @@ -7,6 +7,8 @@ describe InternalId do let(:scope) { { project: project } } let(:init) { ->(s) { s.project.issues.size } } + it_behaves_like 'having unique enum values' + context 'validations' do it { is_expected.to validate_presence_of(:usage) } end diff --git a/spec/models/list_spec.rb b/spec/models/list_spec.rb index 17dc27bd132..a51580f8292 100644 --- a/spec/models/list_spec.rb +++ b/spec/models/list_spec.rb @@ -1,6 +1,8 @@ require 'rails_helper' describe List do + it_behaves_like 'having unique enum values' + describe 'relationships' do it { is_expected.to belong_to(:board) } it { is_expected.to belong_to(:label) } diff --git a/spec/models/notification_setting_spec.rb b/spec/models/notification_setting_spec.rb index e545b674b4f..771d834c4bc 100644 --- a/spec/models/notification_setting_spec.rb +++ b/spec/models/notification_setting_spec.rb @@ -1,6 +1,8 @@ require 'rails_helper' RSpec.describe NotificationSetting do + it_behaves_like 'having unique enum values' + describe "Associations" do it { is_expected.to belong_to(:user) } it { is_expected.to belong_to(:source) } diff --git a/spec/models/project_auto_devops_spec.rb b/spec/models/project_auto_devops_spec.rb index 342798f730b..7ff64c76e37 100644 --- a/spec/models/project_auto_devops_spec.rb +++ b/spec/models/project_auto_devops_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' describe ProjectAutoDevops do set(:project) { build(:project) } + it_behaves_like 'having unique enum values' + it { is_expected.to belong_to(:project) } it { is_expected.to define_enum_for(:deploy_strategy) } diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index af5b0939ca2..3f99fd12e38 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -4,6 +4,8 @@ describe Project do include ProjectForksHelper include GitHelpers + it_behaves_like 'having unique enum values' + describe 'associations' do it { is_expected.to belong_to(:group) } it { is_expected.to belong_to(:namespace) } diff --git a/spec/models/prometheus_metric_spec.rb b/spec/models/prometheus_metric_spec.rb index a83a31ae88c..3692fe9a559 100644 --- a/spec/models/prometheus_metric_spec.rb +++ b/spec/models/prometheus_metric_spec.rb @@ -6,6 +6,8 @@ describe PrometheusMetric do subject { build(:prometheus_metric) } let(:other_project) { build(:project) } + it_behaves_like 'having unique enum values' + it { is_expected.to belong_to(:project) } it { is_expected.to validate_presence_of(:title) } it { is_expected.to validate_presence_of(:query) } diff --git a/spec/models/push_event_payload_spec.rb b/spec/models/push_event_payload_spec.rb index a049ad35584..69a4922b6fd 100644 --- a/spec/models/push_event_payload_spec.rb +++ b/spec/models/push_event_payload_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' describe PushEventPayload do + it_behaves_like 'having unique enum values' + describe 'saving payloads' do it 'does not allow commit messages longer than 70 characters' do event = create(:push_event) diff --git a/spec/models/resource_label_event_spec.rb b/spec/models/resource_label_event_spec.rb index da6e1b5610d..e7e3f7376e6 100644 --- a/spec/models/resource_label_event_spec.rb +++ b/spec/models/resource_label_event_spec.rb @@ -7,6 +7,8 @@ RSpec.describe ResourceLabelEvent, type: :model do let(:issue) { create(:issue) } let(:merge_request) { create(:merge_request) } + it_behaves_like 'having unique enum values' + describe 'associations' do it { is_expected.to belong_to(:user) } it { is_expected.to belong_to(:issue) } diff --git a/spec/models/user_callout_spec.rb b/spec/models/user_callout_spec.rb index 64ba17c81fe..d54355afe12 100644 --- a/spec/models/user_callout_spec.rb +++ b/spec/models/user_callout_spec.rb @@ -3,6 +3,8 @@ require 'rails_helper' describe UserCallout do let!(:callout) { create(:user_callout) } + it_behaves_like 'having unique enum values' + describe 'relationships' do it { is_expected.to belong_to(:user) } end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 7bd6dccd0ad..e5490e0a156 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -4,6 +4,8 @@ describe User do include ProjectForksHelper include TermsHelper + it_behaves_like 'having unique enum values' + describe 'modules' do subject { described_class } diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb index c71eae9164a..0dc459d9b5a 100644 --- a/spec/requests/git_http_spec.rb +++ b/spec/requests/git_http_spec.rb @@ -302,7 +302,7 @@ describe 'Git HTTP requests' do it 'rejects pushes with 403 Forbidden' do upload(path, env) do |response| expect(response).to have_gitlab_http_status(:forbidden) - expect(response.body).to eq(change_access_error(:push_code)) + expect(response.body).to eq('You are not allowed to push code to this project.') end end diff --git a/spec/routing/project_routing_spec.rb b/spec/routing/project_routing_spec.rb index bdfb12dc5df..5c3b37ef11c 100644 --- a/spec/routing/project_routing_spec.rb +++ b/spec/routing/project_routing_spec.rb @@ -36,36 +36,33 @@ describe 'project routing' do shared_examples 'RESTful project resources' do let(:actions) { [:index, :create, :new, :edit, :show, :update, :destroy] } let(:controller_path) { controller } - let(:id) { { id: '1' } } - let(:format) { {} } # response format, e.g. { format: :html } - let(:params) { { namespace_id: 'gitlab', project_id: 'gitlabhq' } } it 'to #index' do - expect(get("/gitlab/gitlabhq/#{controller_path}")).to route_to("projects/#{controller}#index", params) if actions.include?(:index) + expect(get("/gitlab/gitlabhq/#{controller_path}")).to route_to("projects/#{controller}#index", namespace_id: 'gitlab', project_id: 'gitlabhq') if actions.include?(:index) end it 'to #create' do - expect(post("/gitlab/gitlabhq/#{controller_path}")).to route_to("projects/#{controller}#create", params) if actions.include?(:create) + expect(post("/gitlab/gitlabhq/#{controller_path}")).to route_to("projects/#{controller}#create", namespace_id: 'gitlab', project_id: 'gitlabhq') if actions.include?(:create) end it 'to #new' do - expect(get("/gitlab/gitlabhq/#{controller_path}/new")).to route_to("projects/#{controller}#new", params) if actions.include?(:new) + expect(get("/gitlab/gitlabhq/#{controller_path}/new")).to route_to("projects/#{controller}#new", namespace_id: 'gitlab', project_id: 'gitlabhq') if actions.include?(:new) end it 'to #edit' do - expect(get("/gitlab/gitlabhq/#{controller_path}/1/edit")).to route_to("projects/#{controller}#edit", params.merge(**id, **format)) if actions.include?(:edit) + expect(get("/gitlab/gitlabhq/#{controller_path}/1/edit")).to route_to("projects/#{controller}#edit", namespace_id: 'gitlab', project_id: 'gitlabhq', id: '1') if actions.include?(:edit) end it 'to #show' do - expect(get("/gitlab/gitlabhq/#{controller_path}/1")).to route_to("projects/#{controller}#show", params.merge(**id, **format)) if actions.include?(:show) + expect(get("/gitlab/gitlabhq/#{controller_path}/1")).to route_to("projects/#{controller}#show", namespace_id: 'gitlab', project_id: 'gitlabhq', id: '1') if actions.include?(:show) end it 'to #update' do - expect(put("/gitlab/gitlabhq/#{controller_path}/1")).to route_to("projects/#{controller}#update", params.merge(id)) if actions.include?(:update) + expect(put("/gitlab/gitlabhq/#{controller_path}/1")).to route_to("projects/#{controller}#update", namespace_id: 'gitlab', project_id: 'gitlabhq', id: '1') if actions.include?(:update) end it 'to #destroy' do - expect(delete("/gitlab/gitlabhq/#{controller_path}/1")).to route_to("projects/#{controller}#destroy", params.merge(**id, **format)) if actions.include?(:destroy) + expect(delete("/gitlab/gitlabhq/#{controller_path}/1")).to route_to("projects/#{controller}#destroy", namespace_id: 'gitlab', project_id: 'gitlabhq', id: '1') if actions.include?(:destroy) end end @@ -154,13 +151,12 @@ describe 'project routing' do end it 'to #history' do - expect(get('/gitlab/gitlabhq/wikis/1/history')).to route_to('projects/wikis#history', namespace_id: 'gitlab', project_id: 'gitlabhq', id: '1', format: :html) + expect(get('/gitlab/gitlabhq/wikis/1/history')).to route_to('projects/wikis#history', namespace_id: 'gitlab', project_id: 'gitlabhq', id: '1') end it_behaves_like 'RESTful project resources' do let(:actions) { [:create, :edit, :show, :destroy] } let(:controller) { 'wikis' } - let(:format) { { format: :html } } end end diff --git a/spec/support/active_record_enum.rb b/spec/support/active_record_enum.rb new file mode 100644 index 00000000000..fb1189c7f17 --- /dev/null +++ b/spec/support/active_record_enum.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +shared_examples 'having unique enum values' do + described_class.defined_enums.each do |name, enum| + it "has unique values in #{name.inspect}" do + duplicated = enum.group_by(&:last).select { |key, value| value.size > 1 } + + expect(duplicated).to be_empty, + "Duplicated values detected: #{duplicated.values.map(&Hash.method(:[]))}" + end + end +end diff --git a/spec/support/features/discussion_comments_shared_example.rb b/spec/support/features/discussion_comments_shared_example.rb index 18cf08f0b9e..922f3df144d 100644 --- a/spec/support/features/discussion_comments_shared_example.rb +++ b/spec/support/features/discussion_comments_shared_example.rb @@ -142,6 +142,14 @@ shared_examples 'discussion comments' do |resource_name| find(comments_selector, match: :first) end + def submit_reply(text) + find("#{comments_selector} .js-vue-discussion-reply").click + find("#{comments_selector} .note-textarea").send_keys(text) + + click_button "Comment" + wait_for_requests + end + it 'clicking "Start discussion" will post a discussion' do new_comment = all(comments_selector).last @@ -149,16 +157,29 @@ shared_examples 'discussion comments' do |resource_name| expect(new_comment).to have_selector '.discussion' end + if resource_name =~ /(issue|merge request)/ + it 'can be replied to' do + submit_reply('some text') + + expect(page).to have_css('.discussion-notes .note', count: 2) + expect(page).to have_content 'Collapse replies' + end + + it 'can be collapsed' do + submit_reply('another text') + + find('.js-collapse-replies').click + expect(page).to have_css('.discussion-notes .note', count: 1) + expect(page).to have_content '1 reply' + end + end + if resource_name == 'merge request' let(:note_id) { find("#{comments_selector} .note:first-child", match: :first)['data-note-id'] } let(:reply_id) { find("#{comments_selector} .note:last-child", match: :first)['data-note-id'] } it 'shows resolved discussion when toggled' do - find("#{comments_selector} .js-vue-discussion-reply").click - find("#{comments_selector} .note-textarea").send_keys('a') - - click_button "Comment" - wait_for_requests + submit_reply('a') click_button "Resolve discussion" wait_for_requests diff --git a/spec/support/helpers/git_http_helpers.rb b/spec/support/helpers/git_http_helpers.rb index b8289e6c5f1..9a5845af90c 100644 --- a/spec/support/helpers/git_http_helpers.rb +++ b/spec/support/helpers/git_http_helpers.rb @@ -60,9 +60,4 @@ module GitHttpHelpers message = Gitlab::GitAccessWiki::ERROR_MESSAGES[error_key] message || raise("GitAccessWiki error message key '#{error_key}' not found") end - - def change_access_error(error_key) - message = Gitlab::Checks::ChangeAccess::ERROR_MESSAGES[error_key] - message || raise("ChangeAccess error message key '#{error_key}' not found") - end end diff --git a/spec/support/helpers/stub_configuration.rb b/spec/support/helpers/stub_configuration.rb index 776119564ec..2851cd9733c 100644 --- a/spec/support/helpers/stub_configuration.rb +++ b/spec/support/helpers/stub_configuration.rb @@ -27,6 +27,11 @@ module StubConfiguration allow(Gitlab.config.gitlab).to receive_messages(to_settings(messages)) end + def stub_default_url_options(host: "localhost", protocol: "http") + url_options = { host: host, protocol: protocol } + allow(Rails.application.routes).to receive(:default_url_options).and_return(url_options) + end + def stub_gravatar_setting(messages) allow(Gitlab.config.gravatar).to receive_messages(to_settings(messages)) end diff --git a/spec/support/shared_contexts/change_access_checks_shared_context.rb b/spec/support/shared_contexts/change_access_checks_shared_context.rb new file mode 100644 index 00000000000..aca18b0c73b --- /dev/null +++ b/spec/support/shared_contexts/change_access_checks_shared_context.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +shared_context 'change access checks context' do + let(:user) { create(:user) } + let(:project) { create(:project, :repository) } + let(:user_access) { Gitlab::UserAccess.new(user, project: project) } + let(:oldrev) { 'be93687618e4b132087f430a4d8fc3a609c9b77c' } + let(:newrev) { '54fcc214b94e78d7a41a9a8fe6d87a5e59500e51' } + let(:ref) { 'refs/heads/master' } + let(:changes) { { oldrev: oldrev, newrev: newrev, ref: ref } } + let(:protocol) { 'ssh' } + let(:timeout) { Gitlab::GitAccess::INTERNAL_TIMEOUT } + let(:logger) { Gitlab::Checks::TimedLogger.new(timeout: timeout) } + let(:change_access) do + Gitlab::Checks::ChangeAccess.new( + changes, + project: project, + user_access: user_access, + protocol: protocol, + logger: logger + ) + end + + subject { described_class.new(change_access) } + + before do + project.add_developer(user) + end +end diff --git a/spec/support/shared_examples/ci_trace_shared_examples.rb b/spec/support/shared_examples/ci_trace_shared_examples.rb index 377bd82b67e..c603421d748 100644 --- a/spec/support/shared_examples/ci_trace_shared_examples.rb +++ b/spec/support/shared_examples/ci_trace_shared_examples.rb @@ -180,10 +180,9 @@ shared_examples_for 'common trace features' do end context 'runners token' do - let(:token) { 'my_secret_token' } + let(:token) { build.project.runners_token } before do - build.project.update(runners_token: token) trace.set(token) end @@ -193,10 +192,9 @@ shared_examples_for 'common trace features' do end context 'hides build token' do - let(:token) { 'my_secret_token' } + let(:token) { build.token } before do - build.update(token: token) trace.set(token) end diff --git a/spec/workers/pipeline_schedule_worker_spec.rb b/spec/workers/pipeline_schedule_worker_spec.rb index c5a60e9855b..bebcbe01009 100644 --- a/spec/workers/pipeline_schedule_worker_spec.rb +++ b/spec/workers/pipeline_schedule_worker_spec.rb @@ -11,6 +11,7 @@ describe PipelineScheduleWorker do end before do + stub_application_setting(auto_devops_enabled: false) stub_ci_pipeline_to_return_yaml_file pipeline_schedule.update_column(:next_run_at, 1.day.ago) |