diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/api/groups.rb | 1 | ||||
| -rw-r--r-- | lib/api/issues.rb | 3 | ||||
| -rw-r--r-- | lib/api/runners.rb | 23 | ||||
| -rw-r--r-- | lib/api/v3/runners.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/auth/blocked_user_tracker.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/preloader.rb | 28 | ||||
| -rw-r--r-- | lib/gitlab/database/count.rb | 48 | ||||
| -rw-r--r-- | lib/gitlab/git/commit.rb | 25 | ||||
| -rw-r--r-- | lib/gitlab/git/repository.rb | 65 | ||||
| -rw-r--r-- | lib/gitlab/import_export/relation_factory.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/metrics/web_transaction.rb | 18 | ||||
| -rw-r--r-- | lib/gitlab/project_search_results.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/untrusted_regexp.rb | 19 | ||||
| -rw-r--r-- | lib/tasks/gitlab/test.rake | 1 | ||||
| -rw-r--r-- | lib/tasks/migrate/setup_postgresql.rake | 2 | ||||
| -rw-r--r-- | lib/tasks/spinach.rake | 60 |
16 files changed, 124 insertions, 179 deletions
diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 3cb580bae25..03b6b30a0d8 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -165,6 +165,7 @@ module API group = find_group!(params[:id]) authorize! :admin_group, group + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/46285') destroy_conditionally!(group) do |group| ::Groups::DestroyService.new(group, current_user).async_execute end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 12ff2a1398b..2f50f94c897 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -2,7 +2,7 @@ module API class Issues < Grape::API include PaginationParams - before { authenticate! } + before { authenticate_non_get! } helpers ::Gitlab::IssuableMetadata @@ -70,6 +70,7 @@ module API desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`' end get do + authenticate! unless params[:scope] == 'all' issues = paginate(find_issues) options = { diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 5f2a9567605..5cb96d467c0 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -14,7 +14,7 @@ module API use :pagination end get do - runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: %w(specific shared)) + runners = filter_runners(current_user.ci_owned_runners, params[:scope], without: %w(specific shared)) present paginate(runners), with: Entities::Runner end @@ -184,40 +184,35 @@ module API def authenticate_show_runner!(runner) return if runner.is_shared || current_user.admin? - forbidden!("No access granted") unless user_can_access_runner?(runner) + forbidden!("No access granted") unless can?(current_user, :read_runner, runner) end def authenticate_update_runner!(runner) return if current_user.admin? - forbidden!("Runner is shared") if runner.is_shared? - forbidden!("No access granted") unless user_can_access_runner?(runner) + forbidden!("No access granted") unless can?(current_user, :update_runner, runner) end def authenticate_delete_runner!(runner) return if current_user.admin? - forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner associated with more than one project") if runner.projects.count > 1 - forbidden!("No access granted") unless user_can_access_runner?(runner) + forbidden!("No access granted") unless can?(current_user, :delete_runner, runner) end def authenticate_enable_runner!(runner) - forbidden!("Runner is shared") if runner.is_shared? - forbidden!("Runner is locked") if runner.locked? + forbidden!("Runner is a group runner") if runner.group_type? + return if current_user.admin? - forbidden!("No access granted") unless user_can_access_runner?(runner) + forbidden!("Runner is locked") if runner.locked? + forbidden!("No access granted") unless can?(current_user, :assign_runner, runner) end def authenticate_list_runners_jobs!(runner) return if current_user.admin? - forbidden!("No access granted") unless user_can_access_runner?(runner) - end - - def user_can_access_runner?(runner) - current_user.ci_authorized_runners.exists?(runner.id) + forbidden!("No access granted") unless can?(current_user, :read_runner, runner) end end end diff --git a/lib/api/v3/runners.rb b/lib/api/v3/runners.rb index c6d9957d452..8a5c46805bd 100644 --- a/lib/api/v3/runners.rb +++ b/lib/api/v3/runners.rb @@ -58,7 +58,7 @@ module API end def user_can_access_runner?(runner) - current_user.ci_authorized_runners.exists?(runner.id) + current_user.ci_owned_runners.exists?(runner.id) end end end diff --git a/lib/gitlab/auth/blocked_user_tracker.rb b/lib/gitlab/auth/blocked_user_tracker.rb index dae03a179e4..7609a7b04f6 100644 --- a/lib/gitlab/auth/blocked_user_tracker.rb +++ b/lib/gitlab/auth/blocked_user_tracker.rb @@ -17,7 +17,9 @@ module Gitlab # message passed along by Warden. return unless message == User::BLOCKED_MESSAGE - login = env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login') + # Check for either LDAP or regular GitLab account logins + login = env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'username') || + env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login') return unless login.present? diff --git a/lib/gitlab/ci/pipeline/preloader.rb b/lib/gitlab/ci/pipeline/preloader.rb new file mode 100644 index 00000000000..e7a2e5511cf --- /dev/null +++ b/lib/gitlab/ci/pipeline/preloader.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + # Class for preloading data associated with pipelines such as commit + # authors. + module Preloader + def self.preload(pipelines) + # This ensures that all the pipeline commits are eager loaded before we + # start using them. + pipelines.each(&:commit) + + pipelines.each do |pipeline| + # This preloads the author of every commit. We're using "lazy_author" + # here since "author" immediately loads the data on the first call. + pipeline.commit.try(:lazy_author) + + # This preloads the number of warnings for every pipeline, ensuring + # that Ci::Pipeline#has_warnings? doesn't execute any additional + # queries. + pipeline.number_of_warnings + end + end + end + end + end +end diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb new file mode 100644 index 00000000000..3374203960e --- /dev/null +++ b/lib/gitlab/database/count.rb @@ -0,0 +1,48 @@ +# For large tables, PostgreSQL can take a long time to count rows due to MVCC. +# We can optimize this by using the reltuples count as described in https://wiki.postgresql.org/wiki/Slow_Counting. +module Gitlab + module Database + module Count + CONNECTION_ERRORS = + if defined?(PG) + [ + ActionView::Template::Error, + ActiveRecord::StatementInvalid, + PG::Error + ].freeze + else + [ + ActionView::Template::Error, + ActiveRecord::StatementInvalid + ].freeze + end + + def self.approximate_count(model) + return model.count unless Gitlab::Database.postgresql? + + execute_estimate_if_updated_recently(model) || model.count + end + + def self.execute_estimate_if_updated_recently(model) + ActiveRecord::Base.connection.select_value(postgresql_estimate_query(model)).to_i if reltuples_updated_recently?(model) + rescue *CONNECTION_ERRORS + end + + def self.reltuples_updated_recently?(model) + time = "to_timestamp(#{1.hour.ago.to_i})" + query = <<~SQL + SELECT 1 FROM pg_stat_user_tables WHERE relname = '#{model.table_name}' AND + (last_vacuum > #{time} OR last_autovacuum > #{time} OR last_analyze > #{time} OR last_autoanalyze > #{time}) + SQL + + ActiveRecord::Base.connection.select_all(query).count > 0 + rescue *CONNECTION_ERRORS + false + end + + def self.postgresql_estimate_query(model) + "SELECT reltuples::bigint AS estimate FROM pg_class where relname = '#{model.table_name}'" + end + end + end +end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index fabcd46c8e9..d79a4dbeee4 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -342,21 +342,6 @@ module Gitlab parent_ids.first end - # Shows the diff between the commit's parent and the commit. - # - # Cuts out the header and stats from #to_patch and returns only the diff. - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324 - def to_diff - Gitlab::GitalyClient.migrate(:commit_patch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - @repository.gitaly_commit_client.patch(id) - else - rugged_diff_from_parent.patch - end - end - end - # Returns a diff object for the changes from this commit's first parent. # If there is no parent, then the diff is between this commit and an # empty repo. See Repository#diff for keys allowed in the +options+ @@ -432,16 +417,6 @@ module Gitlab Gitlab::Git::CommitStats.new(@repository, self) end - def to_patch(options = {}) - begin - rugged_commit.to_mbox(options) - rescue Rugged::InvalidError => ex - if ex.message =~ /commit \w+ is a merge commit/i - 'Patch format is not currently supported for merge commits.' - end - end - end - # Get ref names collection # # Ex. diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 29a3a35812c..061865a7acf 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -776,13 +776,9 @@ module Gitlab end def add_branch(branch_name, user:, target:) - gitaly_migrate(:operation_user_create_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_add_branch(branch_name, user, target) - else - rugged_add_branch(branch_name, user, target) - end - end + gitaly_operation_client.user_create_branch(branch_name, user, target) + rescue GRPC::FailedPrecondition => ex + raise InvalidRef, ex end def add_tag(tag_name, user:, target:, message: nil) @@ -1467,25 +1463,11 @@ module Gitlab end def branch_names_contains_sha(sha) - gitaly_migrate(:branch_names_contains_sha, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_client.branch_names_contains_sha(sha) - else - refs_contains_sha('refs/heads/', sha) - end - end + gitaly_ref_client.branch_names_contains_sha(sha) end def tag_names_contains_sha(sha) - gitaly_migrate(:tag_names_contains_sha, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_client.tag_names_contains_sha(sha) - else - refs_contains_sha('refs/tags/', sha) - end - end + gitaly_ref_client.tag_names_contains_sha(sha) end def search_files_by_content(query, ref) @@ -1620,27 +1602,6 @@ module Gitlab end end - def refs_contains_sha(refs_prefix, sha) - refs_prefix << "/" unless refs_prefix.ends_with?('/') - - # By forcing the output to %(refname) each line wiht a ref will start with - # the ref prefix. All other lines can be discarded. - args = %W(for-each-ref --contains=#{sha} --format=%(refname) #{refs_prefix}) - names, code = run_git(args) - - return [] unless code.zero? - - refs = [] - left_slice_count = refs_prefix.length - names.lines.each do |line| - next unless line.start_with?(refs_prefix) - - refs << encode_utf8(line.rstrip[left_slice_count..-1]) - end - - refs - end - def rugged_write_config(full_path:) rugged.config['gitlab.fullpath'] = full_path end @@ -2232,22 +2193,6 @@ module Gitlab end end - def gitaly_add_branch(branch_name, user, target) - gitaly_operation_client.user_create_branch(branch_name, user, target) - rescue GRPC::FailedPrecondition => ex - raise InvalidRef, ex - end - - def rugged_add_branch(branch_name, user, target) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object - - OperationService.new(user, self).add_branch(branch_name, target_object.oid) - find_branch(branch_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex - end - def rugged_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) OperationService.new(user, self).with_branch( branch_name, diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index e3e9f156fb4..4a41a69840b 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -28,7 +28,7 @@ module Gitlab IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index 89ff02a96d6..3799aaebf1c 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -4,18 +4,6 @@ module Gitlab CONTROLLER_KEY = 'action_controller.instance'.freeze ENDPOINT_KEY = 'api.endpoint'.freeze - CONTENT_TYPES = { - 'text/html' => :html, - 'text/plain' => :txt, - 'application/json' => :json, - 'text/js' => :js, - 'application/atom+xml' => :atom, - 'image/png' => :png, - 'image/jpeg' => :jpeg, - 'image/gif' => :gif, - 'image/svg+xml' => :svg - }.freeze - def initialize(env) super() @env = env @@ -40,7 +28,11 @@ module Gitlab controller = @env[CONTROLLER_KEY] action = "#{controller.action_name}" - suffix = CONTENT_TYPES[controller.content_type] + + # Devise exposes a method called "request_format" that does the below. + # However, this method is not available to all controllers (e.g. certain + # Doorkeeper controllers). As such we use the underlying code directly. + suffix = controller.request.format.try(:ref) if suffix && suffix != :html action += ".#{suffix}" diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 390efda326a..2e9b6e302f5 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -59,7 +59,7 @@ module Gitlab startline = 0 result.each_line.each_with_index do |line, index| - prefix ||= line.match(/^(?<ref>[^:]*):(?<filename>.*)\x00(?<startline>\d+)\x00/)&.tap do |matches| + prefix ||= line.match(/^(?<ref>[^:]*):(?<filename>[^\x00]*)\x00(?<startline>\d+)\x00/)&.tap do |matches| ref = matches[:ref] filename = matches[:filename] startline = matches[:startline] diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index 7ce2e9d636e..75ba0799058 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -11,7 +11,11 @@ module Gitlab class UntrustedRegexp delegate :===, to: :regexp - def initialize(pattern) + def initialize(pattern, multiline: false) + if multiline + pattern = "(?m)#{pattern}" + end + @regexp = RE2::Regexp.new(pattern, log_errors: false) raise RegexpError.new(regexp.error) unless regexp.ok? @@ -31,6 +35,19 @@ module Gitlab RE2.Replace(text, regexp, rewrite) end + # Handles regular expressions with the preferred RE2 library where possible + # via UntustedRegex. Falls back to Ruby's built-in regular expression library + # when the syntax would be invalid in RE2. + # + # One difference between these is `(?m)` multi-line mode. Ruby regex enables + # this by default, but also handles `^` and `$` differently. + # See: https://www.regular-expressions.info/modifiers.html + def self.with_fallback(pattern, multiline: false) + UntrustedRegexp.new(pattern, multiline: multiline) + rescue RegexpError + Regexp.new(pattern) + end + private attr_reader :regexp diff --git a/lib/tasks/gitlab/test.rake b/lib/tasks/gitlab/test.rake index 523b0fa055b..2222807fe13 100644 --- a/lib/tasks/gitlab/test.rake +++ b/lib/tasks/gitlab/test.rake @@ -4,7 +4,6 @@ namespace :gitlab do cmds = [ %w(rake brakeman), %w(rake rubocop), - %w(rake spinach), %w(rake spec), %w(rake karma) ] diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index af30ecb0e9b..e7aab50e42a 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -8,6 +8,7 @@ task setup_postgresql: :environment do require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb') + require Rails.root.join('db/migrate/20180504195842_project_name_lower_index.rb') require Rails.root.join('db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb') NamespacesProjectsPathLowerIndexes.new.up @@ -18,5 +19,6 @@ task setup_postgresql: :environment do IndexRedirectRoutesPathForLike.new.up AddIndexOnNamespacesLowerName.new.up UsersNameLowerIndex.new.up + ProjectNameLowerIndex.new.up AddPathIndexToRedirectRoutes.new.up end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake deleted file mode 100644 index 19ff13f06c0..00000000000 --- a/lib/tasks/spinach.rake +++ /dev/null @@ -1,60 +0,0 @@ -Rake::Task["spinach"].clear if Rake::Task.task_defined?('spinach') - -namespace :spinach do - namespace :project do - desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features" - task :half do - run_spinach_tests('@project_commits,@project_issues,@project_merge_requests') - end - - desc "GitLab | Spinach | Run remaining project spinach features" - task :rest do - run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests') - end - end - - desc "GitLab | Spinach | Run project spinach features" - task :project do - run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets') - end - - desc "GitLab | Spinach | Run other spinach features" - task :other do - run_spinach_tests('@admin,@dashboard,@profile,@public,@snippets') - end - - desc "GitLab | Spinach | Run other spinach features" - task :builds do - run_spinach_tests('@builds') - end -end - -desc "GitLab | Run spinach" -task :spinach do - run_spinach_tests(nil) -end - -def run_system_command(cmd) - system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) -end - -def run_spinach_command(args) - run_system_command(%w(spinach -r rerun) + args) -end - -def run_spinach_tests(tags) - success = run_spinach_command(%W(--tags #{tags})) - 3.times do |_| - break if success - break unless File.exist?('tmp/spinach-rerun.txt') - - tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp) - puts '' - puts "Spinach tests for #{tags}: Retrying tests... #{tests}".color(:red) - puts '' - sleep(3) - success = run_spinach_command(tests) - end - - raise("spinach tests for #{tags} failed!") unless success -end |