diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/helpers/pagination.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb | 40 | ||||
-rw-r--r-- | lib/gitlab/background_migration/migrate_legacy_uploads.rb | 128 | ||||
-rw-r--r-- | lib/gitlab/ci/templates/Serverless.gitlab-ci.yml | 18 | ||||
-rw-r--r-- | lib/gitlab/danger/helper.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/danger/teammate.rb | 27 | ||||
-rw-r--r-- | lib/gitlab/lets_encrypt/challenge.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/lets_encrypt/order.rb | 11 | ||||
-rw-r--r-- | lib/gitlab/metrics/samplers/ruby_sampler.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/omniauth_initializer.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/setup_helper.rb | 8 |
11 files changed, 220 insertions, 28 deletions
diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 94b58a64d26..2a9b17ad22a 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -23,7 +23,7 @@ module API def base_request_uri @base_request_uri ||= URI.parse(request.url).tap do |uri| uri.host = Gitlab.config.gitlab.host - uri.port = nil + uri.port = Gitlab.config.gitlab.port end end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb new file mode 100644 index 00000000000..0e93b2cb2fa --- /dev/null +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # save validity time pages domain + class FillValidTimeForPagesDomainCertificate + # define PagesDomain with only needed code + class PagesDomain < ActiveRecord::Base + self.table_name = 'pages_domains' + + def x509 + return unless certificate.present? + + @x509 ||= OpenSSL::X509::Certificate.new(certificate) + rescue OpenSSL::X509::CertificateError + nil + end + end + + def perform(start_id, stop_id) + PagesDomain.where(id: start_id..stop_id).find_each do |domain| + if Gitlab::Database.mysql? + domain.update_columns( + certificate_valid_not_before: domain.x509&.not_before, + certificate_valid_not_after: domain.x509&.not_after + ) + else + # for some reason activerecord doesn't append timezone, iso8601 forces this + domain.update_columns( + certificate_valid_not_before: domain.x509&.not_before&.iso8601, + certificate_valid_not_after: domain.x509&.not_after&.iso8601 + ) + end + rescue => e + Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" + end + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_legacy_uploads.rb b/lib/gitlab/background_migration/migrate_legacy_uploads.rb new file mode 100644 index 00000000000..af1ad930aed --- /dev/null +++ b/lib/gitlab/background_migration/migrate_legacy_uploads.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration takes all legacy uploads (that were uploaded using AttachmentUploader) + # and migrate them to the new (FileUploader) location (=under projects). + # + # We have dependencies (uploaders) in this migration because extracting code would add a lot of complexity + # and possible errors could appear as the logic in the uploaders is not trivial. + # + # This migration will be removed in 12.4 in order to get rid of a migration that depends on + # the application code. + class MigrateLegacyUploads + include Database::MigrationHelpers + include ::Gitlab::Utils::StrongMemoize + + # This class takes a legacy upload and migrates it to the correct location + class UploadMover + include Gitlab::Utils::StrongMemoize + + attr_reader :upload, :project, :note + + def initialize(upload) + @upload = upload + @note = Note.find_by(id: upload.model_id) + @project = note&.project + end + + def execute + return unless upload + + if !project + # if we don't have models associated with the upload we can not move it + say "MigrateLegacyUploads: Deleting upload due to model not found: #{upload.inspect}" + destroy_legacy_upload + elsif note.is_a?(LegacyDiffNote) + handle_legacy_note_upload + elsif !legacy_file_exists? + # if we can not find the file we just remove the upload record + say "MigrateLegacyUploads: Deleting upload due to file not found: #{upload.inspect}" + destroy_legacy_upload + else + migrate_upload + end + end + + private + + def migrate_upload + return unless copy_upload_to_project + + add_upload_link_to_note_text + destroy_legacy_file + destroy_legacy_upload + end + + # we should proceed and log whenever one upload copy fails, no matter the reasons + # rubocop: disable Lint/RescueException + def copy_upload_to_project + @uploader = FileUploader.copy_to(legacy_file_uploader, project) + + say "MigrateLegacyUploads: Copied file #{legacy_file_uploader.file.path} -> #{@uploader.file.path}" + true + rescue Exception => e + say "MigrateLegacyUploads: File #{legacy_file_uploader.file.path} couldn't be copied to project uploads. Error: #{e.message}" + false + end + # rubocop: enable Lint/RescueException + + def destroy_legacy_upload + note.remove_attachment = true + note.save + + if upload.destroy + say "MigrateLegacyUploads: Upload #{upload.inspect} was destroyed." + else + say "MigrateLegacyUploads: Upload #{upload.inspect} destroy failed." + end + end + + def destroy_legacy_file + legacy_file_uploader.file.delete + end + + def add_upload_link_to_note_text + new_text = "#{note.note} \n #{@uploader.markdown_link}" + note.update!( + note: new_text + ) + end + + def legacy_file_uploader + strong_memoize(:legacy_file_uploader) do + uploader = upload.build_uploader + uploader.retrieve_from_store!(File.basename(upload.path)) + uploader + end + end + + def legacy_file_exists? + legacy_file_uploader.file.exists? + end + + def handle_legacy_note_upload + note.note += "\n \n Attachment ##{upload.id} with URL \"#{note.attachment.url}\" failed to migrate \ + for model class #{note.class}. See #{help_doc_link}." + note.save + + say "MigrateLegacyUploads: LegacyDiffNote ##{note.id} found, can't move the file: #{upload.inspect} for upload ##{upload.id}. See #{help_doc_link}." + end + + def say(message) + Rails.logger.info(message) + end + + def help_doc_link + 'https://docs.gitlab.com/ee/administration/troubleshooting/migrations.html#legacy-upload-migration' + end + end + + def perform(start_id, end_id) + Upload.where(id: start_id..end_id, uploader: 'AttachmentUploader').find_each do |upload| + UploadMover.new(upload).execute + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml index 0fb7c57ab72..3f46bb89e94 100644 --- a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml @@ -8,25 +8,13 @@ stages: - deploy .serverless:build:image: - variables: - DOCKERFILE: "Dockerfile" stage: build - image: - name: gcr.io/kaniko-project/executor:debug - entrypoint: [""] - only: - refs: - - master - script: - - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/$DOCKERFILE --destination $CI_REGISTRY_IMAGE + image: registry.gitlab.com/gitlab-org/gitlabktl:latest + script: /usr/bin/gitlabktl app build .serverless:deploy:image: stage: deploy - image: gcr.io/triggermesh/tm@sha256:e3ee74db94d215bd297738d93577481f3e4db38013326c90d57f873df7ab41d5 - only: - refs: - - master + image: gcr.io/triggermesh/tm@sha256:3cfdd470a66b741004fb02354319d79f1598c70117ce79978d2e07e192bfb336 # v0.0.11 environment: development script: - echo "$CI_REGISTRY_IMAGE" diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 7effb802678..7a0fb419f8e 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -72,7 +72,8 @@ module Gitlab CATEGORY_LABELS = { docs: "~Documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now. none: "", - qa: "~QA" + qa: "~QA", + test: "~test for `spec/features/*`" }.freeze CATEGORIES = { %r{\Adoc/} => :none, # To reinstate roulette for documentation, set to `:docs`. @@ -104,6 +105,7 @@ module Gitlab %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, %r{\A(ee/)?(bin|config|danger|generator_templates|lib|rubocop|scripts)/} => :backend, + %r{\A(ee/)?spec/features/} => :test, %r{\A(ee/)?spec/(?!javascripts|frontend)[^/]+} => :backend, %r{\A(ee/)?vendor/(?!assets)[^/]+} => :backend, %r{\A(ee/)?vendor/(languages\.yml|licenses\.csv)\z} => :backend, diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index c4e66da8ed1..b44f134f2c1 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -3,11 +3,12 @@ module Gitlab module Danger class Teammate - attr_reader :name, :username, :projects + attr_reader :name, :username, :role, :projects def initialize(options = {}) @username = options['username'] @name = options['name'] || @username + @role = options['role'] @projects = options['projects'] end @@ -20,20 +21,32 @@ module Gitlab end # Traintainers also count as reviewers - def reviewer?(project, category) - capabilities(project).include?("reviewer #{category}") || traintainer?(project, category) + def reviewer?(project, category, labels) + has_capability?(project, category, :reviewer, labels) || + traintainer?(project, category, labels) end - def traintainer?(project, category) - capabilities(project).include?("trainee_maintainer #{category}") + def traintainer?(project, category, labels) + has_capability?(project, category, :trainee_maintainer, labels) end - def maintainer?(project, category) - capabilities(project).include?("maintainer #{category}") + def maintainer?(project, category, labels) + has_capability?(project, category, :maintainer, labels) end private + def has_capability?(project, category, kind, labels) + case category + when :test + area = role[/Test Automation Engineer, (\w+)/, 1] + + area && labels.any?(area) if kind == :reviewer + else + capabilities(project).include?("#{kind} #{category}") + end + end + def capabilities(project) Array(projects.fetch(project, [])) end diff --git a/lib/gitlab/lets_encrypt/challenge.rb b/lib/gitlab/lets_encrypt/challenge.rb index 6a7f5e965c5..136268c974b 100644 --- a/lib/gitlab/lets_encrypt/challenge.rb +++ b/lib/gitlab/lets_encrypt/challenge.rb @@ -7,7 +7,7 @@ module Gitlab @acme_challenge = acme_challenge end - delegate :url, :token, :file_content, :status, :request_validation, to: :acme_challenge + delegate :token, :file_content, :status, :request_validation, to: :acme_challenge private diff --git a/lib/gitlab/lets_encrypt/order.rb b/lib/gitlab/lets_encrypt/order.rb index 5109b5e9843..9c2365f98a8 100644 --- a/lib/gitlab/lets_encrypt/order.rb +++ b/lib/gitlab/lets_encrypt/order.rb @@ -13,7 +13,16 @@ module Gitlab ::Gitlab::LetsEncrypt::Challenge.new(challenge) end - delegate :url, :status, to: :acme_order + def request_certificate(domain:, private_key:) + csr = ::Acme::Client::CertificateRequest.new( + private_key: OpenSSL::PKey.read(private_key), + subject: { common_name: domain } + ) + + acme_order.finalize(csr: csr) + end + + delegate :url, :status, :expires, :certificate, to: :acme_order private diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 4d9c43f37e7..17eacbd21d8 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -77,10 +77,10 @@ module Gitlab end def worker_label + return { worker: 'sidekiq' } if Sidekiq.server? return {} unless defined?(Unicorn::Worker) worker_no = ::Prometheus::Client::Support::Unicorn.worker_id - if worker_no { worker: worker_no } else diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 2a2083ebae0..83204fa5d18 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -63,6 +63,12 @@ module Gitlab { remote_sign_out_handler: authentiq_signout_handler } when 'shibboleth' { fail_with_empty_uid: true } + when 'openid_connect' + # If a name argument is omitted, OmniAuth will expect that the + # matching route is /auth/users/openidconnect instead of + # /auth/users/openid_connect because of + # https://gitlab.com/gitlab-org/gitlab-ce/issues/62208#note_178780341. + { name: 'openid_connect' } else {} end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 61fcb562106..0d3e78c0a66 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -43,7 +43,13 @@ module Gitlab config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } config[:bin_dir] = Gitlab.config.gitaly.client_path - config[:git] = { catfile_cache_size: 5 } + + if Rails.env.test? + # Compared to production, tests run in constrained environments. This + # number is meant to grow with the number of concurrent rails requests / + # sidekiq jobs, and concurrency will be low anyway in test. + config[:git] = { catfile_cache_size: 5 } + end TomlRB.dump(config) end |