summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/api/helpers/pagination.rb2
-rw-r--r--lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb40
-rw-r--r--lib/gitlab/background_migration/migrate_legacy_uploads.rb128
-rw-r--r--lib/gitlab/ci/templates/Serverless.gitlab-ci.yml18
-rw-r--r--lib/gitlab/danger/helper.rb4
-rw-r--r--lib/gitlab/danger/teammate.rb27
-rw-r--r--lib/gitlab/lets_encrypt/challenge.rb2
-rw-r--r--lib/gitlab/lets_encrypt/order.rb11
-rw-r--r--lib/gitlab/metrics/samplers/ruby_sampler.rb2
-rw-r--r--lib/gitlab/omniauth_initializer.rb6
-rw-r--r--lib/gitlab/setup_helper.rb8
11 files changed, 220 insertions, 28 deletions
diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb
index 94b58a64d26..2a9b17ad22a 100644
--- a/lib/api/helpers/pagination.rb
+++ b/lib/api/helpers/pagination.rb
@@ -23,7 +23,7 @@ module API
def base_request_uri
@base_request_uri ||= URI.parse(request.url).tap do |uri|
uri.host = Gitlab.config.gitlab.host
- uri.port = nil
+ uri.port = Gitlab.config.gitlab.port
end
end
diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb
new file mode 100644
index 00000000000..0e93b2cb2fa
--- /dev/null
+++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # save validity time pages domain
+ class FillValidTimeForPagesDomainCertificate
+ # define PagesDomain with only needed code
+ class PagesDomain < ActiveRecord::Base
+ self.table_name = 'pages_domains'
+
+ def x509
+ return unless certificate.present?
+
+ @x509 ||= OpenSSL::X509::Certificate.new(certificate)
+ rescue OpenSSL::X509::CertificateError
+ nil
+ end
+ end
+
+ def perform(start_id, stop_id)
+ PagesDomain.where(id: start_id..stop_id).find_each do |domain|
+ if Gitlab::Database.mysql?
+ domain.update_columns(
+ certificate_valid_not_before: domain.x509&.not_before,
+ certificate_valid_not_after: domain.x509&.not_after
+ )
+ else
+ # for some reason activerecord doesn't append timezone, iso8601 forces this
+ domain.update_columns(
+ certificate_valid_not_before: domain.x509&.not_before&.iso8601,
+ certificate_valid_not_after: domain.x509&.not_after&.iso8601
+ )
+ end
+ rescue => e
+ Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}"
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/migrate_legacy_uploads.rb b/lib/gitlab/background_migration/migrate_legacy_uploads.rb
new file mode 100644
index 00000000000..af1ad930aed
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_legacy_uploads.rb
@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # This migration takes all legacy uploads (that were uploaded using AttachmentUploader)
+ # and migrate them to the new (FileUploader) location (=under projects).
+ #
+ # We have dependencies (uploaders) in this migration because extracting code would add a lot of complexity
+ # and possible errors could appear as the logic in the uploaders is not trivial.
+ #
+ # This migration will be removed in 12.4 in order to get rid of a migration that depends on
+ # the application code.
+ class MigrateLegacyUploads
+ include Database::MigrationHelpers
+ include ::Gitlab::Utils::StrongMemoize
+
+ # This class takes a legacy upload and migrates it to the correct location
+ class UploadMover
+ include Gitlab::Utils::StrongMemoize
+
+ attr_reader :upload, :project, :note
+
+ def initialize(upload)
+ @upload = upload
+ @note = Note.find_by(id: upload.model_id)
+ @project = note&.project
+ end
+
+ def execute
+ return unless upload
+
+ if !project
+ # if we don't have models associated with the upload we can not move it
+ say "MigrateLegacyUploads: Deleting upload due to model not found: #{upload.inspect}"
+ destroy_legacy_upload
+ elsif note.is_a?(LegacyDiffNote)
+ handle_legacy_note_upload
+ elsif !legacy_file_exists?
+ # if we can not find the file we just remove the upload record
+ say "MigrateLegacyUploads: Deleting upload due to file not found: #{upload.inspect}"
+ destroy_legacy_upload
+ else
+ migrate_upload
+ end
+ end
+
+ private
+
+ def migrate_upload
+ return unless copy_upload_to_project
+
+ add_upload_link_to_note_text
+ destroy_legacy_file
+ destroy_legacy_upload
+ end
+
+ # we should proceed and log whenever one upload copy fails, no matter the reasons
+ # rubocop: disable Lint/RescueException
+ def copy_upload_to_project
+ @uploader = FileUploader.copy_to(legacy_file_uploader, project)
+
+ say "MigrateLegacyUploads: Copied file #{legacy_file_uploader.file.path} -> #{@uploader.file.path}"
+ true
+ rescue Exception => e
+ say "MigrateLegacyUploads: File #{legacy_file_uploader.file.path} couldn't be copied to project uploads. Error: #{e.message}"
+ false
+ end
+ # rubocop: enable Lint/RescueException
+
+ def destroy_legacy_upload
+ note.remove_attachment = true
+ note.save
+
+ if upload.destroy
+ say "MigrateLegacyUploads: Upload #{upload.inspect} was destroyed."
+ else
+ say "MigrateLegacyUploads: Upload #{upload.inspect} destroy failed."
+ end
+ end
+
+ def destroy_legacy_file
+ legacy_file_uploader.file.delete
+ end
+
+ def add_upload_link_to_note_text
+ new_text = "#{note.note} \n #{@uploader.markdown_link}"
+ note.update!(
+ note: new_text
+ )
+ end
+
+ def legacy_file_uploader
+ strong_memoize(:legacy_file_uploader) do
+ uploader = upload.build_uploader
+ uploader.retrieve_from_store!(File.basename(upload.path))
+ uploader
+ end
+ end
+
+ def legacy_file_exists?
+ legacy_file_uploader.file.exists?
+ end
+
+ def handle_legacy_note_upload
+ note.note += "\n \n Attachment ##{upload.id} with URL \"#{note.attachment.url}\" failed to migrate \
+ for model class #{note.class}. See #{help_doc_link}."
+ note.save
+
+ say "MigrateLegacyUploads: LegacyDiffNote ##{note.id} found, can't move the file: #{upload.inspect} for upload ##{upload.id}. See #{help_doc_link}."
+ end
+
+ def say(message)
+ Rails.logger.info(message)
+ end
+
+ def help_doc_link
+ 'https://docs.gitlab.com/ee/administration/troubleshooting/migrations.html#legacy-upload-migration'
+ end
+ end
+
+ def perform(start_id, end_id)
+ Upload.where(id: start_id..end_id, uploader: 'AttachmentUploader').find_each do |upload|
+ UploadMover.new(upload).execute
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml
index 0fb7c57ab72..3f46bb89e94 100644
--- a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml
@@ -8,25 +8,13 @@ stages:
- deploy
.serverless:build:image:
- variables:
- DOCKERFILE: "Dockerfile"
stage: build
- image:
- name: gcr.io/kaniko-project/executor:debug
- entrypoint: [""]
- only:
- refs:
- - master
- script:
- - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
- - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/$DOCKERFILE --destination $CI_REGISTRY_IMAGE
+ image: registry.gitlab.com/gitlab-org/gitlabktl:latest
+ script: /usr/bin/gitlabktl app build
.serverless:deploy:image:
stage: deploy
- image: gcr.io/triggermesh/tm@sha256:e3ee74db94d215bd297738d93577481f3e4db38013326c90d57f873df7ab41d5
- only:
- refs:
- - master
+ image: gcr.io/triggermesh/tm@sha256:3cfdd470a66b741004fb02354319d79f1598c70117ce79978d2e07e192bfb336 # v0.0.11
environment: development
script:
- echo "$CI_REGISTRY_IMAGE"
diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb
index 7effb802678..7a0fb419f8e 100644
--- a/lib/gitlab/danger/helper.rb
+++ b/lib/gitlab/danger/helper.rb
@@ -72,7 +72,8 @@ module Gitlab
CATEGORY_LABELS = {
docs: "~Documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now.
none: "",
- qa: "~QA"
+ qa: "~QA",
+ test: "~test for `spec/features/*`"
}.freeze
CATEGORIES = {
%r{\Adoc/} => :none, # To reinstate roulette for documentation, set to `:docs`.
@@ -104,6 +105,7 @@ module Gitlab
%r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend,
%r{\A(ee/)?(bin|config|danger|generator_templates|lib|rubocop|scripts)/} => :backend,
+ %r{\A(ee/)?spec/features/} => :test,
%r{\A(ee/)?spec/(?!javascripts|frontend)[^/]+} => :backend,
%r{\A(ee/)?vendor/(?!assets)[^/]+} => :backend,
%r{\A(ee/)?vendor/(languages\.yml|licenses\.csv)\z} => :backend,
diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb
index c4e66da8ed1..b44f134f2c1 100644
--- a/lib/gitlab/danger/teammate.rb
+++ b/lib/gitlab/danger/teammate.rb
@@ -3,11 +3,12 @@
module Gitlab
module Danger
class Teammate
- attr_reader :name, :username, :projects
+ attr_reader :name, :username, :role, :projects
def initialize(options = {})
@username = options['username']
@name = options['name'] || @username
+ @role = options['role']
@projects = options['projects']
end
@@ -20,20 +21,32 @@ module Gitlab
end
# Traintainers also count as reviewers
- def reviewer?(project, category)
- capabilities(project).include?("reviewer #{category}") || traintainer?(project, category)
+ def reviewer?(project, category, labels)
+ has_capability?(project, category, :reviewer, labels) ||
+ traintainer?(project, category, labels)
end
- def traintainer?(project, category)
- capabilities(project).include?("trainee_maintainer #{category}")
+ def traintainer?(project, category, labels)
+ has_capability?(project, category, :trainee_maintainer, labels)
end
- def maintainer?(project, category)
- capabilities(project).include?("maintainer #{category}")
+ def maintainer?(project, category, labels)
+ has_capability?(project, category, :maintainer, labels)
end
private
+ def has_capability?(project, category, kind, labels)
+ case category
+ when :test
+ area = role[/Test Automation Engineer, (\w+)/, 1]
+
+ area && labels.any?(area) if kind == :reviewer
+ else
+ capabilities(project).include?("#{kind} #{category}")
+ end
+ end
+
def capabilities(project)
Array(projects.fetch(project, []))
end
diff --git a/lib/gitlab/lets_encrypt/challenge.rb b/lib/gitlab/lets_encrypt/challenge.rb
index 6a7f5e965c5..136268c974b 100644
--- a/lib/gitlab/lets_encrypt/challenge.rb
+++ b/lib/gitlab/lets_encrypt/challenge.rb
@@ -7,7 +7,7 @@ module Gitlab
@acme_challenge = acme_challenge
end
- delegate :url, :token, :file_content, :status, :request_validation, to: :acme_challenge
+ delegate :token, :file_content, :status, :request_validation, to: :acme_challenge
private
diff --git a/lib/gitlab/lets_encrypt/order.rb b/lib/gitlab/lets_encrypt/order.rb
index 5109b5e9843..9c2365f98a8 100644
--- a/lib/gitlab/lets_encrypt/order.rb
+++ b/lib/gitlab/lets_encrypt/order.rb
@@ -13,7 +13,16 @@ module Gitlab
::Gitlab::LetsEncrypt::Challenge.new(challenge)
end
- delegate :url, :status, to: :acme_order
+ def request_certificate(domain:, private_key:)
+ csr = ::Acme::Client::CertificateRequest.new(
+ private_key: OpenSSL::PKey.read(private_key),
+ subject: { common_name: domain }
+ )
+
+ acme_order.finalize(csr: csr)
+ end
+
+ delegate :url, :status, :expires, :certificate, to: :acme_order
private
diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb
index 4d9c43f37e7..17eacbd21d8 100644
--- a/lib/gitlab/metrics/samplers/ruby_sampler.rb
+++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb
@@ -77,10 +77,10 @@ module Gitlab
end
def worker_label
+ return { worker: 'sidekiq' } if Sidekiq.server?
return {} unless defined?(Unicorn::Worker)
worker_no = ::Prometheus::Client::Support::Unicorn.worker_id
-
if worker_no
{ worker: worker_no }
else
diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb
index 2a2083ebae0..83204fa5d18 100644
--- a/lib/gitlab/omniauth_initializer.rb
+++ b/lib/gitlab/omniauth_initializer.rb
@@ -63,6 +63,12 @@ module Gitlab
{ remote_sign_out_handler: authentiq_signout_handler }
when 'shibboleth'
{ fail_with_empty_uid: true }
+ when 'openid_connect'
+ # If a name argument is omitted, OmniAuth will expect that the
+ # matching route is /auth/users/openidconnect instead of
+ # /auth/users/openid_connect because of
+ # https://gitlab.com/gitlab-org/gitlab-ce/issues/62208#note_178780341.
+ { name: 'openid_connect' }
else
{}
end
diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb
index 61fcb562106..0d3e78c0a66 100644
--- a/lib/gitlab/setup_helper.rb
+++ b/lib/gitlab/setup_helper.rb
@@ -43,7 +43,13 @@ module Gitlab
config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby
config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
config[:bin_dir] = Gitlab.config.gitaly.client_path
- config[:git] = { catfile_cache_size: 5 }
+
+ if Rails.env.test?
+ # Compared to production, tests run in constrained environments. This
+ # number is meant to grow with the number of concurrent rails requests /
+ # sidekiq jobs, and concurrency will be low anyway in test.
+ config[:git] = { catfile_cache_size: 5 }
+ end
TomlRB.dump(config)
end