diff options
Diffstat (limited to 'lib/gitlab')
83 files changed, 1142 insertions, 299 deletions
diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 7ceb96f502b..47d63eb53cf 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -75,7 +75,8 @@ module Gitlab encryption: options['encryption'], filter: omniauth_user_filter, name_proc: name_proc, - disable_verify_certificates: !options['verify_certificates'] + disable_verify_certificates: !options['verify_certificates'], + tls_options: tls_options ) if has_auth? @@ -85,9 +86,6 @@ module Gitlab ) end - opts[:ca_file] = options['ca_file'] if options['ca_file'].present? - opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? - opts end @@ -196,24 +194,28 @@ module Gitlab end def encryption_options - method = translate_method(options['encryption']) - return nil unless method + method = translate_method + return unless method { method: method, - tls_options: tls_options(method) + tls_options: tls_options } end - def translate_method(method_from_config) - NET_LDAP_ENCRYPTION_METHOD[method_from_config.to_sym] + def translate_method + NET_LDAP_ENCRYPTION_METHOD[options['encryption']&.to_sym] end - def tls_options(method) - return { verify_mode: OpenSSL::SSL::VERIFY_NONE } unless method + def tls_options + return @tls_options if defined?(@tls_options) + + method = translate_method + return unless method - opts = if options['verify_certificates'] - OpenSSL::SSL::SSLContext::DEFAULT_PARAMS + opts = if options['verify_certificates'] && method != 'plain' + # Dup so we don't accidentally overwrite the constant + OpenSSL::SSL::SSLContext::DEFAULT_PARAMS.dup else # It is important to explicitly set verify_mode for two reasons: # 1. The behavior of OpenSSL is undefined when verify_mode is not set. @@ -222,10 +224,35 @@ module Gitlab { verify_mode: OpenSSL::SSL::VERIFY_NONE } end - opts[:ca_file] = options['ca_file'] if options['ca_file'].present? - opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + opts.merge!(custom_tls_options) - opts + @tls_options = opts + end + + def custom_tls_options + return {} unless options['tls_options'] + + # Dup so we don't overwrite the original value + custom_options = options['tls_options'].dup.delete_if { |_, value| value.nil? || value.blank? } + custom_options.symbolize_keys! + + if custom_options[:cert] + begin + custom_options[:cert] = OpenSSL::X509::Certificate.new(custom_options[:cert]) + rescue OpenSSL::X509::CertificateError => e + Rails.logger.error "LDAP TLS Options 'cert' is invalid for provider #{provider}: #{e.message}" + end + end + + if custom_options[:key] + begin + custom_options[:key] = OpenSSL::PKey.read(custom_options[:key]) + rescue OpenSSL::PKey::PKeyError => e + Rails.logger.error "LDAP TLS Options 'key' is invalid for provider #{provider}: #{e.message}" + end + end + + custom_options end def auth_options diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index 48d134f91b0..13d67e0f871 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -112,7 +112,7 @@ module Gitlab attributes = Array(config.attributes[attribute.to_s]) selected_attr = attributes.find { |attr| entry.respond_to?(attr) } - return nil unless selected_attr + return unless selected_attr entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend end diff --git a/lib/gitlab/auth/omniauth_identity_linker_base.rb b/lib/gitlab/auth/omniauth_identity_linker_base.rb index 253445570f2..c620fc5d6bd 100644 --- a/lib/gitlab/auth/omniauth_identity_linker_base.rb +++ b/lib/gitlab/auth/omniauth_identity_linker_base.rb @@ -12,7 +12,7 @@ module Gitlab end def link - save if identity.new_record? + save if unlinked? end def changed? @@ -35,6 +35,10 @@ module Gitlab @changed = identity.save end + def unlinked? + identity.new_record? + end + # rubocop: disable CodeReuse/ActiveRecord def identity @identity ||= current_user.identities diff --git a/lib/gitlab/auth/saml/auth_hash.rb b/lib/gitlab/auth/saml/auth_hash.rb index 1af9fa40c3a..b0df9757bbd 100644 --- a/lib/gitlab/auth/saml/auth_hash.rb +++ b/lib/gitlab/auth/saml/auth_hash.rb @@ -10,11 +10,11 @@ module Gitlab def authn_context response_object = auth_hash.extra[:response_object] - return nil if response_object.blank? + return if response_object.blank? document = response_object.decrypted_document document ||= response_object.document - return nil if document.blank? + return if document.blank? extract_authn_context(document) end diff --git a/lib/gitlab/background_migration/encrypt_columns.rb b/lib/gitlab/background_migration/encrypt_columns.rb index b9ad8267e37..173543b7c25 100644 --- a/lib/gitlab/background_migration/encrypt_columns.rb +++ b/lib/gitlab/background_migration/encrypt_columns.rb @@ -91,7 +91,8 @@ module Gitlab # No need to do anything if the plaintext is nil, or an encrypted # value already exists - return nil unless plaintext.present? && !ciphertext.present? + return unless plaintext.present? + return if ciphertext.present? # attr_encrypted will calculate and set the expected value for us instance.public_send("#{plain_column}=", plaintext) # rubocop:disable GitlabSecurity/PublicSend diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb index 38fecac1bfe..42fcaa87e66 100644 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -24,7 +24,7 @@ module Gitlab def commit_title commit = commits.last - return nil unless commit && commit[:message] + return unless commit && commit[:message] index = commit[:message].index("\n") message = index ? commit[:message][0..index] : commit[:message] diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index 4a9a62aaeb5..a84f794bfae 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -127,7 +127,7 @@ module Gitlab full_path = matchd[1] project = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Project.find_by_full_path(full_path) - return nil unless project + return unless project project.id end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 3cd327f5109..144ba2ec031 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -108,7 +108,7 @@ module Gitlab end def find_or_create_groups - return nil unless group_path.present? + return unless group_path.present? log " * Using namespace: #{group_path}" diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 75a3f17f549..441fdec8a1e 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -47,7 +47,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def find_user_id(username) - return nil unless username + return unless username return users[username] if users.key?(username) diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index dbbedd5dcbe..4a789ae457f 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -65,7 +65,7 @@ module Gitlab end def find_user_id(email) - return nil unless email + return unless email return users[email] if users.key?(email) diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index d06b2df36f2..ad926739752 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -9,13 +9,17 @@ module Gitlab non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project maintainer or owner can delete a protected branch.', non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.', merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.', - push_protected_branch: 'You are not allowed to push code to protected branches on this project.' + push_protected_branch: 'You are not allowed to push code to protected branches on this project.', + create_protected_branch: 'You are not allowed to create protected branches on this project.', + invalid_commit_create_protected_branch: 'You can only use an existing protected branch ref as the basis of a new protected branch.', + non_web_create_protected_branch: 'You can only create protected branches using the web interface and API.' }.freeze LOG_MESSAGES = { delete_default_branch_check: "Checking if default branch is being deleted...", protected_branch_checks: "Checking if you are force pushing to a protected branch...", protected_branch_push_checks: "Checking if you are allowed to push to the protected branch...", + protected_branch_creation_checks: "Checking if you are allowed to create a protected branch...", protected_branch_deletion_checks: "Checking if you are allowed to delete the protected branch..." }.freeze @@ -42,13 +46,33 @@ module Gitlab end end - if deletion? + if project.empty_repo? + protected_branch_push_checks + elsif creation? && protected_branch_creation_enabled? + protected_branch_creation_checks + elsif deletion? protected_branch_deletion_checks else protected_branch_push_checks end end + def protected_branch_creation_checks + logger.log_timed(LOG_MESSAGES[:protected_branch_creation_checks]) do + unless user_access.can_merge_to_branch?(branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_branch] + end + + unless safe_commit_for_new_protected_branch? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:invalid_commit_create_protected_branch] + end + + unless updated_from_web? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_create_protected_branch] + end + end + end + def protected_branch_deletion_checks logger.log_timed(LOG_MESSAGES[:protected_branch_deletion_checks]) do unless user_access.can_delete_branch?(branch_name) @@ -98,6 +122,10 @@ module Gitlab Gitlab::Routing.url_helpers.project_project_members_url(project) end + def protected_branch_creation_enabled? + Feature.enabled?(:protected_branch_creation, project, default_enabled: true) + end + def matching_merge_request? Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? end @@ -105,6 +133,10 @@ module Gitlab def forced_push? Gitlab::Checks::ForcePush.force_push?(project, oldrev, newrev) end + + def safe_commit_for_new_protected_branch? + ProtectedBranch.any_protected?(project, project.repository.branch_names_contains_sha(newrev)) + end end end end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 4dcb3869d4f..fba0de20ced 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -313,7 +313,7 @@ module Gitlab def get_term_color_class(color_index, prefix) color_name = COLOR[color_index] - return nil if color_name.nil? + return if color_name.nil? get_color_class(["term", prefix, color_name]) end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 08dac756cc1..d45a044686e 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -103,7 +103,7 @@ module Gitlab def read_string(gz) string_size = read_uint32(gz) - return nil unless string_size + return unless string_size gz.read(string_size) end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index d0a80518ae8..80e69cdcc95 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -44,7 +44,7 @@ module Gitlab end def parent - return nil unless has_parent? + return unless has_parent? self.class.new(@path.to_s.chomp(basename), @entries) end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index 0e9bb5c94bb..df5f5ffc253 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -29,8 +29,8 @@ module Gitlab def matches_pattern?(pattern, pipeline) return true if pipeline.tag? && pattern == 'tags' return true if pipeline.branch? && pattern == 'branches' - return true if pipeline.source == pattern - return true if pipeline.source&.pluralize == pattern + return true if sanitized_source_name(pipeline) == pattern + return true if sanitized_source_name(pipeline)&.pluralize == pattern # patterns can be matched only when branch or tag is used # the pattern matching does not work for merge requests pipelines @@ -42,6 +42,10 @@ module Gitlab end end end + + def sanitized_source_name(pipeline) + @sanitized_source_name ||= pipeline&.source&.delete_suffix('_event') + end end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 5875479183e..15643fa03ac 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -84,7 +84,8 @@ module Gitlab Config::External::Processor.new(config, project: project, sha: sha || project.repository.root_ref_sha, - user: user).perform + user: user, + expandset: Set.new).perform end end end diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index a747886093c..2ffbb214a92 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -12,7 +12,7 @@ module Gitlab YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze - Context = Struct.new(:project, :sha, :user) + Context = Struct.new(:project, :sha, :user, :expandset) def initialize(params, context) @params = params @@ -43,13 +43,27 @@ module Gitlab end def to_hash - @hash ||= Gitlab::Config::Loader::Yaml.new(content).load! - rescue Gitlab::Config::Loader::FormatError - nil + expanded_content_hash end protected + def expanded_content_hash + return unless content_hash + + strong_memoize(:expanded_content_yaml) do + expand_includes(content_hash) + end + end + + def content_hash + strong_memoize(:content_yaml) do + Gitlab::Config::Loader::Yaml.new(content).load! + end + rescue Gitlab::Config::Loader::FormatError + nil + end + def validate! validate_location! validate_content! if errors.none? @@ -73,6 +87,14 @@ module Gitlab errors.push("Included file `#{location}` does not have valid YAML syntax!") end end + + def expand_includes(hash) + External::Processor.new(hash, **expand_context).perform + end + + def expand_context + { project: nil, sha: nil, user: nil, expandset: context.expandset } + end end end end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index 2535d178ba8..229a06451e8 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -31,6 +31,13 @@ module Gitlab def fetch_local_content context.project.repository.blob_data_at(context.sha, location) end + + def expand_context + super.merge( + project: context.project, + sha: context.sha, + user: context.user) + end end end end diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index e75540dbe5a..b828f77835c 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -64,6 +64,13 @@ module Gitlab project.commit(ref_name).try(:sha) end end + + def expand_context + super.merge( + project: project, + sha: sha, + user: context.user) + end end end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 108bfd5eb43..aff5c5b9651 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -7,6 +7,8 @@ module Gitlab class Mapper include Gitlab::Utils::StrongMemoize + MAX_INCLUDES = 50 + FILE_CLASSES = [ External::File::Remote, External::File::Template, @@ -14,25 +16,34 @@ module Gitlab External::File::Project ].freeze - AmbigiousSpecificationError = Class.new(StandardError) + Error = Class.new(StandardError) + AmbigiousSpecificationError = Class.new(Error) + DuplicateIncludesError = Class.new(Error) + TooManyIncludesError = Class.new(Error) + + def initialize(values, project:, sha:, user:, expandset:) + raise Error, 'Expanded needs to be `Set`' unless expandset.is_a?(Set) - def initialize(values, project:, sha:, user:) @locations = Array.wrap(values.fetch(:include, [])) @project = project @sha = sha @user = user + @expandset = expandset end def process + return [] if locations.empty? + locations .compact .map(&method(:normalize_location)) + .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) end private - attr_reader :locations, :project, :sha, :user + attr_reader :locations, :project, :sha, :user, :expandset # convert location if String to canonical form def normalize_location(location) @@ -51,6 +62,23 @@ module Gitlab end end + def verify_duplicates!(location) + if expandset.count >= MAX_INCLUDES + raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!" + end + + # We scope location to context, as this allows us to properly support + # relative incldues, and similarly looking relative in another project + # does not trigger duplicate error + scoped_location = location.merge( + context_project: project, + context_sha: sha) + + unless expandset.add?(scoped_location) + raise DuplicateIncludesError, "Include `#{location.to_json}` was already included!" + end + end + def select_first_matching(location) matching = FILE_CLASSES.map do |file_class| file_class.new(location, context) @@ -63,7 +91,7 @@ module Gitlab def context strong_memoize(:context) do - External::File::Base::Context.new(project, sha, user) + External::File::Base::Context.new(project, sha, user, expandset) end end end diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb index 69bc164a039..1dd2d42016a 100644 --- a/lib/gitlab/ci/config/external/processor.rb +++ b/lib/gitlab/ci/config/external/processor.rb @@ -7,11 +7,11 @@ module Gitlab class Processor IncludeError = Class.new(StandardError) - def initialize(values, project:, sha:, user:) + def initialize(values, project:, sha:, user:, expandset:) @values = values - @external_files = External::Mapper.new(values, project: project, sha: sha, user: user).process + @external_files = External::Mapper.new(values, project: project, sha: sha, user: user, expandset: expandset).process @content = {} - rescue External::Mapper::AmbigiousSpecificationError => e + rescue External::Mapper::Error => e raise IncludeError, e.message end diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index c0d4d4400b3..6c99e20e7af 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -81,6 +81,7 @@ build: - /build/build.sh only: - branches + - tags test: services: @@ -95,6 +96,7 @@ test: - /bin/herokuish buildpack test only: - branches + - tags except: variables: - $TEST_DISABLED @@ -112,6 +114,7 @@ code_quality: paths: [gl-code-quality-report.json] only: - branches + - tags except: variables: - $CODE_QUALITY_DISABLED @@ -129,6 +132,7 @@ license_management: only: refs: - branches + - tags variables: - $GITLAB_FEATURES =~ /\blicense_management\b/ except: @@ -151,6 +155,7 @@ performance: only: refs: - branches + - tags kubernetes: active except: variables: @@ -171,6 +176,7 @@ sast: only: refs: - branches + - tags variables: - $GITLAB_FEATURES =~ /\bsast\b/ except: @@ -192,6 +198,7 @@ dependency_scanning: only: refs: - branches + - tags variables: - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ except: @@ -212,6 +219,7 @@ container_scanning: only: refs: - branches + - tags variables: - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ except: @@ -231,6 +239,7 @@ dast: only: refs: - branches + - tags kubernetes: active variables: - $GITLAB_FEATURES =~ /\bdast\b/ @@ -260,6 +269,7 @@ review: only: refs: - branches + - tags kubernetes: active except: refs: @@ -283,6 +293,7 @@ stop_review: only: refs: - branches + - tags kubernetes: active except: refs: @@ -488,8 +499,13 @@ rollout 100%: [[ "$TRACE" ]] && set -x auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} export DATABASE_URL=${DATABASE_URL-$auto_database_url} - export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG - export CI_APPLICATION_TAG=$CI_COMMIT_SHA + if [[ -z "$CI_COMMIT_TAG" ]]; then + export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG + export CI_APPLICATION_TAG=$CI_COMMIT_SHA + else + export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE + export CI_APPLICATION_TAG=$CI_COMMIT_TAG + fi export TILLER_NAMESPACE=$KUBE_NAMESPACE # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml new file mode 100644 index 00000000000..805df26b957 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -0,0 +1,44 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/project/merge_requests/dependency_scanning.html +# +# Configure the scanning tool through the environment variables. +# List of the variables: https://gitlab.com/gitlab-org/security-products/dependency-scanning#settings +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +stages: + - test + +dependency_scanning: + stage: test + image: docker:stable + variables: + DOCKER_DRIVER: overlay2 + allow_failure: true + services: + - docker:stable-dind + script: + - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} + - | + docker run \ + --env DS_ANALYZER_IMAGES \ + --env DS_ANALYZER_IMAGE_PREFIX \ + --env DS_ANALYZER_IMAGE_TAG \ + --env DS_DEFAULT_ANALYZERS \ + --env DEP_SCAN_DISABLE_REMOTE_CHECKS \ + --env DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ + --env DS_PULL_ANALYZER_IMAGE_TIMEOUT \ + --env DS_RUN_ANALYZER_TIMEOUT \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_VERSION" /code + artifacts: + reports: + dependency_scanning: gl-dependency-scanning-report.json + dependencies: [] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + except: + variables: + - $DEPENDENCY_SCANNING_DISABLED diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml new file mode 100644 index 00000000000..4f3d08d98fe --- /dev/null +++ b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml @@ -0,0 +1,41 @@ +# GitLab Serverless template + +image: alpine:latest + +stages: + - build + - deploy + +.serverless:build:image: + variables: + DOCKERFILE: "Dockerfile" + stage: build + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + only: + refs: + - master + script: + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/$DOCKERFILE --destination $CI_REGISTRY_IMAGE + +.serverless:deploy:image: + stage: deploy + image: gcr.io/triggermesh/tm@sha256:e3ee74db94d215bd297738d93577481f3e4db38013326c90d57f873df7ab41d5 + only: + refs: + - master + environment: development + script: + - echo "$CI_REGISTRY_IMAGE" + - tm -n "$KUBE_NAMESPACE" --config "$KUBECONFIG" deploy service "$CI_PROJECT_NAME" --from-image "$CI_REGISTRY_IMAGE" --wait + +.serverless:deploy:functions: + stage: deploy + environment: development + image: gcr.io/triggermesh/tm:v0.0.9 + script: + - tm -n "$KUBE_NAMESPACE" set registry-auth gitlab-registry --registry "$CI_REGISTRY" --username "$CI_REGISTRY_USER" --password "$CI_JOB_TOKEN" --push + - tm -n "$KUBE_NAMESPACE" set registry-auth gitlab-registry --registry "$CI_REGISTRY" --username "$CI_DEPLOY_USER" --password "$CI_DEPLOY_PASSWORD" --pull + - tm -n "$KUBE_NAMESPACE" deploy --wait diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml new file mode 100644 index 00000000000..245e6bec60a --- /dev/null +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -0,0 +1,28 @@ +# This is a very simple template that mainly relies on FastLane to build and distribute your app. +# Read more about how to use this template on the blog post https://about.gitlab.com/2019/03/06/ios-publishing-with-gitlab-and-fastlane/ +# You will also need fastlane and signing configuration for this to work, along with a MacOS runner. +# These details are provided in the blog post. + +# Note that when you're using the shell executor for MacOS builds, the +# build and tests run as the identity of the runner logged in user, directly on +# the build host. This is less secure than using container executors, so please +# take a look at our security implications documentation at +# https://docs.gitlab.com/runner/security/#usage-of-shell-executor for additional +# detail on what to keep in mind in this scenario. + +stages: + - build + +variables: + LC_ALL: "en_US.UTF-8" + LANG: "en_US.UTF-8" + GIT_STRATEGY: clone + +build: + stage: build + script: + - bundle install + - bundle exec fastlane build + artifacts: + paths: + - ./*.ipa diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index a7b4e0348c2..f7bbb58df7e 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -17,6 +17,8 @@ module Gitlab end def concat(resources) + return self if resources.nil? + tap { resources.each { |variable| self.append(variable) } } end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index e3e4e62cc02..833aa75adb5 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -5,12 +5,12 @@ module Gitlab module Variables class Collection class Item - def initialize(key:, value:, public: true, file: false) + def initialize(key:, value:, public: true, file: false, masked: false) raise ArgumentError, "`#{key}` must be of type String or nil value, while it was: #{value.class}" unless value.is_a?(String) || value.nil? @variable = { - key: key, value: value, public: public, file: file + key: key, value: value, public: public, file: file, masked: masked } end @@ -27,9 +27,13 @@ module Gitlab # don't expose `file` attribute at all (stems from what the runner # expects). # + # If the `variable_masking` feature is enabled we expose the `masked` + # attribute, otherwise it's not exposed. + # def to_runner_variable @variable.reject do |hash_key, hash_value| - hash_key == :file && hash_value == false + (hash_key == :file && hash_value == false) || + (hash_key == :masked && !Feature.enabled?(:variable_masking)) end end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 5ed6427072a..f7d046600e8 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -49,6 +49,7 @@ module Gitlab Event.contributions.where(author_id: contributor.id) .where(created_at: date.beginning_of_day..date.end_of_day) .where(project_id: projects) + .with_associations end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index db8ac3becea..aeca9d00156 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -40,11 +40,11 @@ module Gitlab end def first_time_reference_commit(event) - return nil unless event && merge_request_diff_commits + return unless event && merge_request_diff_commits commits = merge_request_diff_commits[event['id'].to_i] - return nil if commits.blank? + return if commits.blank? commits.find do |commit| next unless commit[:committed_date] && event['first_mentioned_in_commit_at'] diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index d3c86fdb629..d2b7ca015d4 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -123,6 +123,7 @@ module Gitlab # Files that don't fit into any category are marked with :none %r{\A(ee/)?changelogs/} => :none, + %r{\Alocale/gitlab\.pot\z} => :none, # Fallbacks in case the above patterns miss anything %r{\.rb\z} => :backend, diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb index cf1cf054dbf..fedf6ca4fe1 100644 --- a/lib/gitlab/database/count/tablesample_count_strategy.rb +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -36,7 +36,7 @@ module Gitlab def perform_count(model, estimate) # If we estimate 0, we may not have statistics at all. Don't use them. - return nil unless estimate && estimate > 0 + return unless estimate && estimate > 0 if estimate < EXACT_COUNT_THRESHOLD # The table is considered small, the assumption here is that diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index ac2efe598b4..ffad00fa7d7 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -4,6 +4,7 @@ module Gitlab module DependencyLinker class BaseLinker URL_REGEX = %r{https?://[^'" ]+}.freeze + GIT_INVALID_URL_REGEX = /^git\+#{URL_REGEX}/.freeze REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze class_attribute :file_type @@ -29,8 +30,25 @@ module Gitlab highlighted_lines.join.html_safe end + def external_url(name, external_ref) + return if external_ref =~ GIT_INVALID_URL_REGEX + + case external_ref + when /\A#{URL_REGEX}\z/ + external_ref + when /\A#{REPO_REGEX}\z/ + github_url(external_ref) + else + package_url(name) + end + end + private + def package_url(_name) + raise NotImplementedError + end + def link_dependencies raise NotImplementedError end diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb index 22d2bead891..4b8862b31ee 100644 --- a/lib/gitlab/dependency_linker/composer_json_linker.rb +++ b/lib/gitlab/dependency_linker/composer_json_linker.rb @@ -8,8 +8,8 @@ module Gitlab private def link_packages - link_packages_at_key("require", &method(:package_url)) - link_packages_at_key("require-dev", &method(:package_url)) + link_packages_at_key("require") + link_packages_at_key("require-dev") end def package_url(name) diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb index 8ab219c4962..c6e02248b0a 100644 --- a/lib/gitlab/dependency_linker/gemfile_linker.rb +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -3,8 +3,14 @@ module Gitlab module DependencyLinker class GemfileLinker < MethodLinker + class_attribute :package_keyword + + self.package_keyword = :gem self.file_type = :gemfile + GITHUB_REGEX = /(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/.freeze + GIT_REGEX = /(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/.freeze + private def link_dependencies @@ -14,21 +20,35 @@ module Gitlab def link_urls # Link `github: "user/repo"` to https://github.com/user/repo - link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/, &method(:github_url)) + link_regex(GITHUB_REGEX, &method(:github_url)) # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo - link_regex(/(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/, &:itself) + link_regex(GIT_REGEX, &:itself) # Link `source "https://rubygems.org"` to https://rubygems.org link_method_call('source', URL_REGEX, &:itself) end def link_packages - # Link `gem "package_name"` to https://rubygems.org/gems/package_name - link_method_call('gem') do |name| - "https://rubygems.org/gems/#{name}" + packages = parse_packages + + return if packages.blank? + + packages.each do |package| + link_method_call('gem', package.name) do + external_url(package.name, package.external_ref) + end end end + + def package_url(name) + "https://rubygems.org/gems/#{name}" + end + + def parse_packages + parser = Gitlab::DependencyLinker::Parser::Gemfile.new(plain_text) + parser.parse(keyword: self.class.package_keyword) + end end end end diff --git a/lib/gitlab/dependency_linker/gemspec_linker.rb b/lib/gitlab/dependency_linker/gemspec_linker.rb index b924ea86d89..94c2b375cf9 100644 --- a/lib/gitlab/dependency_linker/gemspec_linker.rb +++ b/lib/gitlab/dependency_linker/gemspec_linker.rb @@ -11,7 +11,7 @@ module Gitlab link_method_call('homepage', URL_REGEX, &:itself) link_method_call('license', &method(:license_url)) - link_method_call(%w[name add_dependency add_runtime_dependency add_development_dependency]) do |name| + link_method_call(%w[add_dependency add_runtime_dependency add_development_dependency]) do |name| "https://rubygems.org/gems/#{name}" end end diff --git a/lib/gitlab/dependency_linker/method_linker.rb b/lib/gitlab/dependency_linker/method_linker.rb index d4d85bb3390..33899a931c6 100644 --- a/lib/gitlab/dependency_linker/method_linker.rb +++ b/lib/gitlab/dependency_linker/method_linker.rb @@ -23,18 +23,22 @@ module Gitlab # link_method_call('name') # # Will link `package` in `self.name = "package"` def link_method_call(method_name, value = nil, &url_proc) + regex = method_call_regex(method_name, value) + + link_regex(regex, &url_proc) + end + + def method_call_regex(method_name, value = nil) method_name = regexp_for_value(method_name) value = regexp_for_value(value) - regex = %r{ + %r{ #{method_name} # Method name \s* # Whitespace [(=]? # Opening brace or equals sign \s* # Whitespace ['"](?<name>#{value})['"] # Package name in quotes }x - - link_regex(regex, &url_proc) end end end diff --git a/lib/gitlab/dependency_linker/package.rb b/lib/gitlab/dependency_linker/package.rb new file mode 100644 index 00000000000..8a509bbd562 --- /dev/null +++ b/lib/gitlab/dependency_linker/package.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module DependencyLinker + class Package + attr_reader :name, :git_ref, :github_ref + + def initialize(name, git_ref, github_ref) + @name = name + @git_ref = git_ref + @github_ref = github_ref + end + + def external_ref + @git_ref || @github_ref + end + end + end +end diff --git a/lib/gitlab/dependency_linker/package_json_linker.rb b/lib/gitlab/dependency_linker/package_json_linker.rb index 578e25f806a..6857f2a4fa2 100644 --- a/lib/gitlab/dependency_linker/package_json_linker.rb +++ b/lib/gitlab/dependency_linker/package_json_linker.rb @@ -8,7 +8,6 @@ module Gitlab private def link_dependencies - link_json('name', json["name"], &method(:package_url)) link_json('license', &method(:license_url)) link_json(%w[homepage url], URL_REGEX, &:itself) @@ -16,25 +15,19 @@ module Gitlab end def link_packages - link_packages_at_key("dependencies", &method(:package_url)) - link_packages_at_key("devDependencies", &method(:package_url)) + link_packages_at_key("dependencies") + link_packages_at_key("devDependencies") end - def link_packages_at_key(key, &url_proc) + def link_packages_at_key(key) dependencies = json[key] return unless dependencies dependencies.each do |name, version| - link_json(name, version, link: :key, &url_proc) - - link_json(name) do |value| - case value - when /\A#{URL_REGEX}\z/ - value - when /\A#{REPO_REGEX}\z/ - github_url(value) - end - end + external_url = external_url(name, version) + + link_json(name, version, link: :key) { external_url } + link_json(name) { external_url } end end diff --git a/lib/gitlab/dependency_linker/parser/gemfile.rb b/lib/gitlab/dependency_linker/parser/gemfile.rb new file mode 100644 index 00000000000..7f755375cea --- /dev/null +++ b/lib/gitlab/dependency_linker/parser/gemfile.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module DependencyLinker + module Parser + class Gemfile < MethodLinker + GIT_REGEX = Gitlab::DependencyLinker::GemfileLinker::GIT_REGEX + GITHUB_REGEX = Gitlab::DependencyLinker::GemfileLinker::GITHUB_REGEX + + def initialize(plain_text) + @plain_text = plain_text + end + + # Returns a list of Gitlab::DependencyLinker::Package + # + # keyword - The package definition keyword, e.g. `:gem` for + # Gemfile parsing, `:pod` for Podfile. + def parse(keyword:) + plain_lines.each_with_object([]) do |line, packages| + name = fetch(line, method_call_regex(keyword)) + + next unless name + + git_ref = fetch(line, GIT_REGEX) + github_ref = fetch(line, GITHUB_REGEX) + + packages << Gitlab::DependencyLinker::Package.new(name, git_ref, github_ref) + end + end + + private + + def fetch(line, regex, group: :name) + match = line.match(regex) + match[group] if match + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/podfile_linker.rb b/lib/gitlab/dependency_linker/podfile_linker.rb index def9b04cca9..a20d285da79 100644 --- a/lib/gitlab/dependency_linker/podfile_linker.rb +++ b/lib/gitlab/dependency_linker/podfile_linker.rb @@ -5,12 +5,21 @@ module Gitlab class PodfileLinker < GemfileLinker include Cocoapods + self.package_keyword = :pod self.file_type = :podfile private def link_packages - link_method_call('pod', &method(:package_url)) + packages = parse_packages + + return unless packages + + packages.each do |package| + link_method_call('pod', package.name) do + external_url(package.name, package.external_ref) + end + end end end end diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb index 6b1758c5a43..14abd3999c4 100644 --- a/lib/gitlab/dependency_linker/podspec_linker.rb +++ b/lib/gitlab/dependency_linker/podspec_linker.rb @@ -19,7 +19,7 @@ module Gitlab link_method_call('license', &method(:license_url)) link_regex(/license\s*=\s*\{\s*(type:|:type\s*=>)\s*#{STRING_REGEX}/, &method(:license_url)) - link_method_call(%w[name dependency], &method(:package_url)) + link_method_call('dependency', &method(:package_url)) end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index c9d89d56884..dbee47a19ee 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -75,7 +75,7 @@ module Gitlab end def line_for_position(pos) - return nil unless pos.position_type == 'text' + return unless pos.position_type == 'text' # This method is normally used to find which line the diff was # commented on, and in this context, it's normally the raw diff persisted @@ -329,6 +329,16 @@ module Gitlab lines end + def fully_expanded? + return true if binary? + + lines = diff_lines_for_serializer + + return true if lines.nil? + + lines.none? { |line| line.type.to_s == 'match' } + end + private def total_blob_lines(blob) diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 2743f011ca6..dc44e9d7481 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -61,7 +61,7 @@ module Gitlab # Force encoding to UTF-8 on a Mail::Message or Mail::Part def fix_charset(object) - return nil if object.nil? + return if object.nil? if object.charset object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 431911d1eee..2c53f9b026d 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -239,7 +239,7 @@ module Gitlab res = ::Projects::DownloadService.new(project, link).execute - return nil if res.nil? + return if res.nil? res[:markdown] end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 259a2b7911a..10df4ed72d9 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -23,6 +23,10 @@ module Gitlab class << self def find(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) + tree_entry(repository, sha, path, limit) + end + + def tree_entry(repository, sha, path, limit) return unless path path = path.sub(%r{\A/*}, '') @@ -179,3 +183,5 @@ module Gitlab end end end + +Gitlab::Git::Blob.singleton_class.prepend Gitlab::Git::RuggedImpl::Blob::ClassMethods diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 5863815ca85..491e4b47196 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -5,6 +5,7 @@ module Gitlab module Git class Commit include Gitlab::EncodingHelper + prepend Gitlab::Git::RuggedImpl::Commit extend Gitlab::Git::WrapsGitalyErrors attr_accessor :raw_commit, :head @@ -57,20 +58,24 @@ module Gitlab return commit_id if commit_id.is_a?(Gitlab::Git::Commit) # Some weird thing? - return nil unless commit_id.is_a?(String) + return unless commit_id.is_a?(String) # This saves us an RPC round trip. - return nil if commit_id.include?(':') + return if commit_id.include?(':') - commit = wrapped_gitaly_errors do - repo.gitaly_commit_client.find_commit(commit_id) - end + commit = find_commit(repo, commit_id) decorate(repo, commit) if commit rescue Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository, ArgumentError nil end + def find_commit(repo, commit_id) + wrapped_gitaly_errors do + repo.gitaly_commit_client.find_commit(commit_id) + end + end + # Get last commit for HEAD # # Ex. @@ -185,6 +190,10 @@ module Gitlab @repository = repository @head = head + init_commit(raw_commit) + end + + def init_commit(raw_commit) case raw_commit when Hash init_from_hash(raw_commit) @@ -400,3 +409,5 @@ module Gitlab end end end + +Gitlab::Git::Commit.singleton_class.prepend Gitlab::Git::RuggedImpl::Commit::ClassMethods diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index eec91194949..47cfb483509 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -4,6 +4,7 @@ module Gitlab module Git class Ref include Gitlab::EncodingHelper + include Gitlab::Git::RuggedImpl::Ref # Branch or tag name # without "refs/tags|heads" prefix diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index aea132a3dd9..35dd042ba6a 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -11,6 +11,7 @@ module Gitlab include Gitlab::Git::WrapsGitalyErrors include Gitlab::EncodingHelper include Gitlab::Utils::StrongMemoize + prepend Gitlab::Git::RuggedImpl::Repository SEARCH_CONTEXT_LINES = 3 REV_LIST_COMMIT_LIMIT = 2_000 @@ -275,7 +276,7 @@ module Gitlab # senddata response. def archive_file_path(storage_path, sha, name, format = "tar.gz") # Build file path - return nil unless name + return unless name extension = case format @@ -852,17 +853,20 @@ module Gitlab true end + # rubocop:disable Metrics/ParameterLists def multi_action( user, branch_name:, message:, actions:, author_email: nil, author_name: nil, - start_branch_name: nil, start_repository: self) + start_branch_name: nil, start_repository: self, + force: false) wrapped_gitaly_errors do gitaly_operation_client.user_commit_files(user, branch_name, message, actions, author_email, author_name, - start_branch_name, start_repository) + start_branch_name, start_repository, force) end end + # rubocop:enable Metrics/ParameterLists def write_config(full_path:) return unless full_path.present? diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb new file mode 100644 index 00000000000..11ee4ebda4b --- /dev/null +++ b/lib/gitlab/git/rugged_impl/blob.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +module Gitlab + module Git + module RuggedImpl + module Blob + module ClassMethods + extend ::Gitlab::Utils::Override + + override :tree_entry + def tree_entry(repository, sha, path, limit) + if Feature.enabled?(:rugged_tree_entry) + rugged_tree_entry(repository, sha, path, limit) + else + super + end + end + + private + + def rugged_tree_entry(repository, sha, path, limit) + return unless path + + # Strip any leading / characters from the path + path = path.sub(%r{\A/*}, '') + + rugged_commit = repository.lookup(sha) + root_tree = rugged_commit.tree + + blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/')) + + return unless blob_entry + + if blob_entry[:type] == :commit + submodule_blob(blob_entry, path, sha) + else + blob = repository.lookup(blob_entry[:oid]) + + if blob + new( + id: blob.oid, + name: blob_entry[:name], + size: blob.size, + # Rugged::Blob#content is expensive; don't call it if we don't have to. + data: limit.zero? ? '' : blob.content(limit), + mode: blob_entry[:filemode].to_s(8), + path: path, + commit_id: sha, + binary: blob.binary? + ) + end + end + rescue Rugged::ReferenceError + nil + end + + # Recursive search of blob id by path + # + # Ex. + # blog/ # oid: 1a + # app/ # oid: 2a + # models/ # oid: 3a + # file.rb # oid: 4a + # + # + # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a' + # + def find_entry_by_path(repository, root_id, *path_parts) + root_tree = repository.lookup(root_id) + + entry = root_tree.find do |entry| + entry[:name] == path_parts[0] + end + + return unless entry + + if path_parts.size > 1 + return unless entry[:type] == :tree + + path_parts.shift + find_entry_by_path(repository, entry[:oid], *path_parts) + else + [:blob, :commit].include?(entry[:type]) ? entry : nil + end + end + + def submodule_blob(blob_entry, path, sha) + new( + id: blob_entry[:oid], + name: blob_entry[:name], + size: 0, + data: '', + path: path, + commit_id: sha + ) + end + end + end + end + end +end diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb new file mode 100644 index 00000000000..251802878c3 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Gitlab + module Git + module RuggedImpl + module Commit + module ClassMethods + extend ::Gitlab::Utils::Override + + def rugged_find(repo, commit_id) + obj = repo.rev_parse_target(commit_id) + + obj.is_a?(::Rugged::Commit) ? obj : nil + rescue ::Rugged::Error + nil + end + + override :find_commit + def find_commit(repo, commit_id) + if Feature.enabled?(:rugged_find_commit) + rugged_find(repo, commit_id) + else + super + end + end + end + + extend ::Gitlab::Utils::Override + + override :init_commit + def init_commit(raw_commit) + case raw_commit + when ::Rugged::Commit + init_from_rugged(raw_commit) + else + super + end + end + + def init_from_rugged(commit) + author = commit.author + committer = commit.committer + + @raw_commit = commit + @id = commit.oid + @message = commit.message + @authored_date = author[:time] + @committed_date = committer[:time] + @author_name = author[:name] + @author_email = author[:email] + @committer_name = committer[:name] + @committer_email = committer[:email] + @parent_ids = commit.parents.map(&:oid) + end + end + end + end +end +# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/git/rugged_impl/ref.rb b/lib/gitlab/git/rugged_impl/ref.rb new file mode 100644 index 00000000000..b553e82dc47 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/ref.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +module Gitlab + module Git + module RuggedImpl + module Ref + def self.dereference_object(object) + object = object.target while object.is_a?(::Rugged::Tag::Annotation) + + object + end + end + end + end +end diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb new file mode 100644 index 00000000000..fe0120b1199 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Gitlab + module Git + module RuggedImpl + module Repository + extend ::Gitlab::Utils::Override + + FEATURE_FLAGS = %i(rugged_find_commit rugged_tree_entries rugged_tree_entry rugged_commit_is_ancestor).freeze + + def alternate_object_directories + relative_object_directories.map { |d| File.join(path, d) } + end + + ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[ + GIT_OBJECT_DIRECTORY_RELATIVE + GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE + ].freeze + + def relative_object_directories + Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact + end + + def rugged + @rugged ||= ::Rugged::Repository.new(path, alternates: alternate_object_directories) + rescue ::Rugged::RepositoryError, ::Rugged::OSError + raise ::Gitlab::Git::Repository::NoRepository.new('no repository for such path') + end + + def cleanup + @rugged&.close + end + + # Return the object that +revspec+ points to. If +revspec+ is an + # annotated tag, then return the tag's target instead. + def rev_parse_target(revspec) + obj = rugged.rev_parse(revspec) + Ref.dereference_object(obj) + end + + override :ancestor? + def ancestor?(from, to) + if Feature.enabled?(:rugged_commit_is_ancestor) + rugged_is_ancestor?(from, to) + else + super + end + end + + def rugged_is_ancestor?(ancestor_id, descendant_id) + return false if ancestor_id.nil? || descendant_id.nil? + + rugged_merge_base(ancestor_id, descendant_id) == ancestor_id + rescue Rugged::OdbError + false + end + + def rugged_merge_base(from, to) + rugged.merge_base(from, to) + rescue Rugged::ReferenceError + nil + end + + # Lookup for rugged object by oid or ref name + def lookup(oid_or_ref_name) + rugged.rev_parse(oid_or_ref_name) + end + end + end + end +end +# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb new file mode 100644 index 00000000000..0ebfd496695 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +module Gitlab + module Git + module RuggedImpl + module Tree + module ClassMethods + extend ::Gitlab::Utils::Override + + override :tree_entries + def tree_entries(repository, sha, path, recursive) + if Feature.enabled?(:rugged_tree_entries) + tree_entries_from_rugged(repository, sha, path, recursive) + else + super + end + end + + def tree_entries_from_rugged(repository, sha, path, recursive) + current_path_entries = get_tree_entries_from_rugged(repository, sha, path) + ordered_entries = [] + + current_path_entries.each do |entry| + ordered_entries << entry + + if recursive && entry.dir? + ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true)) + end + end + + # This was an optimization to reduce N+1 queries for Gitaly + # (https://gitlab.com/gitlab-org/gitaly/issues/530). It + # used to be done lazily in the view via + # TreeHelper#flatten_tree, so it's possible there's a + # performance impact by loading this eagerly. + rugged_populate_flat_path(repository, sha, path, ordered_entries) + end + + def rugged_populate_flat_path(repository, sha, path, entries) + entries.each do |entry| + entry.flat_path = entry.path + + next unless entry.dir? + + entry.flat_path = + if path + File.join(path, rugged_flatten_tree(repository, sha, entry, path)) + else + rugged_flatten_tree(repository, sha, entry, path) + end + end + end + + # Returns the relative path of the first subdir that doesn't have only one directory descendant + def rugged_flatten_tree(repository, sha, tree, root_path) + subtree = tree_entries_from_rugged(repository, sha, tree.path, false) + + if subtree.count == 1 && subtree.first.dir? + File.join(tree.name, rugged_flatten_tree(repository, sha, subtree.first, root_path)) + else + tree.name + end + end + + def get_tree_entries_from_rugged(repository, sha, path) + commit = repository.lookup(sha) + root_tree = commit.tree + + tree = if path + id = find_id_by_path(repository, root_tree.oid, path) + if id + repository.lookup(id) + else + [] + end + else + root_tree + end + + tree.map do |entry| + current_path = path ? File.join(path, entry[:name]) : entry[:name] + + new( + id: entry[:oid], + root_id: root_tree.oid, + name: entry[:name], + type: entry[:type], + mode: entry[:filemode].to_s(8), + path: current_path, + commit_id: sha + ) + end + rescue Rugged::ReferenceError + [] + end + end + end + end + end +end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index 51542bcaaa2..7e072c5db50 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -18,6 +18,10 @@ module Gitlab def where(repository, sha, path = nil, recursive = false) path = nil if path == '' || path == '/' + tree_entries(repository, sha, path, recursive) + end + + def tree_entries(repository, sha, path, recursive) wrapped_gitaly_errors do repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive) end @@ -44,7 +48,7 @@ module Gitlab entry[:name] == path_arr[0] && entry[:type] == :tree end - return nil unless entry + return unless entry if path_arr.size > 1 path_arr.shift @@ -95,3 +99,5 @@ module Gitlab end end end + +Gitlab::Git::Tree.singleton_class.prepend Gitlab::Git::RuggedImpl::Tree::ClassMethods diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 5aeedb0f50d..48c113a8b14 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -164,8 +164,6 @@ module Gitlab kwargs = yield(kwargs) if block_given? stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend - rescue GRPC::Unavailable => ex - handle_grpc_unavailable!(ex) ensure duration = Gitlab::Metrics::System.monotonic_time - start @@ -178,27 +176,6 @@ module Gitlab add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc) end - def self.handle_grpc_unavailable!(ex) - status = ex.to_status - raise ex unless status.details == 'Endpoint read failed' - - # There is a bug in grpc 1.8.x that causes a client process to get stuck - # always raising '14:Endpoint read failed'. The only thing that we can - # do to recover is to restart the process. - # - # See https://gitlab.com/gitlab-org/gitaly/issues/1029 - - if Sidekiq.server? - raise Gitlab::SidekiqMiddleware::Shutdown::WantShutdown.new(ex.to_s) - else - # SIGQUIT requests a Unicorn worker to shut down gracefully after the current request. - Process.kill('QUIT', Process.pid) - end - - raise ex - end - private_class_method :handle_grpc_unavailable! - def self.current_transaction_labels Gitlab::Metrics::Transaction.current&.labels || {} end @@ -251,7 +228,7 @@ module Gitlab result end - SERVER_FEATURE_FLAGS = %w[].freeze + SERVER_FEATURE_FLAGS = %w[go-find-all-tags].freeze def self.server_feature_flags SERVER_FEATURE_FLAGS.map do |f| @@ -267,7 +244,9 @@ module Gitlab end def self.feature_enabled?(feature_name) - Feature.enabled?("gitaly_#{feature_name}") + Feature::FlipperFeature.table_exists? && Feature.enabled?("gitaly_#{feature_name}") + rescue ActiveRecord::NoDatabaseError + false end # Ensures that Gitaly is not being abuse through n+1 misuse etc @@ -407,13 +386,13 @@ module Gitlab # Returns the stacks that calls Gitaly the most times. Used for n+1 detection def self.max_stacks - return nil unless Gitlab::SafeRequestStore.active? + return unless Gitlab::SafeRequestStore.active? stack_counter = Gitlab::SafeRequestStore[:stack_counter] - return nil unless stack_counter + return unless stack_counter max = max_call_count - return nil if max.zero? + return if max.zero? stack_counter.select { |_, v| v == max }.keys end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 39547328210..6b8e58e6199 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -27,7 +27,7 @@ module Gitlab data << msg.data end - return nil if blob.oid.blank? + return if blob.oid.blank? data = data.join diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index d172c798da2..2528208440e 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -62,7 +62,7 @@ module Gitlab end branch = response.branch - return nil unless branch + return unless branch target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) Gitlab::Git::Branch.new(@repository, branch.name, target_commit.id, target_commit) @@ -277,14 +277,14 @@ module Gitlab end end + # rubocop:disable Metrics/ParameterLists def user_commit_files( user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository) - + start_branch_name, start_repository, force = false) req_enum = Enumerator.new do |y| header = user_commit_files_request_header(user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository) + start_branch_name, start_repository, force) y.yield Gitaly::UserCommitFilesRequest.new(header: header) @@ -319,6 +319,7 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end + # rubocop:enable Metrics/ParameterLists def user_commit_patches(user, branch_name, patches) header = Gitaly::UserApplyPatchRequest::Header.new( @@ -382,9 +383,10 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end + # rubocop:disable Metrics/ParameterLists def user_commit_files_request_header( user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository) + start_branch_name, start_repository, force) Gitaly::UserCommitFilesRequestHeader.new( repository: @gitaly_repo, @@ -394,9 +396,11 @@ module Gitlab commit_author_name: encode_binary(author_name), commit_author_email: encode_binary(author_email), start_branch_name: encode_binary(start_branch_name), - start_repository: start_repository.gitaly_repository + start_repository: start_repository.gitaly_repository, + force: force ) end + # rubocop:enable Metrics/ParameterLists def user_commit_files_action_header(action) Gitaly::UserCommitFilesActionHeader.new( diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index a7e20d9429e..a08bfd0e25b 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -324,8 +324,8 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end - def search_files_by_content(ref, query, chunked_response: true) - request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query, chunked_response: chunked_response) + def search_files_by_content(ref, query) + request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request) search_results_from_response(response) @@ -340,18 +340,11 @@ module Gitlab gitaly_response.each do |message| next if message.nil? - # Old client will ignore :chunked_response flag - # and return messages with `matches` key. - # This code path will be removed post 12.0 release - if message.matches.any? - matches += message.matches - else - current_match << message.match_data - - if message.end_of_match - matches << current_match - current_match = +"" - end + current_match << message.match_data + + if message.end_of_match + matches << current_match + current_match = +"" end end diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 754cccb6b3f..78ef6bfc0ec 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -32,11 +32,19 @@ module Gitlab end def self.disk_access_denied? + return false if rugged_enabled? + !temporarily_allowed?(ALLOW_KEY) && GitalyClient.feature_enabled?(DISK_ACCESS_DENIED_FLAG) rescue false # Err on the side of caution, don't break gitlab for people end + def self.rugged_enabled? + Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.any? do |flag| + Feature.enabled?(flag) + end + end + def initialize(storage) raise InvalidConfigurationError, "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash) raise InvalidConfigurationError, INVALID_STORAGE_MESSAGE unless storage.has_key?('path') diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb index 87cf2c8b598..71ff7465d9b 100644 --- a/lib/gitlab/github_import/importer/milestones_importer.rb +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -42,6 +42,7 @@ module Gitlab description: milestone.description, project_id: project.id, state: state_for(milestone), + due_date: milestone.due_on&.to_date, created_at: milestone.created_at, updated_at: milestone.updated_at } diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 3235d3ccc4e..e00309e7946 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -25,6 +25,7 @@ module Gitlab gon.test_env = Rails.env.test? gon.suggested_label_colors = LabelsHelper.suggested_colors gon.first_day_of_week = current_user&.first_day_of_week || Gitlab::CurrentSettings.first_day_of_week + gon.ee = Gitlab.ee? if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index bf463077dcc..7046b4e2a43 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -13,10 +13,18 @@ module Gitlab # # @param [Integer] start first project id for the range # @param [Integer] finish last project id for the range - def bulk_schedule(start:, finish:) + def bulk_schedule_migration(start:, finish:) ::HashedStorage::MigratorWorker.perform_async(start, finish) end + # Schedule a range of projects to be bulk rolledback with #bulk_rollback asynchronously + # + # @param [Integer] start first project id for the range + # @param [Integer] finish last project id for the range + def bulk_schedule_rollback(start:, finish:) + ::HashedStorage::RollbackerWorker.perform_async(start, finish) + end + # Start migration of projects from specified range # # Flagging a project to be migrated is a synchronous action @@ -34,6 +42,23 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # Start rollback of projects from specified range + # + # Flagging a project to be rolled back is a synchronous action + # but the rollback runs through async jobs + # + # @param [Integer] start first project id for the range + # @param [Integer] finish last project id for the range + # rubocop: disable CodeReuse/ActiveRecord + def bulk_rollback(start:, finish:) + projects = build_relation(start, finish) + + projects.with_route.find_each(batch_size: BATCH_SIZE) do |project| + rollback(project) + end + end + # rubocop: enable CodeReuse/ActiveRecord + # Flag a project to be migrated to Hashed Storage # # @param [Project] project that will be migrated @@ -45,8 +70,15 @@ module Gitlab Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end + # Flag a project to be rolled-back to Legacy Storage + # + # @param [Project] project that will be rolled-back def rollback(project) - # TODO: implement rollback strategy + Rails.logger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..." + + project.rollback_to_legacy_storage! + rescue => err + Rails.logger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end private diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 38f552fab03..87a31a37e3f 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -24,7 +24,7 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def self.project_id_batches(&block) + def self.project_id_batches_migration(&block) Project.with_unmigrated_storage.in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches ids = relation.pluck(:id) @@ -34,6 +34,16 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord + def self.project_id_batches_rollback(&block) + Project.with_storage_feature(:repository).in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches + ids = relation.pluck(:id) + + yield ids.min, ids.max + end + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord def self.legacy_attachments_relation Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments]) JOIN projects diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index a4e60bbd828..381f1dd4e55 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -39,7 +39,7 @@ module Gitlab private def custom_language - return nil unless @language + return unless @language Rouge::Lexer.find_fancy(@language) end diff --git a/lib/gitlab/i18n/metadata_entry.rb b/lib/gitlab/i18n/metadata_entry.rb index 3764e379681..4facd10bfc8 100644 --- a/lib/gitlab/i18n/metadata_entry.rb +++ b/lib/gitlab/i18n/metadata_entry.rb @@ -15,7 +15,7 @@ module Gitlab end def expected_forms - return nil unless plural_information + return unless plural_information plural_information['nplurals'].to_i end diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index 477499e1688..b145f37c052 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -67,7 +67,7 @@ module Gitlab # +value+ existing model to be included in the hash # +parsed_hash+ the original hash def parse_hash(value) - return nil if already_contains_methods?(value) + return if already_contains_methods?(value) @attributes_finder.parse(value) do |hash| { include: hash_or_merge(value, hash) } diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index 040a70d6775..deb2f59f05f 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -20,6 +20,17 @@ module Gitlab create_target_branch unless branch_exists?(@merge_request.target_branch) end + # The merge_request_diff associated with the current @merge_request might + # be invalid. Than means, when the @merge_request object is saved, the + # @merge_request.merge_request_diff won't. This can leave the merge request + # in an invalid state, because a merge request must have an associated + # merge request diff. + # In this change, if the associated merge request diff is invalid, we set + # it to nil. This change, in association with the after callback + # :ensure_merge_request_diff in the MergeRequest class, makes that + # when the merge request is going to be created and it doesn't have + # one, a default one will be generated. + @merge_request.merge_request_diff = nil unless @merge_request.merge_request_diff&.valid? @merge_request end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 099b488f68e..61a1aa6da5a 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -75,7 +75,7 @@ module Gitlab # the relation_hash, updating references with new object IDs, mapping users using # the "members_mapper" object, also updating notes if required. def create - return nil if unknown_service? + return if unknown_service? setup_models diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index cc0c633b943..8b346f6d7d2 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -57,7 +57,7 @@ module Gitlab def address_regex wildcard_address = config.address - return nil unless wildcard_address + return unless wildcard_address regex = Regexp.escape(wildcard_address) regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)') diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb index 1adf83739ad..24daad638f4 100644 --- a/lib/gitlab/json_cache.rb +++ b/lib/gitlab/json_cache.rb @@ -71,7 +71,21 @@ module Gitlab end def parse_entry(raw, klass) - klass.new(raw) if valid_entry?(raw, klass) + return unless valid_entry?(raw, klass) + return klass.new(raw) unless klass.ancestors.include?(ActiveRecord::Base) + + # When the cached value is a persisted instance of ActiveRecord::Base in + # some cases a relation can return an empty collection becauses scope.none! + # is being applied on ActiveRecord::Associations::CollectionAssociation#scope + # when the new_record? method incorrectly returns false. + # + # See https://gitlab.com/gitlab-org/gitlab-ee/issues/9903#note_145329964 + attributes = klass.attributes_builder.build_from_database(raw, {}) + klass.allocate.init_with("attributes" => attributes, "new_record" => new_record?(raw, klass)) + end + + def new_record?(raw, klass) + raw.fetch(klass.primary_key, nil).blank? end def valid_entry?(raw, klass) diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index f931248b747..e33ba9305ce 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -7,7 +7,8 @@ module Gitlab include BaseCommand include ClientCommand - attr_reader :name, :files, :chart, :version, :repository, :preinstall, :postinstall + attr_reader :name, :files, :chart, :repository, :preinstall, :postinstall + attr_accessor :version def initialize(name:, chart:, files:, rbac:, version: nil, repository: nil, preinstall: nil, postinstall: nil) @name = name diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 624c2c67551..de14df56555 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -82,6 +82,8 @@ module Gitlab def initialize(api_prefix, **kubeclient_options) @api_prefix = api_prefix @kubeclient_options = kubeclient_options.merge(http_max_redirects: 0) + + validate_url! end def create_or_update_cluster_role_binding(resource) @@ -118,6 +120,12 @@ module Gitlab private + def validate_url! + return if Gitlab::CurrentSettings.allow_local_requests_from_hooks_and_services? + + Gitlab::UrlBlocker.validate!(api_prefix, allow_local_network: false) + end + def cluster_role_binding_exists?(resource) get_cluster_role_binding(resource.metadata.name) rescue ::Kubeclient::ResourceNotFoundError diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index bc952147667..bbdd094e33b 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -68,7 +68,7 @@ module Gitlab end def user(login) - return nil unless login.present? + return unless login.present? return @users[login] if @users.key?(login) @users[login] = api.user(login) diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index ec0e221b1ff..889e6aaa968 100644 --- a/lib/gitlab/legacy_github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -25,7 +25,7 @@ module Gitlab end def find_by_email - return nil unless email + return unless email User.find_by_any_email(email) .try(:id) @@ -33,7 +33,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def find_by_external_uid - return nil unless id + return unless id identities = ::Identity.arel_table diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index 259345b8a9a..e7bfcb16582 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -48,6 +48,8 @@ module Gitlab def execute(context, arg) return if noop? || !available?(context) + count_commands_executed_in(context) + execute_block(action_block, context, arg) end @@ -73,6 +75,13 @@ module Gitlab private + def count_commands_executed_in(context) + return unless context.respond_to?(:commands_executed_count=) + + context.commands_executed_count ||= 0 + context.commands_executed_count += 1 + end + def execute_block(block, context, arg) if arg.present? parsed = parse_params(arg, context) diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb new file mode 100644 index 00000000000..ed2c7ee9a2d --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class MemoryKiller + # Default the RSS limit to 0, meaning the MemoryKiller is disabled + MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i + # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit + GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i + # Wait 30 seconds for running jobs to finish during graceful shutdown + SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + + # Create a mutex used to ensure there will be only one thread waiting to + # shut Sidekiq down + MUTEX = Mutex.new + + def call(worker, job, queue) + yield + + current_rss = get_rss + + return unless MAX_RSS > 0 && current_rss > MAX_RSS + + Thread.new do + # Return if another thread is already waiting to shut Sidekiq down + next unless MUTEX.try_lock + + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ + " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + + # Wait `GRACE_TIME` to give the memory intensive job time to finish. + # Then, tell Sidekiq to stop fetching new jobs. + wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs') + + # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. + # Then, tell Sidekiq to gracefully shut down by giving jobs a few more + # moments to finish, killing and requeuing them if they didn't, and + # then terminating itself. Sidekiq will replicate the TERM to all its + # children if it can. + wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') + + # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. + # Kill the whole pgroup, so we can be sure no children are left behind + wait_and_signal_pgroup(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') + end + end + + private + + def get_rss + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) + return 0 unless status.zero? + + output.to_i + end + + # If this sidekiq process is pgroup leader, signal to the whole pgroup + def wait_and_signal_pgroup(time, signal, explanation) + return wait_and_signal(time, signal, explanation) unless Process.getpgrp == pid + + Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + sleep(time) + + Sidekiq.logger.warn "sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + Process.kill(signal, "-#{pid}") + end + + def wait_and_signal(time, signal, explanation) + Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + sleep(time) + + Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + Process.kill(signal, pid) + end + + def pid + Process.pid + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb deleted file mode 100644 index 19f3be83bce..00000000000 --- a/lib/gitlab/sidekiq_middleware/shutdown.rb +++ /dev/null @@ -1,135 +0,0 @@ -# frozen_string_literal: true - -require 'mutex_m' - -module Gitlab - module SidekiqMiddleware - class Shutdown - extend Mutex_m - - # Default the RSS limit to 0, meaning the MemoryKiller is disabled - MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i - # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit - GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i - # Wait 30 seconds for running jobs to finish during graceful shutdown - SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i - - # This exception can be used to request that the middleware start shutting down Sidekiq - WantShutdown = Class.new(StandardError) - - ShutdownWithoutRaise = Class.new(WantShutdown) - private_constant :ShutdownWithoutRaise - - # For testing only, to avoid race conditions (?) in Rspec mocks. - attr_reader :trace - - # We store the shutdown thread in a class variable to ensure that there - # can be only one shutdown thread in the process. - def self.create_shutdown_thread - mu_synchronize do - break unless @shutdown_thread.nil? - - @shutdown_thread = Thread.new { yield } - end - end - - # For testing only: so we can wait for the shutdown thread to finish. - def self.shutdown_thread - mu_synchronize { @shutdown_thread } - end - - # For testing only: so that we can reset the global state before each test. - def self.clear_shutdown_thread - mu_synchronize { @shutdown_thread = nil } - end - - def initialize - @trace = Queue.new if Rails.env.test? - end - - def call(worker, job, queue) - shutdown_exception = nil - - begin - yield - check_rss! - rescue WantShutdown => ex - shutdown_exception = ex - end - - return unless shutdown_exception - - self.class.create_shutdown_thread do - do_shutdown(worker, job, shutdown_exception) - end - - raise shutdown_exception unless shutdown_exception.is_a?(ShutdownWithoutRaise) - end - - private - - def do_shutdown(worker, job, shutdown_exception) - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} shutting down because of #{shutdown_exception} after job "\ - "#{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" - - # Wait `GRACE_TIME` to give the memory intensive job time to finish. - # Then, tell Sidekiq to stop fetching new jobs. - wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs') - - # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. - # Then, tell Sidekiq to gracefully shut down by giving jobs a few more - # moments to finish, killing and requeuing them if they didn't, and - # then terminating itself. - wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') - - # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. - wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') - end - - def check_rss! - return unless MAX_RSS > 0 - - current_rss = get_rss - return unless current_rss > MAX_RSS - - raise ShutdownWithoutRaise.new("current RSS #{current_rss} exceeds maximum RSS #{MAX_RSS}") - end - - def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status.zero? - - output.to_i - end - - def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - sleep(time) - - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - kill(signal, pid) - end - - def pid - Process.pid - end - - def sleep(time) - if Rails.env.test? - @trace << [:sleep, time] - else - Kernel.sleep(time) - end - end - - def kill(signal, pid) - if Rails.env.test? - @trace << [:kill, signal, pid] - else - Process.kill(signal, pid) - end - end - end - end -end diff --git a/lib/gitlab/sidekiq_signals.rb b/lib/gitlab/sidekiq_signals.rb new file mode 100644 index 00000000000..82462544d07 --- /dev/null +++ b/lib/gitlab/sidekiq_signals.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + # As a process group leader, we can ensure that children of sidekiq are killed + # at the same time as sidekiq itself, to stop long-lived children from being + # reparented to init and "escaping". To do this, we override the default + # handlers used by sidekiq for INT and TERM signals + module SidekiqSignals + REPLACE_SIGNALS = %w[INT TERM].freeze + + SIDEKIQ_CHANGED_MESSAGE = + "Intercepting signal handlers: #{REPLACE_SIGNALS.join(", ")} failed. " \ + "Sidekiq should have registered them, but appears not to have done so." + + def self.install!(sidekiq_handlers) + # This only works if we're process group leader + return unless Process.getpgrp == Process.pid + + raise SIDEKIQ_CHANGED_MESSAGE unless + REPLACE_SIGNALS == sidekiq_handlers.keys & REPLACE_SIGNALS + + REPLACE_SIGNALS.each do |signal| + old_handler = sidekiq_handlers[signal] + sidekiq_handlers[signal] = ->(cli) do + blindly_signal_pgroup!(signal) + old_handler.call(cli) + end + end + end + + # The process group leader can forward INT and TERM signals to the whole + # group. However, the forwarded signal is *also* received by the leader, + # which could lead to an infinite loop. We can avoid this by temporarily + # ignoring the forwarded signal. This may cause us to miss some repeated + # signals from outside the process group, but that isn't fatal. + def self.blindly_signal_pgroup!(signal) + old_trap = trap(signal, 'IGNORE') + begin + Process.kill(signal, 0) + ensure + trap(signal, old_trap) + end + end + end +end diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 92388262035..07d0acdbae9 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -33,7 +33,7 @@ module Gitlab # `LOWER(column) = query` instead of using `ILIKE`. def fuzzy_arel_match(column, query, lower_exact_match: false) query = query.squish - return nil unless query.present? + return unless query.present? words = select_fuzzy_words(query) diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index fbefb5f7f0e..3e2bb11c35f 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -28,11 +28,6 @@ module Gitlab def finder(project = nil) Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) end - - def dropdown_names(context) - categories = context == 'autodeploy' ? ['Auto deploy'] : %w(General Pages) - super().slice(*categories) - end end end end diff --git a/lib/gitlab/tracing/jaeger_factory.rb b/lib/gitlab/tracing/jaeger_factory.rb index 2682007302a..93520d5667b 100644 --- a/lib/gitlab/tracing/jaeger_factory.rb +++ b/lib/gitlab/tracing/jaeger_factory.rb @@ -60,7 +60,7 @@ module Gitlab elsif udp_endpoint.present? sender = get_udp_sender(encoder, udp_endpoint) else - return nil + return end Jaeger::Reporters::RemoteReporter.new( diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index 453d78e2f7b..8518a13cd1c 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -95,7 +95,7 @@ module Gitlab end def cache_commit(commit) - return nil unless commit.present? + return unless commit.present? resolved_commits[commit.id] ||= commit end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 980a8014409..9ef23cf849f 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -118,8 +118,8 @@ module Gitlab protected_refs: project.protected_tags) end - request_cache def protected?(kind, project, ref) - kind.protected?(project, ref) + request_cache def protected?(kind, project, refs) + kind.protected?(project, refs) end end end |