diff options
Diffstat (limited to 'lib/gitlab')
71 files changed, 657 insertions, 979 deletions
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 6eb08f674c2..7ef9f7ef630 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -29,6 +29,10 @@ module Gitlab MAINTAINER_PROJECT_ACCESS = 1 DEVELOPER_MAINTAINER_PROJECT_ACCESS = 2 + # Default subgroup creation level + OWNER_SUBGROUP_ACCESS = 0 + MAINTAINER_SUBGROUP_ACCESS = 1 + class << self delegate :values, to: :options @@ -106,6 +110,13 @@ module Gitlab def project_creation_level_name(name) project_creation_options.key(name) end + + def subgroup_creation_options + { + s_('SubgroupCreationlevel|Owners') => OWNER_SUBGROUP_ACCESS, + s_('SubgroupCreationlevel|Maintainers') => MAINTAINER_SUBGROUP_ACCESS + } + end end def human_access diff --git a/lib/gitlab/action_rate_limiter.rb b/lib/gitlab/action_rate_limiter.rb index c442211e073..fdb06d00548 100644 --- a/lib/gitlab/action_rate_limiter.rb +++ b/lib/gitlab/action_rate_limiter.rb @@ -33,16 +33,48 @@ module Gitlab # Increments the given key and returns true if the action should # be throttled. # - # key - An array of ActiveRecord instances - # threshold_value - The maximum number of times this action should occur in the given time interval + # key - An array of ActiveRecord instances or strings + # threshold_value - The maximum number of times this action should occur in the given time interval. If number is zero is considered disabled. def throttled?(key, threshold_value) - self.increment(key) > threshold_value + threshold_value > 0 && + self.increment(key) > threshold_value + end + + # Logs request into auth.log + # + # request - Web request to be logged + # type - A symbol key that represents the request. + # current_user - Current user of the request, it can be nil. + def log_request(request, type, current_user) + request_information = { + message: 'Action_Rate_Limiter_Request', + env: type, + ip: request.ip, + request_method: request.request_method, + fullpath: request.fullpath + } + + if current_user + request_information.merge!({ + user_id: current_user.id, + username: current_user.username + }) + end + + Gitlab::AuthLogger.error(request_information) end private def action_key(key) - serialized = key.map { |obj| "#{obj.class.model_name.to_s.underscore}:#{obj.id}" }.join(":") + serialized = key.map do |obj| + if obj.is_a?(String) + "#{obj}" + else + "#{obj.class.model_name.to_s.underscore}:#{obj.id}" + end + end.join(":") + "action_rate_limiter:#{action}:#{serialized}" end end diff --git a/lib/gitlab/auth/activity.rb b/lib/gitlab/auth/activity.rb index 558628b5422..988ff196193 100644 --- a/lib/gitlab/auth/activity.rb +++ b/lib/gitlab/auth/activity.rb @@ -37,14 +37,17 @@ module Gitlab def user_authenticated! self.class.user_authenticated_counter_increment! + + case @opts[:message] + when :two_factor_authenticated + self.class.user_two_factor_authenticated_counter_increment! + end end def user_session_override! self.class.user_session_override_counter_increment! case @opts[:message] - when :two_factor_authenticated - self.class.user_two_factor_authenticated_counter_increment! when :sessionless_sign_in self.class.user_sessionless_authentication_counter_increment! end diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 72a187377d0..91b9ddc0d00 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -60,8 +60,7 @@ module Gitlab def get_info(key) value = info[key] - Gitlab::Utils.force_utf8(value) if value - value + value.is_a?(String) ? Gitlab::Utils.force_utf8(value) : value end def username_and_email diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index a5efe33bdc6..bba7e2cbb3c 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -90,8 +90,8 @@ module Gitlab def find_personal_access_token token = current_request.params[PRIVATE_TOKEN_PARAM].presence || - current_request.env[PRIVATE_TOKEN_HEADER].presence - + current_request.env[PRIVATE_TOKEN_HEADER].presence || + parsed_oauth_token return unless token # Expiration, revocation and scopes are verified in `validate_access_token!` @@ -99,9 +99,12 @@ module Gitlab end def find_oauth_access_token - token = Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) + token = parsed_oauth_token return unless token + # PATs with OAuth headers are not handled by OauthAccessToken + return if matches_personal_access_token_length?(token) + # Expiration, revocation and scopes are verified in `validate_access_token!` oauth_token = OauthAccessToken.by_token(token) raise UnauthorizedError unless oauth_token @@ -110,6 +113,14 @@ module Gitlab oauth_token end + def parsed_oauth_token + Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) + end + + def matches_personal_access_token_length?(token) + token.length == PersonalAccessToken::TOKEN_LENGTH + end + # Check if the request is GET/HEAD, or if CSRF token is valid. def verified_request? Gitlab::RequestForgeryProtection.verified?(current_request.env) diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index c8d83cc1803..1d9aa050041 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -40,7 +40,7 @@ module Gitlab end def reload! - @shards = Hash[*Shard.all.map { |shard| [shard.name, shard.id] }.flatten] + @shards = Hash[*Shard.all.flat_map { |shard| [shard.name, shard.id] }] end end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb index 6046d33aeac..4016b807f21 100644 --- a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -19,18 +19,11 @@ module Gitlab def perform(start_id, stop_id) PagesDomain.where(id: start_id..stop_id).find_each do |domain| - if Gitlab::Database.mysql? - domain.update_columns( - certificate_valid_not_before: domain.x509&.not_before, - certificate_valid_not_after: domain.x509&.not_after - ) - else - # for some reason activerecord doesn't append timezone, iso8601 forces this - domain.update_columns( - certificate_valid_not_before: domain.x509&.not_before&.iso8601, - certificate_valid_not_after: domain.x509&.not_after&.iso8601 - ) - end + # for some reason activerecord doesn't append timezone, iso8601 forces this + domain.update_columns( + certificate_valid_not_before: domain.x509&.not_before&.iso8601, + certificate_valid_not_after: domain.x509&.not_after&.iso8601 + ) rescue => e Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" # rubocop:disable Gitlab/RailsLogger end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index 1924f2ffee2..f5fb33f1660 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -176,23 +176,12 @@ module Gitlab self.table_name = 'projects' def self.find_by_full_path(path) - binary = Gitlab::Database.mysql? ? 'BINARY' : '' - order_sql = "(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" + order_sql = "(CASE WHEN routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" where_full_path_in(path).reorder(order_sql).take end def self.where_full_path_in(path) - cast_lower = Gitlab::Database.postgresql? - - path = connection.quote(path) - - where = - if cast_lower - "(LOWER(routes.path) = LOWER(#{path}))" - else - "(routes.path = #{path})" - end - + where = "(LOWER(routes.path) = LOWER(#{connection.quote(path)}))" joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'").where(where) end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index cce2a82c098..806c3a7a369 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -55,7 +55,7 @@ module Gitlab def ensure_temporary_tracking_table_exists table_name = :untracked_files_for_uploads - unless ActiveRecord::Base.connection.data_source_exists?(table_name) + unless ActiveRecord::Base.connection.table_exists?(table_name) UntrackedFile.connection.create_table table_name do |t| t.string :path, limit: 600, null: false t.index :path, unique: true @@ -133,12 +133,9 @@ module Gitlab def insert_sql(file_paths) if postgresql_pre_9_5? "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" - elsif postgresql? + else "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\ " ON CONFLICT DO NOTHING;" - else # MySQL - "INSERT IGNORE INTO"\ - " #{table_columns_and_values_for_insert(file_paths)};" end end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 7f7cc62c8ef..15cccc6f287 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -14,7 +14,7 @@ module Gitlab @ref = ref @job = job - @pipeline = @project.ci_pipelines.latest_successful_for(@ref) + @pipeline = @project.ci_pipelines.latest_successful_for_ref(@ref) end def entity diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 191f5d09645..99356226ef9 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -46,7 +46,7 @@ module Gitlab parallelized_job_names = @parallelized_jobs.keys.map(&:to_s) parallelized_config.each_with_object({}) do |(job_name, config), hash| if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any? - parallelized_deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten + parallelized_deps = intersection.flat_map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) } deps = config[:dependencies] - intersection + parallelized_deps hash[job_name] = config.merge(dependencies: deps) else diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index d01b09f1398..df572188194 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -10,7 +10,7 @@ module Gitlab image: 'illustrations/manual_action.svg', size: 'svg-394', title: _('This job requires a manual action'), - content: _('This job depends on a user to trigger its process. Often they are used to deploy code to production environments') + content: _('This job requires manual intervention to start. Before starting this job, you can add variables below for last-minute configuration changes.') } end diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index 3446644eff8..2a0bf060c9b 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -34,11 +34,9 @@ module Gitlab def extended_statuses return @extended_statuses if defined?(@extended_statuses) - groups = self.class.extended_statuses.map do |group| + @extended_statuses = self.class.extended_statuses.flat_map do |group| Array(group).find { |status| status.matches?(@subject, @user) } - end - - @extended_statuses = groups.flatten.compact + end.compact end def self.extended_statuses diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index cf3d261c1cb..5c1c0c142e5 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -50,13 +50,12 @@ variables: POSTGRES_DB: $CI_ENVIRONMENT_SLUG POSTGRES_VERSION: 9.6.2 - KUBERNETES_VERSION: 1.11.10 - HELM_VERSION: 2.14.0 - DOCKER_DRIVER: overlay2 ROLLOUT_RESOURCE_TYPE: deployment + DOCKER_TLS_CERTDIR: "" # https://gitlab.com/gitlab-org/gitlab-runner/issues/4501 + stages: - build - test diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index a09217e8cf0..b0a79950667 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -2,6 +2,8 @@ performance: stage: performance image: docker:stable allow_failure: true + variables: + DOCKER_TLS_CERTDIR: "" services: - docker:stable-dind script: diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 18f7290e1d9..8061da968ed 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,6 +1,8 @@ build: stage: build image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image/master:stable" + variables: + DOCKER_TLS_CERTDIR: "" services: - docker:stable-dind script: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 005ea4b7a46..3adc6a72874 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -6,6 +6,7 @@ code_quality: - docker:stable-dind variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" script: - | if ! docker info &>/dev/null; then diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 108f0119ae1..a8ec2d4781d 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,14 +1,17 @@ +.auto-deploy: + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.1.0" + review: + extends: .auto-deploy stage: review script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy persist_environment_url environment: name: review/$CI_COMMIT_REF_NAME url: http://$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -27,13 +30,13 @@ review: - $REVIEW_DISABLED stop_review: + extends: .auto-deploy stage: cleanup variables: GIT_STRATEGY: none script: - - install_dependencies - - initialize_tiller - - delete + - auto-deploy initialize_tiller + - auto-deploy delete environment: name: review/$CI_COMMIT_REF_NAME action: stop @@ -57,15 +60,15 @@ stop_review: # STAGING_ENABLED. staging: + extends: .auto-deploy stage: staging script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy environment: name: staging url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN @@ -81,15 +84,15 @@ staging: # CANARY_ENABLED. canary: + extends: .auto-deploy stage: canary script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy canary + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy canary environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -102,18 +105,18 @@ canary: - $CANARY_ENABLED .production: &production_template + extends: .auto-deploy stage: production script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy - - delete canary - - delete rollout - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy delete canary + - auto-deploy delete rollout + - auto-deploy persist_environment_url environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -152,17 +155,17 @@ production_manual: # This job implements incremental rollout on for every push to `master`. .rollout: &rollout_template + extends: .auto-deploy script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy rollout $ROLLOUT_PERCENTAGE - - scale stable $((100-ROLLOUT_PERCENTAGE)) - - delete canary - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE + - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) + - auto-deploy delete canary + - auto-deploy persist_environment_url environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -240,330 +243,3 @@ rollout 100%: <<: *manual_rollout_template <<: *production_template allow_failure: false - -.deploy_helpers: &deploy_helpers | - [[ "$TRACE" ]] && set -x - auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} - export DATABASE_URL=${DATABASE_URL-$auto_database_url} - export TILLER_NAMESPACE=$KUBE_NAMESPACE - - function get_replicas() { - track="${1:-stable}" - percentage="${2:-100}" - - env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' ) - env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' ) - - if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then - # for stable track get number of replicas from `PRODUCTION_REPLICAS` - eval new_replicas=\$${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - new_replicas=$REPLICAS - fi - else - # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` - eval new_replicas=\$${env_track}_${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - eval new_replicas=\${env_track}_REPLICAS - fi - fi - - replicas="${new_replicas:-1}" - replicas="$(($replicas * $percentage / 100))" - - # always return at least one replicas - if [[ $replicas -gt 0 ]]; then - echo "$replicas" - else - echo 1 - fi - } - - # Extracts variables prefixed with K8S_SECRET_ - # and creates a Kubernetes secret. - # - # e.g. If we have the following environment variables: - # K8S_SECRET_A=value1 - # K8S_SECRET_B=multi\ word\ value - # - # Then we will create a secret with the following key-value pairs: - # data: - # A: dmFsdWUxCg== - # B: bXVsdGkgd29yZCB2YWx1ZQo= - function create_application_secret() { - track="${1-stable}" - export APPLICATION_SECRET_NAME=$(application_secret_name "$track") - - env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables - - kubectl create secret \ - -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \ - --from-env-file k8s_prefixed_variables -o yaml --dry-run | - kubectl replace -n "$KUBE_NAMESPACE" --force -f - - - export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1) - - rm k8s_prefixed_variables - } - - function deploy_name() { - name="$CI_ENVIRONMENT_SLUG" - track="${1-stable}" - - if [[ "$track" != "stable" ]]; then - name="$name-$track" - fi - - echo $name - } - - function application_secret_name() { - track="${1-stable}" - name=$(deploy_name "$track") - - echo "${name}-secret" - } - - function deploy() { - track="${1-stable}" - percentage="${2:-100}" - name=$(deploy_name "$track") - - if [[ -z "$CI_COMMIT_TAG" ]]; then - image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG} - image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA} - else - image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE} - image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} - fi - - service_enabled="true" - postgres_enabled="$POSTGRES_ENABLED" - - # if track is different than stable, - # re-use all attached resources - if [[ "$track" != "stable" ]]; then - service_enabled="false" - postgres_enabled="false" - fi - - replicas=$(get_replicas "$track" "$percentage") - - if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then - secret_name='gitlab-registry' - else - secret_name='' - fi - - create_application_secret "$track" - - env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]') - eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS - if [ -n "$env_ADDITIONAL_HOSTS" ]; then - additional_hosts="{$env_ADDITIONAL_HOSTS}" - elif [ -n "$ADDITIONAL_HOSTS" ]; then - additional_hosts="{$ADDITIONAL_HOSTS}" - fi - - if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then - echo "Deploying first release with database initialization..." - helm upgrade --install \ - --wait \ - --set service.enabled="$service_enabled" \ - --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ - --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ - --set image.repository="$image_repository" \ - --set image.tag="$image_tag" \ - --set image.pullPolicy=IfNotPresent \ - --set image.secrets[0].name="$secret_name" \ - --set application.track="$track" \ - --set application.database_url="$DATABASE_URL" \ - --set application.secretName="$APPLICATION_SECRET_NAME" \ - --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \ - --set service.url="$CI_ENVIRONMENT_URL" \ - --set service.additionalHosts="$additional_hosts" \ - --set replicaCount="$replicas" \ - --set postgresql.enabled="$postgres_enabled" \ - --set postgresql.nameOverride="postgres" \ - --set postgresql.postgresUser="$POSTGRES_USER" \ - --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ - --set postgresql.postgresDatabase="$POSTGRES_DB" \ - --set postgresql.imageTag="$POSTGRES_VERSION" \ - --set application.initializeCommand="$DB_INITIALIZE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - - echo "Deploying second release..." - helm upgrade --reuse-values \ - --wait \ - --set application.initializeCommand="" \ - --set application.migrateCommand="$DB_MIGRATE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - else - echo "Deploying new release..." - helm upgrade --install \ - --wait \ - --set service.enabled="$service_enabled" \ - --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ - --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ - --set image.repository="$image_repository" \ - --set image.tag="$image_tag" \ - --set image.pullPolicy=IfNotPresent \ - --set image.secrets[0].name="$secret_name" \ - --set application.track="$track" \ - --set application.database_url="$DATABASE_URL" \ - --set application.secretName="$APPLICATION_SECRET_NAME" \ - --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \ - --set service.url="$CI_ENVIRONMENT_URL" \ - --set service.additionalHosts="$additional_hosts" \ - --set replicaCount="$replicas" \ - --set postgresql.enabled="$postgres_enabled" \ - --set postgresql.nameOverride="postgres" \ - --set postgresql.postgresUser="$POSTGRES_USER" \ - --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ - --set postgresql.postgresDatabase="$POSTGRES_DB" \ - --set postgresql.imageTag="$POSTGRES_VERSION" \ - --set application.migrateCommand="$DB_MIGRATE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - fi - - if [[ -z "$ROLLOUT_STATUS_DISABLED" ]]; then - kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name" - fi - } - - function scale() { - track="${1-stable}" - percentage="${2-100}" - name=$(deploy_name "$track") - - replicas=$(get_replicas "$track" "$percentage") - - if [[ -n "$(helm ls -q "^$name$")" ]]; then - helm upgrade --reuse-values \ - --wait \ - --set replicaCount="$replicas" \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - fi - } - - function install_dependencies() { - apk add -U openssl curl tar gzip bash ca-certificates git - curl -sSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub - curl -sSL -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk - apk add glibc-2.28-r0.apk - rm glibc-2.28-r0.apk - - curl -sS "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx - mv linux-amd64/helm /usr/bin/ - mv linux-amd64/tiller /usr/bin/ - helm version --client - tiller -version - - curl -sSL -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" - chmod +x /usr/bin/kubectl - kubectl version --client - } - - function download_chart() { - if [[ ! -d chart ]]; then - auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app} - auto_chart_name=$(basename $auto_chart) - auto_chart_name=${auto_chart_name%.tgz} - auto_chart_name=${auto_chart_name%.tar.gz} - else - auto_chart="chart" - auto_chart_name="chart" - fi - - helm init --client-only - helm repo add ${AUTO_DEVOPS_CHART_REPOSITORY_NAME:-gitlab} ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} ${AUTO_DEVOPS_CHART_REPOSITORY_USERNAME:+"--username" "$AUTO_DEVOPS_CHART_REPOSITORY_USERNAME"} ${AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD:+"--password" "$AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD"} - if [[ ! -d "$auto_chart" ]]; then - helm fetch ${auto_chart} --untar - fi - if [ "$auto_chart_name" != "chart" ]; then - mv ${auto_chart_name} chart - fi - - helm dependency update chart/ - helm dependency build chart/ - } - - function ensure_namespace() { - kubectl get namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" - } - - function check_kube_domain() { - if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then - echo "In order to deploy or use Review Apps," - echo "KUBE_INGRESS_BASE_DOMAIN variables must be set" - echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings" - echo "or by defining a variable at group or project level." - echo "You can also manually add it in .gitlab-ci.yml" - false - else - true - fi - } - - function initialize_tiller() { - echo "Checking Tiller..." - - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 & - echo "Tiller is listening on ${HELM_HOST}" - - if ! helm version --debug; then - echo "Failed to init Tiller." - return 1 - fi - echo "" - } - - function create_secret() { - echo "Create secret..." - if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then - return - fi - - kubectl create secret -n "$KUBE_NAMESPACE" \ - docker-registry gitlab-registry \ - --docker-server="$CI_REGISTRY" \ - --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \ - --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \ - --docker-email="$GITLAB_USER_EMAIL" \ - -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f - - } - - function persist_environment_url() { - echo $CI_ENVIRONMENT_URL > environment_url.txt - } - - function delete() { - track="${1-stable}" - name=$(deploy_name "$track") - - if [[ -n "$(helm ls -q "^$name$")" ]]; then - helm delete --purge "$name" - fi - - secret_name=$(application_secret_name "$track") - kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name" - } - -before_script: - - *deploy_helpers diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 89eccce69f6..15b84f1540d 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -9,6 +9,7 @@ dependency_scanning: image: docker:stable variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" allow_failure: true services: - docker:stable-dind @@ -41,6 +42,7 @@ dependency_scanning: DS_PULL_ANALYZER_IMAGE_TIMEOUT \ DS_RUN_ANALYZER_TIMEOUT \ DS_PYTHON_VERSION \ + DS_PIP_DEPENDENCY_PATH \ PIP_INDEX_URL \ PIP_EXTRA_INDEX_URL \ ) \ diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 0a97a16b83c..4190de73e1f 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -9,6 +9,7 @@ sast: image: docker:stable variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" allow_failure: true services: - docker:stable-dind diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index ce5857965bf..cb617080c76 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -63,7 +63,15 @@ module Gitlab end def exist? - trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present? + archived_trace_exist? || live_trace_exist? + end + + def archived_trace_exist? + trace_artifact&.exists? + end + + def live_trace_exist? + job.trace_chunks.any? || current_path.present? || old_trace.present? end def read @@ -167,7 +175,7 @@ module Gitlab def clone_file!(src_stream, temp_dir) FileUtils.mkdir_p(temp_dir) - Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path| + Dir.mktmpdir("tmp-trace-#{job.id}", temp_dir) do |dir_path| temp_path = File.join(dir_path, "job.log") FileUtils.touch(temp_path) size = IO.copy_stream(src_stream, temp_path) diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 8c6fd56493f..e99889f4a25 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -166,6 +166,13 @@ module Gitlab end def destroy! + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless build.has_archived_trace? + Sidekiq.logger.warn(message: 'The job does not have archived trace but going to be destroyed.', + job_id: build.id) + end + trace_chunks.fast_destroy_all @tell = @size = 0 ensure diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index 96aa799e864..07ae430c45e 100644 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -4,6 +4,7 @@ module Gitlab module CycleAnalytics class BaseEventFetcher include BaseQuery + include GroupProjectsProvider attr_reader :projections, :query, :stage, :order, :options @@ -73,18 +74,6 @@ module Gitlab def serialization_context {} end - - def projects - group ? Project.inside_path(group.full_path) : [project] - end - - def group - @group ||= options.fetch(:group, nil) - end - - def project - @project ||= options.fetch(:project, nil) - end end end end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index 678a891e941..1cd54238bb4 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -4,6 +4,7 @@ module Gitlab module CycleAnalytics class BaseStage include BaseQuery + include GroupProjectsProvider attr_reader :options @@ -77,18 +78,6 @@ module Gitlab def event_options options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) end - - def projects - group ? Project.inside_path(group.full_path) : [project] - end - - def group - @group ||= options.fetch(:group, nil) - end - - def project - @project ||= options.fetch(:project, nil) - end end end end diff --git a/lib/gitlab/cycle_analytics/group_projects_provider.rb b/lib/gitlab/cycle_analytics/group_projects_provider.rb new file mode 100644 index 00000000000..1287a48daaa --- /dev/null +++ b/lib/gitlab/cycle_analytics/group_projects_provider.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module GroupProjectsProvider + def projects + group ? projects_for_group : [project] + end + + def group + @group ||= options.fetch(:group, nil) + end + + def project + @project ||= options.fetch(:project, nil) + end + + private + + def projects_for_group + projects = Project.inside_path(group.full_path) + projects = projects.where(id: options[:projects]) if options[:projects] + projects + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/group_stage_summary.rb b/lib/gitlab/cycle_analytics/group_stage_summary.rb index 7b5c74e1a1b..a1fc941495d 100644 --- a/lib/gitlab/cycle_analytics/group_stage_summary.rb +++ b/lib/gitlab/cycle_analytics/group_stage_summary.rb @@ -3,15 +3,18 @@ module Gitlab module CycleAnalytics class GroupStageSummary - def initialize(group, from:, current_user:) + attr_reader :group, :from, :current_user, :options + + def initialize(group, options:) @group = group - @from = from - @current_user = current_user + @from = options[:from] + @current_user = options[:current_user] + @options = options end def data - [serialize(Summary::Group::Issue.new(group: @group, from: @from, current_user: @current_user)), - serialize(Summary::Group::Deploy.new(group: @group, from: @from))] + [serialize(Summary::Group::Issue.new(group: group, from: from, current_user: current_user, options: options)), + serialize(Summary::Group::Deploy.new(group: group, from: from, options: options))] end private diff --git a/lib/gitlab/cycle_analytics/summary/group/base.rb b/lib/gitlab/cycle_analytics/summary/group/base.rb index 7f18b61d309..48d8164bde1 100644 --- a/lib/gitlab/cycle_analytics/summary/group/base.rb +++ b/lib/gitlab/cycle_analytics/summary/group/base.rb @@ -5,9 +5,12 @@ module Gitlab module Summary module Group class Base - def initialize(group:, from:) + attr_reader :group, :from, :options + + def initialize(group:, from:, options:) @group = group @from = from + @options = options end def title diff --git a/lib/gitlab/cycle_analytics/summary/group/deploy.rb b/lib/gitlab/cycle_analytics/summary/group/deploy.rb index d8fcd8f2ce4..78d677cf558 100644 --- a/lib/gitlab/cycle_analytics/summary/group/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/group/deploy.rb @@ -5,22 +5,23 @@ module Gitlab module Summary module Group class Deploy < Group::Base + include GroupProjectsProvider + def title n_('Deploy', 'Deploys', value) end def value - @value ||= Deployment.joins(:project) - .where(projects: { id: projects }) - .where("deployments.created_at > ?", @from) - .success - .count + @value ||= find_deployments end private - def projects - Project.inside_path(@group.full_path).ids + def find_deployments + deployments = Deployment.joins(:project).merge(Project.inside_path(group.full_path)) + deployments = deployments.where(projects: { id: options[:projects] }) if options[:projects] + deployments = deployments.where("deployments.created_at > ?", from) + deployments.success.count end end end diff --git a/lib/gitlab/cycle_analytics/summary/group/issue.rb b/lib/gitlab/cycle_analytics/summary/group/issue.rb index 70073e6d843..9daae8531d8 100644 --- a/lib/gitlab/cycle_analytics/summary/group/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/group/issue.rb @@ -5,10 +5,13 @@ module Gitlab module Summary module Group class Issue < Group::Base - def initialize(group:, from:, current_user:) + attr_reader :group, :from, :current_user, :options + + def initialize(group:, from:, current_user:, options:) @group = group @from = from @current_user = current_user + @options = options end def title @@ -16,7 +19,15 @@ module Gitlab end def value - @value ||= IssuesFinder.new(@current_user, group_id: @group.id, include_subgroups: true, created_after: @from).execute.count + @value ||= find_issues + end + + private + + def find_issues + issues = IssuesFinder.new(current_user, group_id: group.id, include_subgroups: true, created_after: from).execute + issues = issues.where(projects: { id: options[:projects] }) if options[:projects] + issues.count end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 3e4c720b49a..eef63536de4 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -4,13 +4,13 @@ module Gitlab module Database include Gitlab::Metrics::Methods - # The max value of INTEGER type is the same between MySQL and PostgreSQL: # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html - # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html MAX_INT_VALUE = 2147483647 + # The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz: # https://www.postgresql.org/docs/9.1/static/datatype-datetime.html # https://dev.mysql.com/doc/refman/5.7/en/datetime.html + # FIXME: this should just be the max value of timestampz MAX_TIMESTAMP_VALUE = Time.at((1 << 31) - 1).freeze # Minimum schema version from which migrations are supported @@ -39,11 +39,11 @@ module Gitlab end def self.human_adapter_name - postgresql? ? 'PostgreSQL' : 'MySQL' - end - - def self.mysql? - adapter_name.casecmp('mysql2').zero? + if postgresql? + 'PostgreSQL' + else + 'Unknown' + end end def self.postgresql? @@ -60,15 +60,14 @@ module Gitlab # Check whether the underlying database is in read-only mode def self.db_read_only? - if postgresql? - pg_is_in_recovery = - ActiveRecord::Base.connection.execute('SELECT pg_is_in_recovery()') - .first.fetch('pg_is_in_recovery') + pg_is_in_recovery = + ActiveRecord::Base + .connection + .execute('SELECT pg_is_in_recovery()') + .first + .fetch('pg_is_in_recovery') - Gitlab::Utils.to_boolean(pg_is_in_recovery) - else - false - end + Gitlab::Utils.to_boolean(pg_is_in_recovery) end def self.db_read_write? @@ -118,51 +117,23 @@ module Gitlab end def self.nulls_last_order(field, direction = 'ASC') - order = "#{field} #{direction}" - - if postgresql? - order = "#{order} NULLS LAST" - else - # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL - # columns. In the (default) ascending order, `0` comes first. - order = "#{field} IS NULL, #{order}" if direction == 'ASC' - end - - Arel.sql(order) + Arel.sql("#{field} #{direction} NULLS LAST") end def self.nulls_first_order(field, direction = 'ASC') - order = "#{field} #{direction}" - - if postgresql? - order = "#{order} NULLS FIRST" - else - # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL - # columns. In the (default) ascending order, `0` comes first. - order = "#{field} IS NULL, #{order}" if direction == 'DESC' - end - - Arel.sql(order) + Arel.sql("#{field} #{direction} NULLS FIRST") end def self.random - postgresql? ? "RANDOM()" : "RAND()" + "RANDOM()" end def self.true_value - if postgresql? - "'t'" - else - 1 - end + "'t'" end def self.false_value - if postgresql? - "'f'" - else - 0 - end + "'f'" end def self.with_connection_pool(pool_size) @@ -182,7 +153,7 @@ module Gitlab # rows - An Array of Hash instances, each mapping the columns to their # values. # return_ids - When set to true the return value will be an Array of IDs of - # the inserted rows, this only works on PostgreSQL. + # the inserted rows # disable_quote - A key or an Array of keys to exclude from quoting (You # become responsible for protection from SQL injection for # these keys!) @@ -191,7 +162,6 @@ module Gitlab keys = rows.first.keys columns = keys.map { |key| connection.quote_column_name(key) } - return_ids = false if mysql? disable_quote = Array(disable_quote).to_set tuples = rows.map do |row| @@ -258,11 +228,7 @@ module Gitlab def self.database_version row = connection.execute("SELECT VERSION()").first - if postgresql? - row['version'] - else - row.first - end + row['version'] end private_class_method :database_version diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index f3d37ccd72a..eac61254bdf 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -37,16 +37,14 @@ module Gitlab # @return [Hash] of Model -> count mapping def self.approximate_counts(models, strategies: [TablesampleCountStrategy, ReltuplesCountStrategy, ExactCountStrategy]) strategies.each_with_object({}) do |strategy, counts_by_model| - if strategy.enabled? - models_with_missing_counts = models - counts_by_model.keys + models_with_missing_counts = models - counts_by_model.keys - break counts_by_model if models_with_missing_counts.empty? + break counts_by_model if models_with_missing_counts.empty? - counts = strategy.new(models_with_missing_counts).count + counts = strategy.new(models_with_missing_counts).count - counts.each do |model, count| - counts_by_model[model] = count - end + counts.each do |model, count| + counts_by_model[model] = count end end end diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb index fa6951eda22..0b8fe640bf8 100644 --- a/lib/gitlab/database/count/exact_count_strategy.rb +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -23,10 +23,6 @@ module Gitlab rescue *CONNECTION_ERRORS {} end - - def self.enabled? - true - end end end end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 695f6fa766e..6cd90c01ab2 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -31,10 +31,6 @@ module Gitlab {} end - def self.enabled? - Gitlab::Database.postgresql? - end - private # Models using single-type inheritance (STI) don't work with diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb index 7777f31f702..e9387a91a14 100644 --- a/lib/gitlab/database/count/tablesample_count_strategy.rb +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -28,10 +28,6 @@ module Gitlab {} end - def self.enabled? - Gitlab::Database.postgresql? && Feature.enabled?(:tablesample_counts) - end - private def perform_count(model, estimate) diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb index 79d2caff151..1392b397012 100644 --- a/lib/gitlab/database/date_time.rb +++ b/lib/gitlab/database/date_time.rb @@ -7,8 +7,7 @@ module Gitlab # the first of the `start_time_attrs` that isn't NULL. `SELECT` the resulting interval # along with an alias specified by the `as` parameter. # - # Note: For MySQL, the interval is returned in seconds. - # For PostgreSQL, the interval is returned as an INTERVAL type. + # Note: the interval is returned as an INTERVAL type. def subtract_datetimes(query_so_far, start_time_attrs, end_time_attrs, as) diff_fn = subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) @@ -16,17 +15,10 @@ module Gitlab end def subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) - if Gitlab::Database.postgresql? - Arel::Nodes::Subtraction.new( - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) - elsif Gitlab::Database.mysql? - Arel::Nodes::NamedFunction.new( - "TIMESTAMPDIFF", - [Arel.sql('second'), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) - end + Arel::Nodes::Subtraction.new( + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)) + ) end end end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index b8d895dee7d..391c1e85a7d 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -17,13 +17,9 @@ module Gitlab def extract_median(results) result = results.compact.first - if Gitlab::Database.postgresql? - result = result.first.presence + result = result.first.presence - result['median']&.to_f if result - elsif Gitlab::Database.mysql? - result.to_a.flatten.first - end + result['median']&.to_f if result end def extract_medians(results) @@ -34,31 +30,6 @@ module Gitlab end end - def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - query = arel_table.from - .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) - .project(average([arel_table[column_sym]], 'median')) - .where( - Arel::Nodes::Between.new( - Arel.sql("(select @row_id := @row_id + 1)"), - Arel::Nodes::And.new( - [Arel.sql('@ct/2.0'), - Arel.sql('@ct/2.0 + 1')] - ) - ) - ). - # Disallow negative values - where(arel_table[column_sym].gteq(0)) - - [ - Arel.sql("CREATE TEMPORARY TABLE IF NOT EXISTS #{query_so_far.to_sql}"), - Arel.sql("set @ct := (select count(1) from #{arel_table.table_name});"), - Arel.sql("set @row_id := 0;"), - query.to_sql, - Arel.sql("DROP TEMPORARY TABLE IF EXISTS #{arel_table.table_name};") - ] - end - def pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column = nil) # Create a CTE with the column we're operating on, row number (after sorting by the column # we're operating on), and count of the table we're operating on (duplicated across) all rows @@ -113,18 +84,8 @@ module Gitlab private - def median_queries(arel_table, query_so_far, column_sym, partition_column = nil) - if Gitlab::Database.postgresql? - pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) - elsif Gitlab::Database.mysql? - raise NotSupportedError, "partition_column is not supported for MySQL" if partition_column - - mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - end - end - def execute_queries(arel_table, query_so_far, column_sym, partition_column = nil) - queries = median_queries(arel_table, query_so_far, column_sym, partition_column) + queries = pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) Array.wrap(queries).map { |query| ActiveRecord::Base.connection.execute(query) } end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 2ae807697bc..4bd09163bf2 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -68,10 +68,7 @@ module Gitlab end end - # Creates a new index, concurrently when supported - # - # On PostgreSQL this method creates an index concurrently, on MySQL this - # creates a regular index. + # Creates a new index, concurrently # # Example: # @@ -85,9 +82,7 @@ module Gitlab 'in the body of your migration class' end - if Database.postgresql? - options = options.merge({ algorithm: :concurrently }) - end + options = options.merge({ algorithm: :concurrently }) if index_exists?(table_name, column_name, options) Rails.logger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" # rubocop:disable Gitlab/RailsLogger @@ -99,9 +94,7 @@ module Gitlab end end - # Removes an existed index, concurrently when supported - # - # On PostgreSQL this method removes an index concurrently. + # Removes an existed index, concurrently # # Example: # @@ -129,9 +122,7 @@ module Gitlab end end - # Removes an existing index, concurrently when supported - # - # On PostgreSQL this method removes an index concurrently. + # Removes an existing index, concurrently # # Example: # @@ -170,8 +161,7 @@ module Gitlab # Adds a foreign key with only minimal locking on the tables involved. # - # This method only requires minimal locking when using PostgreSQL. When - # using MySQL this method will use Rails' default `add_foreign_key`. + # This method only requires minimal locking # # source - The source table containing the foreign key. # target - The target table the key points to. @@ -187,27 +177,7 @@ module Gitlab raise 'add_concurrent_foreign_key can not be run inside a transaction' end - # While MySQL does allow disabling of foreign keys it has no equivalent - # of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall - # back to the normal foreign key procedure. - if Database.mysql? - if foreign_key_exists?(source, target, column: column) - Rails.logger.warn "Foreign key not created because it exists already " \ - "(this may be due to an aborted migration or similar): " \ - "source: #{source}, target: #{target}, column: #{column}" - return - end - - key_options = { column: column, on_delete: on_delete } - - # The MySQL adapter tries to create a foreign key without a name when - # `:name` is nil, instead of generating a name for us. - key_options[:name] = name if name - - return add_foreign_key(source, target, key_options) - else - on_delete = 'SET NULL' if on_delete == :nullify - end + on_delete = 'SET NULL' if on_delete == :nullify key_name = name || concurrent_foreign_key_name(source, column) @@ -265,7 +235,7 @@ module Gitlab # Long-running migrations may take more than the timeout allowed by # the database. Disable the session's statement timeout to ensure - # migrations don't get killed prematurely. (PostgreSQL only) + # migrations don't get killed prematurely. # # There are two possible ways to disable the statement timeout: # @@ -277,15 +247,6 @@ module Gitlab # otherwise the statement will still be disabled until connection is dropped # or `RESET ALL` is executed def disable_statement_timeout - # bypass disabled_statement logic when not using postgres, but still execute block when one is given - unless Database.postgresql? - if block_given? - yield - end - - return - end - if block_given? begin execute('SET statement_timeout TO 0') @@ -535,13 +496,12 @@ module Gitlab quoted_old = quote_column_name(old_column) quoted_new = quote_column_name(new_column) - if Database.postgresql? - install_rename_triggers_for_postgresql(trigger_name, quoted_table, - quoted_old, quoted_new) - else - install_rename_triggers_for_mysql(trigger_name, quoted_table, - quoted_old, quoted_new) - end + install_rename_triggers_for_postgresql( + trigger_name, + quoted_table, + quoted_old, + quoted_new + ) end # Changes the type of a column concurrently. @@ -584,11 +544,7 @@ module Gitlab check_trigger_permissions!(table) - if Database.postgresql? - remove_rename_triggers_for_postgresql(table, trigger_name) - else - remove_rename_triggers_for_mysql(trigger_name) - end + remove_rename_triggers_for_postgresql(table, trigger_name) remove_column(table, old) end @@ -801,38 +757,12 @@ module Gitlab EOF end - # Installs the triggers necessary to perform a concurrent column rename on - # MySQL. - def install_rename_triggers_for_mysql(trigger, table, old, new) - execute <<-EOF.strip_heredoc - CREATE TRIGGER #{trigger}_insert - BEFORE INSERT - ON #{table} - FOR EACH ROW - SET NEW.#{new} = NEW.#{old} - EOF - - execute <<-EOF.strip_heredoc - CREATE TRIGGER #{trigger}_update - BEFORE UPDATE - ON #{table} - FOR EACH ROW - SET NEW.#{new} = NEW.#{old} - EOF - end - # Removes the triggers used for renaming a PostgreSQL column concurrently. def remove_rename_triggers_for_postgresql(table, trigger) execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}") execute("DROP FUNCTION IF EXISTS #{trigger}()") end - # Removes the triggers used for renaming a MySQL column concurrently. - def remove_rename_triggers_for_mysql(trigger) - execute("DROP TRIGGER IF EXISTS #{trigger}_insert") - execute("DROP TRIGGER IF EXISTS #{trigger}_update") - end - # Returns the (base) name to use for triggers when renaming columns. def rename_trigger_name(table, old, new) 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12) @@ -882,8 +812,6 @@ module Gitlab order: index.orders } - # These options are not supported by MySQL, so we only add them if - # they were previously set. options[:using] = index.using if index.using options[:where] = index.where if index.where @@ -923,26 +851,16 @@ module Gitlab end # This will replace the first occurrence of a string in a column with - # the replacement - # On postgresql we can use `regexp_replace` for that. - # On mysql we find the location of the pattern, and overwrite it - # with the replacement + # the replacement using `regexp_replace` def replace_sql(column, pattern, replacement) quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s) quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) - if Database.mysql? - locate = Arel::Nodes::NamedFunction - .new('locate', [quoted_pattern, column]) - insert_in_place = Arel::Nodes::NamedFunction - .new('insert', [column, locate, pattern.size, quoted_replacement]) + replace = Arel::Nodes::NamedFunction.new( + "regexp_replace", [column, quoted_pattern, quoted_replacement] + ) - Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) - else - replace = Arel::Nodes::NamedFunction - .new("regexp_replace", [column, quoted_pattern, quoted_replacement]) - Arel::Nodes::SqlLiteral.new(replace.to_sql) - end + Arel::Nodes::SqlLiteral.new(replace.to_sql) end def remove_foreign_key_if_exists(*args) @@ -984,11 +902,7 @@ database (#{dbname}) using a super user and running: ALTER #{user} WITH SUPERUSER -For MySQL you instead need to run: - - GRANT ALL PRIVILEGES ON #{dbname}.* TO #{user}@'%' - -Both queries will grant the user super user permissions, ensuring you don't run +This query will grant the user super user permissions, ensuring you don't run into similar problems in the future (e.g. when new tables are created). EOF end @@ -1091,10 +1005,6 @@ into similar problems in the future (e.g. when new tables are created). # This will include indexes using an expression on the column, for example: # `CREATE INDEX CONCURRENTLY index_name ON table (LOWER(column));` # - # For mysql, it falls back to the default ActiveRecord implementation that - # will not find custom indexes. But it will select by name without passing - # a column. - # # We can remove this when upgrading to Rails 5 with an updated `index_exists?`: # - https://github.com/rails/rails/commit/edc2b7718725016e988089b5fb6d6fb9d6e16882 # @@ -1105,10 +1015,8 @@ into similar problems in the future (e.g. when new tables are created). # does not find indexes without passing a column name. if indexes(table).map(&:name).include?(index.to_s) true - elsif Gitlab::Database.postgresql? - postgres_exists_by_name?(table, index) else - false + postgres_exists_by_name?(table, index) end end @@ -1124,10 +1032,6 @@ into similar problems in the future (e.g. when new tables are created). connection.select_value(index_sql).to_i > 0 end - def mysql_compatible_index_length - Gitlab::Database.mysql? ? 20 : nil - end - private def validate_timestamp_column_name!(column_name) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 60afa4bcd52..565f34b78b7 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -51,14 +51,10 @@ module Gitlab quoted_old_full_path = quote_string(old_full_path) quoted_old_wildcard_path = quote_string("#{old_full_path}/%") - filter = if Database.mysql? - "lower(routes.path) = lower('#{quoted_old_full_path}') "\ - "OR routes.path LIKE '#{quoted_old_wildcard_path}'" - else - "routes.id IN "\ - "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ - "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" - end + filter = + "routes.id IN "\ + "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ + "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" replace_statement = replace_sql(Route.arel_table[:path], old_full_path, diff --git a/lib/gitlab/database/subquery.rb b/lib/gitlab/database/subquery.rb index 10971d2b274..2a6f39c6a27 100644 --- a/lib/gitlab/database/subquery.rb +++ b/lib/gitlab/database/subquery.rb @@ -6,11 +6,7 @@ module Gitlab class << self def self_join(relation) t = relation.arel_table - # Work around a bug in Rails 5, where LIMIT causes trouble - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/51729 - r = relation.limit(nil).arel - r.take(relation.limit_value) if relation.limit_value - t2 = r.as('t2') + t2 = relation.arel.as('t2') relation.unscoped.joins(t.join(t2).on(t[:id].eq(t2[:id])).join_sources.first) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 6e8aa5d578e..27032602828 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -55,6 +55,10 @@ module Gitlab @name = @relative_path.split("/").last end + def to_s + "<#{self.class.name}: #{self.gl_project_path}>" + end + def ==(other) other.is_a?(self.class) && [storage, relative_path] == [other.storage, other.relative_path] end diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb index 9aea736527b..5c73c0c66a9 100644 --- a/lib/gitlab/git/rugged_impl/blob.rb +++ b/lib/gitlab/git/rugged_impl/blob.rb @@ -16,7 +16,7 @@ module Gitlab override :tree_entry def tree_entry(repository, sha, path, limit) if use_rugged?(repository, :rugged_tree_entry) - wrap_rugged_call { rugged_tree_entry(repository, sha, path, limit) } + execute_rugged_call(:rugged_tree_entry, repository, sha, path, limit) else super end diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb index 29ae9bdd851..0eff35ab1c4 100644 --- a/lib/gitlab/git/rugged_impl/commit.rb +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -36,7 +36,7 @@ module Gitlab override :find_commit def find_commit(repo, commit_id) if use_rugged?(repo, :rugged_find_commit) - wrap_rugged_call { rugged_find(repo, commit_id) } + execute_rugged_call(:rugged_find, repo, commit_id) else super end @@ -45,7 +45,7 @@ module Gitlab override :batch_by_oid def batch_by_oid(repo, oids) if use_rugged?(repo, :rugged_list_commits_by_oid) - wrap_rugged_call { rugged_batch_by_oid(repo, oids) } + execute_rugged_call(:rugged_batch_by_oid, repo, oids) else super end @@ -68,7 +68,7 @@ module Gitlab override :commit_tree_entry def commit_tree_entry(path) if use_rugged?(@repository, :rugged_commit_tree_entry) - wrap_rugged_call { rugged_tree_entry(path) } + execute_rugged_call(:rugged_tree_entry, path) else super end diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb index 7bed553393c..8fde93e71e2 100644 --- a/lib/gitlab/git/rugged_impl/repository.rb +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -48,7 +48,7 @@ module Gitlab override :ancestor? def ancestor?(from, to) if use_rugged?(self, :rugged_commit_is_ancestor) - wrap_rugged_call { rugged_is_ancestor?(from, to) } + execute_rugged_call(:rugged_is_ancestor?, from, to) else super end diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index 479c5f9d8b7..389c9d32ccb 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -16,7 +16,7 @@ module Gitlab override :tree_entries def tree_entries(repository, sha, path, recursive) if use_rugged?(repository, :rugged_tree_entries) - wrap_rugged_call { tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive) } + execute_rugged_call(:tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive) else super end diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb index badf943e39c..80b75689334 100644 --- a/lib/gitlab/git/rugged_impl/use_rugged.rb +++ b/lib/gitlab/git/rugged_impl/use_rugged.rb @@ -11,17 +11,23 @@ module Gitlab Gitlab::GitalyClient.can_use_disk?(repo.storage) end - def wrap_rugged_call(&block) + def execute_rugged_call(method_name, *args) Gitlab::GitalyClient::StorageSettings.allow_disk_access do start = Gitlab::Metrics::System.monotonic_time - result = yield + result = send(method_name, *args) # rubocop:disable GitlabSecurity/PublicSend duration = Gitlab::Metrics::System.monotonic_time - start if Gitlab::RuggedInstrumentation.active? Gitlab::RuggedInstrumentation.increment_query_count Gitlab::RuggedInstrumentation.query_time += duration + + Gitlab::RuggedInstrumentation.add_call_details( + feature: method_name, + args: args, + duration: duration, + backtrace: Gitlab::Profiler.clean_backtrace(caller)) end result diff --git a/lib/gitlab/git_logger.rb b/lib/gitlab/git_logger.rb index dac4ddd320f..ded5349be01 100644 --- a/lib/gitlab/git_logger.rb +++ b/lib/gitlab/git_logger.rb @@ -1,13 +1,9 @@ # frozen_string_literal: true module Gitlab - class GitLogger < Gitlab::Logger + class GitLogger < JsonLogger def self.file_name_noext 'githost' end - - def format_message(severity, timestamp, progname, msg) - "#{timestamp.to_s(:long)} -> #{severity} -> #{msg}\n" - end end end diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb index 5b3f30d894a..aaade39dd62 100644 --- a/lib/gitlab/import/database_helpers.rb +++ b/lib/gitlab/import/database_helpers.rb @@ -6,9 +6,7 @@ module Gitlab # Inserts a raw row and returns the ID of the inserted row. # # attributes - The attributes/columns to set. - # relation - An ActiveRecord::Relation to use for finding the ID of the row - # when using MySQL. - # rubocop: disable CodeReuse/ActiveRecord + # relation - An ActiveRecord::Relation to use for finding the table name def insert_and_return_id(attributes, relation) # We use bulk_insert here so we can bypass any queries executed by # callbacks or validation rules, as doing this wouldn't scale when @@ -16,12 +14,8 @@ module Gitlab result = Gitlab::Database .bulk_insert(relation.table_name, [attributes], return_ids: true) - # MySQL doesn't support returning the IDs of a bulk insert in a way that - # is not a pain, so in this case we'll issue an extra query instead. - result.first || - relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first + result.first end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index f63a5ece71e..bb46bd657e8 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -4,7 +4,9 @@ module Gitlab module ImportExport extend self - # For every version update, the version history in import_export.md has to be kept up to date. + # For every version update the version history in these docs must be kept up to date: + # - development/import_export.md + # - user/project/settings/import_export.md VERSION = '0.2.4'.freeze FILENAME_LIMIT = 50 @@ -28,6 +30,14 @@ module Gitlab "project.bundle" end + def lfs_objects_filename + "lfs-objects.json" + end + + def lfs_objects_storage + "lfs-objects" + end + def config_file Rails.root.join('lib/gitlab/import_export/import_export.yml') end diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 409243e68a5..42cd94add79 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -45,7 +45,7 @@ module Gitlab end def key_from_hash(value) - value.is_a?(Hash) ? value.keys.first : value + value.is_a?(Hash) ? value.first.first : value end end end diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index b145f37c052..a92e3862361 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -27,7 +27,7 @@ module Gitlab # {:merge_requests=>[:merge_request_diff, :notes]} def process_model_objects(model_object_hash) json_config_hash = {} - current_key = model_object_hash.keys.first + current_key = model_object_hash.first.first model_object_hash.values.flatten.each do |model_object| @attributes_finder.parse(current_key) { |hash| json_config_hash[current_key] ||= hash } diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index 345c7880e30..1de8a5bf9ec 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -3,6 +3,10 @@ module Gitlab module ImportExport class LfsRestorer + include Gitlab::Utils::StrongMemoize + + attr_accessor :project, :shared + def initialize(project:, shared:) @project = project @shared = shared @@ -17,7 +21,7 @@ module Gitlab true rescue => e - @shared.error(e) + shared.error(e) false end @@ -29,16 +33,57 @@ module Gitlab lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size) lfs_object.file = File.open(path) unless lfs_object.file&.exists? + lfs_object.save! if lfs_object.changed? - @project.all_lfs_objects << lfs_object + repository_types(oid).each do |repository_type| + LfsObjectsProject.create!( + project: project, + lfs_object: lfs_object, + repository_type: repository_type + ) + end + end + + def repository_types(oid) + # We allow support for imports created before the `lfs-objects.json` + # file was generated. In this case, the restorer will link an LFS object + # with a single `lfs_objects_projects` relation. + # + # This allows us backwards-compatibility without version bumping. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30830#note_192608870 + return ['project'] unless has_lfs_json? + + lfs_json[oid] end def lfs_file_paths @lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*") end + def has_lfs_json? + strong_memoize(:has_lfs_json) do + File.exist?(lfs_json_path) + end + end + + def lfs_json + return {} unless has_lfs_json? + + @lfs_json ||= + begin + json = IO.read(lfs_json_path) + ActiveSupport::JSON.decode(json) + rescue + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + end + def lfs_storage_path - File.join(@shared.export_path, 'lfs-objects') + File.join(shared.export_path, ImportExport.lfs_objects_storage) + end + + def lfs_json_path + File.join(shared.export_path, ImportExport.lfs_objects_filename) end end end diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 954f6f00078..18c590e1ca9 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -5,25 +5,40 @@ module Gitlab class LfsSaver include Gitlab::ImportExport::CommandLineUtil + attr_accessor :lfs_json, :project, :shared + + BATCH_SIZE = 100 + def initialize(project:, shared:) @project = project @shared = shared + @lfs_json = {} end def save - @project.all_lfs_objects.each do |lfs_object| - save_lfs_object(lfs_object) + project.all_lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| + batch.each do |lfs_object| + save_lfs_object(lfs_object) + end + + append_lfs_json_for_batch(batch) if write_lfs_json_enabled? end + write_lfs_json if write_lfs_json_enabled? + true rescue => e - @shared.error(e) + shared.error(e) false end private + def write_lfs_json_enabled? + ::Feature.enabled?(:export_lfs_objects_projects, default_enabled: true) + end + def save_lfs_object(lfs_object) if lfs_object.local_store? copy_file_for_lfs_object(lfs_object) @@ -45,12 +60,36 @@ module Gitlab copy_files(lfs_object.file.path, destination_path_for_object(lfs_object)) end + def append_lfs_json_for_batch(lfs_objects_batch) + lfs_objects_projects = LfsObjectsProject + .select('lfs_objects.oid, array_agg(distinct lfs_objects_projects.repository_type) as repository_types') + .joins(:lfs_object) + .where(project: project, lfs_object: lfs_objects_batch) + .group('lfs_objects.oid') + + lfs_objects_projects.each do |group| + oid = group.oid + + lfs_json[oid] ||= [] + lfs_json[oid] += group.repository_types + end + end + + def write_lfs_json + mkdir_p(shared.export_path) + File.write(lfs_json_path, lfs_json.to_json) + end + def destination_path_for_object(lfs_object) File.join(lfs_export_path, lfs_object.oid) end def lfs_export_path - File.join(@shared.export_path, 'lfs-objects') + File.join(shared.export_path, ImportExport.lfs_objects_storage) + end + + def lfs_json_path + File.join(shared.export_path, ImportExport.lfs_objects_filename) end end end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index a154de5419e..4e976cfca3a 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -35,7 +35,7 @@ module Gitlab end def include?(old_author_id) - map.keys.include?(old_author_id) && map[old_author_id] != default_user_id + map.has_key?(old_author_id) && map[old_author_id] != default_user_id end private @@ -50,6 +50,8 @@ module Gitlab @project.project_members.destroy_all # rubocop: disable DestroyAll ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true) + rescue => e + raise e, "Error adding importer user to project members. #{e.message}" end def add_team_member(member, existing_user = nil) diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 1b545b1d049..0be49e27acb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -185,7 +185,7 @@ module Gitlab return unless EXISTING_OBJECT_CHECK.include?(@relation_name) return unless @relation_hash['group_id'] - @relation_hash['group_id'] = @project.group&.id + @relation_hash['group_id'] = @project.namespace_id end def reset_tokens! diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 79f756c8f8a..1e200db0baf 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -30,18 +30,18 @@ module Gitlab def init_metrics metrics = { - file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), - memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum), + file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels), + memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels), process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), - process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels), process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) } GC.stat.keys.each do |key| - metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) + metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels) end metrics diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 38b32770e90..c06f106ffe1 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -32,11 +32,6 @@ module Gitlab # Returns the maximum depth starting from the base # A base object with no children has a maximum depth of `1` def max_descendants_depth - unless hierarchy_supported? - # This makes the return value consistent with the case where hierarchy is supported - return descendants_base.exists? ? 1 : nil - end - base_and_descendants(with_depth: true).maximum(DEPTH_COLUMN) end @@ -66,8 +61,6 @@ module Gitlab # each parent. # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors(upto: nil, hierarchy_order: nil) - return ancestors_base unless hierarchy_supported? - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order @@ -81,10 +74,6 @@ module Gitlab # When `with_depth` is `true`, a `depth` column is included where it starts with `1` for the base objects # and incremented as we go down the descendant tree def base_and_descendants(with_depth: false) - unless hierarchy_supported? - return with_depth ? descendants_base.select("1 as #{DEPTH_COLUMN}", objects_table[Arel.star]) : descendants_base - end - read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all)) end @@ -112,8 +101,6 @@ module Gitlab # If nested objects are not supported, ancestors_base is returned. # rubocop: disable CodeReuse/ActiveRecord def all_objects - return ancestors_base unless hierarchy_supported? - ancestors = base_and_ancestors_cte descendants = base_and_descendants_cte @@ -135,10 +122,6 @@ module Gitlab private - def hierarchy_supported? - Gitlab::Database.postgresql? - end - # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors_cte(stop_id = nil, hierarchy_order = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index a13b3f9e069..f96466b2b00 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -175,6 +175,10 @@ module Gitlab @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze end + def project_wiki_git_route_regex + @project_wiki_git_route_regex ||= /#{PATH_REGEX_STR}\.wiki/.freeze + end + def full_namespace_path_regex @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z} end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 890228e5e78..615c0ec374c 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -166,7 +166,7 @@ module Gitlab [model, times.count, times.sum] end - summarised_load_times.sort_by(&:last).reverse.each do |(model, query_count, time)| + summarised_load_times.sort_by(&:last).reverse_each do |(model, query_count, time)| logger.info("#{model} total (#{query_count}): #{time.round(2)}ms") end end diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb new file mode 100644 index 00000000000..a9270cd536e --- /dev/null +++ b/lib/gitlab/project_authorizations.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +# This class relies on Common Table Expressions to efficiently get all data, +# including data for nested groups. +module Gitlab + class ProjectAuthorizations + attr_reader :user + + # user - The User object for which to calculate the authorizations. + def initialize(user) + @user = user + end + + def calculate + cte = recursive_cte + cte_alias = cte.table.alias(Group.table_name) + projects = Project.arel_table + links = ProjectGroupLink.arel_table + + relations = [ + # The project a user has direct access to. + user.projects.select_for_project_authorization, + + # The personal projects of the user. + user.personal_projects.select_as_maintainer_for_project_authorization, + + # Projects that belong directly to any of the groups the user has + # access to. + Namespace + .unscoped + .select([alias_as_column(projects[:id], 'project_id'), + cte_alias[:access_level]]) + .from(cte_alias) + .joins(:projects), + + # Projects shared with any of the namespaces the user has access to. + Namespace + .unscoped + .select([ + links[:project_id], + least(cte_alias[:access_level], links[:group_access], 'access_level') + ]) + .from(cte_alias) + .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') + .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') + .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') + .where('p_ns.share_with_group_lock IS FALSE') + ] + + ProjectAuthorization + .unscoped + .with + .recursive(cte.to_arel) + .select_from_union(relations) + end + + private + + # Builds a recursive CTE that gets all the groups the current user has + # access to, including any nested groups. + def recursive_cte + cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) + members = Member.arel_table + namespaces = Namespace.arel_table + + # Namespaces the user is a member of. + cte << user.groups + .select([namespaces[:id], members[:access_level]]) + .except(:order) + + # Sub groups of any groups the user is a member of. + cte << Group.select([ + namespaces[:id], + greatest(members[:access_level], cte.table[:access_level], 'access_level') + ]) + .joins(join_cte(cte)) + .joins(join_members) + .except(:order) + + cte + end + + # Builds a LEFT JOIN to join optional memberships onto the CTE. + def join_members + members = Member.arel_table + namespaces = Namespace.arel_table + + cond = members[:source_id] + .eq(namespaces[:id]) + .and(members[:source_type].eq('Namespace')) + .and(members[:requested_at].eq(nil)) + .and(members[:user_id].eq(user.id)) + + Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) + end + + # Builds an INNER JOIN to join namespaces onto the CTE. + def join_cte(cte) + namespaces = Namespace.arel_table + cond = cte.table[:id].eq(namespaces[:parent_id]) + + Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) + end + + def greatest(left, right, column_alias) + sql_function('GREATEST', [left, right], column_alias) + end + + def least(left, right, column_alias) + sql_function('LEAST', [left, right], column_alias) + end + + def sql_function(name, args, column_alias) + alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) + end + + def alias_as_column(value, alias_to) + Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) + end + end +end diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb deleted file mode 100644 index 2372a316ab0..00000000000 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ /dev/null @@ -1,125 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ProjectAuthorizations - # Calculating new project authorizations when supporting nested groups. - # - # This class relies on Common Table Expressions to efficiently get all data, - # including data for nested groups. As a result this class can only be used - # on PostgreSQL. - class WithNestedGroups - attr_reader :user - - # user - The User object for which to calculate the authorizations. - def initialize(user) - @user = user - end - - def calculate - cte = recursive_cte - cte_alias = cte.table.alias(Group.table_name) - projects = Project.arel_table - links = ProjectGroupLink.arel_table - - relations = [ - # The project a user has direct access to. - user.projects.select_for_project_authorization, - - # The personal projects of the user. - user.personal_projects.select_as_maintainer_for_project_authorization, - - # Projects that belong directly to any of the groups the user has - # access to. - Namespace - .unscoped - .select([alias_as_column(projects[:id], 'project_id'), - cte_alias[:access_level]]) - .from(cte_alias) - .joins(:projects), - - # Projects shared with any of the namespaces the user has access to. - Namespace - .unscoped - .select([links[:project_id], - least(cte_alias[:access_level], - links[:group_access], - 'access_level')]) - .from(cte_alias) - .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') - .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') - .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') - .where('p_ns.share_with_group_lock IS FALSE') - ] - - ProjectAuthorization - .unscoped - .with - .recursive(cte.to_arel) - .select_from_union(relations) - end - - private - - # Builds a recursive CTE that gets all the groups the current user has - # access to, including any nested groups. - def recursive_cte - cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) - members = Member.arel_table - namespaces = Namespace.arel_table - - # Namespaces the user is a member of. - cte << user.groups - .select([namespaces[:id], members[:access_level]]) - .except(:order) - - # Sub groups of any groups the user is a member of. - cte << Group.select([namespaces[:id], - greatest(members[:access_level], - cte.table[:access_level], 'access_level')]) - .joins(join_cte(cte)) - .joins(join_members) - .except(:order) - - cte - end - - # Builds a LEFT JOIN to join optional memberships onto the CTE. - def join_members - members = Member.arel_table - namespaces = Namespace.arel_table - - cond = members[:source_id] - .eq(namespaces[:id]) - .and(members[:source_type].eq('Namespace')) - .and(members[:requested_at].eq(nil)) - .and(members[:user_id].eq(user.id)) - - Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) - end - - # Builds an INNER JOIN to join namespaces onto the CTE. - def join_cte(cte) - namespaces = Namespace.arel_table - cond = cte.table[:id].eq(namespaces[:parent_id]) - - Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) - end - - def greatest(left, right, column_alias) - sql_function('GREATEST', [left, right], column_alias) - end - - def least(left, right, column_alias) - sql_function('LEAST', [left, right], column_alias) - end - - def sql_function(name, args, column_alias) - alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) - end - - def alias_as_column(value, alias_to) - Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) - end - end - end -end diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb deleted file mode 100644 index 50b41b17649..00000000000 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ProjectAuthorizations - # Calculating new project authorizations when not supporting nested groups. - class WithoutNestedGroups - attr_reader :user - - # user - The User object for which to calculate the authorizations. - def initialize(user) - @user = user - end - - def calculate - relations = [ - # Projects the user is a direct member of - user.projects.select_for_project_authorization, - - # Personal projects - user.personal_projects.select_as_maintainer_for_project_authorization, - - # Projects of groups the user is a member of - user.groups_projects.select_for_project_authorization, - - # Projects shared with groups the user is a member of - user.groups.joins(:shared_projects).select_for_project_authorization - ] - - ProjectAuthorization - .unscoped - .select_from_union(relations) - end - end - end -end diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 3137676ba4b..682edfc4259 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -4,7 +4,14 @@ module Gitlab class PushOptions VALID_OPTIONS = HashWithIndifferentAccess.new({ merge_request: { - keys: [:create, :merge_when_pipeline_succeeds, :target] + keys: [ + :create, + :description, + :merge_when_pipeline_succeeds, + :remove_source_branch, + :target, + :title + ] }, ci: { keys: [:skip] diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index e43147a3f37..21614ea003e 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -94,6 +94,12 @@ module Gitlab }mx end + # Based on Jira's project key format + # https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html + def jira_issue_key_regex + @jira_issue_key_regex ||= /[A-Z][A-Z_0-9]+-\d+/ + end + def jira_transition_id_regex @jira_transition_id_regex ||= /\d+/ end diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb index 64593153686..033e451dbee 100644 --- a/lib/gitlab/request_profiler.rb +++ b/lib/gitlab/request_profiler.rb @@ -6,6 +6,21 @@ module Gitlab module RequestProfiler PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles".freeze + def all + Dir["#{PROFILES_DIR}/*.{html,txt}"].map do |path| + Profile.new(File.basename(path)) + end.select(&:valid?) + end + module_function :all + + def find(name) + file_path = File.join(PROFILES_DIR, name) + return unless File.exist?(file_path) + + Profile.new(name) + end + module_function :find + def profile_token Rails.cache.fetch('profile-token') do Devise.friendly_token diff --git a/lib/gitlab/request_profiler/profile.rb b/lib/gitlab/request_profiler/profile.rb index 74f2ec1d083..76c675658b1 100644 --- a/lib/gitlab/request_profiler/profile.rb +++ b/lib/gitlab/request_profiler/profile.rb @@ -7,19 +7,6 @@ module Gitlab alias_method :to_param, :name - def self.all - Dir["#{PROFILES_DIR}/*.{html,txt}"].map do |path| - new(File.basename(path)) - end - end - - def self.find(name) - file_path = File.join(PROFILES_DIR, name) - return unless File.exist?(file_path) - - new(name) - end - def initialize(name) @name = name @file_path = File.join(PROFILES_DIR, name) @@ -27,8 +14,8 @@ module Gitlab set_attributes end - def content - File.read("#{PROFILES_DIR}/#{name}") + def valid? + @request_path.present? end def content_type @@ -43,11 +30,13 @@ module Gitlab private def set_attributes - _, path, timestamp, profile_mode, type = name.split(/(.*)_(\d+)_(.*)\.(html|txt)$/) - @request_path = path.tr('|', '/') - @time = Time.at(timestamp.to_i).utc - @profile_mode = profile_mode - @type = type + matches = name.match(/^(?<path>.*)_(?<timestamp>\d+)(_(?<profile_mode>\w+))?\.(?<type>html|txt)$/) + return unless matches + + @request_path = matches[:path].tr('|', '/') + @time = Time.at(matches[:timestamp].to_i).utc + @profile_mode = matches[:profile_mode] || 'unknown' + @type = matches[:type] end end end diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb index 70c06e8b308..8bb8c547ae1 100644 --- a/lib/gitlab/rugged_instrumentation.rb +++ b/lib/gitlab/rugged_instrumentation.rb @@ -24,7 +24,24 @@ module Gitlab end def self.active? - Gitlab::SafeRequestStore.active? + SafeRequestStore.active? + end + + def self.peek_enabled? + SafeRequestStore[:peek_enabled] + end + + def self.add_call_details(details) + return unless peek_enabled? + + Gitlab::SafeRequestStore[:rugged_call_details] ||= [] + Gitlab::SafeRequestStore[:rugged_call_details] << details + end + + def self.list_call_details + return [] unless peek_enabled? + + Gitlab::SafeRequestStore[:rugged_call_details] || [] end end end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 671d795ec33..4b10f921ed8 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -14,9 +14,12 @@ module Gitlab # shut Sidekiq down MUTEX = Mutex.new + attr_reader :worker + def call(worker, job, queue) yield + @worker = worker current_rss = get_rss return unless MAX_RSS > 0 && current_rss > MAX_RSS @@ -25,9 +28,11 @@ module Gitlab # Return if another thread is already waiting to shut Sidekiq down next unless MUTEX.try_lock - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ - " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + warn("Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ + " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}") + + warn("Sidekiq worker PID-#{pid} will stop fetching new jobs"\ + " in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later") # Wait `GRACE_TIME` to give the memory intensive job time to finish. # Then, tell Sidekiq to stop fetching new jobs. @@ -59,24 +64,28 @@ module Gitlab def wait_and_signal_pgroup(time, signal, explanation) return wait_and_signal(time, signal, explanation) unless Process.getpgrp == pid - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + warn("waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) sleep(time) - Sidekiq.logger.warn "sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + warn("sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) Process.kill(signal, 0) end def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + warn("waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) sleep(time) - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + warn("sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) Process.kill(signal, pid) end def pid Process.pid end + + def warn(message, signal: nil) + Sidekiq.logger.warn(class: worker.class, pid: pid, signal: signal, message: message) + end end end end diff --git a/lib/gitlab/submodule_links.rb b/lib/gitlab/submodule_links.rb index a6c0369d864..18fd604a3b0 100644 --- a/lib/gitlab/submodule_links.rb +++ b/lib/gitlab/submodule_links.rb @@ -9,7 +9,7 @@ module Gitlab end def for(submodule, sha) - submodule_url = submodule_url_for(sha)[submodule.path] + submodule_url = submodule_url_for(sha, submodule.path) SubmoduleHelper.submodule_links_for_url(submodule.id, submodule_url, repository) end @@ -17,10 +17,15 @@ module Gitlab attr_reader :repository - def submodule_url_for(sha) - strong_memoize(:"submodule_links_for_#{sha}") do + def submodule_urls_for(sha) + strong_memoize(:"submodule_urls_for_#{sha}") do repository.submodule_urls_for(sha) end end + + def submodule_url_for(sha, path) + urls = submodule_urls_for(sha) + urls && urls[path] + end end end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index f6b2e2acf16..eab6762cab7 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -45,18 +45,21 @@ module Gitlab ascii_only: ascii_only ) + normalized_hostname = uri.normalized_host hostname = uri.hostname port = get_port(uri) address_info = get_address_info(hostname, port) return [uri, nil] unless address_info - protected_uri_with_hostname = enforce_uri_hostname(address_info, uri, hostname, dns_rebind_protection) + ip_address = ip_address(address_info) + protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, hostname, dns_rebind_protection) # Allow url from the GitLab instance itself but only for the configured hostname and ports return protected_uri_with_hostname if internal?(uri) validate_local_request( + normalized_hostname: normalized_hostname, address_info: address_info, allow_localhost: allow_localhost, allow_local_network: allow_local_network @@ -83,10 +86,7 @@ module Gitlab # # The original hostname is used to validate the SSL, given in that scenario # we'll be making the request to the IP address, instead of using the hostname. - def enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) - address = addrs_info.first - ip_address = address&.ip_address - + def enforce_uri_hostname(ip_address, uri, hostname, dns_rebind_protection) return [uri, nil] unless dns_rebind_protection && ip_address && ip_address != hostname uri = uri.dup @@ -94,6 +94,10 @@ module Gitlab [uri, hostname] end + def ip_address(address_info) + address_info.first&.ip_address + end + def validate_uri(uri:, schemes:, ports:, enforce_sanitization:, enforce_user:, ascii_only:) validate_html_tags(uri) if enforce_sanitization @@ -113,9 +117,19 @@ module Gitlab rescue SocketError end - def validate_local_request(address_info:, allow_localhost:, allow_local_network:) + def validate_local_request( + normalized_hostname:, + address_info:, + allow_localhost:, + allow_local_network:) return if allow_local_network && allow_localhost + ip_whitelist, domain_whitelist = + Gitlab::CurrentSettings.outbound_local_requests_whitelist_arrays + + return if local_domain_whitelisted?(domain_whitelist, normalized_hostname) || + local_ip_whitelisted?(ip_whitelist, ip_address(address_info)) + unless allow_localhost validate_localhost(address_info) validate_loopback(address_info) @@ -231,6 +245,16 @@ module Gitlab (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port) end + def local_ip_whitelisted?(ip_whitelist, ip_string) + ip_obj = Gitlab::Utils.string_to_ip_object(ip_string) + + ip_whitelist.any? { |ip| ip.include?(ip_obj) } + end + + def local_domain_whitelisted?(domain_whitelist, domain_string) + domain_whitelist.include?(domain_string) + end + def config Gitlab.config end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 16ec8a8bb28..c66ce0434a4 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -22,7 +22,7 @@ module Gitlab end def force_utf8(str) - str.force_encoding(Encoding::UTF_8) + str.dup.force_encoding(Encoding::UTF_8) end def ensure_utf8_size(str, bytes:) @@ -131,5 +131,12 @@ module Gitlab data end end + + def string_to_ip_object(str) + return unless str + + IPAddr.new(str) + rescue IPAddr::InvalidAddressError + end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 46a7b5b982a..3b77fe838ae 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -221,7 +221,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis::Queues.with do |redis| + Gitlab::Redis::SharedState.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") |