diff options
Diffstat (limited to 'lib')
32 files changed, 493 insertions, 118 deletions
diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 0ff8fa74a84..979328efe0e 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -142,7 +142,7 @@ module API return not_found!(build) unless build return forbidden!('Build is not retryable') unless build.retryable? - build = Ci::Build.retry(build) + build = Ci::Build.retry(build, current_user) present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :read_build, user_project) @@ -166,6 +166,26 @@ module API present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end + + # Keep the artifacts to prevent them from being deleted + # + # Parameters: + # id (required) - the id of a project + # build_id (required) - The ID of a build + # Example Request: + # POST /projects/:id/builds/:build_id/artifacts/keep + post ':id/builds/:build_id/artifacts/keep' do + authorize_update_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build && build.artifacts? + + build.keep_artifacts! + + status 200 + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end end helpers do diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 14370ac218d..cc29c7ef428 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -88,10 +88,7 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility_level expose :avatar_url - - expose :web_url do |group, options| - Gitlab::Routing.url_helpers.group_url(group) - end + expose :web_url end class GroupDetail < Group diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb index 4aefdf319c6..b703da0557a 100644 --- a/lib/api/project_members.rb +++ b/lib/api/project_members.rb @@ -46,7 +46,7 @@ module API required_attributes! [:user_id, :access_level] # either the user is already a team member or a new one - project_member = user_project.project_member_by_id(params[:user_id]) + project_member = user_project.project_member(params[:user_id]) if project_member.nil? project_member = user_project.project_members.new( user_id: params[:user_id], diff --git a/lib/api/session.rb b/lib/api/session.rb index 56e69b2366f..56c202f1294 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -11,7 +11,7 @@ module API # Example Request: # POST /session post "/session" do - user = Gitlab::Auth.find_in_gitlab_or_ldap(params[:email] || params[:login], params[:password]) + user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password]) return unauthorized! unless user present user, with: Entities::UserLogin diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index c0f503c9af3..45bb66dc99f 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -10,11 +10,11 @@ module Banzai def call return doc unless project - doc.search('a').each do |el| + doc.xpath('descendant-or-self::a[starts-with(@href, "/uploads/")]').each do |el| process_link_attr el.attribute('href') end - doc.search('img').each do |el| + doc.xpath('descendant-or-self::img[starts-with(@src, "/uploads/")]').each do |el| process_link_attr el.attribute('src') end @@ -24,12 +24,7 @@ module Banzai protected def process_link_attr(html_attr) - return if html_attr.blank? - - uri = html_attr.value - if uri.starts_with?("/uploads/") - html_attr.value = build_url(uri).to_s - end + html_attr.value = build_url(html_attr.value).to_s end def build_url(uri) diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb index f2395867658..042fb2e6e14 100644 --- a/lib/banzai/pipeline/description_pipeline.rb +++ b/lib/banzai/pipeline/description_pipeline.rb @@ -1,23 +1,16 @@ module Banzai module Pipeline class DescriptionPipeline < FullPipeline + WHITELIST = Banzai::Filter::SanitizationFilter::LIMITED.deep_dup.merge( + elements: Banzai::Filter::SanitizationFilter::LIMITED[:elements] - %w(pre code img ol ul li) + ) + def self.transform_context(context) super(context).merge( # SanitizationFilter - whitelist: whitelist + whitelist: WHITELIST ) end - - private - - def self.whitelist - # Descriptions are more heavily sanitized, allowing only a few elements. - # See http://git.io/vkuAN - whitelist = Banzai::Filter::SanitizationFilter::LIMITED - whitelist[:elements] -= %w(pre code img ol ul li) - - whitelist - end end end end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 24076e3d9ec..f306079d833 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -25,7 +25,21 @@ module Banzai def issues_for_nodes(nodes) @issues_for_nodes ||= grouped_objects_for_nodes( nodes, - Issue.all.includes(:author, :assignee, :project), + Issue.all.includes( + :author, + :assignee, + { + # These associations are primarily used for checking permissions. + # Eager loading these ensures we don't end up running dozens of + # queries in this process. + project: [ + { namespace: :owner }, + { group: [:owners, :group_members] }, + :invited_groups, + :project_members + ] + } + ), self.class.data_attribute ) end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 607359769d1..9f270f7b387 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -114,6 +114,7 @@ module Ci # id (required) - The ID of a build # token (required) - The build authorization token # file (required) - Artifacts file + # expire_in (optional) - Specify when artifacts should expire (ex. 7d) # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition @@ -145,6 +146,7 @@ module Ci build.artifacts_file = artifacts build.artifacts_metadata = metadata + build.artifacts_expire_in = params['expire_in'] if build.save present(build, with: Entities::BuildDetails) diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index a902ced35d7..3f5bdaba3f5 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -20,7 +20,7 @@ module Ci expose :name, :token, :stage expose :project_id expose :project_name - expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts? } + expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } end class BuildDetails < Build @@ -29,6 +29,7 @@ module Ci expose :before_sha expose :allow_git_fetch expose :token + expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? } expose :options do |model| model.options diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 130f5b0892e..68246497e90 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -2,17 +2,24 @@ module Ci class GitlabCiYamlProcessor class ValidationError < StandardError; end + include Gitlab::Ci::Config::Node::ValidationHelpers + DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache] ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache, - :dependencies, :before_script, :after_script, :variables] + :dependencies, :before_script, :after_script, :variables, + :environment] + ALLOWED_CACHE_KEYS = [:key, :untracked, :paths] + ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in] - attr_reader :before_script, :after_script, :image, :services, :path, :cache + attr_reader :after_script, :image, :services, :path, :cache def initialize(config, path = nil) - @config = Gitlab::Ci::Config.new(config).to_hash + @ci_config = Gitlab::Ci::Config.new(config) + @config = @ci_config.to_hash + @path = path initial_parsing @@ -50,7 +57,6 @@ module Ci private def initial_parsing - @before_script = @config[:before_script] || [] @after_script = @config[:after_script] @image = @config[:image] @services = @config[:services] @@ -78,13 +84,14 @@ module Ci { stage_idx: stages.index(job[:stage]), stage: job[:stage], - commands: [job[:before_script] || @before_script, job[:script]].flatten.join("\n"), + commands: [job[:before_script] || [@ci_config.before_script], job[:script]].flatten.compact.join("\n"), tag_list: job[:tags] || [], name: name, only: job[:only], except: job[:except], allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', + environment: job[:environment], options: { image: job[:image] || @image, services: job[:services] || @services, @@ -97,6 +104,10 @@ module Ci end def validate! + unless @ci_config.valid? + raise ValidationError, @ci_config.errors.first + end + validate_global! @jobs.each do |name, job| @@ -107,10 +118,6 @@ module Ci end def validate_global! - unless validate_array_of_strings(@before_script) - raise ValidationError, "before_script should be an array of strings" - end - unless @after_script.nil? || validate_array_of_strings(@after_script) raise ValidationError, "after_script should be an array of strings" end @@ -135,6 +142,12 @@ module Ci end def validate_global_cache! + @cache.keys.each do |key| + unless ALLOWED_CACHE_KEYS.include? key + raise ValidationError, "#{name} cache unknown parameter #{key}" + end + end + if @cache[:key] && !validate_string(@cache[:key]) raise ValidationError, "cache:key parameter should be a string" end @@ -200,9 +213,13 @@ module Ci raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" end - if job[:when] && !job[:when].in?(%w(on_success on_failure always)) + if job[:when] && !job[:when].in?(%w[on_success on_failure always]) raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" end + + if job[:environment] && !validate_environment(job[:environment]) + raise ValidationError, "#{name} job: environment parameter #{Gitlab::Regex.environment_name_regex_message}" + end end def validate_job_script!(name, job) @@ -233,6 +250,12 @@ module Ci end def validate_job_cache!(name, job) + job[:cache].keys.each do |key| + unless ALLOWED_CACHE_KEYS.include? key + raise ValidationError, "#{name} job: cache unknown parameter #{key}" + end + end + if job[:cache][:key] && !validate_string(job[:cache][:key]) raise ValidationError, "#{name} job: cache:key parameter should be a string" end @@ -247,6 +270,12 @@ module Ci end def validate_job_artifacts!(name, job) + job[:artifacts].keys.each do |key| + unless ALLOWED_ARTIFACTS_KEYS.include? key + raise ValidationError, "#{name} job: artifacts unknown parameter #{key}" + end + end + if job[:artifacts][:name] && !validate_string(job[:artifacts][:name]) raise ValidationError, "#{name} job: artifacts:name parameter should be a string" end @@ -258,6 +287,14 @@ module Ci if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths]) raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings" end + + if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always]) + raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always" + end + + if job[:artifacts][:expire_in] && !validate_duration(job[:artifacts][:expire_in]) + raise ValidationError, "#{name} job: artifacts:expire_in parameter should be a duration" + end end def validate_job_dependencies!(name, job) @@ -276,22 +313,6 @@ module Ci end end - def validate_array_of_strings(values) - values.is_a?(Array) && values.all? { |value| validate_string(value) } - end - - def validate_variables(variables) - variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) } - end - - def validate_string(value) - value.is_a?(String) || value.is_a?(Symbol) - end - - def validate_boolean(value) - value.in?([true, false]) - end - def process?(only_params, except_params, ref, tag, trigger_request) if only_params.present? return false unless matching?(only_params, ref, tag, trigger_request) diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb index 4e20dc4f875..eb5a2596177 100644 --- a/lib/container_registry/blob.rb +++ b/lib/container_registry/blob.rb @@ -18,7 +18,7 @@ module ContainerRegistry end def digest - config['digest'] + config['digest'] || config['blobSum'] end def type diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 4d726692f45..e0b3f14d384 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -47,7 +47,9 @@ module ContainerRegistry conn.request :json conn.headers['Accept'] = MANIFEST_VERSION - conn.response :json, content_type: /\bjson$/ + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' if options[:user] && options[:password] conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 43f8d6dc8c2..7a0929d774e 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -12,6 +12,14 @@ module ContainerRegistry manifest.present? end + def v1? + manifest && manifest['schemaVersion'] == 1 + end + + def v2? + manifest && manifest['schemaVersion'] == 2 + end + def manifest return @manifest if defined?(@manifest) @@ -57,7 +65,9 @@ module ContainerRegistry return @layers if defined?(@layers) return unless manifest - @layers = manifest['layers'].map do |layer| + layers = manifest['layers'] || manifest['fsLayers'] + + @layers = layers.map do |layer| repository.blob(layer) end end @@ -65,7 +75,7 @@ module ContainerRegistry def total_size return unless layers - layers.map(&:size).sum + layers.map(&:size).sum if v2? end def delete diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 076e2af7d38..db1704af75e 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -3,14 +3,14 @@ module Gitlab Result = Struct.new(:user, :type) class << self - def find(login, password, project:, ip:) + def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? result = Result.new if valid_ci_request?(login, password, project) result.type = :ci - elsif result.user = find_in_gitlab_or_ldap(login, password) + elsif result.user = find_with_user_password(login, password) result.type = :gitlab_or_ldap elsif result.user = oauth_access_token_check(login, password) result.type = :oauth @@ -20,7 +20,7 @@ module Gitlab result end - def find_in_gitlab_or_ldap(login, password) + def find_with_user_password(login, password) user = User.by_login(login) # If no user is found, or it's an LDAP server, try LDAP. diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 9e09d2e118d..7e3f5abba62 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -1,5 +1,3 @@ -require_relative 'shell_env' - module Grack class AuthSpawner def self.call(env) @@ -61,11 +59,6 @@ module Grack end @user = authenticate_user(login, password) - - if @user - Gitlab::ShellEnv.set_env(@user) - @env['REMOTE_USER'] = @auth.username - end end def ci_request?(login, password) @@ -95,7 +88,7 @@ module Grack end def authenticate_user(login, password) - user = Gitlab::Auth.find_in_gitlab_or_ldap(login, password) + user = Gitlab::Auth.find_with_user_password(login, password) unless user user = oauth_access_token_check(login, password) diff --git a/lib/gitlab/backend/shell_env.rb b/lib/gitlab/backend/shell_env.rb deleted file mode 100644 index 9f5adee594a..00000000000 --- a/lib/gitlab/backend/shell_env.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Gitlab - # This module provide 2 methods - # to set specific ENV variables for GitLab Shell - module ShellEnv - extend self - - def set_env(user) - # Set GL_ID env variable - if user - ENV['GL_ID'] = gl_id(user) - end - end - - def reset_env - # Reset GL_ID env variable - ENV['GL_ID'] = nil - end - - def gl_id(user) - if user.present? - "user-#{user.id}" - else - # This empty string is used in the render_grack_auth_ok method - "" - end - end - end -end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index ffe633d4b63..b48d3592f16 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -1,11 +1,21 @@ module Gitlab module Ci + ## + # Base GitLab CI Configuration facade + # class Config - class LoaderError < StandardError; end + delegate :valid?, :errors, to: :@global + + ## + # Temporary delegations that should be removed after refactoring + # + delegate :before_script, to: :@global def initialize(config) - loader = Loader.new(config) - @config = loader.load! + @config = Loader.new(config).load! + + @global = Node::Global.new(@config) + @global.process! end def to_hash diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb new file mode 100644 index 00000000000..d60f87f3f94 --- /dev/null +++ b/lib/gitlab/ci/config/node/configurable.rb @@ -0,0 +1,61 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This mixin is responsible for adding DSL, which purpose is to + # simplifly process of adding child nodes. + # + # This can be used only if parent node is a configuration entry that + # holds a hash as a configuration value, for example: + # + # job: + # script: ... + # artifacts: ... + # + module Configurable + extend ActiveSupport::Concern + + def allowed_nodes + self.class.allowed_nodes || {} + end + + private + + def prevalidate! + unless @value.is_a?(Hash) + @errors << 'should be a configuration entry with hash value' + end + end + + def create_node(key, factory) + factory.with(value: @value[key]) + factory.nullify! unless @value.has_key?(key) + factory.create! + end + + class_methods do + def allowed_nodes + Hash[@allowed_nodes.map { |key, factory| [key, factory.dup] }] + end + + private + + def allow_node(symbol, entry_class, metadata) + factory = Node::Factory.new(entry_class) + .with(description: metadata[:description]) + + define_method(symbol) do + raise Entry::InvalidError unless valid? + + @nodes[symbol].try(:value) + end + + (@allowed_nodes ||= {}).merge!(symbol => factory) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb new file mode 100644 index 00000000000..52758a962f3 --- /dev/null +++ b/lib/gitlab/ci/config/node/entry.rb @@ -0,0 +1,77 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Base abstract class for each configuration entry node. + # + class Entry + class InvalidError < StandardError; end + + attr_accessor :description + + def initialize(value) + @value = value + @nodes = {} + @errors = [] + + prevalidate! + end + + def process! + return if leaf? + return unless valid? + + compose! + + nodes.each(&:process!) + nodes.each(&:validate!) + end + + def nodes + @nodes.values + end + + def valid? + errors.none? + end + + def leaf? + allowed_nodes.none? + end + + def errors + @errors + nodes.map(&:errors).flatten + end + + def allowed_nodes + {} + end + + def validate! + raise NotImplementedError + end + + def value + raise NotImplementedError + end + + private + + def prevalidate! + end + + def compose! + allowed_nodes.each do |key, essence| + @nodes[key] = create_node(key, essence) + end + end + + def create_node(key, essence) + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb new file mode 100644 index 00000000000..787ca006f5a --- /dev/null +++ b/lib/gitlab/ci/config/node/factory.rb @@ -0,0 +1,39 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Factory class responsible for fabricating node entry objects. + # + # It uses Fluent Interface pattern to set all necessary attributes. + # + class Factory + class InvalidFactory < StandardError; end + + def initialize(entry_class) + @entry_class = entry_class + @attributes = {} + end + + def with(attributes) + @attributes.merge!(attributes) + self + end + + def nullify! + @entry_class = Node::Null + self + end + + def create! + raise InvalidFactory unless @attributes.has_key?(:value) + + @entry_class.new(@attributes[:value]).tap do |entry| + entry.description = @attributes[:description] + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb new file mode 100644 index 00000000000..044603423d5 --- /dev/null +++ b/lib/gitlab/ci/config/node/global.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents a global entry - root node for entire + # GitLab CI Configuration file. + # + class Global < Entry + include Configurable + + allow_node :before_script, Script, + description: 'Script that will be executed before each job.' + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb new file mode 100644 index 00000000000..4f590f6bec8 --- /dev/null +++ b/lib/gitlab/ci/config/node/null.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents a configuration entry that is not being used + # in configuration file. + # + # This implements Null Object pattern. + # + class Null < Entry + def value + nil + end + + def validate! + nil + end + + def method_missing(*) + nil + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/node/script.rb new file mode 100644 index 00000000000..5072bf0db7d --- /dev/null +++ b/lib/gitlab/ci/config/node/script.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a script. + # + # Each element in the value array is a command that will be executed + # by GitLab Runner. Currently we concatenate these commands with + # new line character as a separator, what is compatible with + # implementation in Runner. + # + class Script < Entry + include ValidationHelpers + + def value + @value.join("\n") + end + + def validate! + unless validate_array_of_strings(@value) + @errors << 'before_script should be an array of strings' + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/validation_helpers.rb new file mode 100644 index 00000000000..3900fc89391 --- /dev/null +++ b/lib/gitlab/ci/config/node/validation_helpers.rb @@ -0,0 +1,38 @@ +module Gitlab + module Ci + class Config + module Node + module ValidationHelpers + private + + def validate_duration(value) + value.is_a?(String) && ChronicDuration.parse(value) + rescue ChronicDuration::DurationParseError + false + end + + def validate_array_of_strings(values) + values.is_a?(Array) && values.all? { |value| validate_string(value) } + end + + def validate_variables(variables) + variables.is_a?(Hash) && + variables.all? { |key, value| validate_string(key) && validate_string(value) } + end + + def validate_string(value) + value.is_a?(String) || value.is_a?(Symbol) + end + + def validate_environment(value) + value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex + end + + def validate_boolean(value) + value.in?([true, false]) + end + end + end + end + end +end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 04fa6a3a5de..d76ecb54017 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -30,6 +30,10 @@ module Gitlab order end + def self.random + Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()" + end + def true_value if Gitlab::Database.postgresql? "'t'" diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 978c3f7896d..dd3ff0ab18b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -31,8 +31,6 @@ module Gitlab # Any data inserted while running this method (or after it has finished # running) is _not_ updated automatically. # - # This method _only_ updates rows where the column's value is set to NULL. - # # table - The name of the table. # column - The name of the column to update. # value - The value for the column. @@ -55,10 +53,10 @@ module Gitlab first['count']. to_i - # Update in batches of 5% + # Update in batches of 5% until we run out of any rows to update. batch_size = ((total / 100.0) * 5.0).ceil - while processed < total + loop do start_row = exec_query(%Q{ SELECT id FROM #{quoted_table} @@ -66,6 +64,9 @@ module Gitlab LIMIT 1 OFFSET #{processed} }).to_hash.first + # There are no more rows to process + break unless start_row + stop_row = exec_query(%Q{ SELECT id FROM #{quoted_table} @@ -126,6 +127,8 @@ module Gitlab begin transaction do update_column_in_batches(table, column, default) + + change_column_null(table, column, false) unless allow_null end # We want to rescue _all_ exceptions here, even those that don't inherit # from StandardError. @@ -134,8 +137,6 @@ module Gitlab raise error end - - change_column_null(table, column, false) unless allow_null end end end diff --git a/lib/gitlab/gl_id.rb b/lib/gitlab/gl_id.rb new file mode 100644 index 00000000000..624fd00367e --- /dev/null +++ b/lib/gitlab/gl_id.rb @@ -0,0 +1,11 @@ +module Gitlab + module GlId + def self.gl_id(user) + if user.present? + "user-#{user.id}" + else + "" + end + end + end +end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 0f115893a15..d81d26754fe 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -56,7 +56,7 @@ module Gitlab end end - # Instruments all public methods of a module. + # Instruments all public and private methods of a module. # # This method optionally takes a block that can be used to determine if a # method should be instrumented or not. The block is passed the receiving @@ -65,7 +65,8 @@ module Gitlab # # mod - The module to instrument. def self.instrument_methods(mod) - mod.public_methods(false).each do |name| + methods = mod.methods(false) + mod.private_methods(false) + methods.each do |name| method = mod.method(name) if method.owner == mod.singleton_class @@ -76,13 +77,14 @@ module Gitlab end end - # Instruments all public instance methods of a module. + # Instruments all public and private instance methods of a module. # # See `instrument_methods` for more information. # # mod - The module to instrument. def self.instrument_instance_methods(mod) - mod.public_instance_methods(false).each do |name| + methods = mod.instance_methods(false) + mod.private_instance_methods(false) + methods.each do |name| method = mod.instance_method(name) if method.owner == mod @@ -149,13 +151,16 @@ module Gitlab trans = Gitlab::Metrics::Instrumentation.transaction if trans - start = Time.now - retval = super - duration = (Time.now - start) * 1000.0 + start = Time.now + cpu_start = Gitlab::Metrics::System.cpu_time + retval = super + duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold + cpu_duration = Gitlab::Metrics::System.cpu_time - cpu_start + trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, - { duration: duration }, + { duration: duration, cpu_duration: cpu_duration }, method: #{label.inspect}) end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 6f179789d3e..3fe27779d03 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -1,8 +1,9 @@ module Gitlab module Metrics - # Rack middleware for tracking Rails requests. + # Rack middleware for tracking Rails and Grape requests. class RackMiddleware CONTROLLER_KEY = 'action_controller.instance' + ENDPOINT_KEY = 'api.endpoint' def initialize(app) @app = app @@ -21,6 +22,8 @@ module Gitlab ensure if env[CONTROLLER_KEY] tag_controller(trans, env) + elsif env[ENDPOINT_KEY] + tag_endpoint(trans, env) end trans.finish @@ -42,6 +45,26 @@ module Gitlab controller = env[CONTROLLER_KEY] trans.action = "#{controller.class.name}##{controller.action_name}" end + + def tag_endpoint(trans, env) + endpoint = env[ENDPOINT_KEY] + path = endpoint_paths_cache[endpoint.route.route_method][endpoint.route.route_path] + trans.action = "Grape##{endpoint.route.route_method} #{path}" + end + + private + + def endpoint_paths_cache + @endpoint_paths_cache ||= Hash.new do |hash, http_method| + hash[http_method] = Hash.new do |inner_hash, raw_path| + inner_hash[raw_path] = endpoint_instrumentable_path(raw_path) + end + end + end + + def endpoint_instrumentable_path(raw_path) + raw_path.sub('(.:format)', '').sub('/:version', '') + end end end end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index fc709222a9b..0000450d9bb 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -66,7 +66,11 @@ module Gitlab def sample_objects sample = Allocations.to_hash counts = sample.each_with_object({}) do |(klass, count), hash| - hash[klass.name] = count + name = klass.name + + next unless name + + hash[name] = count end # Symbols aren't allocated so we'll need to add those manually. diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 1cbd6d945a0..c84c68f96f6 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -100,5 +100,13 @@ module Gitlab def container_registry_reference_regex git_reference_regex end + + def environment_name_regex + @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze + end + + def environment_name_regex_message + "can contain only letters, digits, '-' and '_'." + end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 388f84dbe0e..40e8299c36b 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -8,7 +8,7 @@ module Gitlab class << self def git_http_ok(repository, user) { - 'GL_ID' => Gitlab::ShellEnv.gl_id(user), + 'GL_ID' => Gitlab::GlId.gl_id(user), 'RepoPath' => repository.path_to_repo, } end |