diff options
Diffstat (limited to 'lib/gitlab')
76 files changed, 1357 insertions, 207 deletions
diff --git a/lib/gitlab/chat_commands/base_command.rb b/lib/gitlab/chat_commands/base_command.rb new file mode 100644 index 00000000000..e59d69b72b9 --- /dev/null +++ b/lib/gitlab/chat_commands/base_command.rb @@ -0,0 +1,49 @@ +module Gitlab + module ChatCommands + class BaseCommand + QUERY_LIMIT = 5 + + def self.match(_text) + raise NotImplementedError + end + + def self.help_message + raise NotImplementedError + end + + def self.available?(_project) + raise NotImplementedError + end + + def self.allowed?(_user, _ability) + true + end + + def self.can?(object, action, subject) + Ability.allowed?(object, action, subject) + end + + def execute(_) + raise NotImplementedError + end + + def collection + raise NotImplementedError + end + + attr_accessor :project, :current_user, :params + + def initialize(project, user, params = {}) + @project, @current_user, @params = project, user, params.dup + end + + private + + def find_by_iid(iid) + resource = collection.find_by(iid: iid) + + readable?(resource) ? resource : nil + end + end + end +end diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb new file mode 100644 index 00000000000..0ec358debc7 --- /dev/null +++ b/lib/gitlab/chat_commands/command.rb @@ -0,0 +1,62 @@ +module Gitlab + module ChatCommands + class Command < BaseCommand + COMMANDS = [ + Gitlab::ChatCommands::IssueShow, + Gitlab::ChatCommands::IssueCreate, + Gitlab::ChatCommands::Deploy, + ].freeze + + def execute + command, match = match_command + + if command + if command.allowed?(project, current_user) + present command.new(project, current_user, params).execute(match) + else + access_denied + end + else + help(help_messages) + end + end + + private + + def match_command + match = nil + service = available_commands.find do |klass| + match = klass.match(command) + end + + [service, match] + end + + def help_messages + available_commands.map(&:help_message) + end + + def available_commands + COMMANDS.select do |klass| + klass.available?(project) + end + end + + def command + params[:text] + end + + def help(messages) + Mattermost::Presenter.help(messages, params[:command]) + end + + def access_denied + Mattermost::Presenter.access_denied + end + + def present(resource) + Mattermost::Presenter.present(resource) + end + end + end +end diff --git a/lib/gitlab/chat_commands/deploy.rb b/lib/gitlab/chat_commands/deploy.rb new file mode 100644 index 00000000000..0eed1fce0dc --- /dev/null +++ b/lib/gitlab/chat_commands/deploy.rb @@ -0,0 +1,57 @@ +module Gitlab + module ChatCommands + class Deploy < BaseCommand + include Gitlab::Routing.url_helpers + + def self.match(text) + /\Adeploy\s+(?<from>.*)\s+to+\s+(?<to>.*)\z/.match(text) + end + + def self.help_message + 'deploy <environment> to <target-environment>' + end + + def self.available?(project) + project.builds_enabled? + end + + def self.allowed?(project, user) + can?(user, :create_deployment, project) + end + + def execute(match) + from = match[:from] + to = match[:to] + + actions = find_actions(from, to) + return unless actions.present? + + if actions.one? + play!(from, to, actions.first) + else + Result.new(:error, 'Too many actions defined') + end + end + + private + + def play!(from, to, action) + new_action = action.play(current_user) + + Result.new(:success, "Deployment from #{from} to #{to} started. Follow the progress: #{url(new_action)}.") + end + + def find_actions(from, to) + environment = project.environments.find_by(name: from) + return unless environment + + environment.actions_for(to).select(&:starts_environment?) + end + + def url(subject) + polymorphic_url( + [ subject.project.namespace.becomes(Namespace), subject.project, subject ]) + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_command.rb b/lib/gitlab/chat_commands/issue_command.rb new file mode 100644 index 00000000000..f1bc36239d5 --- /dev/null +++ b/lib/gitlab/chat_commands/issue_command.rb @@ -0,0 +1,17 @@ +module Gitlab + module ChatCommands + class IssueCommand < BaseCommand + def self.available?(project) + project.issues_enabled? && project.default_issues_tracker? + end + + def collection + project.issues + end + + def readable?(issue) + self.class.can?(current_user, :read_issue, issue) + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_create.rb b/lib/gitlab/chat_commands/issue_create.rb new file mode 100644 index 00000000000..99c1382af44 --- /dev/null +++ b/lib/gitlab/chat_commands/issue_create.rb @@ -0,0 +1,26 @@ +module Gitlab + module ChatCommands + class IssueCreate < IssueCommand + def self.match(text) + # we can not match \n with the dot by passing the m modifier as than + # the title and description are not seperated + /\Aissue\s+create\s+(?<title>[^\n]*)\n*(?<description>(.|\n)*)/.match(text) + end + + def self.help_message + 'issue create <title>\n<description>' + end + + def self.allowed?(project, user) + can?(user, :create_issue, project) + end + + def execute(match) + title = match[:title] + description = match[:description].to_s.rstrip + + Issues::CreateService.new(project, current_user, title: title, description: description).execute + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_show.rb b/lib/gitlab/chat_commands/issue_show.rb new file mode 100644 index 00000000000..f5bceb038e5 --- /dev/null +++ b/lib/gitlab/chat_commands/issue_show.rb @@ -0,0 +1,17 @@ +module Gitlab + module ChatCommands + class IssueShow < IssueCommand + def self.match(text) + /\Aissue\s+show\s+(?<iid>\d+)/.match(text) + end + + def self.help_message + "issue show <id>" + end + + def execute(match) + find_by_iid(match[:iid]) + end + end + end +end diff --git a/lib/gitlab/chat_commands/result.rb b/lib/gitlab/chat_commands/result.rb new file mode 100644 index 00000000000..324d7ef43a3 --- /dev/null +++ b/lib/gitlab/chat_commands/result.rb @@ -0,0 +1,5 @@ +module Gitlab + module ChatCommands + Result = Struct.new(:type, :message) + end +end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb new file mode 100644 index 00000000000..1b081aa9b1d --- /dev/null +++ b/lib/gitlab/chat_name_token.rb @@ -0,0 +1,45 @@ +require 'json' + +module Gitlab + class ChatNameToken + attr_reader :token + + TOKEN_LENGTH = 50 + EXPIRY_TIME = 10.minutes + + def initialize(token = new_token) + @token = token + end + + def get + Gitlab::Redis.with do |redis| + data = redis.get(redis_key) + JSON.parse(data, symbolize_names: true) if data + end + end + + def store!(params) + Gitlab::Redis.with do |redis| + params = params.to_json + redis.set(redis_key, params, ex: EXPIRY_TIME) + token + end + end + + def delete + Gitlab::Redis.with do |redis| + redis.del(redis_key) + end + end + + private + + def new_token + Devise.friendly_token(TOKEN_LENGTH) + end + + def redis_key + "gitlab:chat_names:#{token}" + end + end +end diff --git a/lib/gitlab/ci/build/credentials/base.rb b/lib/gitlab/ci/build/credentials/base.rb new file mode 100644 index 00000000000..29a7a27c963 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/base.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Build + module Credentials + class Base + def type + self.class.name.demodulize.underscore + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/credentials/factory.rb b/lib/gitlab/ci/build/credentials/factory.rb new file mode 100644 index 00000000000..2423aa8857d --- /dev/null +++ b/lib/gitlab/ci/build/credentials/factory.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module Build + module Credentials + class Factory + def initialize(build) + @build = build + end + + def create! + credentials.select(&:valid?) + end + + private + + def credentials + providers.map { |provider| provider.new(@build) } + end + + def providers + [Registry] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/credentials/registry.rb b/lib/gitlab/ci/build/credentials/registry.rb new file mode 100644 index 00000000000..55eafcaed10 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/registry.rb @@ -0,0 +1,24 @@ +module Gitlab + module Ci + module Build + module Credentials + class Registry < Base + attr_reader :username, :password + + def initialize(build) + @username = 'gitlab-ci-token' + @password = build.token + end + + def url + Gitlab.config.registry.host_port + end + + def valid? + Gitlab.config.registry.enabled + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index bbfa6cf7d05..f7ff7ea212e 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -4,16 +4,10 @@ module Gitlab # Base GitLab CI Configuration facade # class Config - ## - # Temporary delegations that should be removed after refactoring - # - delegate :before_script, :image, :services, :after_script, :variables, - :stages, :cache, :jobs, to: :@global - def initialize(config) @config = Loader.new(config).load! - @global = Node::Global.new(@config) + @global = Entry::Global.new(@config) @global.compose! end @@ -28,6 +22,41 @@ module Gitlab def to_hash @config end + + ## + # Temporary method that should be removed after refactoring + # + def before_script + @global.before_script_value + end + + def image + @global.image_value + end + + def services + @global.services_value + end + + def after_script + @global.after_script_value + end + + def variables + @global.variables_value + end + + def stages + @global.stages_value + end + + def cache + @global.cache_value + end + + def jobs + @global.jobs_value + end end end end diff --git a/lib/gitlab/ci/config/node/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 844bd2fe998..b756b0d4555 100644 --- a/lib/gitlab/ci/config/node/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a configuration of job artifacts. # - class Artifacts < Entry + class Artifacts < Node include Validatable include Attributable diff --git a/lib/gitlab/ci/config/node/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb index 221b666f9f6..1c8b55ee4c4 100644 --- a/lib/gitlab/ci/config/node/attributable.rb +++ b/lib/gitlab/ci/config/entry/attributable.rb @@ -1,7 +1,7 @@ module Gitlab module Ci class Config - module Node + module Entry module Attributable extend ActiveSupport::Concern diff --git a/lib/gitlab/ci/config/node/boolean.rb b/lib/gitlab/ci/config/entry/boolean.rb index 84b03ee7832..f3357f85b99 100644 --- a/lib/gitlab/ci/config/node/boolean.rb +++ b/lib/gitlab/ci/config/entry/boolean.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a boolean value. # - class Boolean < Entry + class Boolean < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index b4bda2841ac..7653cab668b 100644 --- a/lib/gitlab/ci/config/node/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a cache configuration # - class Cache < Entry + class Cache < Node include Configurable ALLOWED_KEYS = %i[key untracked paths] @@ -14,13 +14,13 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS end - node :key, Node::Key, + entry :key, Entry::Key, description: 'Cache key used to define a cache affinity.' - node :untracked, Node::Boolean, + entry :untracked, Entry::Boolean, description: 'Cache all untracked files.' - node :paths, Node::Paths, + entry :paths, Entry::Paths, description: 'Specify which paths should be cached across builds.' end end diff --git a/lib/gitlab/ci/config/node/commands.rb b/lib/gitlab/ci/config/entry/commands.rb index d7657ae314b..65d19db249c 100644 --- a/lib/gitlab/ci/config/node/commands.rb +++ b/lib/gitlab/ci/config/entry/commands.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a job script. # - class Commands < Entry + class Commands < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb index 6b7ab2fdaf2..833ae4a0ff3 100644 --- a/lib/gitlab/ci/config/node/configurable.rb +++ b/lib/gitlab/ci/config/entry/configurable.rb @@ -1,7 +1,7 @@ module Gitlab module Ci class Config - module Node + module Entry ## # This mixin is responsible for adding DSL, which purpose is to # simplifly process of adding child nodes. @@ -48,8 +48,8 @@ module Gitlab private # rubocop:disable Lint/UselessAccessModifier - def node(key, node, metadata) - factory = Node::Factory.new(node) + def entry(key, entry, metadata) + factory = Entry::Factory.new(entry) .with(description: metadata[:description]) (@nodes ||= {}).merge!(key.to_sym => factory) @@ -66,8 +66,6 @@ module Gitlab @entries[symbol].value end - - alias_method symbol.to_sym, "#{symbol}_value".to_sym end end end diff --git a/lib/gitlab/ci/config/node/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 9a95ef43628..b7b4b91eb51 100644 --- a/lib/gitlab/ci/config/node/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents an environment. # - class Environment < Entry + class Environment < Node include Validatable ALLOWED_KEYS = %i[name url action on_stop] diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/entry/factory.rb index 5387f29ad59..9f5e393d191 100644 --- a/lib/gitlab/ci/config/node/factory.rb +++ b/lib/gitlab/ci/config/entry/factory.rb @@ -1,15 +1,15 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # Factory class responsible for fabricating node entry objects. + # Factory class responsible for fabricating entry objects. # class Factory class InvalidFactory < StandardError; end - def initialize(node) - @node = node + def initialize(entry) + @entry = entry @metadata = {} @attributes = {} end @@ -37,11 +37,11 @@ module Gitlab # See issue #18775. # if @value.nil? - Node::Unspecified.new( + Entry::Unspecified.new( fabricate_unspecified ) else - fabricate(@node, @value) + fabricate(@entry, @value) end end @@ -49,21 +49,21 @@ module Gitlab def fabricate_unspecified ## - # If node has a default value we fabricate concrete node + # If entry has a default value we fabricate concrete node # with default value. # - if @node.default.nil? - fabricate(Node::Undefined) + if @entry.default.nil? + fabricate(Entry::Undefined) else - fabricate(@node, @node.default) + fabricate(@entry, @entry.default) end end - def fabricate(node, value = nil) - node.new(value, @metadata).tap do |entry| - entry.key = @attributes[:key] - entry.parent = @attributes[:parent] - entry.description = @attributes[:description] + def fabricate(entry, value = nil) + entry.new(value, @metadata).tap do |node| + node.key = @attributes[:key] + node.parent = @attributes[:parent] + node.description = @attributes[:description] end end end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/entry/global.rb index 2a2943c9288..a4ec8f0ff2f 100644 --- a/lib/gitlab/ci/config/node/global.rb +++ b/lib/gitlab/ci/config/entry/global.rb @@ -1,36 +1,36 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # This class represents a global entry - root node for entire + # This class represents a global entry - root Entry for entire # GitLab CI Configuration file. # - class Global < Entry + class Global < Node include Configurable - node :before_script, Node::Script, + entry :before_script, Entry::Script, description: 'Script that will be executed before each job.' - node :image, Node::Image, + entry :image, Entry::Image, description: 'Docker image that will be used to execute jobs.' - node :services, Node::Services, + entry :services, Entry::Services, description: 'Docker images that will be linked to the container.' - node :after_script, Node::Script, + entry :after_script, Entry::Script, description: 'Script that will be executed after each job.' - node :variables, Node::Variables, + entry :variables, Entry::Variables, description: 'Environment variables that will be used.' - node :stages, Node::Stages, + entry :stages, Entry::Stages, description: 'Configuration of stages for this pipeline.' - node :types, Node::Stages, + entry :types, Entry::Stages, description: 'Deprecated: stages for this pipeline.' - node :cache, Node::Cache, + entry :cache, Entry::Cache, description: 'Configure caching between build jobs.' helpers :before_script, :image, :services, :after_script, @@ -46,7 +46,7 @@ module Gitlab private def compose_jobs! - factory = Node::Factory.new(Node::Jobs) + factory = Entry::Factory.new(Entry::Jobs) .value(@config.except(*self.class.nodes.keys)) .with(key: :jobs, parent: self, description: 'Jobs definition for this pipeline') diff --git a/lib/gitlab/ci/config/node/hidden.rb b/lib/gitlab/ci/config/entry/hidden.rb index fe4ee8a7fc6..6fc3aa385bc 100644 --- a/lib/gitlab/ci/config/node/hidden.rb +++ b/lib/gitlab/ci/config/entry/hidden.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # Entry that represents a hidden CI/CD job. + # Entry that represents a hidden CI/CD key. # - class Hidden < Entry + class Hidden < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/image.rb b/lib/gitlab/ci/config/entry/image.rb index 5d3c7c5eab0..b5050257688 100644 --- a/lib/gitlab/ci/config/node/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a Docker image. # - class Image < Entry + class Image < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/job.rb b/lib/gitlab/ci/config/entry/job.rb index 603334d6793..a55362f0b6b 100644 --- a/lib/gitlab/ci/config/node/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a concrete CI/CD job. # - class Job < Entry + class Job < Node include Configurable include Attributable @@ -13,12 +13,10 @@ module Gitlab type stage when artifacts cache dependencies before_script after_script variables environment] - attributes :tags, :allow_failure, :when, :dependencies - validations do validates :config, allowed_keys: ALLOWED_KEYS - validates :config, presence: true + validates :script, presence: true validates :name, presence: true validates :name, type: Symbol @@ -34,49 +32,51 @@ module Gitlab end end - node :before_script, Node::Script, + entry :before_script, Entry::Script, description: 'Global before script overridden in this job.' - node :script, Node::Commands, + entry :script, Entry::Commands, description: 'Commands that will be executed in this job.' - node :stage, Node::Stage, + entry :stage, Entry::Stage, description: 'Pipeline stage this job will be executed into.' - node :type, Node::Stage, + entry :type, Entry::Stage, description: 'Deprecated: stage this job will be executed into.' - node :after_script, Node::Script, + entry :after_script, Entry::Script, description: 'Commands that will be executed when finishing job.' - node :cache, Node::Cache, + entry :cache, Entry::Cache, description: 'Cache definition for this job.' - node :image, Node::Image, + entry :image, Entry::Image, description: 'Image that will be used to execute this job.' - node :services, Node::Services, + entry :services, Entry::Services, description: 'Services that will be used to execute this job.' - node :only, Node::Trigger, + entry :only, Entry::Trigger, description: 'Refs policy this job will be executed for.' - node :except, Node::Trigger, + entry :except, Entry::Trigger, description: 'Refs policy this job will be executed for.' - node :variables, Node::Variables, + entry :variables, Entry::Variables, description: 'Environment variables available for this job.' - node :artifacts, Node::Artifacts, + entry :artifacts, Entry::Artifacts, description: 'Artifacts configuration for this job.' - node :environment, Node::Environment, + entry :environment, Entry::Environment, description: 'Environment configuration for this job.' helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, :artifacts, :commands, :environment + attributes :script, :tags, :allow_failure, :when, :dependencies + def compose!(deps = nil) super do if type_defined? && !stage_defined? @@ -108,7 +108,7 @@ module Gitlab self.class.nodes.each_key do |key| global_entry = deps[key] - job_entry = @entries[key] + job_entry = self[key] if global_entry.specified? && !job_entry.specified? @entries[key] = global_entry @@ -118,20 +118,20 @@ module Gitlab def to_hash { name: name, - before_script: before_script, - script: script, + before_script: before_script_value, + script: script_value, commands: commands, - image: image, - services: services, - stage: stage, - cache: cache, - only: only, - except: except, - variables: variables_defined? ? variables : nil, - environment: environment_defined? ? environment : nil, - environment_name: environment_defined? ? environment[:name] : nil, - artifacts: artifacts, - after_script: after_script } + image: image_value, + services: services_value, + stage: stage_value, + cache: cache_value, + only: only_value, + except: except_value, + variables: variables_defined? ? variables_value : nil, + environment: environment_defined? ? environment_value : nil, + environment_name: environment_defined? ? environment_value[:name] : nil, + artifacts: artifacts_value, + after_script: after_script_value } end end end diff --git a/lib/gitlab/ci/config/node/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index d10e80d1a7d..5671a09480b 100644 --- a/lib/gitlab/ci/config/node/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a set of jobs. # - class Jobs < Entry + class Jobs < Node include Validatable validations do @@ -29,9 +29,9 @@ module Gitlab def compose!(deps = nil) super do @config.each do |name, config| - node = hidden?(name) ? Node::Hidden : Node::Job + node = hidden?(name) ? Entry::Hidden : Entry::Job - factory = Node::Factory.new(node) + factory = Entry::Factory.new(node) .value(config || {}) .metadata(name: name) .with(key: name, parent: self, diff --git a/lib/gitlab/ci/config/node/key.rb b/lib/gitlab/ci/config/entry/key.rb index f8b461ca098..0e4c9fe6edc 100644 --- a/lib/gitlab/ci/config/node/key.rb +++ b/lib/gitlab/ci/config/entry/key.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a key. # - class Key < Entry + class Key < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index 0c291efe6a5..f01975aab5c 100644 --- a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -1,7 +1,7 @@ module Gitlab module Ci class Config - module Node + module Entry module LegacyValidationHelpers private diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/entry/node.rb index 8717eabf81e..5eef2868cd6 100644 --- a/lib/gitlab/ci/config/node/entry.rb +++ b/lib/gitlab/ci/config/entry/node.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Base abstract class for each configuration entry node. # - class Entry + class Node class InvalidError < StandardError; end attr_reader :config, :metadata @@ -21,7 +21,7 @@ module Gitlab end def [](key) - @entries[key] || Node::Undefined.new + @entries[key] || Entry::Undefined.new end def compose!(deps = nil) diff --git a/lib/gitlab/ci/config/node/paths.rb b/lib/gitlab/ci/config/entry/paths.rb index 3c6d3a52966..68dad161149 100644 --- a/lib/gitlab/ci/config/node/paths.rb +++ b/lib/gitlab/ci/config/entry/paths.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents an array of paths. # - class Paths < Entry + class Paths < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/entry/script.rb index 39328f0fade..29ecd9995ca 100644 --- a/lib/gitlab/ci/config/node/script.rb +++ b/lib/gitlab/ci/config/entry/script.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a script. # - class Script < Entry + class Script < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/services.rb b/lib/gitlab/ci/config/entry/services.rb index 481e2b66adc..84f8ab780f5 100644 --- a/lib/gitlab/ci/config/node/services.rb +++ b/lib/gitlab/ci/config/entry/services.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a configuration of Docker services. # - class Services < Entry + class Services < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/stage.rb b/lib/gitlab/ci/config/entry/stage.rb index cbc97641f5a..b7afaba1de8 100644 --- a/lib/gitlab/ci/config/node/stage.rb +++ b/lib/gitlab/ci/config/entry/stage.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a stage for a job. # - class Stage < Entry + class Stage < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/stages.rb b/lib/gitlab/ci/config/entry/stages.rb index b1fe45357ff..ec187bd3732 100644 --- a/lib/gitlab/ci/config/node/stages.rb +++ b/lib/gitlab/ci/config/entry/stages.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a configuration for pipeline stages. # - class Stages < Entry + class Stages < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb index d8b31975088..28b0a9ffe01 100644 --- a/lib/gitlab/ci/config/node/trigger.rb +++ b/lib/gitlab/ci/config/entry/trigger.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a trigger policy for the job. # - class Trigger < Entry + class Trigger < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/undefined.rb b/lib/gitlab/ci/config/entry/undefined.rb index 33e78023539..b33b8238230 100644 --- a/lib/gitlab/ci/config/node/undefined.rb +++ b/lib/gitlab/ci/config/entry/undefined.rb @@ -1,13 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # This class represents an undefined node. + # This class represents an undefined entry. # - # Implements the Null Object pattern. - # - class Undefined < Entry + class Undefined < Node def initialize(*) super(nil) end diff --git a/lib/gitlab/ci/config/node/unspecified.rb b/lib/gitlab/ci/config/entry/unspecified.rb index a7d1f6131b8..fbb2551e870 100644 --- a/lib/gitlab/ci/config/node/unspecified.rb +++ b/lib/gitlab/ci/config/entry/unspecified.rb @@ -1,9 +1,9 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # This class represents an unspecified entry node. + # This class represents an unspecified entry. # # It decorates original entry adding method that indicates it is # unspecified. diff --git a/lib/gitlab/ci/config/node/validatable.rb b/lib/gitlab/ci/config/entry/validatable.rb index 085e6e988d1..f7f1b111571 100644 --- a/lib/gitlab/ci/config/node/validatable.rb +++ b/lib/gitlab/ci/config/entry/validatable.rb @@ -1,13 +1,13 @@ module Gitlab module Ci class Config - module Node + module Entry module Validatable extend ActiveSupport::Concern class_methods do def validator - @validator ||= Class.new(Node::Validator).tap do |validator| + @validator ||= Class.new(Entry::Validator).tap do |validator| if defined?(@validations) @validations.each { |rules| validator.class_eval(&rules) } end diff --git a/lib/gitlab/ci/config/node/validator.rb b/lib/gitlab/ci/config/entry/validator.rb index 43c7e102b50..55343005fe3 100644 --- a/lib/gitlab/ci/config/node/validator.rb +++ b/lib/gitlab/ci/config/entry/validator.rb @@ -1,14 +1,14 @@ module Gitlab module Ci class Config - module Node + module Entry class Validator < SimpleDelegator include ActiveModel::Validations - include Node::Validators + include Entry::Validators - def initialize(node) - super(node) - @node = node + def initialize(entry) + super(entry) + @entry = entry end def messages @@ -30,7 +30,7 @@ module Gitlab def key_name if key.blank? - @node.class.name.demodulize.underscore.humanize + @entry.class.name.demodulize.underscore.humanize else key end diff --git a/lib/gitlab/ci/config/node/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index e20908ad3cb..8632dd0e233 100644 --- a/lib/gitlab/ci/config/node/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -1,7 +1,7 @@ module Gitlab module Ci class Config - module Node + module Entry module Validators class AllowedKeysValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) diff --git a/lib/gitlab/ci/config/node/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index 5f813f81f55..c3b0e651c3a 100644 --- a/lib/gitlab/ci/config/node/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents environment variables. # - class Variables < Entry + class Variables < Node include Validatable validations do diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index ef9160d6437..c6bb8f9c8ed 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -23,6 +23,10 @@ module Gitlab settings || fake_application_settings end + def sidekiq_throttling_enabled? + current_application_settings.sidekiq_throttling_enabled? + end + def fake_application_settings OpenStruct.new( default_projects_limit: Settings.gitlab['default_projects_limit'], @@ -50,6 +54,7 @@ module Gitlab repository_checks_enabled: true, container_registry_token_expire_delay: 5, user_default_external: false, + sidekiq_throttling_enabled: false, ) end diff --git a/lib/gitlab/cycle_analytics/base_event.rb b/lib/gitlab/cycle_analytics/base_event.rb new file mode 100644 index 00000000000..53a148ad703 --- /dev/null +++ b/lib/gitlab/cycle_analytics/base_event.rb @@ -0,0 +1,57 @@ +module Gitlab + module CycleAnalytics + class BaseEvent + include MetricsTables + + attr_reader :stage, :start_time_attrs, :end_time_attrs, :projections, :query + + def initialize(project:, options:) + @query = EventsQuery.new(project: project, options: options) + @project = project + @options = options + end + + def fetch + update_author! + + event_result.map do |event| + serialize(event) if has_permission?(event['id']) + end.compact + end + + def custom_query(_base_query); end + + def order + @order || @start_time_attrs + end + + private + + def update_author! + return unless event_result.any? && event_result.first['author_id'] + + Updater.update!(event_result, from: 'author_id', to: 'author', klass: User) + end + + def event_result + @event_result ||= @query.execute(self).to_a + end + + def serialize(_event) + raise NotImplementedError.new("Expected #{self.name} to implement serialize(event)") + end + + def has_permission?(id) + allowed_ids.nil? || allowed_ids.include?(id.to_i) + end + + def allowed_ids + nil + end + + def event_result_ids + event_result.map { |event| event['id'] } + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/code_event.rb b/lib/gitlab/cycle_analytics/code_event.rb new file mode 100644 index 00000000000..2afdf0b8518 --- /dev/null +++ b/lib/gitlab/cycle_analytics/code_event.rb @@ -0,0 +1,28 @@ +module Gitlab + module CycleAnalytics + class CodeEvent < BaseEvent + include MergeRequestAllowed + + def initialize(*args) + @stage = :code + @start_time_attrs = issue_metrics_table[:first_mentioned_in_commit_at] + @end_time_attrs = mr_table[:created_at] + @projections = [mr_table[:title], + mr_table[:iid], + mr_table[:id], + mr_table[:created_at], + mr_table[:state], + mr_table[:author_id]] + @order = mr_table[:created_at] + + super(*args) + end + + private + + def serialize(event) + AnalyticsMergeRequestSerializer.new(project: @project).represent(event).as_json + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/events.rb b/lib/gitlab/cycle_analytics/events.rb new file mode 100644 index 00000000000..2d703d76cbb --- /dev/null +++ b/lib/gitlab/cycle_analytics/events.rb @@ -0,0 +1,38 @@ +module Gitlab + module CycleAnalytics + class Events + def initialize(project:, options:) + @project = project + @options = options + end + + def issue_events + IssueEvent.new(project: @project, options: @options).fetch + end + + def plan_events + PlanEvent.new(project: @project, options: @options).fetch + end + + def code_events + CodeEvent.new(project: @project, options: @options).fetch + end + + def test_events + TestEvent.new(project: @project, options: @options).fetch + end + + def review_events + ReviewEvent.new(project: @project, options: @options).fetch + end + + def staging_events + StagingEvent.new(project: @project, options: @options).fetch + end + + def production_events + ProductionEvent.new(project: @project, options: @options).fetch + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/events_query.rb b/lib/gitlab/cycle_analytics/events_query.rb new file mode 100644 index 00000000000..2418832ccc2 --- /dev/null +++ b/lib/gitlab/cycle_analytics/events_query.rb @@ -0,0 +1,37 @@ +module Gitlab + module CycleAnalytics + class EventsQuery + attr_reader :project + + def initialize(project:, options: {}) + @project = project + @from = options[:from] + @branch = options[:branch] + @fetcher = Gitlab::CycleAnalytics::MetricsFetcher.new(project: project, from: @from, branch: @branch) + end + + def execute(stage_class) + @stage_class = stage_class + + ActiveRecord::Base.connection.exec_query(query.to_sql) + end + + private + + def query + base_query = @fetcher.base_query_for(@stage_class.stage) + diff_fn = @fetcher.subtract_datetimes_diff(base_query, @stage_class.start_time_attrs, @stage_class.end_time_attrs) + + @stage_class.custom_query(base_query) + + base_query.project(extract_epoch(diff_fn).as('total_time'), *@stage_class.projections).order(@stage_class.order.desc) + end + + def extract_epoch(arel_attribute) + return arel_attribute unless Gitlab::Database.postgresql? + + Arel.sql(%Q{EXTRACT(EPOCH FROM (#{arel_attribute.to_sql}))}) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/issue_allowed.rb b/lib/gitlab/cycle_analytics/issue_allowed.rb new file mode 100644 index 00000000000..a7652a70641 --- /dev/null +++ b/lib/gitlab/cycle_analytics/issue_allowed.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module IssueAllowed + def allowed_ids + @allowed_ids ||= IssuesFinder.new(@options[:current_user], project_id: @project.id).execute.where(id: event_result_ids).pluck(:id) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/issue_event.rb b/lib/gitlab/cycle_analytics/issue_event.rb new file mode 100644 index 00000000000..705b7e5ce24 --- /dev/null +++ b/lib/gitlab/cycle_analytics/issue_event.rb @@ -0,0 +1,27 @@ +module Gitlab + module CycleAnalytics + class IssueEvent < BaseEvent + include IssueAllowed + + def initialize(*args) + @stage = :issue + @start_time_attrs = issue_table[:created_at] + @end_time_attrs = [issue_metrics_table[:first_associated_with_milestone_at], + issue_metrics_table[:first_added_to_board_at]] + @projections = [issue_table[:title], + issue_table[:iid], + issue_table[:id], + issue_table[:created_at], + issue_table[:author_id]] + + super(*args) + end + + private + + def serialize(event) + AnalyticsIssueSerializer.new(project: @project).represent(event).as_json + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/merge_request_allowed.rb b/lib/gitlab/cycle_analytics/merge_request_allowed.rb new file mode 100644 index 00000000000..28f6db44759 --- /dev/null +++ b/lib/gitlab/cycle_analytics/merge_request_allowed.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module MergeRequestAllowed + def allowed_ids + @allowed_ids ||= MergeRequestsFinder.new(@options[:current_user], project_id: @project.id).execute.where(id: event_result_ids).pluck(:id) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/metrics_fetcher.rb b/lib/gitlab/cycle_analytics/metrics_fetcher.rb new file mode 100644 index 00000000000..b71e8735e27 --- /dev/null +++ b/lib/gitlab/cycle_analytics/metrics_fetcher.rb @@ -0,0 +1,60 @@ +module Gitlab + module CycleAnalytics + class MetricsFetcher + include Gitlab::Database::Median + include Gitlab::Database::DateTime + include MetricsTables + + DEPLOYMENT_METRIC_STAGES = %i[production staging] + + def initialize(project:, from:, branch:) + @project = project + @project = project + @from = from + @branch = branch + end + + def calculate_metric(name, start_time_attrs, end_time_attrs) + cte_table = Arel::Table.new("cte_table_for_#{name}") + + # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). + # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). + # We compute the (end_time - start_time) interval, and give it an alias based on the current + # cycle analytics stage. + interval_query = Arel::Nodes::As.new( + cte_table, + subtract_datetimes(base_query_for(name), start_time_attrs, end_time_attrs, name.to_s)) + + median_datetime(cte_table, interval_query, name) + end + + # Join table with a row for every <issue,merge_request> pair (where the merge request + # closes the given issue) with issue and merge request metrics included. The metrics + # are loaded with an inner join, so issues / merge requests without metrics are + # automatically excluded. + def base_query_for(name) + # Load issues + query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])). + join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])). + where(issue_table[:project_id].eq(@project.id)). + where(issue_table[:deleted_at].eq(nil)). + where(issue_table[:created_at].gteq(@from)) + + query = query.where(build_table[:ref].eq(@branch)) if name == :test && @branch + + # Load merge_requests + query = query.join(mr_table, Arel::Nodes::OuterJoin). + on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])). + join(mr_metrics_table). + on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) + + if DEPLOYMENT_METRIC_STAGES.include?(name) + # Limit to merge requests that have been deployed to production after `@from` + query.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@from)) + end + + query + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb new file mode 100644 index 00000000000..9d25ef078e8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/metrics_tables.rb @@ -0,0 +1,37 @@ +module Gitlab + module CycleAnalytics + module MetricsTables + def mr_metrics_table + MergeRequest::Metrics.arel_table + end + + def mr_table + MergeRequest.arel_table + end + + def mr_diff_table + MergeRequestDiff.arel_table + end + + def mr_closing_issues_table + MergeRequestsClosingIssues.arel_table + end + + def issue_table + Issue.arel_table + end + + def issue_metrics_table + Issue::Metrics.arel_table + end + + def user_table + User.arel_table + end + + def build_table + ::CommitStatus.arel_table + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb new file mode 100644 index 00000000000..bef3b95ff1b --- /dev/null +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -0,0 +1,44 @@ +module Gitlab + module CycleAnalytics + class Permissions + STAGE_PERMISSIONS = { + issue: :read_issue, + code: :read_merge_request, + test: :read_build, + review: :read_merge_request, + staging: :read_build, + production: :read_issue, + }.freeze + + def self.get(*args) + new(*args).get + end + + def initialize(user:, project:) + @user = user + @project = project + @stage_permission_hash = {} + end + + def get + ::CycleAnalytics::STAGES.each do |stage| + @stage_permission_hash[stage] = authorized_stage?(stage) + end + + @stage_permission_hash + end + + private + + def authorized_stage?(stage) + return false unless authorize_project(:read_cycle_analytics) + + STAGE_PERMISSIONS[stage] ? authorize_project(STAGE_PERMISSIONS[stage]) : true + end + + def authorize_project(permission) + Ability.allowed?(@user, permission, @project) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/plan_event.rb b/lib/gitlab/cycle_analytics/plan_event.rb new file mode 100644 index 00000000000..7c3f0e9989f --- /dev/null +++ b/lib/gitlab/cycle_analytics/plan_event.rb @@ -0,0 +1,46 @@ +module Gitlab + module CycleAnalytics + class PlanEvent < BaseEvent + def initialize(*args) + @stage = :plan + @start_time_attrs = issue_metrics_table[:first_associated_with_milestone_at] + @end_time_attrs = [issue_metrics_table[:first_added_to_board_at], + issue_metrics_table[:first_mentioned_in_commit_at]] + @projections = [mr_diff_table[:st_commits].as('commits'), + issue_metrics_table[:first_mentioned_in_commit_at]] + + super(*args) + end + + def custom_query(base_query) + base_query.join(mr_diff_table).on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) + end + + private + + def serialize(event) + st_commit = first_time_reference_commit(event.delete('commits'), event) + + return unless st_commit + + serialize_commit(event, st_commit, query) + end + + def first_time_reference_commit(commits, event) + return nil if commits.blank? + + YAML.load(commits).find do |commit| + next unless commit[:committed_date] && event['first_mentioned_in_commit_at'] + + commit[:committed_date].to_i == DateTime.parse(event['first_mentioned_in_commit_at'].to_s).to_i + end + end + + def serialize_commit(event, st_commit, query) + commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project) + + AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit).as_json + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/production_event.rb b/lib/gitlab/cycle_analytics/production_event.rb new file mode 100644 index 00000000000..4868c3c6237 --- /dev/null +++ b/lib/gitlab/cycle_analytics/production_event.rb @@ -0,0 +1,26 @@ +module Gitlab + module CycleAnalytics + class ProductionEvent < BaseEvent + include IssueAllowed + + def initialize(*args) + @stage = :production + @start_time_attrs = issue_table[:created_at] + @end_time_attrs = mr_metrics_table[:first_deployed_to_production_at] + @projections = [issue_table[:title], + issue_table[:iid], + issue_table[:id], + issue_table[:created_at], + issue_table[:author_id]] + + super(*args) + end + + private + + def serialize(event) + AnalyticsIssueSerializer.new(project: @project).represent(event).as_json + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/review_event.rb b/lib/gitlab/cycle_analytics/review_event.rb new file mode 100644 index 00000000000..b394a02cc52 --- /dev/null +++ b/lib/gitlab/cycle_analytics/review_event.rb @@ -0,0 +1,25 @@ +module Gitlab + module CycleAnalytics + class ReviewEvent < BaseEvent + include MergeRequestAllowed + + def initialize(*args) + @stage = :review + @start_time_attrs = mr_table[:created_at] + @end_time_attrs = mr_metrics_table[:merged_at] + @projections = [mr_table[:title], + mr_table[:iid], + mr_table[:id], + mr_table[:created_at], + mr_table[:state], + mr_table[:author_id]] + + super(*args) + end + + def serialize(event) + AnalyticsMergeRequestSerializer.new(project: @project).represent(event).as_json + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/staging_event.rb b/lib/gitlab/cycle_analytics/staging_event.rb new file mode 100644 index 00000000000..a1f30b716f6 --- /dev/null +++ b/lib/gitlab/cycle_analytics/staging_event.rb @@ -0,0 +1,31 @@ +module Gitlab + module CycleAnalytics + class StagingEvent < BaseEvent + def initialize(*args) + @stage = :staging + @start_time_attrs = mr_metrics_table[:merged_at] + @end_time_attrs = mr_metrics_table[:first_deployed_to_production_at] + @projections = [build_table[:id]] + @order = build_table[:created_at] + + super(*args) + end + + def fetch + Updater.update!(event_result, from: 'id', to: 'build', klass: ::Ci::Build) + + super + end + + def custom_query(base_query) + base_query.join(build_table).on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) + end + + private + + def serialize(event) + AnalyticsBuildSerializer.new.represent(event['build']).as_json + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/test_event.rb b/lib/gitlab/cycle_analytics/test_event.rb new file mode 100644 index 00000000000..d553d0b5aec --- /dev/null +++ b/lib/gitlab/cycle_analytics/test_event.rb @@ -0,0 +1,13 @@ +module Gitlab + module CycleAnalytics + class TestEvent < StagingEvent + def initialize(*args) + super(*args) + + @stage = :test + @start_time_attrs = mr_metrics_table[:latest_build_started_at] + @end_time_attrs = mr_metrics_table[:latest_build_finished_at] + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/updater.rb b/lib/gitlab/cycle_analytics/updater.rb new file mode 100644 index 00000000000..953268ebd46 --- /dev/null +++ b/lib/gitlab/cycle_analytics/updater.rb @@ -0,0 +1,30 @@ +module Gitlab + module CycleAnalytics + class Updater + def self.update!(*args) + new(*args).update! + end + + def initialize(event_result, from:, to:, klass:) + @event_result = event_result + @klass = klass + @from = from + @to = to + end + + def update! + @event_result.each do |event| + event[@to] = items[event.delete(@from).to_i].first + end + end + + def result_ids + @event_result.map { |event| event[@from] } + end + + def items + @items ||= @klass.find(result_ids).group_by { |item| item['id'] } + end + end + end +end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 55b8f888d53..2d5c9232425 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -35,6 +35,13 @@ module Gitlab order end + def self.serialized_transaction + opts = {} + opts[:isolation] = :serializable unless Rails.env.test? && connection.transaction_open? + + connection.transaction(opts) { yield } + end + def self.random Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()" end diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb index b6a89f715fd..25e56998038 100644 --- a/lib/gitlab/database/date_time.rb +++ b/lib/gitlab/database/date_time.rb @@ -7,21 +7,25 @@ module Gitlab # # Note: For MySQL, the interval is returned in seconds. # For PostgreSQL, the interval is returned as an INTERVAL type. - def subtract_datetimes(query_so_far, end_time_attrs, start_time_attrs, as) - diff_fn = if Gitlab::Database.postgresql? - Arel::Nodes::Subtraction.new( - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) - elsif Gitlab::Database.mysql? - Arel::Nodes::NamedFunction.new( - "TIMESTAMPDIFF", - [Arel.sql('second'), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) - end + def subtract_datetimes(query_so_far, start_time_attrs, end_time_attrs, as) + diff_fn = subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) query_so_far.project(diff_fn.as(as)) end + + def subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) + if Gitlab::Database.postgresql? + Arel::Nodes::Subtraction.new( + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) + elsif Gitlab::Database.mysql? + Arel::Nodes::NamedFunction.new( + "TIMESTAMPDIFF", + [Arel.sql('second'), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) + end + end end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index ce85e5e0123..c6bf25b5874 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -55,6 +55,12 @@ module Gitlab repository.commit(deleted_file ? old_ref : new_ref) end + def old_content_commit + return unless diff_refs + + repository.commit(old_ref) + end + def old_ref diff_refs.try(:base_sha) end @@ -111,13 +117,10 @@ module Gitlab diff_lines.count(&:removed?) end - def old_blob(commit = content_commit) + def old_blob(commit = old_content_commit) return unless commit - parent_id = commit.parent_id - return unless parent_id - - repository.blob_at(parent_id, old_path) + repository.blob_at(commit.id, old_path) end def blob(commit = content_commit) @@ -126,7 +129,7 @@ module Gitlab repository.blob_at(commit.id, file_path) end - def cache_key + def file_identifier "#{file_path}-#{new_file}-#{deleted_file}-#{renamed_file}" end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index dc4d47c878b..fe7adb7bed6 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -39,7 +39,7 @@ module Gitlab # hashes that represent serialized diff lines. # def cache_highlight!(diff_file) - item_key = diff_file.cache_key + item_key = diff_file.file_identifier if highlight_cache[item_key] highlight_diff_file_from_cache!(diff_file, highlight_cache[item_key]) diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index f4d1505ea91..c8e36d8ff4a 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -149,7 +149,7 @@ module Gitlab end def ce_patch_name - @ce_patch_name ||= "#{ce_branch}.patch" + @ce_patch_name ||= patch_name_from_branch(ce_branch) end def ce_patch_full_path @@ -161,13 +161,17 @@ module Gitlab end def ee_patch_name - @ee_patch_name ||= "#{ee_branch}.patch" + @ee_patch_name ||= patch_name_from_branch(ee_branch) end def ee_patch_full_path @ee_patch_full_path ||= patches_dir.join(ee_patch_name) end + def patch_name_from_branch(branch_name) + branch_name.parameterize << '.patch' + end + def step(desc, cmd = nil) puts "\n=> #{desc}\n" diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb new file mode 100644 index 00000000000..a4ca62bfc41 --- /dev/null +++ b/lib/gitlab/email/html_parser.rb @@ -0,0 +1,34 @@ +module Gitlab + module Email + class HTMLParser + def self.parse_reply(raw_body) + new(raw_body).filtered_text + end + + attr_reader :raw_body + def initialize(raw_body) + @raw_body = raw_body + end + + def document + @document ||= Nokogiri::HTML.parse(raw_body) + end + + def filter_replies! + document.xpath('//blockquote').each(&:remove) + document.xpath('//table').each(&:remove) + end + + def filtered_html + @filtered_html ||= begin + filter_replies! + document.inner_html + end + end + + def filtered_text + @filtered_text ||= Html2Text.convert(filtered_html) + end + end + end +end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 3411eb1d9ce..85402c2a278 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -23,19 +23,26 @@ module Gitlab private def select_body(message) - text = message.text_part if message.multipart? - text ||= message if message.content_type !~ /text\/html/ + if message.multipart? + part = message.text_part || message.html_part || message + else + part = message + end - return "" unless text + decoded = fix_charset(part) - text = fix_charset(text) + return "" unless decoded # Certain trigger phrases that means we didn't parse correctly - if text =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/ + if decoded =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/ return "" end - text + if (part.content_type || '').include? 'text/html' + HTMLParser.parse_reply(decoded) + else + decoded + end end # Force encoding to UTF-8 on a Mail::Message or Mail::Part diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb new file mode 100644 index 00000000000..1d93a67dc56 --- /dev/null +++ b/lib/gitlab/file_detector.rb @@ -0,0 +1,63 @@ +require 'set' + +module Gitlab + # Module that can be used to detect if a path points to a special file such as + # a README or a CONTRIBUTING file. + module FileDetector + PATTERNS = { + readme: /\Areadme/i, + changelog: /\A(changelog|history|changes|news)/i, + license: /\A(licen[sc]e|copying)(\..+|\z)/i, + contributing: /\Acontributing/i, + version: 'version', + gitignore: '.gitignore', + koding: '.koding.yml', + gitlab_ci: '.gitlab-ci.yml', + avatar: /\Alogo\.(png|jpg|gif)\z/ + } + + # Returns an Array of file types based on the given paths. + # + # This method can be used to check if a list of file paths (e.g. of changed + # files) involve any special files such as a README or a LICENSE file. + # + # Example: + # + # types_in_paths(%w{README.md foo/bar.txt}) # => [:readme] + def self.types_in_paths(paths) + types = Set.new + + paths.each do |path| + type = type_of(path) + + types << type if type + end + + types.to_a + end + + # Returns the type of a file path, or nil if none could be detected. + # + # Returned types are Symbols such as `:readme`, `:version`, etc. + # + # Example: + # + # type_of('README.md') # => :readme + # type_of('VERSION') # => :version + def self.type_of(path) + name = File.basename(path) + + PATTERNS.each do |type, search| + did_match = if search.is_a?(Regexp) + name =~ search + else + name.casecmp(search) == 0 + end + + return type if did_match + end + + nil + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 90cf38a8513..281b65bdeba 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -20,10 +20,18 @@ module Gitlab end def execute + # The ordering of importing is important here due to the way GitHub structures their data + # 1. Labels are required by other items while not having a dependency on anything else + # so need to be first + # 2. Pull requests must come before issues. Every pull request is also an issue but not + # all issues are pull requests. Only the issue entity has labels defined in GitHub. GitLab + # doesn't structure data like this so we need to make sure that we've created the MRs + # before we attempt to add the labels defined in the GitHub issue for the related, already + # imported, pull request import_labels import_milestones - import_issues import_pull_requests + import_issues import_comments(:issues) import_comments(:pull_requests) import_wiki @@ -79,13 +87,17 @@ module Gitlab issues.each do |raw| gh_issue = IssueFormatter.new(project, raw) - if gh_issue.valid? - begin - issue = gh_issue.create! - apply_labels(issue, raw) - rescue => e - errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } - end + begin + issuable = + if gh_issue.pull_request? + MergeRequest.find_by_iid(gh_issue.number) + else + gh_issue.create! + end + + apply_labels(issuable, raw) + rescue => e + errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } end end end @@ -101,8 +113,7 @@ module Gitlab restore_source_branch(pull_request) unless pull_request.source_branch_exists? restore_target_branch(pull_request) unless pull_request.target_branch_exists? - merge_request = pull_request.create! - apply_labels(merge_request, raw) + pull_request.create! rescue => e errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message } ensure @@ -133,21 +144,14 @@ module Gitlab remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? end - def apply_labels(issuable, raw_issuable) - # GH returns labels for issues but not for pull requests! - labels = if issuable.is_a?(MergeRequest) - client.labels_for_issue(repo, raw_issuable.number) - else - raw_issuable.labels - end + def apply_labels(issuable, raw) + return unless raw.labels.count > 0 - if labels.count > 0 - label_ids = labels - .map { |attrs| @labels[attrs.name] } - .compact + label_ids = raw.labels + .map { |attrs| @labels[attrs.name] } + .compact - issuable.update_attribute(:label_ids, label_ids) - end + issuable.update_attribute(:label_ids, label_ids) end def import_comments(issuable_type) diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 8c32ac59fc5..887690bcc7c 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -32,8 +32,8 @@ module Gitlab raw_data.number end - def valid? - raw_data.pull_request.nil? + def pull_request? + raw_data.pull_request.present? end private diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index f8809db21aa..94678b6ec40 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -21,10 +21,8 @@ module Gitlab return if !commit || !commit.author_email - email = commit.author_email - - identify_with_cache(:email, email) do - User.find_by(email: email) + identify_with_cache(:email, commit.author_email) do + commit.author end end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 8b38cfaefb6..7b05290e5cc 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -89,9 +89,7 @@ module Gitlab end def user_filter(filter = nil) - if config.user_filter.present? - user_filter = Net::LDAP::Filter.construct(config.user_filter) - end + user_filter = config.constructed_user_filter if config.user_filter.present? if user_filter && filter Net::LDAP::Filter.join(filter, user_filter) diff --git a/lib/gitlab/ldap/authentication.rb b/lib/gitlab/ldap/authentication.rb index bad683c6511..4745311402c 100644 --- a/lib/gitlab/ldap/authentication.rb +++ b/lib/gitlab/ldap/authentication.rb @@ -54,11 +54,9 @@ module Gitlab # Apply LDAP user filter if present if config.user_filter.present? - filter = Net::LDAP::Filter.join( - filter, - Net::LDAP::Filter.construct(config.user_filter) - ) + filter = Net::LDAP::Filter.join(filter, config.constructed_user_filter) end + filter end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 6ea069d26df..de52ef3fc65 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -13,7 +13,7 @@ module Gitlab end def self.providers - servers.map {|server| server['provider_name'] } + servers.map { |server| server['provider_name'] } end def self.valid_provider?(provider) @@ -38,13 +38,31 @@ module Gitlab end def adapter_options - { - host: options['host'], - port: options['port'], - encryption: encryption - }.tap do |options| - options.merge!(auth_options) if has_auth? + opts = base_options.merge( + encryption: encryption, + ) + + opts.merge!(auth_options) if has_auth? + + opts + end + + def omniauth_options + opts = base_options.merge( + base: base, + method: options['method'], + filter: omniauth_user_filter, + name_proc: name_proc + ) + + if has_auth? + opts.merge!( + bind_dn: options['bind_dn'], + password: options['password'] + ) end + + opts end def base @@ -68,6 +86,10 @@ module Gitlab options['user_filter'] end + def constructed_user_filter + @constructed_user_filter ||= Net::LDAP::Filter.construct(user_filter) + end + def group_base options['group_base'] end @@ -96,8 +118,27 @@ module Gitlab options['password'] || options['bind_dn'] end + def allow_username_or_email_login + options['allow_username_or_email_login'] + end + + def name_proc + if allow_username_or_email_login + Proc.new { |name| name.gsub(/@.*\z/, '') } + else + Proc.new { |name| name } + end + end + protected + def base_options + { + host: options['host'], + port: options['port'] + } + end + def base_config Gitlab.config.ldap end @@ -126,6 +167,16 @@ module Gitlab } } end + + def omniauth_user_filter + uid_filter = Net::LDAP::Filter.eq(uid, '%{username}') + + if user_filter.present? + Net::LDAP::Filter.join(uid_filter, constructed_user_filter).to_s + else + uid_filter.to_s + end + end end end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index a5220d92312..3503fac40e8 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -31,6 +31,7 @@ module Gitlab config[:ssl] = false if config[:ssl].nil? config[:start_tls] = false if config[:start_tls].nil? config[:mailbox] = 'inbox' if config[:mailbox].nil? + config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] gitlab_redis = Gitlab::Redis.new(rails_env) diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 0a91d3918d5..a8b4dc2a83f 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -102,6 +102,8 @@ module Gitlab Gitlab::LDAP::Config.providers.each do |provider| adapter = Gitlab::LDAP::Adapter.new(provider) @ldap_person = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) + # The `uid` might actually be a DN. Try it next. + @ldap_person ||= Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) break if @ldap_person end @ldap_person diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index b8326a64b22..66e6b29e798 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -5,7 +5,7 @@ module Gitlab def initialize(current_user, project, query, repository_ref = nil) @current_user = current_user @project = project - @repository_ref = repository_ref.presence + @repository_ref = repository_ref.presence || project.default_branch @query = query end @@ -40,10 +40,57 @@ module Gitlab @commits_count ||= commits.count end + def self.parse_search_result(result) + ref = nil + filename = nil + basename = nil + startline = 0 + + result.each_line.each_with_index do |line, index| + if line =~ /^.*:.*:\d+:/ + ref, filename, startline = line.split(':') + startline = startline.to_i - index + extname = Regexp.escape(File.extname(filename)) + basename = filename.sub(/#{extname}$/, '') + break + end + end + + data = "" + + result.each_line do |line| + data << line.sub(ref, '').sub(filename, '').sub(/^:-\d+-/, '').sub(/^::\d+:/, '') + end + + OpenStruct.new( + filename: filename, + basename: basename, + ref: ref, + startline: startline, + data: data + ) + end + private def blobs - @blobs ||= project.repository.search_files(query, repository_ref) + @blobs ||= begin + blobs = project.repository.search_files_by_content(query, repository_ref).first(100) + found_file_names = Set.new + + results = blobs.map do |blob| + blob = self.class.parse_search_result(blob) + found_file_names << blob.filename + + [blob.filename, blob] + end + + project.repository.search_files_by_name(query, repository_ref).first(100).each do |filename| + results << [filename, nil] unless found_file_names.include?(filename) + end + + results.sort_by(&:first) + end end def wiki_blobs diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index cb1659f9cee..a06cf6a989c 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -2,7 +2,16 @@ module Gitlab module Regex extend self - NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_])(?<!\.git|\.atom)'.freeze + # The namespace regex is used in Javascript to validate usernames in the "Register" form. However, Javascript + # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`. + # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to + # allow non-regex validatiions, etc), `NAMESPACE_REGEX_STR_SIMPLE` serves as a Javascript-compatible version of + # `NAMESPACE_REGEX_STR`, with the negative lookbehind assertion removed. This means that the client-side validation + # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. + PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze + NAMESPACE_REGEX_STR_SIMPLE = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze + NAMESPACE_REGEX_STR = '(?:' + NAMESPACE_REGEX_STR_SIMPLE + ')(?<!\.git|\.atom)'.freeze + PROJECT_REGEX_STR = PATH_REGEX_STR + '(?<!\.git|\.atom)'.freeze def namespace_regex @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze @@ -26,16 +35,24 @@ module Gitlab end def project_name_regex - @project_name_regex ||= /\A[\p{Alnum}_][\p{Alnum}\p{Pd}_\. ]*\z/.freeze + @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9c0}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9c0}_\. ]*\z/.freeze end def project_name_regex_message - "can contain only letters, digits, '_', '.', dash and space. " \ - "It must start with letter, digit or '_'." + "can contain only letters, digits, emojis, '_', '.', dash, space. " \ + "It must start with letter, digit, emoji or '_'." end def project_path_regex - @project_path_regex ||= /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\.]*(?<!\.git|\.atom)\z/.freeze + @project_path_regex ||= /\A#{PROJECT_REGEX_STR}\z/.freeze + end + + def project_route_regex + @project_route_regex ||= /#{PROJECT_REGEX_STR}/.freeze + end + + def project_git_route_regex + @project_route_git_regex ||= /#{PATH_REGEX_STR}\.git/.freeze end def project_path_regex_message diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb new file mode 100644 index 00000000000..d4d39a888e7 --- /dev/null +++ b/lib/gitlab/sidekiq_throttler.rb @@ -0,0 +1,23 @@ +module Gitlab + class SidekiqThrottler + class << self + def execute! + if Gitlab::CurrentSettings.sidekiq_throttling_enabled? + Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue| + Sidekiq::Queue[queue].limit = queue_limit + end + end + end + + private + + def queue_limit + @queue_limit ||= + begin + factor = Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_factor + (factor * Sidekiq.options[:concurrency]).ceil + end + end + end + end +end |