diff options
Diffstat (limited to 'lib/gitlab')
152 files changed, 4945 insertions, 754 deletions
diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index fa234284361..0618107e2c3 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -1,5 +1,6 @@ require 'asciidoctor' require 'asciidoctor/converter/html5' +require "asciidoctor-plantuml" module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters @@ -29,6 +30,8 @@ module Gitlab ) asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) + plantuml_setup + html = ::Asciidoctor.convert(input, asciidoc_opts) html = Banzai.post_process(html, context) @@ -36,6 +39,15 @@ module Gitlab html.html_safe end + def self.plantuml_setup + Asciidoctor::PlantUml.configure do |conf| + conf.url = ApplicationSetting.current.plantuml_url + conf.svg_enable = ApplicationSetting.current.plantuml_enabled + conf.png_enable = ApplicationSetting.current.plantuml_enabled + conf.txt_enable = false + end + end + class Html5Converter < Asciidoctor::Converter::Html5Converter extend Asciidoctor::Converter::Config diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 8dda65c71ef..f638905a1e0 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -10,13 +10,16 @@ module Gitlab def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? + # `user_with_password_for_git` should be the last check + # because it's the most expensive, especially when LDAP + # is enabled. result = service_request_check(login, password, project) || build_access_token_check(login, password) || - user_with_password_for_git(login, password) || - oauth_access_token_check(login, password) || lfs_token_check(login, password) || + oauth_access_token_check(login, password) || personal_access_token_check(login, password) || + user_with_password_for_git(login, password) || Gitlab::Auth::Result.new rate_limit!(ip, success: result.success?, login: login) @@ -143,7 +146,9 @@ module Gitlab read_authentication_abilities end - Result.new(actor, nil, token_handler.type, authentication_abilities) if Devise.secure_compare(token_handler.token, password) + if Devise.secure_compare(token_handler.token, password) + Gitlab::Auth::Result.new(actor, nil, token_handler.type, authentication_abilities) + end end def build_access_token_check(login, password) diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 6be7f690676..39b86c61a18 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -9,8 +9,7 @@ module Gitlab def lfs_deploy_token?(for_project) type == :lfs_deploy_token && - actor && - actor.projects.include?(for_project) + actor.try(:has_access_to?, for_project) end def success? diff --git a/lib/gitlab/chat_commands/base_command.rb b/lib/gitlab/chat_commands/base_command.rb index 4fe53ce93a9..25da8474e95 100644 --- a/lib/gitlab/chat_commands/base_command.rb +++ b/lib/gitlab/chat_commands/base_command.rb @@ -42,10 +42,6 @@ module Gitlab def find_by_iid(iid) collection.find_by(iid: iid) end - - def presenter - Gitlab::ChatCommands::Presenter.new - end end end end diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb index 145086755e4..f34ed0f4cf2 100644 --- a/lib/gitlab/chat_commands/command.rb +++ b/lib/gitlab/chat_commands/command.rb @@ -3,7 +3,7 @@ module Gitlab class Command < BaseCommand COMMANDS = [ Gitlab::ChatCommands::IssueShow, - Gitlab::ChatCommands::IssueCreate, + Gitlab::ChatCommands::IssueNew, Gitlab::ChatCommands::IssueSearch, Gitlab::ChatCommands::Deploy, ].freeze @@ -13,51 +13,32 @@ module Gitlab if command if command.allowed?(project, current_user) - present command.new(project, current_user, params).execute(match) + command.new(project, current_user, params).execute(match) else - access_denied + Gitlab::ChatCommands::Presenters::Access.new.access_denied end else - help(help_messages) + Gitlab::ChatCommands::Help.new(project, current_user, params).execute(available_commands, params[:text]) end end def match_command match = nil - service = available_commands.find do |klass| - match = klass.match(command) - end + service = + available_commands.find do |klass| + match = klass.match(params[:text]) + end [service, match] end private - def help_messages - available_commands.map(&:help_message) - end - def available_commands COMMANDS.select do |klass| klass.available?(project) end end - - def command - params[:text] - end - - def help(messages) - presenter.help(messages, params[:command]) - end - - def access_denied - presenter.access_denied - end - - def present(resource) - presenter.present(resource) - end end end end diff --git a/lib/gitlab/chat_commands/deploy.rb b/lib/gitlab/chat_commands/deploy.rb index 6bb854dc080..458d90f84e8 100644 --- a/lib/gitlab/chat_commands/deploy.rb +++ b/lib/gitlab/chat_commands/deploy.rb @@ -1,8 +1,6 @@ module Gitlab module ChatCommands class Deploy < BaseCommand - include Gitlab::Routing.url_helpers - def self.match(text) /\Adeploy\s+(?<from>\S+.*)\s+to+\s+(?<to>\S+.*)\z/.match(text) end @@ -24,34 +22,29 @@ module Gitlab to = match[:to] actions = find_actions(from, to) - return unless actions.present? - if actions.one? - play!(from, to, actions.first) + if actions.none? + Gitlab::ChatCommands::Presenters::Deploy.new(nil).no_actions + elsif actions.one? + action = play!(from, to, actions.first) + Gitlab::ChatCommands::Presenters::Deploy.new(action).present(from, to) else - Result.new(:error, 'Too many actions defined') + Gitlab::ChatCommands::Presenters::Deploy.new(actions).too_many_actions end end private def play!(from, to, action) - new_action = action.play(current_user) - - Result.new(:success, "Deployment from #{from} to #{to} started. Follow the progress: #{url(new_action)}.") + action.play(current_user) end def find_actions(from, to) environment = project.environments.find_by(name: from) - return unless environment + return [] unless environment environment.actions_for(to).select(&:starts_environment?) end - - def url(subject) - polymorphic_url( - [ subject.project.namespace.becomes(Namespace), subject.project, subject ]) - end end end end diff --git a/lib/gitlab/chat_commands/help.rb b/lib/gitlab/chat_commands/help.rb new file mode 100644 index 00000000000..6c0e4d304a4 --- /dev/null +++ b/lib/gitlab/chat_commands/help.rb @@ -0,0 +1,28 @@ +module Gitlab + module ChatCommands + class Help < BaseCommand + # This class has to be used last, as it always matches. It has to match + # because other commands were not triggered and we want to show the help + # command + def self.match(_text) + true + end + + def self.help_message + 'help' + end + + def self.allowed?(_project, _user) + true + end + + def execute(commands, text) + Gitlab::ChatCommands::Presenters::Help.new(commands).present(trigger, text) + end + + def trigger + params[:command] + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_create.rb b/lib/gitlab/chat_commands/issue_new.rb index cefb6775db8..016054ecd46 100644 --- a/lib/gitlab/chat_commands/issue_create.rb +++ b/lib/gitlab/chat_commands/issue_new.rb @@ -1,8 +1,8 @@ module Gitlab module ChatCommands - class IssueCreate < IssueCommand + class IssueNew < IssueCommand def self.match(text) - # we can not match \n with the dot by passing the m modifier as than + # we can not match \n with the dot by passing the m modifier as than # the title and description are not seperated /\Aissue\s+(new|create)\s+(?<title>[^\n]*)\n*(?<description>(.|\n)*)/.match(text) end @@ -19,8 +19,24 @@ module Gitlab title = match[:title] description = match[:description].to_s.rstrip + issue = create_issue(title: title, description: description) + + if issue.persisted? + presenter(issue).present + else + presenter(issue).display_errors + end + end + + private + + def create_issue(title:, description:) Issues::CreateService.new(project, current_user, title: title, description: description).execute end + + def presenter(issue) + Gitlab::ChatCommands::Presenters::IssueNew.new(issue) + end end end end diff --git a/lib/gitlab/chat_commands/issue_search.rb b/lib/gitlab/chat_commands/issue_search.rb index 51bf80c800b..3491b53093e 100644 --- a/lib/gitlab/chat_commands/issue_search.rb +++ b/lib/gitlab/chat_commands/issue_search.rb @@ -10,7 +10,13 @@ module Gitlab end def execute(match) - collection.search(match[:query]).limit(QUERY_LIMIT) + issues = collection.search(match[:query]).limit(QUERY_LIMIT) + + if issues.present? + Presenters::IssueSearch.new(issues).present + else + Presenters::Access.new(issues).not_found + end end end end diff --git a/lib/gitlab/chat_commands/issue_show.rb b/lib/gitlab/chat_commands/issue_show.rb index 2a45d49cf6b..d6013f4d10c 100644 --- a/lib/gitlab/chat_commands/issue_show.rb +++ b/lib/gitlab/chat_commands/issue_show.rb @@ -10,7 +10,13 @@ module Gitlab end def execute(match) - find_by_iid(match[:iid]) + issue = find_by_iid(match[:iid]) + + if issue + Gitlab::ChatCommands::Presenters::IssueShow.new(issue).present + else + Gitlab::ChatCommands::Presenters::Access.new.not_found + end end end end diff --git a/lib/gitlab/chat_commands/presenter.rb b/lib/gitlab/chat_commands/presenter.rb deleted file mode 100644 index caceaa25391..00000000000 --- a/lib/gitlab/chat_commands/presenter.rb +++ /dev/null @@ -1,131 +0,0 @@ -module Gitlab - module ChatCommands - class Presenter - include Gitlab::Routing - - def authorize_chat_name(url) - message = if url - ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{url})." - else - ":sweat_smile: Couldn't identify you, nor can I autorize you!" - end - - ephemeral_response(message) - end - - def help(commands, trigger) - if commands.none? - ephemeral_response("No commands configured") - else - commands.map! { |command| "#{trigger} #{command}" } - message = header_with_list("Available commands", commands) - - ephemeral_response(message) - end - end - - def present(subject) - return not_found unless subject - - if subject.is_a?(Gitlab::ChatCommands::Result) - show_result(subject) - elsif subject.respond_to?(:count) - if subject.many? - multiple_resources(subject) - elsif subject.none? - not_found - else - single_resource(subject) - end - else - single_resource(subject) - end - end - - def access_denied - ephemeral_response("Whoops! That action is not allowed. This incident will be [reported](https://xkcd.com/838/).") - end - - private - - def show_result(result) - case result.type - when :success - in_channel_response(result.message) - else - ephemeral_response(result.message) - end - end - - def not_found - ephemeral_response("404 not found! GitLab couldn't find what you were looking for! :boom:") - end - - def single_resource(resource) - return error(resource) if resource.errors.any? || !resource.persisted? - - message = "#{title(resource)}:" - message << "\n\n#{resource.description}" if resource.try(:description) - - in_channel_response(message) - end - - def multiple_resources(resources) - resources.map! { |resource| title(resource) } - - message = header_with_list("Multiple results were found:", resources) - - ephemeral_response(message) - end - - def error(resource) - message = header_with_list("The action was not successful, because:", resource.errors.messages) - - ephemeral_response(message) - end - - def title(resource) - reference = resource.try(:to_reference) || resource.try(:id) - title = resource.try(:title) || resource.try(:name) - - "[#{reference} #{title}](#{url(resource)})" - end - - def header_with_list(header, items) - message = [header] - - items.each do |item| - message << "- #{item}" - end - - message.join("\n") - end - - def url(resource) - url_for( - [ - resource.project.namespace.becomes(Namespace), - resource.project, - resource - ] - ) - end - - def ephemeral_response(message) - { - response_type: :ephemeral, - text: message, - status: 200 - } - end - - def in_channel_response(message) - { - response_type: :in_channel, - text: message, - status: 200 - } - end - end - end -end diff --git a/lib/gitlab/chat_commands/presenters/access.rb b/lib/gitlab/chat_commands/presenters/access.rb new file mode 100644 index 00000000000..92f4fa17f78 --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/access.rb @@ -0,0 +1,40 @@ +module Gitlab + module ChatCommands + module Presenters + class Access < Presenters::Base + def access_denied + ephemeral_response(text: "Whoops! This action is not allowed. This incident will be [reported](https://xkcd.com/838/).") + end + + def not_found + ephemeral_response(text: "404 not found! GitLab couldn't find what you were looking for! :boom:") + end + + def authorize + message = + if @resource + ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{@resource})." + else + ":sweat_smile: Couldn't identify you, nor can I autorize you!" + end + + ephemeral_response(text: message) + end + + def unknown_command(commands) + ephemeral_response(text: help_message(trigger)) + end + + private + + def help_message(trigger) + header_with_list("Command not found, these are the commands you can use", full_commands(trigger)) + end + + def full_commands(trigger) + @resource.map { |command| "#{trigger} #{command.help_message}" } + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/base.rb b/lib/gitlab/chat_commands/presenters/base.rb new file mode 100644 index 00000000000..2700a5a2ad5 --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/base.rb @@ -0,0 +1,77 @@ +module Gitlab + module ChatCommands + module Presenters + class Base + include Gitlab::Routing.url_helpers + + def initialize(resource = nil) + @resource = resource + end + + def display_errors + message = header_with_list("The action was not successful, because:", @resource.errors.full_messages) + + ephemeral_response(text: message) + end + + private + + def header_with_list(header, items) + message = [header] + + items.each do |item| + message << "- #{item}" + end + + message.join("\n") + end + + def ephemeral_response(message) + response = { + response_type: :ephemeral, + status: 200 + }.merge(message) + + format_response(response) + end + + def in_channel_response(message) + response = { + response_type: :in_channel, + status: 200 + }.merge(message) + + format_response(response) + end + + def format_response(response) + response[:text] = format(response[:text]) if response.has_key?(:text) + + if response.has_key?(:attachments) + response[:attachments].each do |attachment| + attachment[:pretext] = format(attachment[:pretext]) if attachment[:pretext] + attachment[:text] = format(attachment[:text]) if attachment[:text] + end + end + + response + end + + # Convert Markdown to slacks format + def format(string) + Slack::Notifier::LinkFormatter.format(string) + end + + def resource_url + url_for( + [ + @resource.project.namespace.becomes(Namespace), + @resource.project, + @resource + ] + ) + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/deploy.rb b/lib/gitlab/chat_commands/presenters/deploy.rb new file mode 100644 index 00000000000..863d0bf99ca --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/deploy.rb @@ -0,0 +1,21 @@ +module Gitlab + module ChatCommands + module Presenters + class Deploy < Presenters::Base + def present(from, to) + message = "Deployment started from #{from} to #{to}. [Follow its progress](#{resource_url})." + + in_channel_response(text: message) + end + + def no_actions + ephemeral_response(text: "No action found to be executed") + end + + def too_many_actions + ephemeral_response(text: "Too many actions defined") + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/help.rb b/lib/gitlab/chat_commands/presenters/help.rb new file mode 100644 index 00000000000..cd47b7f4c6a --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/help.rb @@ -0,0 +1,27 @@ +module Gitlab + module ChatCommands + module Presenters + class Help < Presenters::Base + def present(trigger, text) + ephemeral_response(text: help_message(trigger, text)) + end + + private + + def help_message(trigger, text) + return "No commands available :thinking_face:" unless @resource.present? + + if text.start_with?('help') + header_with_list("Available commands", full_commands(trigger)) + else + header_with_list("Unknown command, these commands are available", full_commands(trigger)) + end + end + + def full_commands(trigger) + @resource.map { |command| "#{trigger} #{command.help_message}" } + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/issuable.rb b/lib/gitlab/chat_commands/presenters/issuable.rb new file mode 100644 index 00000000000..dfb1c8f6616 --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/issuable.rb @@ -0,0 +1,43 @@ +module Gitlab + module ChatCommands + module Presenters + module Issuable + def color(issuable) + issuable.open? ? '#38ae67' : '#d22852' + end + + def status_text(issuable) + issuable.open? ? 'Open' : 'Closed' + end + + def project + @resource.project + end + + def author + @resource.author + end + + def fields + [ + { + title: "Assignee", + value: @resource.assignee ? @resource.assignee.name : "_None_", + short: true + }, + { + title: "Milestone", + value: @resource.milestone ? @resource.milestone.title : "_None_", + short: true + }, + { + title: "Labels", + value: @resource.labels.any? ? @resource.label_names : "_None_", + short: true + } + ] + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/issue_new.rb b/lib/gitlab/chat_commands/presenters/issue_new.rb new file mode 100644 index 00000000000..a1a3add56c9 --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/issue_new.rb @@ -0,0 +1,50 @@ +module Gitlab + module ChatCommands + module Presenters + class IssueNew < Presenters::Base + include Presenters::Issuable + + def present + in_channel_response(new_issue) + end + + private + + def new_issue + { + attachments: [ + { + title: "#{@resource.title} · #{@resource.to_reference}", + title_link: resource_url, + author_name: author.name, + author_icon: author.avatar_url, + fallback: "New issue #{@resource.to_reference}: #{@resource.title}", + pretext: pretext, + color: color(@resource), + fields: fields, + mrkdwn_in: [ + :title, + :pretext, + :text, + :fields + ] + } + ] + } + end + + def pretext + "I created an issue on #{author_profile_link}'s behalf: **#{@resource.to_reference}** in #{project_link}" + end + + def project_link + "[#{project.name_with_namespace}](#{projects_url(project)})" + end + + def author_profile_link + "[#{author.to_reference}](#{url_for(author)})" + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/issue_search.rb b/lib/gitlab/chat_commands/presenters/issue_search.rb new file mode 100644 index 00000000000..3478359b91d --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/issue_search.rb @@ -0,0 +1,47 @@ +module Gitlab + module ChatCommands + module Presenters + class IssueSearch < Presenters::Base + include Presenters::Issuable + + def present + text = if @resource.count >= 5 + "Here are the first 5 issues I found:" + elsif @resource.one? + "Here is the only issue I found:" + else + "Here are the #{@resource.count} issues I found:" + end + + ephemeral_response(text: text, attachments: attachments) + end + + private + + def attachments + @resource.map do |issue| + url = "[#{issue.to_reference}](#{url_for([namespace, project, issue])})" + + { + color: color(issue), + fallback: "#{issue.to_reference} #{issue.title}", + text: "#{url} · #{issue.title} (#{status_text(issue)})", + + mrkdwn_in: [ + :text + ] + } + end + end + + def project + @project ||= @resource.first.project + end + + def namespace + @namespace ||= project.namespace.becomes(Namespace) + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/issue_show.rb b/lib/gitlab/chat_commands/presenters/issue_show.rb new file mode 100644 index 00000000000..fe5847ccd15 --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/issue_show.rb @@ -0,0 +1,61 @@ +module Gitlab + module ChatCommands + module Presenters + class IssueShow < Presenters::Base + include Presenters::Issuable + + def present + if @resource.confidential? + ephemeral_response(show_issue) + else + in_channel_response(show_issue) + end + end + + private + + def show_issue + { + attachments: [ + { + title: "#{@resource.title} · #{@resource.to_reference}", + title_link: resource_url, + author_name: author.name, + author_icon: author.avatar_url, + fallback: "Issue #{@resource.to_reference}: #{@resource.title}", + pretext: pretext, + text: text, + color: color(@resource), + fields: fields, + mrkdwn_in: [ + :pretext, + :text, + :fields + ] + } + ] + } + end + + def text + message = "**#{status_text(@resource)}**" + + if @resource.upvotes.zero? && @resource.downvotes.zero? && @resource.user_notes_count.zero? + return message + end + + message << " · " + message << ":+1: #{@resource.upvotes} " unless @resource.upvotes.zero? + message << ":-1: #{@resource.downvotes} " unless @resource.downvotes.zero? + message << ":speech_balloon: #{@resource.user_notes_count}" unless @resource.user_notes_count.zero? + + message + end + + def pretext + "Issue *#{@resource.to_reference}* from #{project.name_with_namespace}" + end + end + end + end +end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 3d203017d9f..273118135a9 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -1,14 +1,16 @@ module Gitlab module Checks class ChangeAccess - attr_reader :user_access, :project + attr_reader :user_access, :project, :skip_authorization - def initialize(change, user_access:, project:, env: {}) + def initialize( + change, user_access:, project:, env: {}, skip_authorization: false) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @user_access = user_access @project = project @env = env + @skip_authorization = skip_authorization end def exec @@ -24,12 +26,13 @@ module Gitlab protected def protected_branch_checks + return if skip_authorization return unless @branch_name return unless project.protected_branch?(@branch_name) - if forced_push? && user_access.cannot_do_action?(:force_push_code_to_protected_branches) + if forced_push? return "You are not allowed to force push code to a protected branch on this project." - elsif Gitlab::Git.blank_ref?(@newrev) && user_access.cannot_do_action?(:remove_protected_branches) + elsif Gitlab::Git.blank_ref?(@newrev) return "You are not allowed to delete protected branches from this project." end @@ -49,6 +52,8 @@ module Gitlab end def tag_checks + return if skip_authorization + tag_ref = Gitlab::Git.tag_name(@ref) if tag_ref && protected_tag?(tag_ref) && user_access.cannot_do_action?(:admin_project) @@ -57,6 +62,8 @@ module Gitlab end def push_checks + return if skip_authorization + if user_access.cannot_do_action?(:push_code) "You are not allowed to push code to this project." end diff --git a/lib/gitlab/ci/config/entry/coverage.rb b/lib/gitlab/ci/config/entry/coverage.rb new file mode 100644 index 00000000000..12a063059cb --- /dev/null +++ b/lib/gitlab/ci/config/entry/coverage.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents Coverage settings. + # + class Coverage < Node + include Validatable + + validations do + validates :config, regexp: true + end + + def value + @config[1...-1] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index b7b4b91eb51..f7c530c7d9f 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -33,7 +33,6 @@ module Gitlab validates :url, length: { maximum: 255 }, - addressable_url: true, allow_nil: true validates :action, diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index a55362f0b6b..69a5e6f433d 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,7 +11,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except type image services allow_failure type stage when artifacts cache dependencies before_script - after_script variables environment] + after_script variables environment coverage] validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -71,9 +71,12 @@ module Gitlab entry :environment, Entry::Environment, description: 'Environment configuration for this job.' + entry :coverage, Entry::Coverage, + description: 'Coverage configuration for this job.' + helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands, :environment + :artifacts, :commands, :environment, :coverage attributes :script, :tags, :allow_failure, :when, :dependencies @@ -130,6 +133,7 @@ module Gitlab variables: variables_defined? ? variables_value : nil, environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, + coverage: coverage_defined? ? coverage_value : nil, artifacts: artifacts_value, after_script: after_script_value } end diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index f01975aab5c..9b9a0a8125a 100644 --- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -28,17 +28,21 @@ module Gitlab value.is_a?(String) || value.is_a?(Symbol) end + def validate_regexp(value) + !value.nil? && Regexp.new(value.to_s) && true + rescue RegexpError, TypeError + false + end + def validate_string_or_regexp(value) return true if value.is_a?(Symbol) return false unless value.is_a?(String) if value.first == '/' && value.last == '/' - Regexp.new(value[1...-1]) + validate_regexp(value[1...-1]) else true end - rescue RegexpError - false end def validate_boolean(value) diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb index 28b0a9ffe01..16b234e6c59 100644 --- a/lib/gitlab/ci/config/entry/trigger.rb +++ b/lib/gitlab/ci/config/entry/trigger.rb @@ -9,15 +9,7 @@ module Gitlab include Validatable validations do - include LegacyValidationHelpers - - validate :array_of_strings_or_regexps - - def array_of_strings_or_regexps - unless validate_array_of_strings_or_regexps(config) - errors.add(:config, 'should be an array of strings or regexps') - end - end + validates :config, array_of_strings_or_regexps: true end end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index 8632dd0e233..bd7428b1272 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -54,6 +54,51 @@ module Gitlab end end + class RegexpValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_regexp(value) + record.errors.add(attribute, 'must be a regular expression') + end + end + + private + + def look_like_regexp?(value) + value.is_a?(String) && value.start_with?('/') && + value.end_with?('/') + end + + def validate_regexp(value) + look_like_regexp?(value) && + Regexp.new(value.to_s[1...-1]) && + true + rescue RegexpError + false + end + end + + class ArrayOfStringsOrRegexpsValidator < RegexpValidator + def validate_each(record, attribute, value) + unless validate_array_of_strings_or_regexps(value) + record.errors.add(attribute, 'should be an array of strings or regexps') + end + end + + private + + def validate_array_of_strings_or_regexps(values) + values.is_a?(Array) && values.all?(&method(:validate_string_or_regexp)) + end + + def validate_string_or_regexp(value) + return false unless value.is_a?(String) + return validate_regexp(value) if look_like_regexp?(value) + true + end + end + class TypeValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) type = options[:with] diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index a979fe7d573..67bbc3c4849 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -10,7 +10,7 @@ module Gitlab end def action_icon - 'ban' + 'icon_action_cancel' end def action_path diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index eee9a64120b..38ac6edc9f1 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -4,8 +4,11 @@ module Gitlab module Build class Factory < Status::Factory def self.extended_statuses - [Status::Build::Stop, Status::Build::Play, - Status::Build::Cancelable, Status::Build::Retryable] + [[Status::Build::Cancelable, + Status::Build::Retryable], + [Status::Build::FailedAllowed, + Status::Build::Play, + Status::Build::Stop]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb new file mode 100644 index 00000000000..807afe24bd5 --- /dev/null +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module Status + module Build + class FailedAllowed < SimpleDelegator + include Status::Extended + + def label + 'failed (allowed to fail)' + end + + def icon + 'icon_status_warning' + end + + def group + 'failed_with_warnings' + end + + def self.matches?(build, user) + build.failed? && build.allow_failure? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index 1bf949c96dd..0f4b7b24cef 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -26,17 +26,13 @@ module Gitlab end def action_icon - 'play' + 'icon_action_play' end def action_title 'Play' end - def action_class - 'ci-play-icon' - end - def action_path play_namespace_project_build_path(subject.project.namespace, subject.project, diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 8e38d6a8523..6b362af7634 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -10,7 +10,7 @@ module Gitlab end def action_icon - 'refresh' + 'icon_action_retry' end def action_title diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index e1dfdb76d41..90401cad0d2 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -26,7 +26,7 @@ module Gitlab end def action_icon - 'stop' + 'icon_action_stop' end def action_title diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index 73b6ab5a635..3dd2b9e01f6 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -42,9 +42,6 @@ module Gitlab raise NotImplementedError end - def action_class - end - def action_path raise NotImplementedError end diff --git a/lib/gitlab/ci/status/external/common.rb b/lib/gitlab/ci/status/external/common.rb new file mode 100644 index 00000000000..4969a350862 --- /dev/null +++ b/lib/gitlab/ci/status/external/common.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module External + module Common + def has_details? + subject.target_url.present? && + can?(user, :read_commit_status, subject) + end + + def details_path + subject.target_url + end + + def has_action? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/external/factory.rb b/lib/gitlab/ci/status/external/factory.rb new file mode 100644 index 00000000000..07b15bd8d97 --- /dev/null +++ b/lib/gitlab/ci/status/external/factory.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Status + module External + class Factory < Status::Factory + def self.common_helpers + Status::External::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index ae9ef895df4..15836c699c7 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -5,41 +5,46 @@ module Gitlab def initialize(subject, user) @subject = subject @user = user + @status = subject.status || HasStatus::DEFAULT_STATUS end def fabricate! - if extended_status - extended_status.new(core_status) - else + if extended_statuses.none? core_status + else + compound_extended_status end end - def self.extended_statuses - [] + def core_status + Gitlab::Ci::Status + .const_get(@status.capitalize) + .new(@subject, @user) + .extend(self.class.common_helpers) end - def self.common_helpers - Module.new + def compound_extended_status + extended_statuses.inject(core_status) do |status, extended| + extended.new(status) + end end - private + def extended_statuses + return @extended_statuses if defined?(@extended_statuses) - def simple_status - @simple_status ||= @subject.status || :created + groups = self.class.extended_statuses.map do |group| + Array(group).find { |status| status.matches?(@subject, @user) } + end + + @extended_statuses = groups.flatten.compact end - def core_status - Gitlab::Ci::Status - .const_get(simple_status.capitalize) - .new(@subject, @user) - .extend(self.class.common_helpers) + def self.extended_statuses + [] end - def extended_status - @extended ||= self.class.extended_statuses.find do |status| - status.matches?(@subject, @user) - end + def self.common_helpers + Module.new end end end diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb index 16dcb326be9..13c8343b12a 100644 --- a/lib/gitlab/ci/status/pipeline/factory.rb +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -4,7 +4,7 @@ module Gitlab module Pipeline class Factory < Status::Factory def self.extended_statuses - [Pipeline::SuccessWithWarnings] + [Status::SuccessWarning] end def self.common_helpers diff --git a/lib/gitlab/ci/status/pipeline/success_with_warnings.rb b/lib/gitlab/ci/status/pipeline/success_with_warnings.rb deleted file mode 100644 index 24bf8b869e0..00000000000 --- a/lib/gitlab/ci/status/pipeline/success_with_warnings.rb +++ /dev/null @@ -1,31 +0,0 @@ -module Gitlab - module Ci - module Status - module Pipeline - class SuccessWithWarnings < SimpleDelegator - include Status::Extended - - def text - 'passed' - end - - def label - 'passed with warnings' - end - - def icon - 'icon_status_warning' - end - - def group - 'success_with_warnings' - end - - def self.matches?(pipeline, user) - pipeline.success? && pipeline.has_warnings? - end - end - end - end - end -end diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb index 689a5dd45bc..4c37f084d07 100644 --- a/lib/gitlab/ci/status/stage/factory.rb +++ b/lib/gitlab/ci/status/stage/factory.rb @@ -3,6 +3,10 @@ module Gitlab module Status module Stage class Factory < Status::Factory + def self.extended_statuses + [Status::SuccessWarning] + end + def self.common_helpers Status::Stage::Common end diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb new file mode 100644 index 00000000000..d4cdab6957a --- /dev/null +++ b/lib/gitlab/ci/status/success_warning.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Status + ## + # Extended status used when pipeline or stage passed conditionally. + # This means that failed jobs that are allowed to fail were present. + # + class SuccessWarning < SimpleDelegator + include Status::Extended + + def text + 'passed' + end + + def label + 'passed with warnings' + end + + def icon + 'icon_status_warning' + end + + def group + 'success_with_warnings' + end + + def self.matches?(subject, user) + subject.success? && subject.has_warnings? + end + end + end + end +end diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb index 37e51536e8f..1d7ddeb3e0f 100644 --- a/lib/gitlab/ci/trace_reader.rb +++ b/lib/gitlab/ci/trace_reader.rb @@ -42,6 +42,7 @@ module Gitlab end chunks.join.lines.last(max_lines).join + .force_encoding(Encoding.default_external) end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 9d142f1b82e..e20f5f6f514 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -9,7 +9,9 @@ module Gitlab end def ensure_application_settings! - if connect_to_db? + return fake_application_settings unless connect_to_db? + + unless ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' begin settings = ::ApplicationSetting.current # In case Redis isn't running or the Redis UNIX socket file is not available @@ -20,42 +22,23 @@ module Gitlab settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end - settings || fake_application_settings + settings || in_memory_application_settings end def sidekiq_throttling_enabled? current_application_settings.sidekiq_throttling_enabled? end + def in_memory_application_settings + @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) + # In case migrations the application_settings table is not created yet, + # we fallback to a simple OpenStruct + rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError + fake_application_settings + end + def fake_application_settings - OpenStruct.new( - default_projects_limit: Settings.gitlab['default_projects_limit'], - default_branch_protection: Settings.gitlab['default_branch_protection'], - signup_enabled: Settings.gitlab['signup_enabled'], - signin_enabled: Settings.gitlab['signin_enabled'], - gravatar_enabled: Settings.gravatar['enabled'], - koding_enabled: false, - sign_in_text: nil, - after_sign_up_text: nil, - help_page_text: nil, - shared_runners_text: nil, - restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], - max_attachment_size: Settings.gitlab['max_attachment_size'], - session_expire_delay: Settings.gitlab['session_expire_delay'], - default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], - default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], - domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[gitea github bitbucket gitlab google_code fogbugz git gitlab_project], - shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], - max_artifacts_size: Settings.artifacts['max_size'], - require_two_factor_authentication: false, - two_factor_grace_period: 48, - akismet_enabled: false, - repository_checks_enabled: true, - container_registry_token_expire_delay: 5, - user_default_external: false, - sidekiq_throttling_enabled: false, - ) + OpenStruct.new(::ApplicationSetting.defaults) end private diff --git a/lib/gitlab/cycle_analytics/base_event.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index 53a148ad703..0d8791d396b 100644 --- a/lib/gitlab/cycle_analytics/base_event.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -1,13 +1,13 @@ module Gitlab module CycleAnalytics - class BaseEvent - include MetricsTables + class BaseEventFetcher + include BaseQuery - attr_reader :stage, :start_time_attrs, :end_time_attrs, :projections, :query + attr_reader :projections, :query, :stage, :order - def initialize(project:, options:) - @query = EventsQuery.new(project: project, options: options) + def initialize(project:, stage:, options:) @project = project + @stage = stage @options = options end @@ -19,10 +19,8 @@ module Gitlab end.compact end - def custom_query(_base_query); end - def order - @order || @start_time_attrs + @order || default_order end private @@ -34,7 +32,17 @@ module Gitlab end def event_result - @event_result ||= @query.execute(self).to_a + @event_result ||= ActiveRecord::Base.connection.exec_query(events_query.to_sql).to_a + end + + def events_query + diff_fn = subtract_datetimes_diff(base_query, @options[:start_time_attrs], @options[:end_time_attrs]) + + base_query.project(extract_diff_epoch(diff_fn).as('total_time'), *projections).order(order.desc) + end + + def default_order + [@options[:start_time_attrs]].flatten.first end def serialize(_event) diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb new file mode 100644 index 00000000000..d560dca45c8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -0,0 +1,31 @@ +module Gitlab + module CycleAnalytics + module BaseQuery + include MetricsTables + include Gitlab::Database::Median + include Gitlab::Database::DateTime + + private + + def base_query + @base_query ||= stage_query + end + + def stage_query + query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])). + join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])). + where(issue_table[:project_id].eq(@project.id)). + where(issue_table[:deleted_at].eq(nil)). + where(issue_table[:created_at].gteq(@options[:from])) + + # Load merge_requests + query = query.join(mr_table, Arel::Nodes::OuterJoin). + on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])). + join(mr_metrics_table). + on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) + + query + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb new file mode 100644 index 00000000000..559e3939da6 --- /dev/null +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -0,0 +1,54 @@ +module Gitlab + module CycleAnalytics + class BaseStage + include BaseQuery + + def initialize(project:, options:) + @project = project + @options = options + end + + def events + event_fetcher.fetch + end + + def as_json + AnalyticsStageSerializer.new.represent(self) + end + + def title + name.to_s.capitalize + end + + def median + cte_table = Arel::Table.new("cte_table_for_#{name}") + + # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). + # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). + # We compute the (end_time - start_time) interval, and give it an alias based on the current + # cycle analytics stage. + interval_query = Arel::Nodes::As.new( + cte_table, + subtract_datetimes(base_query.dup, start_time_attrs, end_time_attrs, name.to_s)) + + median_datetime(cte_table, interval_query, name) + end + + def name + raise NotImplementedError.new("Expected #{self.name} to implement name") + end + + private + + def event_fetcher + @event_fetcher ||= Gitlab::CycleAnalytics::EventFetcher[name].new(project: @project, + stage: name, + options: event_options) + end + + def event_options + @options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/code_event.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb index 2afdf0b8518..d5bf6149749 100644 --- a/lib/gitlab/cycle_analytics/code_event.rb +++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb @@ -1,12 +1,9 @@ module Gitlab module CycleAnalytics - class CodeEvent < BaseEvent + class CodeEventFetcher < BaseEventFetcher include MergeRequestAllowed def initialize(*args) - @stage = :code - @start_time_attrs = issue_metrics_table[:first_mentioned_in_commit_at] - @end_time_attrs = mr_table[:created_at] @projections = [mr_table[:title], mr_table[:iid], mr_table[:id], @@ -21,7 +18,7 @@ module Gitlab private def serialize(event) - AnalyticsMergeRequestSerializer.new(project: @project).represent(event).as_json + AnalyticsMergeRequestSerializer.new(project: @project).represent(event) end end end diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb new file mode 100644 index 00000000000..d1bc2055ba8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/code_stage.rb @@ -0,0 +1,21 @@ +module Gitlab + module CycleAnalytics + class CodeStage < BaseStage + def start_time_attrs + @start_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_table[:created_at] + end + + def name + :code + end + + def description + "Time until first merge request" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/event_fetcher.rb b/lib/gitlab/cycle_analytics/event_fetcher.rb new file mode 100644 index 00000000000..50e126cf00b --- /dev/null +++ b/lib/gitlab/cycle_analytics/event_fetcher.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module EventFetcher + def self.[](stage_name) + CycleAnalytics.const_get("#{stage_name.to_s.camelize}EventFetcher") + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/events.rb b/lib/gitlab/cycle_analytics/events.rb deleted file mode 100644 index 2d703d76cbb..00000000000 --- a/lib/gitlab/cycle_analytics/events.rb +++ /dev/null @@ -1,38 +0,0 @@ -module Gitlab - module CycleAnalytics - class Events - def initialize(project:, options:) - @project = project - @options = options - end - - def issue_events - IssueEvent.new(project: @project, options: @options).fetch - end - - def plan_events - PlanEvent.new(project: @project, options: @options).fetch - end - - def code_events - CodeEvent.new(project: @project, options: @options).fetch - end - - def test_events - TestEvent.new(project: @project, options: @options).fetch - end - - def review_events - ReviewEvent.new(project: @project, options: @options).fetch - end - - def staging_events - StagingEvent.new(project: @project, options: @options).fetch - end - - def production_events - ProductionEvent.new(project: @project, options: @options).fetch - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/events_query.rb b/lib/gitlab/cycle_analytics/events_query.rb deleted file mode 100644 index 2418832ccc2..00000000000 --- a/lib/gitlab/cycle_analytics/events_query.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Gitlab - module CycleAnalytics - class EventsQuery - attr_reader :project - - def initialize(project:, options: {}) - @project = project - @from = options[:from] - @branch = options[:branch] - @fetcher = Gitlab::CycleAnalytics::MetricsFetcher.new(project: project, from: @from, branch: @branch) - end - - def execute(stage_class) - @stage_class = stage_class - - ActiveRecord::Base.connection.exec_query(query.to_sql) - end - - private - - def query - base_query = @fetcher.base_query_for(@stage_class.stage) - diff_fn = @fetcher.subtract_datetimes_diff(base_query, @stage_class.start_time_attrs, @stage_class.end_time_attrs) - - @stage_class.custom_query(base_query) - - base_query.project(extract_epoch(diff_fn).as('total_time'), *@stage_class.projections).order(@stage_class.order.desc) - end - - def extract_epoch(arel_attribute) - return arel_attribute unless Gitlab::Database.postgresql? - - Arel.sql(%Q{EXTRACT(EPOCH FROM (#{arel_attribute.to_sql}))}) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/issue_event.rb b/lib/gitlab/cycle_analytics/issue_event.rb deleted file mode 100644 index 705b7e5ce24..00000000000 --- a/lib/gitlab/cycle_analytics/issue_event.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Gitlab - module CycleAnalytics - class IssueEvent < BaseEvent - include IssueAllowed - - def initialize(*args) - @stage = :issue - @start_time_attrs = issue_table[:created_at] - @end_time_attrs = [issue_metrics_table[:first_associated_with_milestone_at], - issue_metrics_table[:first_added_to_board_at]] - @projections = [issue_table[:title], - issue_table[:iid], - issue_table[:id], - issue_table[:created_at], - issue_table[:author_id]] - - super(*args) - end - - private - - def serialize(event) - AnalyticsIssueSerializer.new(project: @project).represent(event).as_json - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/production_event.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb index 4868c3c6237..3df9cbdcfce 100644 --- a/lib/gitlab/cycle_analytics/production_event.rb +++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb @@ -1,12 +1,9 @@ module Gitlab module CycleAnalytics - class ProductionEvent < BaseEvent + class IssueEventFetcher < BaseEventFetcher include IssueAllowed def initialize(*args) - @stage = :production - @start_time_attrs = issue_table[:created_at] - @end_time_attrs = mr_metrics_table[:first_deployed_to_production_at] @projections = [issue_table[:title], issue_table[:iid], issue_table[:id], @@ -19,7 +16,7 @@ module Gitlab private def serialize(event) - AnalyticsIssueSerializer.new(project: @project).represent(event).as_json + AnalyticsIssueSerializer.new(project: @project).represent(event) end end end diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb new file mode 100644 index 00000000000..d2068fbc38f --- /dev/null +++ b/lib/gitlab/cycle_analytics/issue_stage.rb @@ -0,0 +1,22 @@ +module Gitlab + module CycleAnalytics + class IssueStage < BaseStage + def start_time_attrs + @start_time_attrs ||= issue_table[:created_at] + end + + def end_time_attrs + @end_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], + issue_metrics_table[:first_added_to_board_at]] + end + + def name + :issue + end + + def description + "Time before an issue gets scheduled" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/metrics_fetcher.rb b/lib/gitlab/cycle_analytics/metrics_fetcher.rb deleted file mode 100644 index b71e8735e27..00000000000 --- a/lib/gitlab/cycle_analytics/metrics_fetcher.rb +++ /dev/null @@ -1,60 +0,0 @@ -module Gitlab - module CycleAnalytics - class MetricsFetcher - include Gitlab::Database::Median - include Gitlab::Database::DateTime - include MetricsTables - - DEPLOYMENT_METRIC_STAGES = %i[production staging] - - def initialize(project:, from:, branch:) - @project = project - @project = project - @from = from - @branch = branch - end - - def calculate_metric(name, start_time_attrs, end_time_attrs) - cte_table = Arel::Table.new("cte_table_for_#{name}") - - # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). - # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). - # We compute the (end_time - start_time) interval, and give it an alias based on the current - # cycle analytics stage. - interval_query = Arel::Nodes::As.new( - cte_table, - subtract_datetimes(base_query_for(name), start_time_attrs, end_time_attrs, name.to_s)) - - median_datetime(cte_table, interval_query, name) - end - - # Join table with a row for every <issue,merge_request> pair (where the merge request - # closes the given issue) with issue and merge request metrics included. The metrics - # are loaded with an inner join, so issues / merge requests without metrics are - # automatically excluded. - def base_query_for(name) - # Load issues - query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])). - join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])). - where(issue_table[:project_id].eq(@project.id)). - where(issue_table[:deleted_at].eq(nil)). - where(issue_table[:created_at].gteq(@from)) - - query = query.where(build_table[:ref].eq(@branch)) if name == :test && @branch - - # Load merge_requests - query = query.join(mr_table, Arel::Nodes::OuterJoin). - on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])). - join(mr_metrics_table). - on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) - - if DEPLOYMENT_METRIC_STAGES.include?(name) - # Limit to merge requests that have been deployed to production after `@from` - query.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@from)) - end - - query - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/plan_event.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index 7c3f0e9989f..7d342a2d2cb 100644 --- a/lib/gitlab/cycle_analytics/plan_event.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -1,19 +1,17 @@ module Gitlab module CycleAnalytics - class PlanEvent < BaseEvent + class PlanEventFetcher < BaseEventFetcher def initialize(*args) - @stage = :plan - @start_time_attrs = issue_metrics_table[:first_associated_with_milestone_at] - @end_time_attrs = [issue_metrics_table[:first_added_to_board_at], - issue_metrics_table[:first_mentioned_in_commit_at]] @projections = [mr_diff_table[:st_commits].as('commits'), issue_metrics_table[:first_mentioned_in_commit_at]] super(*args) end - def custom_query(base_query) + def events_query base_query.join(mr_diff_table).on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) + + super end private @@ -39,7 +37,7 @@ module Gitlab def serialize_commit(event, st_commit, query) commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project) - AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit).as_json + AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit) end end end diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb new file mode 100644 index 00000000000..3b4dfc6a30e --- /dev/null +++ b/lib/gitlab/cycle_analytics/plan_stage.rb @@ -0,0 +1,22 @@ +module Gitlab + module CycleAnalytics + class PlanStage < BaseStage + def start_time_attrs + @start_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], + issue_metrics_table[:first_added_to_board_at]] + end + + def end_time_attrs + @end_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] + end + + def name + :plan + end + + def description + "Time before an issue starts implementation" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb new file mode 100644 index 00000000000..0fa2e87f673 --- /dev/null +++ b/lib/gitlab/cycle_analytics/production_event_fetcher.rb @@ -0,0 +1,6 @@ +module Gitlab + module CycleAnalytics + class ProductionEventFetcher < IssueEventFetcher + end + end +end diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb new file mode 100644 index 00000000000..d693443bfa4 --- /dev/null +++ b/lib/gitlab/cycle_analytics/production_helper.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module ProductionHelper + def stage_query + super.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@options[:from])) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb new file mode 100644 index 00000000000..2a6bcc80116 --- /dev/null +++ b/lib/gitlab/cycle_analytics/production_stage.rb @@ -0,0 +1,28 @@ +module Gitlab + module CycleAnalytics + class ProductionStage < BaseStage + include ProductionHelper + + def start_time_attrs + @start_time_attrs ||= issue_table[:created_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:first_deployed_to_production_at] + end + + def name + :production + end + + def description + "From issue creation until deploy to production" + end + + def query + # Limit to merge requests that have been deployed to production after `@from` + query.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@from)) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/review_event.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb index b394a02cc52..4c7b3f4467f 100644 --- a/lib/gitlab/cycle_analytics/review_event.rb +++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb @@ -1,12 +1,9 @@ module Gitlab module CycleAnalytics - class ReviewEvent < BaseEvent + class ReviewEventFetcher < BaseEventFetcher include MergeRequestAllowed def initialize(*args) - @stage = :review - @start_time_attrs = mr_table[:created_at] - @end_time_attrs = mr_metrics_table[:merged_at] @projections = [mr_table[:title], mr_table[:iid], mr_table[:id], @@ -18,7 +15,7 @@ module Gitlab end def serialize(event) - AnalyticsMergeRequestSerializer.new(project: @project).represent(event).as_json + AnalyticsMergeRequestSerializer.new(project: @project).represent(event) end end end diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb new file mode 100644 index 00000000000..fbaa3010d81 --- /dev/null +++ b/lib/gitlab/cycle_analytics/review_stage.rb @@ -0,0 +1,21 @@ +module Gitlab + module CycleAnalytics + class ReviewStage < BaseStage + def start_time_attrs + @start_time_attrs ||= mr_table[:created_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:merged_at] + end + + def name + :review + end + + def description + "Time between merge request creation and merge/close" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/stage.rb b/lib/gitlab/cycle_analytics/stage.rb new file mode 100644 index 00000000000..28e0455df59 --- /dev/null +++ b/lib/gitlab/cycle_analytics/stage.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module Stage + def self.[](stage_name) + CycleAnalytics.const_get("#{stage_name.to_s.camelize}Stage") + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb new file mode 100644 index 00000000000..fc77bd86097 --- /dev/null +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -0,0 +1,23 @@ +module Gitlab + module CycleAnalytics + class StageSummary + def initialize(project, from:, current_user:) + @project = project + @from = from + @current_user = current_user + end + + def data + [serialize(Summary::Issue.new(project: @project, from: @from, current_user: @current_user)), + serialize(Summary::Commit.new(project: @project, from: @from)), + serialize(Summary::Deploy.new(project: @project, from: @from))] + end + + private + + def serialize(summary_object) + AnalyticsSummarySerializer.new.represent(summary_object) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/staging_event.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb index a1f30b716f6..36c0260dbfe 100644 --- a/lib/gitlab/cycle_analytics/staging_event.rb +++ b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb @@ -1,10 +1,7 @@ module Gitlab module CycleAnalytics - class StagingEvent < BaseEvent + class StagingEventFetcher < BaseEventFetcher def initialize(*args) - @stage = :staging - @start_time_attrs = mr_metrics_table[:merged_at] - @end_time_attrs = mr_metrics_table[:first_deployed_to_production_at] @projections = [build_table[:id]] @order = build_table[:created_at] @@ -17,14 +14,16 @@ module Gitlab super end - def custom_query(base_query) + def events_query base_query.join(build_table).on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) + + super end private def serialize(event) - AnalyticsBuildSerializer.new.represent(event['build']).as_json + AnalyticsBuildSerializer.new.represent(event['build']) end end end diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb new file mode 100644 index 00000000000..945909a4d62 --- /dev/null +++ b/lib/gitlab/cycle_analytics/staging_stage.rb @@ -0,0 +1,22 @@ +module Gitlab + module CycleAnalytics + class StagingStage < BaseStage + include ProductionHelper + def start_time_attrs + @start_time_attrs ||= mr_metrics_table[:merged_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:first_deployed_to_production_at] + end + + def name + :staging + end + + def description + "From merge request merge until deploy to production" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb new file mode 100644 index 00000000000..43fa3795e5c --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -0,0 +1,20 @@ +module Gitlab + module CycleAnalytics + module Summary + class Base + def initialize(project:, from:) + @project = project + @from = from + end + + def title + self.class.name.demodulize + end + + def value + raise NotImplementedError.new("Expected #{self.name} to implement value") + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb new file mode 100644 index 00000000000..7b8faa4d854 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -0,0 +1,39 @@ +module Gitlab + module CycleAnalytics + module Summary + class Commit < Base + def value + @value ||= count_commits + end + + private + + # Don't use the `Gitlab::Git::Repository#log` method, because it enforces + # a limit. Since we need a commit count, we _can't_ enforce a limit, so + # the easiest way forward is to replicate the relevant portions of the + # `log` function here. + def count_commits + return unless ref + + repository = @project.repository.raw_repository + sha = @project.repository.commit(ref).sha + + cmd = %W(git --git-dir=#{repository.path} log) + cmd << '--format=%H' + cmd << "--after=#{@from.iso8601}" + cmd << sha + + output, status = Gitlab::Popen.popen(cmd) + + raise IOError, output unless status.zero? + + output.lines.count + end + + def ref + @ref ||= @project.default_branch.presence + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb new file mode 100644 index 00000000000..06032e9200e --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -0,0 +1,11 @@ +module Gitlab + module CycleAnalytics + module Summary + class Deploy < Base + def value + @value ||= @project.deployments.where("created_at > ?", @from).count + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb new file mode 100644 index 00000000000..008468f24b9 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -0,0 +1,21 @@ +module Gitlab + module CycleAnalytics + module Summary + class Issue < Base + def initialize(project:, from:, current_user:) + @project = project + @from = from + @current_user = current_user + end + + def title + 'New Issue' + end + + def value + @value ||= IssuesFinder.new(@current_user, project_id: @project.id).execute.created_after(@from).count + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/test_event.rb b/lib/gitlab/cycle_analytics/test_event.rb deleted file mode 100644 index d553d0b5aec..00000000000 --- a/lib/gitlab/cycle_analytics/test_event.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Gitlab - module CycleAnalytics - class TestEvent < StagingEvent - def initialize(*args) - super(*args) - - @stage = :test - @start_time_attrs = mr_metrics_table[:latest_build_started_at] - @end_time_attrs = mr_metrics_table[:latest_build_finished_at] - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/test_event_fetcher.rb b/lib/gitlab/cycle_analytics/test_event_fetcher.rb new file mode 100644 index 00000000000..a2589c6601a --- /dev/null +++ b/lib/gitlab/cycle_analytics/test_event_fetcher.rb @@ -0,0 +1,6 @@ +module Gitlab + module CycleAnalytics + class TestEventFetcher < StagingEventFetcher + end + end +end diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb new file mode 100644 index 00000000000..0079d56e0e4 --- /dev/null +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -0,0 +1,29 @@ +module Gitlab + module CycleAnalytics + class TestStage < BaseStage + def start_time_attrs + @start_time_attrs ||= mr_metrics_table[:latest_build_started_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:latest_build_finished_at] + end + + def name + :test + end + + def description + "Total test time for all commits/merges" + end + + def stage_query + if @options[:branch] + super.where(build_table[:ref].eq(@options[:branch])) + else + super + end + end + end + end +end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 55b8f888d53..dc2537d36aa 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -35,6 +35,20 @@ module Gitlab order end + def self.nulls_first_order(field, direction = 'ASC') + order = "#{field} #{direction}" + + if Gitlab::Database.postgresql? + order << ' NULLS FIRST' + else + # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL + # columns. In the (default) ascending order, `0` comes first. + order.prepend("#{field} IS NULL, ") if direction == 'DESC' + end + + order + end + def self.random Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()" end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 1444d25ebc7..08607c27c09 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -103,6 +103,11 @@ module Gitlab Arel.sql(%Q{EXTRACT(EPOCH FROM "#{arel_attribute.relation.name}"."#{arel_attribute.name}")}) end + def extract_diff_epoch(diff) + return diff unless Gitlab::Database.postgresql? + + Arel.sql(%Q{EXTRACT(EPOCH FROM (#{diff.to_sql}))}) + end # Need to cast '0' to an INTERVAL before we can check if the interval is positive def zero_interval Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 56530448f36..329d12f13d1 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -61,7 +61,10 @@ module Gitlab end def cacheable?(diff_file) - @merge_request_diff.present? && diff_file.blob && diff_file.blob.text? + @merge_request_diff.present? && + diff_file.blob && + diff_file.blob.text? && + @project.repository.diffable?(diff_file.blob) end def cache_key diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index bd3267e2a80..bd2f5d3615e 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -1,10 +1,11 @@ require 'gitlab/email/handler/create_note_handler' require 'gitlab/email/handler/create_issue_handler' +require 'gitlab/email/handler/unsubscribe_handler' module Gitlab module Email module Handler - HANDLERS = [CreateNoteHandler, CreateIssueHandler] + HANDLERS = [UnsubscribeHandler, CreateNoteHandler, CreateIssueHandler] def self.for(mail, mail_key) HANDLERS.find do |klass| diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 7cccf465334..3f6ace0311a 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -9,52 +9,13 @@ module Gitlab @mail_key = mail_key end - def message - @message ||= process_message - end - - def author + def can_execute? raise NotImplementedError end - def project + def execute raise NotImplementedError end - - private - - def validate_permission!(permission) - raise UserNotFoundError unless author - raise UserBlockedError if author.blocked? - raise ProjectNotFound unless author.can?(:read_project, project) - raise UserNotAuthorizedError unless author.can?(permission, project) - end - - def process_message - message = ReplyParser.new(mail).execute.strip - add_attachments(message) - end - - def add_attachments(reply) - attachments = Email::AttachmentUploader.new(mail).execute(project) - - reply + attachments.map do |link| - "\n\n#{link[:markdown]}" - end.join - end - - def verify_record!(record:, invalid_exception:, record_name:) - return if record.persisted? - return if record.errors.key?(:commands_only) - - error_title = "The #{record_name} could not be created for the following reasons:" - - msg = error_title + record.errors.full_messages.map do |error| - "\n\n- #{error}" - end.join - - raise invalid_exception, msg - end end end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 9f90a3ec2b2..b8ec9138c10 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -5,6 +5,7 @@ module Gitlab module Email module Handler class CreateIssueHandler < BaseHandler + include ReplyProcessing attr_reader :project_path, :incoming_email_token def initialize(mail, mail_key) @@ -33,7 +34,7 @@ module Gitlab end def project - @project ||= Project.find_with_namespace(project_path) + @project ||= Project.find_by_full_path(project_path) end private diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 447c7a6a6b9..d87ba427f4b 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -1,10 +1,13 @@ require 'gitlab/email/handler/base_handler' +require 'gitlab/email/handler/reply_processing' module Gitlab module Email module Handler class CreateNoteHandler < BaseHandler + include ReplyProcessing + def can_handle? mail_key =~ /\A\w+\z/ end @@ -24,6 +27,8 @@ module Gitlab record_name: 'comment') end + private + def author sent_notification.recipient end @@ -36,8 +41,6 @@ module Gitlab @sent_notification ||= SentNotification.for(mail_key) end - private - def create_note Notes::CreateService.new( project, diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb new file mode 100644 index 00000000000..32c5caf93e8 --- /dev/null +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -0,0 +1,54 @@ +module Gitlab + module Email + module Handler + module ReplyProcessing + private + + def author + raise NotImplementedError + end + + def project + raise NotImplementedError + end + + def message + @message ||= process_message + end + + def process_message + message = ReplyParser.new(mail).execute.strip + add_attachments(message) + end + + def add_attachments(reply) + attachments = Email::AttachmentUploader.new(mail).execute(project) + + reply + attachments.map do |link| + "\n\n#{link[:markdown]}" + end.join + end + + def validate_permission!(permission) + raise UserNotFoundError unless author + raise UserBlockedError if author.blocked? + raise ProjectNotFound unless author.can?(:read_project, project) + raise UserNotAuthorizedError unless author.can?(permission, project) + end + + def verify_record!(record:, invalid_exception:, record_name:) + return if record.persisted? + return if record.errors.key?(:commands_only) + + error_title = "The #{record_name} could not be created for the following reasons:" + + msg = error_title + record.errors.full_messages.map do |error| + "\n\n- #{error}" + end.join + + raise invalid_exception, msg + end + end + end + end +end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb new file mode 100644 index 00000000000..97d7a8d65ff --- /dev/null +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -0,0 +1,32 @@ +require 'gitlab/email/handler/base_handler' + +module Gitlab + module Email + module Handler + class UnsubscribeHandler < BaseHandler + def can_handle? + mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ + end + + def execute + raise SentNotificationNotFoundError unless sent_notification + return unless sent_notification.unsubscribable? + + noteable = sent_notification.noteable + raise NoteableNotFoundError unless noteable + noteable.unsubscribe(sent_notification.recipient) + end + + private + + def sent_notification + @sent_notification ||= SentNotification.for(reply_key) + end + + def reply_key + mail_key.sub(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX, '') + end + end + end + end +end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index a40c44eb1bc..b64db5d01ae 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -35,6 +35,8 @@ module Gitlab handler.execute end + private + def build_mail Mail::Message.new(@raw) rescue Encoding::UndefinedConversionError, @@ -54,7 +56,24 @@ module Gitlab end def key_from_additional_headers(mail) - Array(mail.references).find do |mail_id| + references = ensure_references_array(mail.references) + + find_key_from_references(references) + end + + def ensure_references_array(references) + case references + when Array + references + when String + # Handle emails from clients which append with commas, + # example clients are Microsoft exchange and iOS app + Gitlab::IncomingEmail.scan_fallback_references(references) + end + end + + def find_key_from_references(references) + references.find do |mail_id| key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id) break key if key end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index f586c5ab062..8c8dd1b9cef 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -13,9 +13,17 @@ module Gitlab encoding = body.encoding - body = discourse_email_trimmer(body) + body = EmailReplyTrimmer.trim(body) - body = EmailReplyParser.parse_reply(body) + return '' unless body + + # not using /\s+$/ here because that deletes empty lines + body = body.gsub(/[ \t]$/, '') + + # NOTE: We currently don't support empty quotes. + # EmailReplyTrimmer allows this as a special case, + # so we detect it manually here. + return "" if body.lines.all? { |l| l.strip.empty? || l.start_with?('>') } body.force_encoding(encoding).encode("UTF-8") end @@ -57,30 +65,6 @@ module Gitlab rescue nil end - - REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date) - REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" }) - - def discourse_email_trimmer(body) - lines = body.scrub.lines.to_a - range_end = 0 - - lines.each_with_index do |l, idx| - # This one might be controversial but so many reply lines have years, times and end with a colon. - # Let's try it and see how well it works. - break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) || - (l =~ /On \w+ \d+,? \d+,?.*wrote:/) - - # Headers on subsequent lines - break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX } - # Headers on the same line - break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3 - - range_end = idx - end - - lines[0..range_end].join.strip - end end end end diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index a7c596dced0..b984492d369 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -76,7 +76,7 @@ module Gitlab if referable.respond_to?(:project) referable.to_reference(target_project) else - referable.to_reference(@source_project, target_project) + referable.to_reference(@source_project, target_project: target_project) end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 3cd515e4a3a..d3df3f1bca1 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -6,7 +6,7 @@ module Gitlab class << self def ref_name(ref) - ref.gsub(/\Arefs\/(tags|heads)\//, '') + ref.sub(/\Arefs\/(tags|heads)\//, '') end def branch_name(ref) diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes.rb new file mode 100644 index 00000000000..42140ecc993 --- /dev/null +++ b/lib/gitlab/git/attributes.rb @@ -0,0 +1,131 @@ +module Gitlab + module Git + # Class for parsing Git attribute files and extracting the attributes for + # file patterns. + # + # Unlike Rugged this parser only needs a single IO call (a call to `open`), + # vastly reducing the time spent in extracting attributes. + # + # This class _only_ supports parsing the attributes file located at + # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files + # (`.gitattributes` is copied to this particular path). + # + # Basic usage: + # + # attributes = Gitlab::Git::Attributes.new(some_repo.path) + # + # attributes.attributes('README.md') # => { "eol" => "lf } + class Attributes + # path - The path to the Git repository. + def initialize(path) + @path = File.expand_path(path) + @patterns = nil + end + + # Returns all the Git attributes for the given path. + # + # path - A path to a file for which to get the attributes. + # + # Returns a Hash. + def attributes(path) + full_path = File.join(@path, path) + + patterns.each do |pattern, attrs| + return attrs if File.fnmatch?(pattern, full_path) + end + + {} + end + + # Returns a Hash containing the file patterns and their attributes. + def patterns + @patterns ||= parse_file + end + + # Parses an attribute string. + # + # These strings can be in the following formats: + # + # text # => { "text" => true } + # -text # => { "text" => false } + # key=value # => { "key" => "value" } + # + # string - The string to parse. + # + # Returns a Hash containing the attributes and their values. + def parse_attributes(string) + values = {} + dash = '-' + equal = '=' + binary = 'binary' + + string.split(/\s+/).each do |chunk| + # Data such as "foo = bar" should be treated as "foo" and "bar" being + # separate boolean attributes. + next if chunk == equal + + key = chunk + + # Input: "-foo" + if chunk.start_with?(dash) + key = chunk.byteslice(1, chunk.length - 1) + value = false + + # Input: "foo=bar" + elsif chunk.include?(equal) + key, value = chunk.split(equal, 2) + + # Input: "foo" + else + value = true + end + + values[key] = value + + # When the "binary" option is set the "diff" option should be set to + # the inverse. If "diff" is later set it should overwrite the + # automatically set value. + values['diff'] = false if key == binary && value + end + + values + end + + # Iterates over every line in the attributes file. + def each_line + full_path = File.join(@path, 'info/attributes') + + return unless File.exist?(full_path) + + File.open(full_path, 'r') do |handle| + handle.each_line do |line| + break unless line.valid_encoding? + + yield line.strip + end + end + end + + private + + # Parses the Git attributes file. + def parse_file + pairs = [] + comment = '#' + + each_line do |line| + next if line.start_with?(comment) || line.empty? + + pattern, attrs = line.split(/\s+/, 2) + + parsed = attrs ? parse_attributes(attrs) : {} + + pairs << [File.join(@path, pattern), parsed] + end + + # Newer entries take precedence over older entries. + pairs.reverse.to_h + end + end + end +end diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb new file mode 100644 index 00000000000..58193391926 --- /dev/null +++ b/lib/gitlab/git/blame.rb @@ -0,0 +1,75 @@ +module Gitlab + module Git + class Blame + include Gitlab::Git::EncodingHelper + + attr_reader :lines, :blames + + def initialize(repository, sha, path) + @repo = repository + @sha = sha + @path = path + @lines = [] + @blames = load_blame + end + + def each + @blames.each do |blame| + yield( + Gitlab::Git::Commit.new(blame.commit), + blame.line + ) + end + end + + private + + def load_blame + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) + # Read in binary mode to ensure ASCII-8BIT + raw_output = IO.popen(cmd, 'rb') {|io| io.read } + output = encode_utf8(raw_output) + process_raw_blame output + end + + def process_raw_blame(output) + lines, final = [], [] + info, commits = {}, {} + + # process the output + output.split("\n").each do |line| + if line[0, 1] == "\t" + lines << line[1, line.size] + elsif m = /^(\w{40}) (\d+) (\d+)/.match(line) + commit_id, old_lineno, lineno = m[1], m[2].to_i, m[3].to_i + commits[commit_id] = nil unless commits.key?(commit_id) + info[lineno] = [commit_id, old_lineno] + end + end + + # load all commits in single call + commits.keys.each do |key| + commits[key] = @repo.lookup(key) + end + + # get it together + info.sort.each do |lineno, (commit_id, old_lineno)| + commit = commits[commit_id] + final << BlameLine.new(lineno, old_lineno, commit, lines[lineno - 1]) + end + + @lines = final + end + end + + class BlameLine + attr_accessor :lineno, :oldlineno, :commit, :line + def initialize(lineno, oldlineno, commit, line) + @lineno = lineno + @oldlineno = oldlineno + @commit = commit + @line = line + end + end + end +end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb new file mode 100644 index 00000000000..b742d9e1e4b --- /dev/null +++ b/lib/gitlab/git/blob.rb @@ -0,0 +1,330 @@ +module Gitlab + module Git + class Blob + include Linguist::BlobHelper + include Gitlab::Git::EncodingHelper + + # This number is the maximum amount of data that we want to display to + # the user. We load as much as we can for encoding detection + # (Linguist) and LFS pointer parsing. All other cases where we need full + # blob data should use load_all_data!. + MAX_DATA_DISPLAY_SIZE = 10485760 + + attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary + + class << self + def find(repository, sha, path) + commit = repository.lookup(sha) + root_tree = commit.tree + + blob_entry = find_entry_by_path(repository, root_tree.oid, path) + + return nil unless blob_entry + + if blob_entry[:type] == :commit + submodule_blob(blob_entry, path, sha) + else + blob = repository.lookup(blob_entry[:oid]) + + if blob + new( + id: blob.oid, + name: blob_entry[:name], + size: blob.size, + data: blob.content(MAX_DATA_DISPLAY_SIZE), + mode: blob_entry[:filemode].to_s(8), + path: path, + commit_id: sha, + binary: blob.binary? + ) + end + end + end + + def raw(repository, sha) + blob = repository.lookup(sha) + + new( + id: blob.oid, + size: blob.size, + data: blob.content(MAX_DATA_DISPLAY_SIZE), + binary: blob.binary? + ) + end + + # Recursive search of blob id by path + # + # Ex. + # blog/ # oid: 1a + # app/ # oid: 2a + # models/ # oid: 3a + # file.rb # oid: 4a + # + # + # Blob.find_entry_by_path(repo, '1a', 'app/file.rb') # => '4a' + # + def find_entry_by_path(repository, root_id, path) + root_tree = repository.lookup(root_id) + # Strip leading slashes + path[/^\/*/] = '' + path_arr = path.split('/') + + entry = root_tree.find do |entry| + entry[:name] == path_arr[0] + end + + return nil unless entry + + if path_arr.size > 1 + return nil unless entry[:type] == :tree + path_arr.shift + find_entry_by_path(repository, entry[:oid], path_arr.join('/')) + else + [:blob, :commit].include?(entry[:type]) ? entry : nil + end + end + + def submodule_blob(blob_entry, path, sha) + new( + id: blob_entry[:oid], + name: blob_entry[:name], + data: '', + path: path, + commit_id: sha, + ) + end + + # Commit file in repository and return commit sha + # + # options should contain next structure: + # file: { + # content: 'Lorem ipsum...', + # path: 'documents/story.txt', + # update: true + # }, + # author: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # committer: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # commit: { + # message: 'Wow such commit', + # branch: 'master', + # update_ref: false + # } + # + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def commit(repository, options, action = :add) + file = options[:file] + update = file[:update].nil? ? true : file[:update] + author = options[:author] + committer = options[:committer] + commit = options[:commit] + repo = repository.rugged + ref = commit[:branch] + update_ref = commit[:update_ref].nil? ? true : commit[:update_ref] + parents = [] + mode = 0o100644 + + unless ref.start_with?('refs/') + ref = 'refs/heads/' + ref + end + + path_name = Gitlab::Git::PathHelper.normalize_path(file[:path]) + # Abort if any invalid characters remain (e.g. ../foo) + raise Gitlab::Git::Repository::InvalidBlobName.new("Invalid path") if path_name.each_filename.to_a.include?('..') + + filename = path_name.to_s + index = repo.index + + unless repo.empty? + rugged_ref = repo.references[ref] + raise Gitlab::Git::Repository::InvalidRef.new("Invalid branch name") unless rugged_ref + last_commit = rugged_ref.target + index.read_tree(last_commit.tree) + parents = [last_commit] + end + + if action == :remove + index.remove(filename) + else + file_entry = index.get(filename) + + if action == :rename + old_path_name = Gitlab::Git::PathHelper.normalize_path(file[:previous_path]) + old_filename = old_path_name.to_s + file_entry = index.get(old_filename) + index.remove(old_filename) unless file_entry.blank? + end + + if file_entry + raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists; update not allowed") unless update + + # Preserve the current file mode if one is available + mode = file_entry[:mode] if file_entry[:mode] + end + + content = file[:content] + detect = CharlockHolmes::EncodingDetector.new.detect(content) if content + + unless detect && detect[:type] == :binary + # When writing to the repo directly as we are doing here, + # the `core.autocrlf` config isn't taken into account. + content.gsub!("\r\n", "\n") if repository.autocrlf + end + + oid = repo.write(content, :blob) + index.add(path: filename, oid: oid, mode: mode) + end + + opts = {} + opts[:tree] = index.write_tree(repo) + opts[:author] = author + opts[:committer] = committer + opts[:message] = commit[:message] + opts[:parents] = parents + opts[:update_ref] = ref if update_ref + + Rugged::Commit.create(repo, opts) + end + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/PerceivedComplexity + + # Remove file from repository and return commit sha + # + # options should contain next structure: + # file: { + # path: 'documents/story.txt' + # }, + # author: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # committer: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # commit: { + # message: 'Remove FILENAME', + # branch: 'master' + # } + # + def remove(repository, options) + commit(repository, options, :remove) + end + + # Rename file from repository and return commit sha + # + # options should contain next structure: + # file: { + # previous_path: 'documents/old_story.txt' + # path: 'documents/story.txt' + # content: 'Lorem ipsum...', + # update: true + # }, + # author: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # committer: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # commit: { + # message: 'Rename FILENAME', + # branch: 'master' + # } + # + def rename(repository, options) + commit(repository, options, :rename) + end + end + + def initialize(options) + %w(id name path size data mode commit_id binary).each do |key| + self.send("#{key}=", options[key.to_sym]) + end + + @loaded_all_data = false + # Retain the actual size before it is encoded + @loaded_size = @data.bytesize if @data + end + + def binary? + @binary.nil? ? super : @binary == true + end + + def empty? + !data || data == '' + end + + def data + encode! @data + end + + # Load all blob data (not just the first MAX_DATA_DISPLAY_SIZE bytes) into + # memory as a Ruby string. + def load_all_data!(repository) + return if @data == '' # don't mess with submodule blobs + return @data if @loaded_all_data + + @loaded_all_data = true + @data = repository.lookup(id).content + @loaded_size = @data.bytesize + end + + def name + encode! @name + end + + # Valid LFS object pointer is a text file consisting of + # version + # oid + # size + # see https://github.com/github/git-lfs/blob/v1.1.0/docs/spec.md#the-pointer + def lfs_pointer? + has_lfs_version_key? && lfs_oid.present? && lfs_size.present? + end + + def lfs_oid + if has_lfs_version_key? + oid = data.match(/(?<=sha256:)([0-9a-f]{64})/) + return oid[1] if oid + end + + nil + end + + def lfs_size + if has_lfs_version_key? + size = data.match(/(?<=size )([0-9]+)/) + return size[1] if size + end + + nil + end + + def truncated? + size && (size > loaded_size) + end + + private + + def has_lfs_version_key? + !empty? && text? && data.start_with?("version https://git-lfs.github.com/spec") + end + end + end +end diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb new file mode 100644 index 00000000000..e98de57fc22 --- /dev/null +++ b/lib/gitlab/git/blob_snippet.rb @@ -0,0 +1,32 @@ +module Gitlab + module Git + class BlobSnippet + include Linguist::BlobHelper + + attr_accessor :ref + attr_accessor :lines + attr_accessor :filename + attr_accessor :startline + + def initialize(ref, lines, startline, filename) + @ref, @lines, @startline, @filename = ref, lines, startline, filename + end + + def data + lines.join("\n") if lines + end + + def name + filename + end + + def size + data.length + end + + def mode + nil + end + end + end +end diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb new file mode 100644 index 00000000000..586380da94a --- /dev/null +++ b/lib/gitlab/git/branch.rb @@ -0,0 +1,6 @@ +module Gitlab + module Git + class Branch < Ref + end + end +end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb new file mode 100644 index 00000000000..d785516ebdd --- /dev/null +++ b/lib/gitlab/git/commit.rb @@ -0,0 +1,310 @@ +# Gitlab::Git::Commit is a wrapper around native Rugged::Commit object +module Gitlab + module Git + class Commit + include Gitlab::Git::EncodingHelper + + attr_accessor :raw_commit, :head, :refs + + SERIALIZE_KEYS = [ + :id, :message, :parent_ids, + :authored_date, :author_name, :author_email, + :committed_date, :committer_name, :committer_email + ].freeze + + attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator + + def ==(other) + return false unless other.is_a?(Gitlab::Git::Commit) + + methods = [:message, :parent_ids, :authored_date, :author_name, + :author_email, :committed_date, :committer_name, + :committer_email] + + methods.all? do |method| + send(method) == other.send(method) + end + end + + class << self + # Get commits collection + # + # Ex. + # Commit.where( + # repo: repo, + # ref: 'master', + # path: 'app/models', + # limit: 10, + # offset: 5, + # ) + # + def where(options) + repo = options.delete(:repo) + raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log) + + repo.log(options).map { |c| decorate(c) } + end + + # Get single commit + # + # Ex. + # Commit.find(repo, '29eda46b') + # + # Commit.find(repo, 'master') + # + def find(repo, commit_id = "HEAD") + return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) + + obj = if commit_id.is_a?(String) + repo.rev_parse_target(commit_id) + else + Gitlab::Git::Ref.dereference_object(commit_id) + end + + return nil unless obj.is_a?(Rugged::Commit) + + decorate(obj) + rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, Gitlab::Git::Repository::NoRepository + nil + end + + # Get last commit for HEAD + # + # Ex. + # Commit.last(repo) + # + def last(repo) + find(repo) + end + + # Get last commit for specified path and ref + # + # Ex. + # Commit.last_for_path(repo, '29eda46b', 'app/models') + # + # Commit.last_for_path(repo, 'master', 'Gemfile') + # + def last_for_path(repo, ref, path = nil) + where( + repo: repo, + ref: ref, + path: path, + limit: 1 + ).first + end + + # Get commits between two revspecs + # See also #repository.commits_between + # + # Ex. + # Commit.between(repo, '29eda46b', 'master') + # + def between(repo, base, head) + repo.commits_between(base, head).map do |commit| + decorate(commit) + end + rescue Rugged::ReferenceError + [] + end + + # Delegate Repository#find_commits + def find_all(repo, options = {}) + repo.find_commits(options) + end + + def decorate(commit, ref = nil) + Gitlab::Git::Commit.new(commit, ref) + end + + # Returns a diff object for the changes introduced by +rugged_commit+. + # If +rugged_commit+ doesn't have a parent, then the diff is between + # this commit and an empty repo. See Repository#diff for the keys + # allowed in the +options+ hash. + def diff_from_parent(rugged_commit, options = {}) + options ||= {} + break_rewrites = options[:break_rewrites] + actual_options = Gitlab::Git::Diff.filter_diff_options(options) + + diff = if rugged_commit.parents.empty? + rugged_commit.diff(actual_options.merge(reverse: true)) + else + rugged_commit.parents[0].diff(rugged_commit, actual_options) + end + + diff.find_similar!(break_rewrites: break_rewrites) + diff + end + end + + def initialize(raw_commit, head = nil) + raise "Nil as raw commit passed" unless raw_commit + + if raw_commit.is_a?(Hash) + init_from_hash(raw_commit) + elsif raw_commit.is_a?(Rugged::Commit) + init_from_rugged(raw_commit) + else + raise "Invalid raw commit type: #{raw_commit.class}" + end + + @head = head + end + + def sha + id + end + + def short_id(length = 10) + id.to_s[0..length] + end + + def safe_message + @safe_message ||= message + end + + def created_at + committed_date + end + + # Was this commit committed by a different person than the original author? + def different_committer? + author_name != committer_name || author_email != committer_email + end + + def parent_id + parent_ids.first + end + + # Shows the diff between the commit's parent and the commit. + # + # Cuts out the header and stats from #to_patch and returns only the diff. + def to_diff(options = {}) + diff_from_parent(options).patch + end + + # Returns a diff object for the changes from this commit's first parent. + # If there is no parent, then the diff is between this commit and an + # empty repo. See Repository#diff for keys allowed in the +options+ + # hash. + def diff_from_parent(options = {}) + Commit.diff_from_parent(raw_commit, options) + end + + def has_zero_stats? + stats.total.zero? + rescue + true + end + + def no_commit_message + "--no commit message" + end + + def to_hash + serialize_keys.map.with_object({}) do |key, hash| + hash[key] = send(key) + end + end + + def date + committed_date + end + + def diffs(options = {}) + Gitlab::Git::DiffCollection.new(diff_from_parent(options), options) + end + + def parents + raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) } + end + + def tree + raw_commit.tree + end + + def stats + Gitlab::Git::CommitStats.new(self) + end + + def to_patch(options = {}) + begin + raw_commit.to_mbox(options) + rescue Rugged::InvalidError => ex + if ex.message =~ /Commit \w+ is a merge commit/ + 'Patch format is not currently supported for merge commits.' + end + end + end + + # Get a collection of Rugged::Reference objects for this commit. + # + # Ex. + # commit.ref(repo) + # + def refs(repo) + repo.refs_hash[id] + end + + # Get ref names collection + # + # Ex. + # commit.ref_names(repo) + # + def ref_names(repo) + refs(repo).map do |ref| + ref.name.sub(%r{^refs/(heads|remotes|tags)/}, "") + end + end + + def message + encode! @message + end + + def author_name + encode! @author_name + end + + def author_email + encode! @author_email + end + + def committer_name + encode! @committer_name + end + + def committer_email + encode! @committer_email + end + + private + + def init_from_hash(hash) + raw_commit = hash.symbolize_keys + + serialize_keys.each do |key| + send("#{key}=", raw_commit[key]) + end + end + + def init_from_rugged(commit) + author = commit.author + committer = commit.committer + + @raw_commit = commit + @id = commit.oid + @message = commit.message + @authored_date = author[:time] + @committed_date = committer[:time] + @author_name = author[:name] + @author_email = author[:email] + @committer_name = committer[:name] + @committer_email = committer[:email] + @parent_ids = commit.parents.map(&:oid) + end + + def serialize_keys + SERIALIZE_KEYS + end + end + end +end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb new file mode 100644 index 00000000000..e9118bbed0e --- /dev/null +++ b/lib/gitlab/git/commit_stats.rb @@ -0,0 +1,26 @@ +# Gitlab::Git::CommitStats counts the additions, deletions, and total changes +# in a commit. +module Gitlab + module Git + class CommitStats + attr_reader :id, :additions, :deletions, :total + + # Instantiate a CommitStats object + def initialize(commit) + @id = commit.id + @additions = 0 + @deletions = 0 + @total = 0 + + diff = commit.diff_from_parent + + diff.each_patch do |p| + # TODO: Use the new Rugged convenience methods when they're released + @additions += p.stat[0] + @deletions += p.stat[1] + @total += p.changes + end + end + end + end +end diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb new file mode 100644 index 00000000000..696a2acd5e3 --- /dev/null +++ b/lib/gitlab/git/compare.rb @@ -0,0 +1,43 @@ +module Gitlab + module Git + class Compare + attr_reader :head, :base, :straight + + def initialize(repository, base, head, straight = false) + @repository = repository + @straight = straight + + unless base && head + @commits = [] + return + end + + @base = Gitlab::Git::Commit.find(repository, base.try(:strip)) + @head = Gitlab::Git::Commit.find(repository, head.try(:strip)) + + @commits = [] unless @base && @head + @commits = [] if same + end + + def same + @base && @head && @base.id == @head.id + end + + def commits + return @commits if defined?(@commits) + + @commits = Gitlab::Git::Commit.between(@repository, @base.id, @head.id) + end + + def diffs(options = {}) + unless @head && @base + return Gitlab::Git::DiffCollection.new([]) + end + + paths = options.delete(:paths) || [] + options[:straight] = @straight + Gitlab::Git::Diff.between(@repository, @head.id, @base.id, options, *paths) + end + end + end +end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb new file mode 100644 index 00000000000..d6b3b5705a9 --- /dev/null +++ b/lib/gitlab/git/diff.rb @@ -0,0 +1,322 @@ +# Gitlab::Git::Diff is a wrapper around native Rugged::Diff object +module Gitlab + module Git + class Diff + class TimeoutError < StandardError; end + include Gitlab::Git::EncodingHelper + + # Diff properties + attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff + + # Stats properties + attr_accessor :new_file, :renamed_file, :deleted_file + + attr_accessor :too_large + + # The maximum size of a diff to display. + DIFF_SIZE_LIMIT = 102400 # 100 KB + + # The maximum size before a diff is collapsed. + DIFF_COLLAPSE_LIMIT = 10240 # 10 KB + + class << self + def between(repo, head, base, options = {}, *paths) + straight = options.delete(:straight) || false + + common_commit = if straight + base + else + # Only show what is new in the source branch + # compared to the target branch, not the other way + # around. The linex below with merge_base is + # equivalent to diff with three dots (git diff + # branch1...branch2) From the git documentation: + # "git diff A...B" is equivalent to "git diff + # $(git-merge-base A B) B" + repo.merge_base_commit(head, base) + end + + options ||= {} + actual_options = filter_diff_options(options) + repo.diff(common_commit, head, actual_options, *paths) + end + + # Return a copy of the +options+ hash containing only keys that can be + # passed to Rugged. Allowed options are: + # + # :max_size :: + # An integer specifying the maximum byte size of a file before a it + # will be treated as binary. The default value is 512MB. + # + # :context_lines :: + # The number of unchanged lines that define the boundary of a hunk + # (and to display before and after the actual changes). The default is + # 3. + # + # :interhunk_lines :: + # The maximum number of unchanged lines between hunk boundaries before + # the hunks will be merged into a one. The default is 0. + # + # :old_prefix :: + # The virtual "directory" to prefix to old filenames in hunk headers. + # The default is "a". + # + # :new_prefix :: + # The virtual "directory" to prefix to new filenames in hunk headers. + # The default is "b". + # + # :reverse :: + # If true, the sides of the diff will be reversed. + # + # :force_text :: + # If true, all files will be treated as text, disabling binary + # attributes & detection. + # + # :ignore_whitespace :: + # If true, all whitespace will be ignored. + # + # :ignore_whitespace_change :: + # If true, changes in amount of whitespace will be ignored. + # + # :ignore_whitespace_eol :: + # If true, whitespace at end of line will be ignored. + # + # :ignore_submodules :: + # if true, submodules will be excluded from the diff completely. + # + # :patience :: + # If true, the "patience diff" algorithm will be used (currenlty + # unimplemented). + # + # :include_ignored :: + # If true, ignored files will be included in the diff. + # + # :include_untracked :: + # If true, untracked files will be included in the diff. + # + # :include_unmodified :: + # If true, unmodified files will be included in the diff. + # + # :recurse_untracked_dirs :: + # Even if +:include_untracked+ is true, untracked directories will + # only be marked with a single entry in the diff. If this flag is set + # to true, all files under ignored directories will be included in the + # diff, too. + # + # :disable_pathspec_match :: + # If true, the given +*paths+ will be applied as exact matches, + # instead of as fnmatch patterns. + # + # :deltas_are_icase :: + # If true, filename comparisons will be made with case-insensitivity. + # + # :include_untracked_content :: + # if true, untracked content will be contained in the the diff patch + # text. + # + # :skip_binary_check :: + # If true, diff deltas will be generated without spending time on + # binary detection. This is useful to improve performance in cases + # where the actual file content difference is not needed. + # + # :include_typechange :: + # If true, type changes for files will not be interpreted as deletion + # of the "old file" and addition of the "new file", but will generate + # typechange records. + # + # :include_typechange_trees :: + # Even if +:include_typechange+ is true, blob -> tree changes will + # still usually be handled as a deletion of the blob. If this flag is + # set to true, blob -> tree changes will be marked as typechanges. + # + # :ignore_filemode :: + # If true, file mode changes will be ignored. + # + # :recurse_ignored_dirs :: + # Even if +:include_ignored+ is true, ignored directories will only be + # marked with a single entry in the diff. If this flag is set to true, + # all files under ignored directories will be included in the diff, + # too. + def filter_diff_options(options, default_options = {}) + allowed_options = [:max_size, :context_lines, :interhunk_lines, + :old_prefix, :new_prefix, :reverse, :force_text, + :ignore_whitespace, :ignore_whitespace_change, + :ignore_whitespace_eol, :ignore_submodules, + :patience, :include_ignored, :include_untracked, + :include_unmodified, :recurse_untracked_dirs, + :disable_pathspec_match, :deltas_are_icase, + :include_untracked_content, :skip_binary_check, + :include_typechange, :include_typechange_trees, + :ignore_filemode, :recurse_ignored_dirs, :paths, + :max_files, :max_lines, :all_diffs, :no_collapse] + + if default_options + actual_defaults = default_options.dup + actual_defaults.keep_if do |key| + allowed_options.include?(key) + end + else + actual_defaults = {} + end + + if options + filtered_opts = options.dup + filtered_opts.keep_if do |key| + allowed_options.include?(key) + end + filtered_opts = actual_defaults.merge(filtered_opts) + else + filtered_opts = actual_defaults + end + + filtered_opts + end + end + + def initialize(raw_diff, collapse: false) + case raw_diff + when Hash + init_from_hash(raw_diff, collapse: collapse) + when Rugged::Patch, Rugged::Diff::Delta + init_from_rugged(raw_diff, collapse: collapse) + when nil + raise "Nil as raw diff passed" + else + raise "Invalid raw diff type: #{raw_diff.class}" + end + end + + def serialize_keys + @serialize_keys ||= %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large) + end + + def to_hash + hash = {} + + keys = serialize_keys + + keys.each do |key| + hash[key] = send(key) + end + + hash + end + + def submodule? + a_mode == '160000' || b_mode == '160000' + end + + def line_count + @line_count ||= Util.count_lines(@diff) + end + + def too_large? + if @too_large.nil? + @too_large = @diff.bytesize >= DIFF_SIZE_LIMIT + else + @too_large + end + end + + def collapsible? + @diff.bytesize >= DIFF_COLLAPSE_LIMIT + end + + def prune_large_diff! + @diff = '' + @line_count = 0 + @too_large = true + end + + def collapsed? + return @collapsed if defined?(@collapsed) + false + end + + def prune_collapsed_diff! + @diff = '' + @line_count = 0 + @collapsed = true + end + + private + + def init_from_rugged(rugged, collapse: false) + if rugged.is_a?(Rugged::Patch) + init_from_rugged_patch(rugged, collapse: collapse) + d = rugged.delta + else + d = rugged + end + + @new_path = encode!(d.new_file[:path]) + @old_path = encode!(d.old_file[:path]) + @a_mode = d.old_file[:mode].to_s(8) + @b_mode = d.new_file[:mode].to_s(8) + @new_file = d.added? + @renamed_file = d.renamed? + @deleted_file = d.deleted? + end + + def init_from_rugged_patch(patch, collapse: false) + # Don't bother initializing diffs that are too large. If a diff is + # binary we're not going to display anything so we skip the size check. + return if !patch.delta.binary? && prune_large_patch(patch, collapse) + + @diff = encode!(strip_diff_headers(patch.to_s)) + end + + def init_from_hash(hash, collapse: false) + raw_diff = hash.symbolize_keys + + serialize_keys.each do |key| + send(:"#{key}=", raw_diff[key.to_sym]) + end + + prune_large_diff! if too_large? + prune_collapsed_diff! if collapse && collapsible? + end + + # If the patch surpasses any of the diff limits it calls the appropiate + # prune method and returns true. Otherwise returns false. + def prune_large_patch(patch, collapse) + size = 0 + + patch.each_hunk do |hunk| + hunk.each_line do |line| + size += line.content.bytesize + + if size >= DIFF_SIZE_LIMIT + prune_large_diff! + return true + end + end + end + + if collapse && size >= DIFF_COLLAPSE_LIMIT + prune_collapsed_diff! + return true + end + + false + end + + # Strip out the information at the beginning of the patch's text to match + # Grit's output + def strip_diff_headers(diff_text) + # Delete everything up to the first line that starts with '---' or + # 'Binary' + diff_text.sub!(/\A.*?^(---|Binary)/m, '\1') + + if diff_text.start_with?('---', 'Binary') + diff_text + else + # If the diff_text did not contain a line starting with '---' or + # 'Binary', return the empty string. No idea why; we are just + # preserving behavior from before the refactor. + '' + end + end + end + end +end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb new file mode 100644 index 00000000000..65e06f5065d --- /dev/null +++ b/lib/gitlab/git/diff_collection.rb @@ -0,0 +1,129 @@ +module Gitlab + module Git + class DiffCollection + include Enumerable + + DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze + + def initialize(iterator, options = {}) + @iterator = iterator + @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) + @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) + @max_bytes = @max_files * 5120 # Average 5 KB per file + @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min + @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min + @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file + @all_diffs = !!options.fetch(:all_diffs, false) + @no_collapse = !!options.fetch(:no_collapse, true) + @deltas_only = !!options.fetch(:deltas_only, false) + + @line_count = 0 + @byte_count = 0 + @overflow = false + @array = Array.new + end + + def each(&block) + if @populated + # @iterator.each is slower than just iterating the array in place + @array.each(&block) + elsif @deltas_only + each_delta(&block) + else + each_patch(&block) + end + end + + def empty? + !@iterator.any? + end + + def overflow? + populate! + !!@overflow + end + + def size + @size ||= count # forces a loop using each method + end + + def real_size + populate! + + if @overflow + "#{size}+" + else + size.to_s + end + end + + def decorate! + collection = each_with_index do |element, i| + @array[i] = yield(element) + end + @populated = true + collection + end + + private + + def populate! + return if @populated + + each { nil } # force a loop through all diffs + @populated = true + nil + end + + def over_safe_limits?(files) + files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes + end + + def each_delta + @iterator.each_delta.with_index do |delta, i| + diff = Gitlab::Git::Diff.new(delta) + + yield @array[i] = diff + end + end + + def each_patch + @iterator.each_with_index do |raw, i| + # First yield cached Diff instances from @array + if @array[i] + yield @array[i] + next + end + + # We have exhausted @array, time to create new Diff instances or stop. + break if @overflow + + if !@all_diffs && i >= @max_files + @overflow = true + break + end + + collapse = !@all_diffs && !@no_collapse + + diff = Gitlab::Git::Diff.new(raw, collapse: collapse) + + if collapse && over_safe_limits?(i) + diff.prune_collapsed_diff! + end + + @line_count += diff.line_count + @byte_count += diff.diff.bytesize + + if !@all_diffs && (@line_count >= @max_lines || @byte_count >= @max_bytes) + # This last Diff instance pushes us over the lines limit. We stop and + # discard it. + @overflow = true + break + end + + yield @array[i] = diff + end + end + end + end +end diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb new file mode 100644 index 00000000000..e57d228e688 --- /dev/null +++ b/lib/gitlab/git/encoding_helper.rb @@ -0,0 +1,58 @@ +module Gitlab + module Git + module EncodingHelper + extend self + + # This threshold is carefully tweaked to prevent usage of encodings detected + # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, + # we're better off sticking with utf8 encoding. + # Reason: git diff can return strings with invalid utf8 byte sequences if it + # truncates a diff in the middle of a multibyte character. In this case + # CharlockHolmes will try to guess the encoding and will likely suggest an + # obscure encoding with low confidence. + # There is a lot more info with this merge request: + # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 + ENCODING_CONFIDENCE_THRESHOLD = 40 + + def encode!(message) + return nil unless message.respond_to? :force_encoding + + # if message is utf-8 encoding, just return it + message.force_encoding("UTF-8") + return message if message.valid_encoding? + + # return message if message type is binary + detect = CharlockHolmes::EncodingDetector.detect(message) + return message.force_encoding("BINARY") if detect && detect[:type] == :binary + + # force detected encoding if we have sufficient confidence. + if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD + message.force_encoding(detect[:encoding]) + end + + # encode and clean the bad chars + message.replace clean(message) + rescue + encoding = detect ? detect[:encoding] : "unknown" + "--broken encoding: #{encoding}" + end + + def encode_utf8(message) + detect = CharlockHolmes::EncodingDetector.detect(message) + if detect + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + else + clean(message) + end + end + + private + + def clean(message) + message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") + .encode("UTF-8") + .gsub("\0".encode("UTF-8"), "") + end + end + end +end diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb new file mode 100644 index 00000000000..0148cd8df05 --- /dev/null +++ b/lib/gitlab/git/path_helper.rb @@ -0,0 +1,16 @@ +module Gitlab + module Git + class PathHelper + class << self + def normalize_path(filename) + # Strip all leading slashes so that //foo -> foo + filename[/^\/*/] = '' + + # Expand relative paths (e.g. foo/../bar) + filename = Pathname.new(filename) + filename.relative_path_from(Pathname.new('')) + end + end + end + end +end diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb new file mode 100644 index 00000000000..df9ca3ee5ac --- /dev/null +++ b/lib/gitlab/git/popen.rb @@ -0,0 +1,26 @@ +require 'open3' + +module Gitlab + module Git + module Popen + def popen(cmd, path) + unless cmd.is_a?(Array) + raise "System commands must be given as an array of strings" + end + + vars = { "PWD" => path } + options = { chdir: path } + + @cmd_output = "" + @cmd_status = 0 + Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| + @cmd_output << stdout.read + @cmd_output << stderr.read + @cmd_status = wait_thr.value.exitstatus + end + + [@cmd_output, @cmd_status] + end + end + end +end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb new file mode 100644 index 00000000000..37ef6836742 --- /dev/null +++ b/lib/gitlab/git/ref.rb @@ -0,0 +1,49 @@ +module Gitlab + module Git + class Ref + include Gitlab::Git::EncodingHelper + + # Branch or tag name + # without "refs/tags|heads" prefix + attr_reader :name + + # Target sha. + # Usually it is commit sha but in case + # when tag reference on other tag it can be tag sha + attr_reader :target + + # Dereferenced target + # Commit object to which the Ref points to + attr_reader :dereferenced_target + + # Extract branch name from full ref path + # + # Ex. + # Ref.extract_branch_name('refs/heads/master') #=> 'master' + def self.extract_branch_name(str) + str.gsub(/\Arefs\/heads\//, '') + end + + def self.dereference_object(object) + object = object.target while object.is_a?(Rugged::Tag::Annotation) + + object + end + + def initialize(repository, name, target) + encode! name + @name = name.gsub(/\Arefs\/(tags|heads)\//, '') + @dereferenced_target = Gitlab::Git::Commit.find(repository, target) + @target = if target.respond_to?(:oid) + target.oid + elsif target.respond_to?(:name) + target.name + elsif target.is_a? String + target + else + nil + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb new file mode 100644 index 00000000000..7068e68a855 --- /dev/null +++ b/lib/gitlab/git/repository.rb @@ -0,0 +1,1251 @@ +# Gitlab::Git::Repository is a wrapper around native Rugged::Repository object +require 'forwardable' +require 'tempfile' +require 'forwardable' +require "rubygems/package" + +module Gitlab + module Git + class Repository + extend Forwardable + include Gitlab::Git::Popen + + SEARCH_CONTEXT_LINES = 3 + + class NoRepository < StandardError; end + class InvalidBlobName < StandardError; end + class InvalidRef < StandardError; end + + # Full path to repo + attr_reader :path + + # Directory name of repo + attr_reader :name + + # Rugged repo object + attr_reader :rugged + + # 'path' must be the path to a _bare_ git repository, e.g. + # /path/to/my-repo.git + def initialize(path) + @path = path + @name = path.split("/").last + @attributes = Gitlab::Git::Attributes.new(path) + end + + # Default branch in the repository + def root_ref + @root_ref ||= discover_default_branch + end + + # Alias to old method for compatibility + def raw + rugged + end + + def rugged + @rugged ||= Rugged::Repository.new(path) + rescue Rugged::RepositoryError, Rugged::OSError + raise NoRepository.new('no repository for such path') + end + + # Returns an Array of branch names + # sorted by name ASC + def branch_names + branches.map(&:name) + end + + # Returns an Array of Branches + def branches + rugged.branches.map do |rugged_ref| + begin + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact.sort_by(&:name) + end + + def reload_rugged + @rugged = nil + end + + # Directly find a branch with a simple name (e.g. master) + # + # force_reload causes a new Rugged repository to be instantiated + # + # This is to work around a bug in libgit2 that causes in-memory refs to + # be stale/invalid when packed-refs is changed. + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333 + def find_branch(name, force_reload = false) + reload_rugged if force_reload + + rugged_ref = rugged.branches[name] + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref + end + + def local_branches + rugged.branches.each(:local).map do |branch| + Gitlab::Git::Branch.new(self, branch.name, branch.target) + end + end + + # Returns the number of valid branches + def branch_count + rugged.branches.count do |ref| + begin + ref.name && ref.target # ensures the branch is valid + + true + rescue Rugged::ReferenceError + false + end + end + end + + # Returns an Array of tag names + def tag_names + rugged.tags.map { |t| t.name } + end + + # Returns an Array of Tags + def tags + rugged.references.each("refs/tags/*").map do |ref| + message = nil + + if ref.target.is_a?(Rugged::Tag::Annotation) + tag_message = ref.target.message + + if tag_message.respond_to?(:chomp) + message = tag_message.chomp + end + end + + Gitlab::Git::Tag.new(self, ref.name, ref.target, message) + end.sort_by(&:name) + end + + # Returns true if the given tag exists + # + # name - The name of the tag as a String. + def tag_exists?(name) + !!rugged.tags[name] + end + + # Returns true if the given branch exists + # + # name - The name of the branch as a String. + def branch_exists?(name) + rugged.branches.exists?(name) + + # If the branch name is invalid (e.g. ".foo") Rugged will raise an error. + # Whatever code calls this method shouldn't have to deal with that so + # instead we just return `false` (which is true since a branch doesn't + # exist when it has an invalid name). + rescue Rugged::ReferenceError + false + end + + # Returns an Array of branch and tag names + def ref_names + branch_names + tag_names + end + + # Deprecated. Will be removed in 5.2 + def heads + rugged.references.each("refs/heads/*").map do |head| + Gitlab::Git::Ref.new(self, head.name, head.target) + end.sort_by(&:name) + end + + def has_commits? + !empty? + end + + def empty? + rugged.empty? + end + + def bare? + rugged.bare? + end + + def repo_exists? + !!rugged + end + + # Discovers the default branch based on the repository's available branches + # + # - If no branches are present, returns nil + # - If one branch is present, returns its name + # - If two or more branches are present, returns current HEAD or master or first branch + def discover_default_branch + names = branch_names + + return if names.empty? + + return names[0] if names.length == 1 + + if rugged_head + extracted_name = Ref.extract_branch_name(rugged_head.name) + + return extracted_name if names.include?(extracted_name) + end + + if names.include?('master') + 'master' + else + names[0] + end + end + + def rugged_head + rugged.head + rescue Rugged::ReferenceError + nil + end + + def archive_metadata(ref, storage_path, format = "tar.gz") + ref ||= root_ref + commit = Gitlab::Git::Commit.find(self, ref) + return {} if commit.nil? + + project_name = self.name.chomp('.git') + prefix = "#{project_name}-#{ref}-#{commit.id}" + + { + 'RepoPath' => path, + 'ArchivePrefix' => prefix, + 'ArchivePath' => archive_file_path(prefix, storage_path, format), + 'CommitId' => commit.id, + } + end + + def archive_file_path(name, storage_path, format = "tar.gz") + # Build file path + return nil unless name + + extension = + case format + when "tar.bz2", "tbz", "tbz2", "tb2", "bz2" + "tar.bz2" + when "tar" + "tar" + when "zip" + "zip" + else + # everything else should fall back to tar.gz + "tar.gz" + end + + file_name = "#{name}.#{extension}" + File.join(storage_path, self.name, file_name) + end + + # Return repo size in megabytes + def size + size = popen(%w(du -sk), path).first.strip.to_i + (size.to_f / 1024).round(2) + end + + # Returns an array of BlobSnippets for files at the specified +ref+ that + # contain the +query+ string. + def search_files(query, ref = nil) + greps = [] + ref ||= root_ref + + populated_index(ref).each do |entry| + # Discard submodules + next if submodule?(entry) + + blob = Gitlab::Git::Blob.raw(self, entry[:oid]) + + # Skip binary files + next if blob.data.encoding == Encoding::ASCII_8BIT + + blob.load_all_data!(self) + greps += build_greps(blob.data, query, ref, entry[:path]) + end + + greps + end + + # Use the Rugged Walker API to build an array of commits. + # + # Usage. + # repo.log( + # ref: 'master', + # path: 'app/models', + # limit: 10, + # offset: 5, + # after: Time.new(2016, 4, 21, 14, 32, 10) + # ) + # + def log(options) + default_options = { + limit: 10, + offset: 0, + path: nil, + follow: false, + skip_merges: false, + disable_walk: false, + after: nil, + before: nil + } + + options = default_options.merge(options) + options[:limit] ||= 0 + options[:offset] ||= 0 + actual_ref = options[:ref] || root_ref + begin + sha = sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + if log_using_shell?(options) + log_by_shell(sha, options) + else + log_by_walk(sha, options) + end + end + + def log_using_shell?(options) + options[:path].present? || + options[:disable_walk] || + options[:skip_merges] || + options[:after] || + options[:before] + end + + def log_by_walk(sha, options) + walk_options = { + show: sha, + sort: Rugged::SORT_DATE, + limit: options[:limit], + offset: options[:offset] + } + Rugged::Walker.walk(rugged, walk_options).to_a + end + + def log_by_shell(sha, options) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} log) + cmd += %W(-n #{options[:limit].to_i}) + cmd += %w(--format=%H) + cmd += %W(--skip=#{options[:offset].to_i}) + cmd += %w(--follow) if options[:follow] + cmd += %w(--no-merges) if options[:skip_merges] + cmd += %W(--after=#{options[:after].iso8601}) if options[:after] + cmd += %W(--before=#{options[:before].iso8601}) if options[:before] + cmd += [sha] + cmd += %W(-- #{options[:path]}) if options[:path].present? + + raw_output = IO.popen(cmd) {|io| io.read } + + log = raw_output.lines.map do |c| + Rugged::Commit.new(rugged, c.strip) + end + + log.is_a?(Array) ? log : [] + end + + def sha_from_ref(ref) + rev_parse_target(ref).oid + end + + # Return the object that +revspec+ points to. If +revspec+ is an + # annotated tag, then return the tag's target instead. + def rev_parse_target(revspec) + obj = rugged.rev_parse(revspec) + Ref.dereference_object(obj) + end + + # Return a collection of Rugged::Commits between the two revspec arguments. + # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for + # a detailed list of valid arguments. + def commits_between(from, to) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_DATE | Rugged::SORT_REVERSE) + + sha_from = sha_from_ref(from) + sha_to = sha_from_ref(to) + + walker.push(sha_to) + walker.hide(sha_from) + + commits = walker.to_a + walker.reset + + commits + end + + # Counts the amount of commits between `from` and `to`. + def count_commits_between(from, to) + commits_between(from, to).size + end + + # Returns the SHA of the most recent common ancestor of +from+ and +to+ + def merge_base_commit(from, to) + rugged.merge_base(from, to) + end + + # Return an array of Diff objects that represent the diff + # between +from+ and +to+. See Diff::filter_diff_options for the allowed + # diff options. The +options+ hash can also include :break_rewrites to + # split larger rewrites into delete/add pairs. + def diff(from, to, options = {}, *paths) + Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options) + end + + # Returns commits collection + # + # Ex. + # repo.find_commits( + # ref: 'master', + # max_count: 10, + # skip: 5, + # order: :date + # ) + # + # +options+ is a Hash of optional arguments to git + # :ref is the ref from which to begin (SHA1 or name) + # :contains is the commit contained by the refs from which to begin (SHA1 or name) + # :max_count is the maximum number of commits to fetch + # :skip is the number of commits to skip + # :order is the commits order and allowed value is :date(default) or :topo + # + def find_commits(options = {}) + actual_options = options.dup + + allowed_options = [:ref, :max_count, :skip, :contains, :order] + + actual_options.keep_if do |key| + allowed_options.include?(key) + end + + default_options = { skip: 0 } + actual_options = default_options.merge(actual_options) + + walker = Rugged::Walker.new(rugged) + + if actual_options[:ref] + walker.push(rugged.rev_parse_oid(actual_options[:ref])) + elsif actual_options[:contains] + branches_contains(actual_options[:contains]).each do |branch| + walker.push(branch.target_id) + end + else + rugged.references.each("refs/heads/*") do |ref| + walker.push(ref.target_id) + end + end + + if actual_options[:order] == :topo + walker.sorting(Rugged::SORT_TOPO) + else + walker.sorting(Rugged::SORT_DATE) + end + + commits = [] + offset = actual_options[:skip] + limit = actual_options[:max_count] + walker.each(offset: offset, limit: limit) do |commit| + gitlab_commit = Gitlab::Git::Commit.decorate(commit) + commits.push(gitlab_commit) + end + + walker.reset + + commits + rescue Rugged::OdbError + [] + end + + # Returns branch names collection that contains the special commit(SHA1 + # or name) + # + # Ex. + # repo.branch_names_contains('master') + # + def branch_names_contains(commit) + branches_contains(commit).map { |c| c.name } + end + + # Returns branch collection that contains the special commit(SHA1 or name) + # + # Ex. + # repo.branch_names_contains('master') + # + def branches_contains(commit) + commit_obj = rugged.rev_parse(commit) + parent = commit_obj.parents.first unless commit_obj.parents.empty? + + walker = Rugged::Walker.new(rugged) + + rugged.branches.select do |branch| + walker.push(branch.target_id) + walker.hide(parent) if parent + result = walker.any? { |c| c.oid == commit_obj.oid } + walker.reset + + result + end + end + + # Get refs hash which key is SHA1 + # and value is a Rugged::Reference + def refs_hash + # Initialize only when first call + if @refs_hash.nil? + @refs_hash = Hash.new { |h, k| h[k] = [] } + + rugged.references.each do |r| + # Symbolic/remote references may not have an OID; skip over them + target_oid = r.target.try(:oid) + if target_oid + sha = rev_parse_target(target_oid).oid + @refs_hash[sha] << r + end + end + end + @refs_hash + end + + # Lookup for rugged object by oid or ref name + def lookup(oid_or_ref_name) + rugged.rev_parse(oid_or_ref_name) + end + + # Return hash with submodules info for this repository + # + # Ex. + # { + # "rack" => { + # "id" => "c67be4624545b4263184c4a0e8f887efd0a66320", + # "path" => "rack", + # "url" => "git://github.com/chneukirchen/rack.git" + # }, + # "encoding" => { + # "id" => .... + # } + # } + # + def submodules(ref) + commit = rev_parse_target(ref) + return {} unless commit + + begin + content = blob_content(commit, ".gitmodules") + rescue InvalidBlobName + return {} + end + + parse_gitmodules(commit, content) + end + + # Return total commits count accessible from passed ref + def commit_count(ref) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) + oid = rugged.rev_parse_oid(ref) + walker.push(oid) + walker.count + end + + # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or + # tag name or a commit SHA. Valid +reset_type+ values are: + # + # [:soft] + # the head will be moved to the commit. + # [:mixed] + # will trigger a +:soft+ reset, plus the index will be replaced + # with the content of the commit tree. + # [:hard] + # will trigger a +:mixed+ reset and the working directory will be + # replaced with the content of the index. (Untracked and ignored files + # will be left alone) + def reset(ref, reset_type) + rugged.reset(ref, reset_type) + end + + # Mimic the `git clean` command and recursively delete untracked files. + # Valid keys that can be passed in the +options+ hash are: + # + # :d - Remove untracked directories + # :f - Remove untracked directories that are managed by a different + # repository + # :x - Remove ignored files + # + # The value in +options+ must evaluate to true for an option to take + # effect. + # + # Examples: + # + # repo.clean(d: true, f: true) # Enable the -d and -f options + # + # repo.clean(d: false, x: true) # -x is enabled, -d is not + def clean(options = {}) + strategies = [:remove_untracked] + strategies.push(:force) if options[:f] + strategies.push(:remove_ignored) if options[:x] + + # TODO: implement this method + end + + # Check out the specified ref. Valid options are: + # + # :b - Create a new branch at +start_point+ and set HEAD to the new + # branch. + # + # * These options are passed to the Rugged::Repository#checkout method: + # + # :progress :: + # A callback that will be executed for checkout progress notifications. + # Up to 3 parameters are passed on each execution: + # + # - The path to the last updated file (or +nil+ on the very first + # invocation). + # - The number of completed checkout steps. + # - The number of total checkout steps to be performed. + # + # :notify :: + # A callback that will be executed for each checkout notification + # types specified with +:notify_flags+. Up to 5 parameters are passed + # on each execution: + # + # - An array containing the +:notify_flags+ that caused the callback + # execution. + # - The path of the current file. + # - A hash describing the baseline blob (or +nil+ if it does not + # exist). + # - A hash describing the target blob (or +nil+ if it does not exist). + # - A hash describing the workdir blob (or +nil+ if it does not + # exist). + # + # :strategy :: + # A single symbol or an array of symbols representing the strategies + # to use when performing the checkout. Possible values are: + # + # :none :: + # Perform a dry run (default). + # + # :safe :: + # Allow safe updates that cannot overwrite uncommitted data. + # + # :safe_create :: + # Allow safe updates plus creation of missing files. + # + # :force :: + # Allow all updates to force working directory to look like index. + # + # :allow_conflicts :: + # Allow checkout to make safe updates even if conflicts are found. + # + # :remove_untracked :: + # Remove untracked files not in index (that are not ignored). + # + # :remove_ignored :: + # Remove ignored files not in index. + # + # :update_only :: + # Only update existing files, don't create new ones. + # + # :dont_update_index :: + # Normally checkout updates index entries as it goes; this stops + # that. + # + # :no_refresh :: + # Don't refresh index/config/etc before doing checkout. + # + # :disable_pathspec_match :: + # Treat pathspec as simple list of exact match file paths. + # + # :skip_locked_directories :: + # Ignore directories in use, they will be left empty. + # + # :skip_unmerged :: + # Allow checkout to skip unmerged files (NOT IMPLEMENTED). + # + # :use_ours :: + # For unmerged files, checkout stage 2 from index (NOT IMPLEMENTED). + # + # :use_theirs :: + # For unmerged files, checkout stage 3 from index (NOT IMPLEMENTED). + # + # :update_submodules :: + # Recursively checkout submodules with same options (NOT + # IMPLEMENTED). + # + # :update_submodules_if_changed :: + # Recursively checkout submodules if HEAD moved in super repo (NOT + # IMPLEMENTED). + # + # :disable_filters :: + # If +true+, filters like CRLF line conversion will be disabled. + # + # :dir_mode :: + # Mode for newly created directories. Default: +0755+. + # + # :file_mode :: + # Mode for newly created files. Default: +0755+ or +0644+. + # + # :file_open_flags :: + # Mode for opening files. Default: + # <code>IO::CREAT | IO::TRUNC | IO::WRONLY</code>. + # + # :notify_flags :: + # A single symbol or an array of symbols representing the cases in + # which the +:notify+ callback should be invoked. Possible values are: + # + # :none :: + # Do not invoke the +:notify+ callback (default). + # + # :conflict :: + # Invoke the callback for conflicting paths. + # + # :dirty :: + # Invoke the callback for "dirty" files, i.e. those that do not need + # an update but no longer match the baseline. + # + # :updated :: + # Invoke the callback for any file that was changed. + # + # :untracked :: + # Invoke the callback for untracked files. + # + # :ignored :: + # Invoke the callback for ignored files. + # + # :all :: + # Invoke the callback for all these cases. + # + # :paths :: + # A glob string or an array of glob strings specifying which paths + # should be taken into account for the checkout operation. +nil+ will + # match all files. Default: +nil+. + # + # :baseline :: + # A Rugged::Tree that represents the current, expected contents of the + # workdir. Default: +HEAD+. + # + # :target_directory :: + # A path to an alternative workdir directory in which the checkout + # should be performed. + def checkout(ref, options = {}, start_point = "HEAD") + if options[:b] + rugged.branches.create(ref, start_point) + options.delete(:b) + end + default_options = { strategy: [:recreate_missing, :safe] } + rugged.checkout(ref, default_options.merge(options)) + end + + # Delete the specified branch from the repository + def delete_branch(branch_name) + rugged.branches.delete(branch_name) + end + + # Create a new branch named **ref+ based on **stat_point+, HEAD by default + # + # Examples: + # create_branch("feature") + # create_branch("other-feature", "master") + def create_branch(ref, start_point = "HEAD") + rugged_ref = rugged.branches.create(ref, start_point) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + rescue Rugged::ReferenceError => e + raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Invalid reference #{start_point}") + end + + # Return an array of this repository's remote names + def remote_names + rugged.remotes.each_name.to_a + end + + # Delete the specified remote from this repository. + def remote_delete(remote_name) + rugged.remotes.delete(remote_name) + end + + # Add a new remote to this repository. Returns a Rugged::Remote object + def remote_add(remote_name, url) + rugged.remotes.create(remote_name, url) + end + + # Update the specified remote using the values in the +options+ hash + # + # Example + # repo.update_remote("origin", url: "path/to/repo") + def remote_update(remote_name, options = {}) + # TODO: Implement other remote options + rugged.remotes.set_url(remote_name, options[:url]) if options[:url] + end + + # Fetch the specified remote + def fetch(remote_name) + rugged.remotes[remote_name].fetch + end + + # Push +*refspecs+ to the remote identified by +remote_name+. + def push(remote_name, *refspecs) + rugged.remotes[remote_name].push(refspecs) + end + + # Merge the +source_name+ branch into the +target_name+ branch. This is + # equivalent to `git merge --no_ff +source_name+`, since a merge commit + # is always created. + def merge(source_name, target_name, options = {}) + our_commit = rugged.branches[target_name].target + their_commit = rugged.branches[source_name].target + + raise "Invalid merge target" if our_commit.nil? + raise "Invalid merge source" if their_commit.nil? + + merge_index = rugged.merge_commits(our_commit, their_commit) + return false if merge_index.conflicts? + + actual_options = options.merge( + parents: [our_commit, their_commit], + tree: merge_index.write_tree(rugged), + update_ref: "refs/heads/#{target_name}" + ) + Rugged::Commit.create(rugged, actual_options) + end + + def commits_since(from_date) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_DATE | Rugged::SORT_REVERSE) + + rugged.references.each("refs/heads/*") do |ref| + walker.push(ref.target_id) + end + + commits = [] + walker.each do |commit| + break if commit.author[:time].to_date < from_date + commits.push(commit) + end + + commits + end + + AUTOCRLF_VALUES = { + "true" => true, + "false" => false, + "input" => :input + }.freeze + + def autocrlf + AUTOCRLF_VALUES[rugged.config['core.autocrlf']] + end + + def autocrlf=(value) + rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value] + end + + # Create a new directory with a .gitkeep file. Creates + # all required nested directories (i.e. mkdir -p behavior) + # + # options should contain next structure: + # author: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # committer: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # commit: { + # message: 'Wow such commit', + # branch: 'master', + # update_ref: false + # } + def mkdir(path, options = {}) + # Check if this directory exists; if it does, then don't bother + # adding .gitkeep file. + ref = options[:commit][:branch] + path = Gitlab::Git::PathHelper.normalize_path(path).to_s + rugged_ref = rugged.ref(ref) + + raise InvalidRef.new("Invalid ref") if rugged_ref.nil? + + target_commit = rugged_ref.target + + raise InvalidRef.new("Invalid target commit") if target_commit.nil? + + entry = tree_entry(target_commit, path) + + if entry + if entry[:type] == :blob + raise InvalidBlobName.new("Directory already exists as a file") + else + raise InvalidBlobName.new("Directory already exists") + end + end + + options[:file] = { + content: '', + path: "#{path}/.gitkeep", + update: true + } + + Gitlab::Git::Blob.commit(self, options) + end + + # Returns result like "git ls-files" , recursive and full file path + # + # Ex. + # repo.ls_files('master') + # + def ls_files(ref) + actual_ref = ref || root_ref + + begin + sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-tree) + cmd += %w(-r) + cmd += %w(--full-tree) + cmd += %w(--full-name) + cmd += %W(-- #{actual_ref}) + + raw_output = IO.popen(cmd, &:read).split("\n").map do |f| + stuff, path = f.split("\t") + _mode, type, _sha = stuff.split(" ") + path if type == "blob" + # Contain only blob type + end + + raw_output.compact + end + + def copy_gitattributes(ref) + begin + commit = lookup(ref) + rescue Rugged::ReferenceError + raise InvalidRef.new("Ref #{ref} is invalid") + end + + # Create the paths + info_dir_path = File.join(path, 'info') + info_attributes_path = File.join(info_dir_path, 'attributes') + + begin + # Retrieve the contents of the blob + gitattributes_content = blob_content(commit, '.gitattributes') + rescue InvalidBlobName + # No .gitattributes found. Should now remove any info/attributes and return + File.delete(info_attributes_path) if File.exist?(info_attributes_path) + return + end + + # Create the info directory if needed + Dir.mkdir(info_dir_path) unless File.directory?(info_dir_path) + + # Write the contents of the .gitattributes file to info/attributes + # Use binary mode to prevent Rails from converting ASCII-8BIT to UTF-8 + File.open(info_attributes_path, "wb") do |file| + file.write(gitattributes_content) + end + end + + # Checks if the blob should be diffable according to its attributes + def diffable?(blob) + attributes(blob.path).fetch('diff') { blob.text? } + end + + # Returns the Git attributes for the given file path. + # + # See `Gitlab::Git::Attributes` for more information. + def attributes(path) + @attributes.attributes(path) + end + + private + + # Get the content of a blob for a given commit. If the blob is a commit + # (for submodules) then return the blob's OID. + def blob_content(commit, blob_name) + blob_entry = tree_entry(commit, blob_name) + + unless blob_entry + raise InvalidBlobName.new("Invalid blob name: #{blob_name}") + end + + case blob_entry[:type] + when :commit + blob_entry[:oid] + when :tree + raise InvalidBlobName.new("#{blob_name} is a tree, not a blob") + when :blob + rugged.lookup(blob_entry[:oid]).content + end + end + + # Parses the contents of a .gitmodules file and returns a hash of + # submodule information. + def parse_gitmodules(commit, content) + results = {} + + current = "" + content.split("\n").each do |txt| + if txt =~ /^\s*\[/ + current = txt.match(/(?<=").*(?=")/)[0] + results[current] = {} + else + next unless results[current] + match_data = txt.match(/(\w+)\s*=\s*(.*)/) + next unless match_data + target = match_data[2].chomp + results[current][match_data[1]] = target + + if match_data[1] == "path" + begin + results[current]["id"] = blob_content(commit, target) + rescue InvalidBlobName + results.delete(current) + end + end + end + end + + results + end + + # Returns true if +commit+ introduced changes to +path+, using commit + # trees to make that determination. Uses the history simplification + # rules that `git log` uses by default, where a commit is omitted if it + # is TREESAME to any parent. + # + # If the +follow+ option is true and the file specified by +path+ was + # renamed, then the path value is set to the old path. + def commit_touches_path?(commit, path, follow, walker) + entry = tree_entry(commit, path) + + if commit.parents.empty? + # This is the root commit, return true if it has +path+ in its tree + return !entry.nil? + end + + num_treesame = 0 + commit.parents.each do |parent| + parent_entry = tree_entry(parent, path) + + # Only follow the first TREESAME parent for merge commits + if num_treesame > 0 + walker.hide(parent) + next + end + + if entry.nil? && parent_entry.nil? + num_treesame += 1 + elsif entry && parent_entry && entry[:oid] == parent_entry[:oid] + num_treesame += 1 + end + end + + case num_treesame + when 0 + detect_rename(commit, commit.parents.first, path) if follow + true + else false + end + end + + # Find the entry for +path+ in the tree for +commit+ + def tree_entry(commit, path) + pathname = Pathname.new(path) + first = true + tmp_entry = nil + + pathname.each_filename do |dir| + if first + tmp_entry = commit.tree[dir] + first = false + elsif tmp_entry.nil? + return nil + else + tmp_entry = rugged.lookup(tmp_entry[:oid]) + return nil unless tmp_entry.type == :tree + tmp_entry = tmp_entry[dir] + end + end + + tmp_entry + end + + # Compare +commit+ and +parent+ for +path+. If +path+ is a file and was + # renamed in +commit+, then set +path+ to the old filename. + def detect_rename(commit, parent, path) + diff = parent.diff(commit, paths: [path], disable_pathspec_match: true) + + # If +path+ is a filename, not a directory, then we should only have + # one delta. We don't need to follow renames for directories. + return nil if diff.each_delta.count > 1 + + delta = diff.each_delta.first + if delta.added? + full_diff = parent.diff(commit) + full_diff.find_similar! + + full_diff.each_delta do |full_delta| + if full_delta.renamed? && path == full_delta.new_file[:path] + # Look for the old path in ancestors + path.replace(full_delta.old_file[:path]) + end + end + end + end + + def archive_to_file(treeish = 'master', filename = 'archive.tar.gz', format = nil, compress_cmd = %w(gzip -n)) + git_archive_cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} archive) + + # Put files into a directory before archiving + prefix = "#{archive_name(treeish)}/" + git_archive_cmd << "--prefix=#{prefix}" + + # Format defaults to tar + git_archive_cmd << "--format=#{format}" if format + + git_archive_cmd += %W(-- #{treeish}) + + open(filename, 'w') do |file| + # Create a pipe to act as the '|' in 'git archive ... | gzip' + pipe_rd, pipe_wr = IO.pipe + + # Get the compression process ready to accept data from the read end + # of the pipe + compress_pid = spawn(*nice(compress_cmd), in: pipe_rd, out: file) + # The read end belongs to the compression process now; we should + # close our file descriptor for it. + pipe_rd.close + + # Start 'git archive' and tell it to write into the write end of the + # pipe. + git_archive_pid = spawn(*nice(git_archive_cmd), out: pipe_wr) + # The write end belongs to 'git archive' now; close it. + pipe_wr.close + + # When 'git archive' and the compression process are finished, we are + # done. + Process.waitpid(git_archive_pid) + raise "#{git_archive_cmd.join(' ')} failed" unless $?.success? + Process.waitpid(compress_pid) + raise "#{compress_cmd.join(' ')} failed" unless $?.success? + end + end + + def nice(cmd) + nice_cmd = %w(nice -n 20) + unless unsupported_platform? + nice_cmd += %w(ionice -c 2 -n 7) + end + nice_cmd + cmd + end + + def unsupported_platform? + %w[darwin freebsd solaris].map { |platform| RUBY_PLATFORM.include?(platform) }.any? + end + + # Returns true if the index entry has the special file mode that denotes + # a submodule. + def submodule?(index_entry) + index_entry[:mode] == 57344 + end + + # Return a Rugged::Index that has read from the tree at +ref_name+ + def populated_index(ref_name) + commit = rev_parse_target(ref_name) + index = rugged.index + index.read_tree(commit.tree) + index + end + + # Return an array of BlobSnippets for lines in +file_contents+ that match + # +query+ + def build_greps(file_contents, query, ref, filename) + # The file_contents string is potentially huge so we make sure to loop + # through it one line at a time. This gives Ruby the chance to GC lines + # we are not interested in. + # + # We need to do a little extra work because we are not looking for just + # the lines that matches the query, but also for the context + # (surrounding lines). We will use Enumerable#each_cons to efficiently + # loop through the lines while keeping surrounding lines on hand. + # + # First, we turn "foo\nbar\nbaz" into + # [ + # [nil, -3], [nil, -2], [nil, -1], + # ['foo', 0], ['bar', 1], ['baz', 3], + # [nil, 4], [nil, 5], [nil, 6] + # ] + lines_with_index = Enumerator.new do |yielder| + # Yield fake 'before' lines for the first line of file_contents + (-SEARCH_CONTEXT_LINES..-1).each do |i| + yielder.yield [nil, i] + end + + # Yield the actual file contents + count = 0 + file_contents.each_line do |line| + line.chomp! + yielder.yield [line, count] + count += 1 + end + + # Yield fake 'after' lines for the last line of file_contents + (count + 1..count + SEARCH_CONTEXT_LINES).each do |i| + yielder.yield [nil, i] + end + end + + greps = [] + + # Loop through consecutive blocks of lines with indexes + lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block| + # Get the 'middle' line and index from the block + line, _ = line_block[SEARCH_CONTEXT_LINES] + + next unless line && line.match(/#{Regexp.escape(query)}/i) + + # Yay, 'line' contains a match! + # Get an array with just the context lines (no indexes) + match_with_context = line_block.map(&:first) + # Remove 'nil' lines in case we are close to the first or last line + match_with_context.compact! + + # Get the line number (1-indexed) of the first context line + first_context_line_number = line_block[0][1] + 1 + + greps << Gitlab::Git::BlobSnippet.new( + ref, + match_with_context, + first_context_line_number, + filename + ) + end + + greps + end + + # Return the Rugged patches for the diff between +from+ and +to+. + def diff_patches(from, to, options = {}, *paths) + options ||= {} + break_rewrites = options[:break_rewrites] + actual_options = Gitlab::Git::Diff.filter_diff_options(options.merge(paths: paths)) + + diff = rugged.diff(from, to, actual_options) + diff.find_similar!(break_rewrites: break_rewrites) + diff.each_patch + end + end + end +end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb new file mode 100644 index 00000000000..b5342c3d310 --- /dev/null +++ b/lib/gitlab/git/tag.rb @@ -0,0 +1,17 @@ +module Gitlab + module Git + class Tag < Ref + attr_reader :object_sha + + def initialize(repository, name, target, message = nil) + super(repository, name, target) + + @message = message + end + + def message + encode! @message + end + end + end +end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb new file mode 100644 index 00000000000..f7450e8b58f --- /dev/null +++ b/lib/gitlab/git/tree.rb @@ -0,0 +1,104 @@ +module Gitlab + module Git + class Tree + include Gitlab::Git::EncodingHelper + + attr_accessor :id, :root_id, :name, :path, :type, + :mode, :commit_id, :submodule_url + + class << self + # Get list of tree objects + # for repository based on commit sha and path + # Uses rugged for raw objects + def where(repository, sha, path = nil) + path = nil if path == '' || path == '/' + + commit = repository.lookup(sha) + root_tree = commit.tree + + tree = if path + id = find_id_by_path(repository, root_tree.oid, path) + if id + repository.lookup(id) + else + [] + end + else + root_tree + end + + tree.map do |entry| + new( + id: entry[:oid], + root_id: root_tree.oid, + name: entry[:name], + type: entry[:type], + mode: entry[:filemode], + path: path ? File.join(path, entry[:name]) : entry[:name], + commit_id: sha, + ) + end + end + + # Recursive search of tree id for path + # + # Ex. + # blog/ # oid: 1a + # app/ # oid: 2a + # models/ # oid: 3a + # views/ # oid: 4a + # + # + # Tree.find_id_by_path(repo, '1a', 'app/models') # => '3a' + # + def find_id_by_path(repository, root_id, path) + root_tree = repository.lookup(root_id) + path_arr = path.split('/') + + entry = root_tree.find do |entry| + entry[:name] == path_arr[0] && entry[:type] == :tree + end + + return nil unless entry + + if path_arr.size > 1 + path_arr.shift + find_id_by_path(repository, entry[:oid], path_arr.join('/')) + else + entry[:oid] + end + end + end + + def initialize(options) + %w(id root_id name path type mode commit_id).each do |key| + self.send("#{key}=", options[key.to_sym]) + end + end + + def name + encode! @name + end + + def dir? + type == :tree + end + + def file? + type == :blob + end + + def submodule? + type == :commit + end + + def readme? + name =~ /^readme/i + end + + def contributing? + name =~ /^contributing/i + end + end + end +end diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb new file mode 100644 index 00000000000..7973da2e8f8 --- /dev/null +++ b/lib/gitlab/git/util.rb @@ -0,0 +1,18 @@ +module Gitlab + module Git + module Util + LINE_SEP = "\n".freeze + + def self.count_lines(string) + case string[-1] + when nil + 0 + when LINE_SEP + string.count(LINE_SEP) + else + string.count(LINE_SEP) + 1 + end + end + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index c6b6efda360..7e1484613f2 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -7,7 +7,8 @@ module Gitlab ERROR_MESSAGES = { upload: 'You are not allowed to upload code for this project.', download: 'You are not allowed to download code from this project.', - deploy_key: 'Deploy keys are not allowed to push code.', + deploy_key_upload: + 'This deploy key does not have write access to this project.', no_repo: 'A repository for this project does not exist yet.' } @@ -31,12 +32,13 @@ module Gitlab check_active_user! check_project_accessibility! check_command_existence!(cmd) + check_repository_existence! case cmd when *DOWNLOAD_COMMANDS - download_access_check + check_download_access! when *PUSH_COMMANDS - push_access_check(changes) + check_push_access!(changes) end build_status_object(true) @@ -44,32 +46,10 @@ module Gitlab build_status_object(false, ex.message) end - def download_access_check - if user - user_download_access_check - elsif deploy_key.nil? && !guest_can_downlod_code? - raise UnauthorizedError, ERROR_MESSAGES[:download] - end - end - - def push_access_check(changes) - if user - user_push_access_check(changes) - else - raise UnauthorizedError, ERROR_MESSAGES[deploy_key ? :deploy_key : :upload] - end - end - - def guest_can_downlod_code? + def guest_can_download_code? Guest.can?(:download_code, project) end - def user_download_access_check - unless user_can_download_code? || build_can_download_code? - raise UnauthorizedError, ERROR_MESSAGES[:download] - end - end - def user_can_download_code? authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code) end @@ -78,35 +58,6 @@ module Gitlab authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) end - def user_push_access_check(changes) - unless authentication_abilities.include?(:push_code) - raise UnauthorizedError, ERROR_MESSAGES[:upload] - end - - if changes.blank? - return # Allow access. - end - - unless project.repository.exists? - raise UnauthorizedError, ERROR_MESSAGES[:no_repo] - end - - changes_list = Gitlab::ChangesList.new(changes) - - # Iterate over all changes to find if user allowed all of them to be applied - changes_list.each do |change| - status = change_access_check(change) - unless status.allowed? - # If user does not have access to make at least one change - cancel all push - raise UnauthorizedError, status.message - end - end - end - - def change_access_check(change) - Checks::ChangeAccess.new(change, user_access: user_access, project: project, env: @env).exec - end - def protocol_allowed? Gitlab::ProtocolAccess.allowed?(protocol) end @@ -120,6 +71,8 @@ module Gitlab end def check_active_user! + return if deploy_key? + if user && !user_access.allowed? raise UnauthorizedError, "Your account has been blocked." end @@ -137,33 +90,92 @@ module Gitlab end end - def matching_merge_request?(newrev, branch_name) - Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? + def check_repository_existence! + unless project.repository.exists? + raise UnauthorizedError, ERROR_MESSAGES[:no_repo] + end end - def deploy_key - actor if actor.is_a?(DeployKey) + def check_download_access! + return if deploy_key? + + passed = user_can_download_code? || + build_can_download_code? || + guest_can_download_code? + + unless passed + raise UnauthorizedError, ERROR_MESSAGES[:download] + end end - def deploy_key_can_read_project? + def check_push_access!(changes) if deploy_key - return true if project.public? - deploy_key.projects.include?(project) + check_deploy_key_push_access! + elsif user + check_user_push_access! else - false + raise UnauthorizedError, ERROR_MESSAGES[:upload] end + + return if changes.blank? # Allow access. + + check_change_access!(changes) end - def can_read_project? - if user - user_access.can_read_project? - elsif deploy_key - deploy_key_can_read_project? - else - Guest.can?(:read_project, project) + def check_user_push_access! + unless authentication_abilities.include?(:push_code) + raise UnauthorizedError, ERROR_MESSAGES[:upload] end end + def check_deploy_key_push_access! + unless deploy_key.can_push_to?(project) + raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] + end + end + + def check_change_access!(changes) + changes_list = Gitlab::ChangesList.new(changes) + + # Iterate over all changes to find if user allowed all of them to be applied + changes_list.each do |change| + status = check_single_change_access(change) + unless status.allowed? + # If user does not have access to make at least one change - cancel all push + raise UnauthorizedError, status.message + end + end + end + + def check_single_change_access(change) + Checks::ChangeAccess.new( + change, + user_access: user_access, + project: project, + env: @env, + skip_authorization: deploy_key?).exec + end + + def matching_merge_request?(newrev, branch_name) + Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? + end + + def deploy_key + actor if deploy_key? + end + + def deploy_key? + actor.is_a?(DeployKey) + end + + def can_read_project? + if deploy_key + deploy_key.has_access_to?(project) + elsif user + user.can?(:read_project, project) + end || Guest.can?(:read_project, project) + end + protected def user diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 2c06c4ff1ef..67eaa5e088d 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,6 +1,6 @@ module Gitlab class GitAccessWiki < GitAccess - def guest_can_downlod_code? + def guest_can_download_code? Guest.can?(:download_wiki_code, project) end @@ -8,7 +8,7 @@ module Gitlab authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code) end - def change_access_check(change) + def check_single_change_access(change) if user_access.can_do_action?(:create_wiki) build_status_object(true) else diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index d32bdd86427..6babea144c7 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -30,11 +30,11 @@ module Gitlab def retrieve_project_and_type @type = :project - @project = Project.find_with_namespace(@repo_path) + @project = Project.find_by_full_path(@repo_path) if @repo_path.end_with?('.wiki') && !@project @type = :wiki - @project = Project.find_with_namespace(@repo_path.gsub(/\.wiki\z/, '')) + @project = Project.find_by_full_path(@repo_path.gsub(/\.wiki\z/, '')) end end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 3f635be22ba..a55adc9b1c8 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,6 +1,8 @@ module Gitlab module GithubImport class ProjectCreator + include Gitlab::CurrentSettings + attr_reader :repo, :name, :namespace, :current_user, :session_data, :type def initialize(repo, name, namespace, current_user, session_data, type: 'github') @@ -34,7 +36,7 @@ module Gitlab end def visibility_level - repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility + repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility end # diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 4d4e04e9e35..b8a5ac907a4 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -13,6 +13,7 @@ module Gitlab if current_user gon.current_user_id = current_user.id + gon.current_username = current_user.username end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index eb667a85b78..d679edec36b 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.5' + VERSION = '0.1.6' FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index e6ecd118609..416194e57d7 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -6,6 +6,7 @@ project_tree: - :events - issues: - :events + - :timelogs - notes: - :author - :events @@ -27,6 +28,7 @@ project_tree: - :events - :merge_request_diff - :events + - :timelogs - label_links: - label: :priorities @@ -37,7 +39,6 @@ project_tree: - :author - :events - :statuses - - :variables - :triggers - :deploy_keys - :services diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index b790733f4a7..8b8e48aac76 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -1,13 +1,10 @@ module Gitlab module ImportExport class MembersMapper - attr_reader :missing_author_ids - def initialize(exported_members:, user:, project:) - @exported_members = exported_members + @exported_members = user.admin? ? exported_members : [] @user = user @project = project - @missing_author_ids = [] # This needs to run first, as second call would be from #map # which means project members already exist. @@ -35,16 +32,21 @@ module Gitlab @user.id end + def include?(old_author_id) + map.keys.include?(old_author_id) && map[old_author_id] != default_user_id + end + private def missing_keys_tracking_hash Hash.new do |_, key| - @missing_author_ids << key default_user_id end end def ensure_default_member! + @project.project_members.destroy_all + ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) end @@ -64,7 +66,7 @@ module Gitlab end def find_project_user_query(member) - user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email'])) + user_arel[:email].eq(member['user']['email']).or(user_arel[:username].eq(member['user']['username'])) end def user_arel diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 2fbf437ec26..b79be62245b 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -5,8 +5,9 @@ module Gitlab attr_reader :full_path - def initialize(project:, shared:) + def initialize(project:, current_user:, shared:) @project = project + @current_user = current_user @shared = shared @full_path = File.join(@shared.export_path, ImportExport.project_filename) end @@ -24,7 +25,29 @@ module Gitlab private def project_json_tree - @project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree) + project_json['project_members'] += group_members_json + + project_json.to_json + end + + def project_json + @project_json ||= @project.as_json(reader.project_tree) + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end + + def group_members_json + group_members.as_json(reader.group_members_tree).each do |group_member| + group_member['source_type'] = 'Project' # Make group members project members of the future import + end + end + + def group_members + return [] unless @current_user.can?(:admin_group, @project.group) + + MembersFinder.new(@project.project_members, @project.group).execute(@current_user) end end end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 5021a1a14ce..a1e7159fe42 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -21,6 +21,10 @@ module Gitlab false end + def group_members_tree + @attributes_finder.find_included(:project_members).merge(include: @attributes_finder.find(:user)) + end + private # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 7a649f28340..fae792237d9 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -4,7 +4,6 @@ module Gitlab OVERRIDES = { snippets: :project_snippets, pipelines: 'Ci::Pipeline', statuses: 'commit_status', - variables: 'Ci::Variable', triggers: 'Ci::Trigger', builds: 'Ci::Build', hooks: 'ProjectHook', @@ -14,7 +13,7 @@ module Gitlab priorities: :label_priorities, label: :project_label }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id resolved_by_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze @@ -24,6 +23,8 @@ module Gitlab EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze + TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze + def self.create(*args) new(*args).create end @@ -61,7 +62,9 @@ module Gitlab update_project_references handle_group_label if group_label? - reset_ci_tokens if @relation_name == 'Ci::Trigger' + reset_tokens! + remove_encrypted_attributes! + @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] set_st_diffs if @relation_name == :merge_request_diff end @@ -80,17 +83,13 @@ module Gitlab # is left. def set_note_author old_author_id = @relation_hash['author_id'] - - # Users with admin access can map users - @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id - author = @relation_hash.delete('author') - update_note_for_missing_author(author['name']) if missing_author?(old_author_id) + update_note_for_missing_author(author['name']) unless has_author?(old_author_id) end - def missing_author?(old_author_id) - !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id) + def has_author?(old_author_id) + admin_user? && @members_mapper.include?(old_author_id) end def missing_author_note(updated_at, author_name) @@ -144,11 +143,22 @@ module Gitlab end end - def reset_ci_tokens - return unless Gitlab::ImportExport.reset_tokens? + def reset_tokens! + return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s) # If we import/export a project to the same instance, tokens will have to be reset. - @relation_hash['token'] = nil + # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across. + relation_class.attribute_names.select { |name| name.include?('token') }.each do |token| + @relation_hash[token] = nil + end + end + + def remove_encrypted_attributes! + return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any? + + relation_class.encrypted_attributes.each_key do |key| + @relation_hash[key.to_s] = nil + end end def relation_class diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 45958710c13..52276cbcd9a 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -5,8 +5,6 @@ # module Gitlab module ImportSources - extend CurrentSettings - ImportSource = Struct.new(:name, :title, :importer) ImportTable = [ diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 801dfde9a36..c9122a23568 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -1,10 +1,9 @@ module Gitlab module IncomingEmail + UNSUBSCRIBE_SUFFIX = '+unsubscribe'.freeze WILDCARD_PLACEHOLDER = '%{key}'.freeze class << self - FALLBACK_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze - def enabled? config.enabled && config.address end @@ -18,7 +17,11 @@ module Gitlab end def reply_address(key) - config.address.gsub(WILDCARD_PLACEHOLDER, key) + config.address.sub(WILDCARD_PLACEHOLDER, key) + end + + def unsubscribe_address(key) + config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") end def key_from_address(address) @@ -32,10 +35,14 @@ module Gitlab end def key_from_fallback_message_id(mail_id) - match = mail_id.match(FALLBACK_MESSAGE_ID_REGEX) - return unless match + message_id_regexp = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\z/ - match[1] + mail_id[message_id_regexp, 1] + end + + def scan_fallback_references(references) + # It's looking for each <...> + references.scan(/(?!<)[^<>]+(?=>)/) end def config @@ -49,7 +56,7 @@ module Gitlab return nil unless wildcard_address regex = Regexp.escape(wildcard_address) - regex = regex.gsub(Regexp.escape('%{key}'), "(.+)") + regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)') Regexp.new(regex).freeze end end diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb new file mode 100644 index 00000000000..8db91d25a4b --- /dev/null +++ b/lib/gitlab/job_waiter.rb @@ -0,0 +1,27 @@ +module Gitlab + # JobWaiter can be used to wait for a number of Sidekiq jobs to complete. + class JobWaiter + # The sleep interval between checking keys, in seconds. + INTERVAL = 0.1 + + # jobs - The job IDs to wait for. + def initialize(jobs) + @jobs = jobs + end + + # Waits for all the jobs to be completed. + # + # timeout - The maximum amount of seconds to block the caller for. This + # ensures we don't indefinitely block a caller in case a job takes + # long to process, or is never processed. + def wait(timeout = 60) + start = Time.current + + while (Time.current - start) <= timeout + break if SidekiqStatus.all_completed?(@jobs) + + sleep(INTERVAL) # to not overload Redis too much. + end + end + end +end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index 288771c1c12..3a7af363548 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -43,10 +43,10 @@ module Gitlab end end - def add_terminal_auth(terminal, token, ca_pem = nil) + def add_terminal_auth(terminal, token:, max_session_time:, ca_pem: nil) terminal[:headers]['Authorization'] << "Bearer #{token}" + terminal[:max_session_time] = max_session_time terminal[:ca_pem] = ca_pem if ca_pem.present? - terminal end def container_exec_url(api_url, namespace, pod_name, container_name) diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index 7e06bd2b0fb..54a5b1d31cd 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -34,21 +34,21 @@ module Gitlab def allowed? if ldap_user unless ldap_config.active_directory - user.activate if user.ldap_blocked? + unblock_user(user, 'is available again') if user.ldap_blocked? return true end # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) - user.ldap_block + block_user(user, 'is disabled in Active Directory') false else - user.activate if user.ldap_blocked? + unblock_user(user, 'is not disabled anymore') if user.ldap_blocked? true end else # Block the user if they no longer exist in LDAP/AD - user.ldap_block + block_user(user, 'does not exist anymore') false end end @@ -64,6 +64,24 @@ module Gitlab def ldap_user @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) end + + def block_user(user, reason) + user.ldap_block + + Gitlab::AppLogger.info( + "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "blocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end + + def unblock_user(user, reason) + user.activate + + Gitlab::AppLogger.info( + "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "unblocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end end end end diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb index bf4dd9542d5..95378e5a769 100644 --- a/lib/gitlab/ldap/auth_hash.rb +++ b/lib/gitlab/ldap/auth_hash.rb @@ -25,7 +25,7 @@ module Gitlab end def get_raw(key) - auth_hash.extra[:raw_info][key] + auth_hash.extra[:raw_info][key] if auth_hash.extra end def ldap_config diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index de52ef3fc65..28129198438 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -107,7 +107,7 @@ module Gitlab end def attributes - options['attributes'] + default_attributes.merge(options['attributes']) end def timeout @@ -130,6 +130,16 @@ module Gitlab end end + def default_attributes + { + 'username' => %w(uid userid sAMAccountName), + 'email' => %w(mail email userPrincipalName), + 'name' => 'cn', + 'first_name' => 'givenName', + 'last_name' => 'sn' + } + end + protected def base_options diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index b81f3e8e8f5..7084fd1767d 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -28,7 +28,7 @@ module Gitlab end def name - entry.cn.first + attribute_value(:name).first end def uid @@ -40,7 +40,7 @@ module Gitlab end def email - entry.try(:mail) + attribute_value(:email) end def dn @@ -56,6 +56,19 @@ module Gitlab def config @config ||= Gitlab::LDAP::Config.new(provider) end + + # Using the LDAP attributes configuration, find and return the first + # attribute with a value. For example, by default, when given 'email', + # this method looks for 'mail', 'email' and 'userPrincipalName' and + # returns the first with a value. + def attribute_value(attribute) + attributes = Array(config.attributes[attribute.to_s]) + selected_attr = attributes.find { |attr| entry.respond_to?(attr) } + + return nil unless selected_attr + + entry.public_send(selected_attr) + end end end end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 91fb0bb317a..47f88727fc8 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -70,8 +70,19 @@ module Gitlab def tag_endpoint(trans, env) endpoint = env[ENDPOINT_KEY] - path = endpoint_paths_cache[endpoint.route.request_method][endpoint.route.path] - trans.action = "Grape##{endpoint.route.request_method} #{path}" + + begin + route = endpoint.route + rescue + # endpoint.route is calling env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info] + # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response + # so we're rescuing exceptions and bailing out + end + + if route + path = endpoint_paths_cache[route.request_method][route.path] + trans.action = "Grape##{route.request_method} #{path}" + end end private diff --git a/lib/gitlab/middleware/webpack_proxy.rb b/lib/gitlab/middleware/webpack_proxy.rb new file mode 100644 index 00000000000..3fe32adeade --- /dev/null +++ b/lib/gitlab/middleware/webpack_proxy.rb @@ -0,0 +1,24 @@ +# This Rack middleware is intended to proxy the webpack assets directory to the +# webpack-dev-server. It is only intended for use in development. + +module Gitlab + module Middleware + class WebpackProxy < Rack::Proxy + def initialize(app = nil, opts = {}) + @proxy_host = opts.fetch(:proxy_host, 'localhost') + @proxy_port = opts.fetch(:proxy_port, 3808) + @proxy_path = opts[:proxy_path] if opts[:proxy_path] + super(app, opts) + end + + def perform_request(env) + unless @proxy_path && env['PATH_INFO'].start_with?("/#{@proxy_path}") + return @app.call(env) + end + + env['HTTP_HOST'] = "#{@proxy_host}:#{@proxy_port}" + super(env) + end + end + end +end diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb new file mode 100644 index 00000000000..fb215f27cbd --- /dev/null +++ b/lib/gitlab/pages_transfer.rb @@ -0,0 +1,7 @@ +module Gitlab + class PagesTransfer < ProjectTransfer + def root_dir + Gitlab.config.pages.path + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 6bdf3db9cb8..db325c00705 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -71,6 +71,14 @@ module Gitlab ) end + def single_commit_result? + commits_count == 1 && total_result_count == 1 + end + + def total_result_count + issues_count + merge_requests_count + milestones_count + notes_count + blobs_count + wiki_blobs_count + commits_count + end + private def blobs @@ -114,7 +122,25 @@ module Gitlab end def commits - @commits ||= project.repository.find_commits_by_message(query) + @commits ||= find_commits(query) + end + + def find_commits(query) + return [] unless Ability.allowed?(@current_user, :download_code, @project) + + commits = find_commits_by_message(query) + commit_by_sha = find_commit_by_sha(query) + commits |= [commit_by_sha] if commit_by_sha + commits + end + + def find_commits_by_message(query) + project.repository.find_commits_by_message(query) + end + + def find_commit_by_sha(query) + key = query.strip + project.repository.commit(key) if Commit.valid_hash?(key) end def project_ids_relation diff --git a/lib/gitlab/project_transfer.rb b/lib/gitlab/project_transfer.rb new file mode 100644 index 00000000000..1bba0b78e2f --- /dev/null +++ b/lib/gitlab/project_transfer.rb @@ -0,0 +1,35 @@ +module Gitlab + class ProjectTransfer + def move_project(project_path, namespace_path_was, namespace_path) + new_namespace_folder = File.join(root_dir, namespace_path) + FileUtils.mkdir_p(new_namespace_folder) unless Dir.exist?(new_namespace_folder) + from = File.join(root_dir, namespace_path_was, project_path) + to = File.join(root_dir, namespace_path, project_path) + move(from, to, "") + end + + def rename_project(path_was, path, namespace_path) + base_dir = File.join(root_dir, namespace_path) + move(path_was, path, base_dir) + end + + def rename_namespace(path_was, path) + move(path_was, path) + end + + def root_dir + raise NotImplementedError + end + + private + + def move(path_was, path, base_dir = nil) + base_dir = root_dir unless base_dir + from = File.join(base_dir, path_was) + to = File.join(base_dir, path) + FileUtils.mv(from, to) + rescue Errno::ENOENT + false + end + end +end diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index 70e7f25d518..4bc76ea033f 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -10,5 +10,9 @@ module Gitlab true end end + + def self.enabled? + current_application_settings.recaptcha_enabled + end end end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 9226da2d6b1..9384102acec 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -42,7 +42,7 @@ module Gitlab return @_raw_config if defined?(@_raw_config) begin - @_raw_config = File.read(CONFIG_FILE).freeze + @_raw_config = ERB.new(File.read(CONFIG_FILE)).result.freeze rescue Errno::ENOENT @_raw_config = false end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 9e0b0e5ea98..a3fa7c1331a 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -61,11 +61,11 @@ module Gitlab end def file_name_regex - @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@]*\z/.freeze + @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze end def file_name_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'." + "can contain only letters, digits, '_', '-', '@', '+' and '.'." end def file_path_regex diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb index 786e1d49f5e..ef42b0557e0 100644 --- a/lib/gitlab/request_profiler/middleware.rb +++ b/lib/gitlab/request_profiler/middleware.rb @@ -1,5 +1,4 @@ require 'ruby-prof' -require_dependency 'gitlab/request_profiler' module Gitlab module RequestProfiler @@ -20,7 +19,7 @@ module Gitlab header_token = env['HTTP_X_PROFILE_TOKEN'] return unless header_token.present? - profile_token = RequestProfiler.profile_token + profile_token = Gitlab::RequestProfiler.profile_token return unless profile_token.present? header_token == profile_token diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb new file mode 100644 index 00000000000..72d00abfcc2 --- /dev/null +++ b/lib/gitlab/route_map.rb @@ -0,0 +1,50 @@ +module Gitlab + class RouteMap + class FormatError < StandardError; end + + def initialize(data) + begin + entries = YAML.safe_load(data) + rescue + raise FormatError, 'Route map is not valid YAML' + end + + raise FormatError, 'Route map is not an array' unless entries.is_a?(Array) + + @map = entries.map { |entry| parse_entry(entry) } + end + + def public_path_for_source_path(path) + mapping = @map.find { |mapping| mapping[:source] === path } + return unless mapping + + path.sub(mapping[:source], mapping[:public]) + end + + private + + def parse_entry(entry) + raise FormatError, 'Route map entry is not a hash' unless entry.is_a?(Hash) + raise FormatError, 'Route map entry does not have a source key' unless entry.has_key?('source') + raise FormatError, 'Route map entry does not have a public key' unless entry.has_key?('public') + + source_pattern = entry['source'] + public_path = entry['public'] + + if source_pattern.start_with?('/') && source_pattern.end_with?('/') + source_pattern = source_pattern[1...-1].gsub('\/', '/') + + begin + source_pattern = /\A#{source_pattern}\z/ + rescue RegexpError => e + raise FormatError, "Route map entry source is not a valid regular expression: #{e}" + end + end + + { + source: source_pattern, + public: public_path + } + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 35212992698..c9c65f76f4b 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -43,6 +43,10 @@ module Gitlab @milestones_count ||= milestones.count end + def single_commit_result? + false + end + private def projects diff --git a/lib/gitlab/serialize/ci/variables.rb b/lib/gitlab/serializer/ci/variables.rb index 3a9443bfcd9..c059c454eac 100644 --- a/lib/gitlab/serialize/ci/variables.rb +++ b/lib/gitlab/serializer/ci/variables.rb @@ -1,5 +1,5 @@ module Gitlab - module Serialize + module Serializer module Ci # This serializer could make sure our YAML variables' keys and values # are always strings. This is more for legacy build data because diff --git a/lib/gitlab/serializer/pagination.rb b/lib/gitlab/serializer/pagination.rb new file mode 100644 index 00000000000..bf2c0acc729 --- /dev/null +++ b/lib/gitlab/serializer/pagination.rb @@ -0,0 +1,36 @@ +module Gitlab + module Serializer + class Pagination + class InvalidResourceError < StandardError; end + include ::API::Helpers::Pagination + + def initialize(request, response) + @request = request + @response = response + end + + def paginate(resource) + if resource.respond_to?(:page) + super(resource) + else + raise InvalidResourceError + end + end + + private + + # Methods needed by `API::Helpers::Pagination` + # + + attr_reader :request + + def params + @request.query_parameters + end + + def header(header, value) + @response.headers[header] = value + end + end + end +end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/shell.rb index 82e194c1af1..82e194c1af1 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/shell.rb diff --git a/lib/gitlab/backend/shell_adapter.rb b/lib/gitlab/shell_adapter.rb index fbe2a7a0d72..fbe2a7a0d72 100644 --- a/lib/gitlab/backend/shell_adapter.rb +++ b/lib/gitlab/shell_adapter.rb diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb new file mode 100644 index 00000000000..aadc401ff8d --- /dev/null +++ b/lib/gitlab/sidekiq_status.rb @@ -0,0 +1,66 @@ +module Gitlab + # The SidekiqStatus module and its child classes can be used for checking if a + # Sidekiq job has been processed or not. + # + # To check if a job has been completed, simply pass the job ID to the + # `completed?` method: + # + # job_id = SomeWorker.perform_async(...) + # + # if Gitlab::SidekiqStatus.completed?(job_id) + # ... + # end + # + # For each job ID registered a separate key is stored in Redis, making lookups + # much faster than using Sidekiq's built-in job finding/status API. These keys + # expire after a certain period of time to prevent storing too many keys in + # Redis. + module SidekiqStatus + STATUS_KEY = 'gitlab-sidekiq-status:%s'.freeze + + # The default time (in seconds) after which a status key is expired + # automatically. The default of 30 minutes should be more than sufficient + # for most jobs. + DEFAULT_EXPIRATION = 30.minutes.to_i + + # Starts tracking of the given job. + # + # jid - The Sidekiq job ID + # expire - The expiration time of the Redis key. + def self.set(jid, expire = DEFAULT_EXPIRATION) + Sidekiq.redis do |redis| + redis.set(key_for(jid), 1, ex: expire) + end + end + + # Stops the tracking of the given job. + # + # jid - The Sidekiq job ID to remove. + def self.unset(jid) + Sidekiq.redis do |redis| + redis.del(key_for(jid)) + end + end + + # Returns true if all the given job have been completed. + # + # jids - The Sidekiq job IDs to check. + # + # Returns true or false. + def self.all_completed?(jids) + keys = jids.map { |jid| key_for(jid) } + + responses = Sidekiq.redis do |redis| + redis.pipelined do + keys.each { |key| redis.exists(key) } + end + end + + responses.all? { |value| !value } + end + + def self.key_for(jid) + STATUS_KEY % jid + end + end +end diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb new file mode 100644 index 00000000000..779a9998b22 --- /dev/null +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -0,0 +1,10 @@ +module Gitlab + module SidekiqStatus + class ClientMiddleware + def call(_, job, _, _) + SidekiqStatus.set(job['jid']) + yield + end + end + end +end diff --git a/lib/gitlab/sidekiq_status/server_middleware.rb b/lib/gitlab/sidekiq_status/server_middleware.rb new file mode 100644 index 00000000000..31dfa46ff9d --- /dev/null +++ b/lib/gitlab/sidekiq_status/server_middleware.rb @@ -0,0 +1,13 @@ +module Gitlab + module SidekiqStatus + class ServerMiddleware + def call(worker, job, queue) + ret = yield + + SidekiqStatus.unset(job['jid']) + + ret + end + end + end +end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index d19b0a52043..9d2ecee9756 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -15,7 +15,7 @@ module Gitlab { 'General' => '', 'Pages' => 'Pages', - 'Autodeploy' => 'autodeploy' + 'Auto deploy' => 'autodeploy' } end @@ -28,7 +28,7 @@ module Gitlab end def dropdown_names(context) - categories = context == 'autodeploy' ? ['Autodeploy'] : ['General', 'Pages'] + categories = context == 'autodeploy' ? ['Auto deploy'] : ['General', 'Pages'] super().slice(*categories) end end diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index d4020af76f9..19ab76ae80f 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -15,7 +15,7 @@ module Gitlab Theme.new(1, 'Graphite', 'ui_graphite'), Theme.new(2, 'Charcoal', 'ui_charcoal'), Theme.new(3, 'Green', 'ui_green'), - Theme.new(4, 'Gray', 'ui_gray'), + Theme.new(4, 'Black', 'ui_black'), Theme.new(5, 'Violet', 'ui_violet'), Theme.new(6, 'Blue', 'ui_blue') ].freeze diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb new file mode 100644 index 00000000000..d615c24149a --- /dev/null +++ b/lib/gitlab/time_tracking_formatter.rb @@ -0,0 +1,34 @@ +module Gitlab + module TimeTrackingFormatter + extend self + + def parse(string) + with_custom_config do + string.sub!(/\A-/, '') + + seconds = ChronicDuration.parse(string, default_unit: 'hours') rescue nil + seconds *= -1 if seconds && Regexp.last_match + seconds + end + end + + def output(seconds) + with_custom_config do + ChronicDuration.output(seconds, format: :short, limit_to_hours: false, weeks: true) rescue nil + end + end + + def with_custom_config + # We may want to configure it through project settings in a future version. + ChronicDuration.hours_per_day = 8 + ChronicDuration.days_per_week = 5 + + result = yield + + ChronicDuration.hours_per_day = 24 + ChronicDuration.days_per_week = 7 + + result + end + end +end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index f3567f3ef85..e78d0c34a02 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -61,7 +61,7 @@ module Gitlab "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}), "Install gems" => %W(bundle), "Migrate DB" => %W(bundle exec rake db:migrate), - "Recompile assets" => %W(bundle exec rake assets:clean assets:precompile), + "Recompile assets" => %W(bundle exec rake gitlab:assets:clean gitlab:assets:compile), "Clear cache" => %W(bundle exec rake cache:clear) } end diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb index be8fcc7b2d2..81701831a6a 100644 --- a/lib/gitlab/uploads_transfer.rb +++ b/lib/gitlab/uploads_transfer.rb @@ -1,33 +1,5 @@ module Gitlab - class UploadsTransfer - def move_project(project_path, namespace_path_was, namespace_path) - new_namespace_folder = File.join(root_dir, namespace_path) - FileUtils.mkdir_p(new_namespace_folder) unless Dir.exist?(new_namespace_folder) - from = File.join(root_dir, namespace_path_was, project_path) - to = File.join(root_dir, namespace_path, project_path) - move(from, to, "") - end - - def rename_project(path_was, path, namespace_path) - base_dir = File.join(root_dir, namespace_path) - move(path_was, path, base_dir) - end - - def rename_namespace(path_was, path) - move(path_was, path) - end - - private - - def move(path_was, path, base_dir = nil) - base_dir = root_dir unless base_dir - from = File.join(base_dir, path_was) - to = File.join(base_dir, path) - FileUtils.mv(from, to) - rescue Errno::ENOENT - false - end - + class UploadsTransfer < ProjectTransfer def root_dir File.join(Rails.root, "public", "uploads") end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 9858d2e7d83..6ce9b229294 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -8,6 +8,8 @@ module Gitlab end def can_do_action?(action) + return false if no_user_or_blocked? + @permission_cache ||= {} @permission_cache[action] ||= user.can?(action, project) end @@ -17,7 +19,7 @@ module Gitlab end def allowed? - return false if user.blank? || user.blocked? + return false if no_user_or_blocked? if user.requires_ldap_check? && user.try_obtain_ldap_lease return false unless Gitlab::LDAP::Access.allowed?(user) @@ -27,20 +29,22 @@ module Gitlab end def can_push_to_branch?(ref) - return false unless user + return false if no_user_or_blocked? if project.protected_branch?(ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) access_levels = project.protected_branches.matching(ref).map(&:push_access_levels).flatten - access_levels.any? { |access_level| access_level.check_access(user) } + has_access = access_levels.any? { |access_level| access_level.check_access(user) } + + has_access || !project.repository.branch_exists?(ref) && can_merge_to_branch?(ref) else user.can?(:push_code, project) end end def can_merge_to_branch?(ref) - return false unless user + return false if no_user_or_blocked? if project.protected_branch?(ref) access_levels = project.protected_branches.matching(ref).map(&:merge_access_levels).flatten @@ -51,9 +55,15 @@ module Gitlab end def can_read_project? - return false unless user + return false if no_user_or_blocked? user.can?(:read_project, project) end + + private + + def no_user_or_blocked? + user.nil? || user.blocked? + end end end diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb new file mode 100644 index 00000000000..dbfe0941e4d --- /dev/null +++ b/lib/gitlab/view/presenter/base.rb @@ -0,0 +1,30 @@ +module Gitlab + module View + module Presenter + CannotOverrideMethodError = Class.new(StandardError) + + module Base + extend ActiveSupport::Concern + + include Gitlab::Routing + include Gitlab::Allowable + + attr_reader :subject + + def can?(user, action, overriden_subject = nil) + super(user, action, overriden_subject || subject) + end + + class_methods do + def presenter? + true + end + + def presents(name) + define_method(name) { subject } + end + end + end + end + end +end diff --git a/lib/gitlab/view/presenter/delegated.rb b/lib/gitlab/view/presenter/delegated.rb new file mode 100644 index 00000000000..387ff0f5d43 --- /dev/null +++ b/lib/gitlab/view/presenter/delegated.rb @@ -0,0 +1,23 @@ +module Gitlab + module View + module Presenter + class Delegated < SimpleDelegator + include Gitlab::View::Presenter::Base + + def initialize(subject, **attributes) + @subject = subject + + attributes.each do |key, value| + if subject.respond_to?(key) + raise CannotOverrideMethodError.new("#{subject} already respond to #{key}!") + end + + define_singleton_method(key) { value } + end + + super(subject) + end + end + end + end +end diff --git a/lib/gitlab/view/presenter/factory.rb b/lib/gitlab/view/presenter/factory.rb new file mode 100644 index 00000000000..d172d61e2c9 --- /dev/null +++ b/lib/gitlab/view/presenter/factory.rb @@ -0,0 +1,24 @@ +module Gitlab + module View + module Presenter + class Factory + def initialize(subject, **attributes) + @subject = subject + @attributes = attributes + end + + def fabricate! + presenter_class.new(subject, attributes) + end + + private + + attr_reader :subject, :attributes + + def presenter_class + "#{subject.class.name}Presenter".constantize + end + end + end + end +end diff --git a/lib/gitlab/view/presenter/simple.rb b/lib/gitlab/view/presenter/simple.rb new file mode 100644 index 00000000000..b7653a0f3cc --- /dev/null +++ b/lib/gitlab/view/presenter/simple.rb @@ -0,0 +1,17 @@ +module Gitlab + module View + module Presenter + class Simple + include Gitlab::View::Presenter::Base + + def initialize(subject, **attributes) + @subject = subject + + attributes.each do |key, value| + define_singleton_method(key) { value } + end + end + end + end + end +end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 9462f3368e6..a4e966e4016 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -11,8 +11,21 @@ module Gitlab included do scope :public_only, -> { where(visibility_level: PUBLIC) } scope :public_and_internal_only, -> { where(visibility_level: [PUBLIC, INTERNAL] ) } - - scope :public_to_user, -> (user) { user && !user.external ? public_and_internal_only : public_only } + scope :non_public_only, -> { where.not(visibility_level: PUBLIC) } + + scope :public_to_user, -> (user) do + if user + if user.admin? + all + elsif !user.external? + public_and_internal_only + else + public_only + end + else + public_only + end + end end PRIVATE = 0 unless const_defined?(:PRIVATE) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index d28bb583fe7..c8872df8a93 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -15,10 +15,17 @@ module Gitlab class << self def git_http_ok(repository, user) - { + params = { GL_ID: Gitlab::GlId.gl_id(user), RepoPath: repository.path_to_repo, } + + params.merge!( + GitalySocketPath: Gitlab.config.gitaly.socket_path, + GitalyResourcePath: "/projects/#{repository.project.id}/git-http/info-refs", + ) if Gitlab.config.gitaly.socket_path.present? + + params end def lfs_upload_ok(oid, size) @@ -100,7 +107,8 @@ module Gitlab 'Terminal' => { 'Subprotocols' => terminal[:subprotocols], 'Url' => terminal[:url], - 'Header' => terminal[:headers] + 'Header' => terminal[:headers], + 'MaxSessionTime' => terminal[:max_session_time], } } details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) |