diff options
Diffstat (limited to 'lib')
39 files changed, 990 insertions, 367 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 40671e2517c..fe1bf8a4816 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -52,5 +52,6 @@ module API mount Labels mount Settings mount Keys + mount Tags end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 20cadae2291..9f337bc3cc6 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -62,7 +62,7 @@ module API expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace - expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :snippets_enabled, :created_at, :last_activity_at + expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :created_at, :last_activity_at expose :creator_id expose :namespace expose :forked_from_project, using: Entities::ForkedFromProject, if: lambda{ | project, options | project.forked? } @@ -95,25 +95,6 @@ module API end end - class RepoTag < Grape::Entity - expose :name - expose :message do |repo_obj, _options| - if repo_obj.respond_to?(:message) - repo_obj.message - else - nil - end - end - - expose :commit do |repo_obj, options| - if repo_obj.respond_to?(:commit) - repo_obj.commit - elsif options[:project] - options[:project].repository.commit(repo_obj.target) - end - end - end - class RepoObject < Grape::Entity expose :name @@ -181,7 +162,9 @@ module API end class MergeRequest < ProjectEntity - expose :target_branch, :source_branch, :upvotes, :downvotes + expose :target_branch, :source_branch + # deprecated, always returns 0 + expose :upvotes, :downvotes expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id expose :label_names, as: :labels @@ -211,6 +194,7 @@ module API expose :author, using: Entities::UserBasic expose :created_at expose :system?, as: :system + # upvote? and downvote? are deprecated, always return false expose :upvote?, as: :upvote expose :downvote?, as: :downvote end @@ -341,5 +325,35 @@ module API expose :user_oauth_applications expose :after_sign_out_path end + + class Release < Grape::Entity + expose :tag, as: :tag_name + expose :description + end + + class RepoTag < Grape::Entity + expose :name + expose :message do |repo_obj, _options| + if repo_obj.respond_to?(:message) + repo_obj.message + else + nil + end + end + + expose :commit do |repo_obj, options| + if repo_obj.respond_to?(:commit) + repo_obj.commit + elsif options[:project] + options[:project].repository.commit(repo_obj.target) + end + end + + expose :release, using: Entities::Release do |repo_obj, options| + if options[:project] + options[:project].releases.find_by(tag: repo_obj.name) + end + end + end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 67ee66a2058..2b4ada6e2eb 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -75,6 +75,7 @@ module API # description (optional) - short project description # issues_enabled (optional) # merge_requests_enabled (optional) + # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) # namespace_id (optional) - defaults to user namespace @@ -90,6 +91,7 @@ module API :description, :issues_enabled, :merge_requests_enabled, + :builds_enabled, :wiki_enabled, :snippets_enabled, :namespace_id, @@ -117,6 +119,7 @@ module API # default_branch (optional) - 'master' by default # issues_enabled (optional) # merge_requests_enabled (optional) + # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 @@ -132,6 +135,7 @@ module API :default_branch, :issues_enabled, :merge_requests_enabled, + :builds_enabled, :wiki_enabled, :snippets_enabled, :public, @@ -172,6 +176,7 @@ module API # description (optional) - short project description # issues_enabled (optional) # merge_requests_enabled (optional) + # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 @@ -185,6 +190,7 @@ module API :default_branch, :issues_enabled, :merge_requests_enabled, + :builds_enabled, :wiki_enabled, :snippets_enabled, :public, diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 20d568cf462..d7c48639eba 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -16,41 +16,6 @@ module API end end - # Get a project repository tags - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/repository/tags - get ":id/repository/tags" do - present user_project.repo.tags.sort_by(&:name).reverse, - with: Entities::RepoTag, project: user_project - end - - # Create tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # ref (required) - Create tag from commit sha or branch - # message (optional) - Specifying a message creates an annotated tag. - # Example Request: - # POST /projects/:id/repository/tags - post ':id/repository/tags' do - authorize_push_project - message = params[:message] || nil - result = CreateTagService.new(user_project, current_user). - execute(params[:tag_name], params[:ref], message) - - if result[:status] == :success - present result[:tag], - with: Entities::RepoTag, - project: user_project - else - render_api_error!(result[:message], 400) - end - end - # Get a project repository tree # # Parameters: diff --git a/lib/api/tags.rb b/lib/api/tags.rb new file mode 100644 index 00000000000..47621f443e6 --- /dev/null +++ b/lib/api/tags.rb @@ -0,0 +1,86 @@ +module API + # Git Tags API + class Tags < Grape::API + before { authenticate! } + before { authorize! :download_code, user_project } + + resource :projects do + # Get a project repository tags + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # GET /projects/:id/repository/tags + get ":id/repository/tags" do + present user_project.repo.tags.sort_by(&:name).reverse, + with: Entities::RepoTag, project: user_project + end + + # Create tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # ref (required) - Create tag from commit sha or branch + # message (optional) - Specifying a message creates an annotated tag. + # Example Request: + # POST /projects/:id/repository/tags + post ':id/repository/tags' do + authorize_push_project + message = params[:message] || nil + result = CreateTagService.new(user_project, current_user). + execute(params[:tag_name], params[:ref], message, params[:release_description]) + + if result[:status] == :success + present result[:tag], + with: Entities::RepoTag, + project: user_project + else + render_api_error!(result[:message], 400) + end + end + + # Add release notes to tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # description (required) - Release notes with markdown support + # Example Request: + # POST /projects/:id/repository/tags/:tag_name/release + post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.*/ } do + authorize_push_project + required_attributes! [:description] + result = CreateReleaseService.new(user_project, current_user). + execute(params[:tag_name], params[:description]) + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + + # Updates a release notes of a tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # description (required) - Release notes with markdown support + # Example Request: + # PUT /projects/:id/repository/tags/:tag_name/release + put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.*/ } do + authorize_push_project + required_attributes! [:description] + result = UpdateReleaseService.new(user_project, current_user). + execute(params[:tag_name], params[:description]) + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + end + end +end diff --git a/lib/award_emoji.rb b/lib/award_emoji.rb new file mode 100644 index 00000000000..d58a196c4ef --- /dev/null +++ b/lib/award_emoji.rb @@ -0,0 +1,12 @@ +class AwardEmoji + EMOJI_LIST = [ + "+1", "-1", "100", "blush", "heart", "smile", "rage", + "beers", "disappointed", "ok_hand", + "helicopter", "shit", "airplane", "alarm_clock", + "ambulance", "anguished", "two_hearts", "wink" + ] + + def self.path_to_emoji_image(name) + "emoji/#{Emoji.emoji_filename(name)}.png" + end +end diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb new file mode 100644 index 00000000000..4153467fbee --- /dev/null +++ b/lib/backup/lfs.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Lfs < Files + def initialize + super('lfs', Settings.lfs.storage_path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 9e15d5411a1..099062eeb8b 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -150,17 +150,15 @@ module Backup private def backup_contents - folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"] + folders_to_backup + archives_to_backup + ["backup_information.yml"] end - def folders_to_backup - folders = %w{repositories db} - - if ENV["SKIP"] - return folders.reject{ |folder| ENV["SKIP"].include?(folder) } - end + def archives_to_backup + %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + end - folders + def folders_to_backup + %w{repositories db}.reject{ |name| skipped?(name) } end def settings diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 0a586672807..15faa6edd84 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -58,6 +58,7 @@ module Ci # POST /builds/:id/artifacts/authorize post ":id/artifacts/authorize" do require_gitlab_workhorse! + not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) not_found! unless build authenticate_build_token!(build) @@ -91,6 +92,7 @@ module Ci # POST /builds/:id/artifacts post ":id/artifacts" do require_gitlab_workhorse! + not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) not_found! unless build authenticate_build_token!(build) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 2e2209031ee..3beafcad117 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -4,10 +4,10 @@ module Ci DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' - ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables] - ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts] + ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables, :cache] + ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache] - attr_reader :before_script, :image, :services, :variables, :path + attr_reader :before_script, :image, :services, :variables, :path, :cache def initialize(config, path = nil) @config = YAML.load(config) @@ -46,6 +46,7 @@ module Ci @services = @config[:services] @stages = @config[:stages] || @config[:types] @variables = @config[:variables] || {} + @cache = @config[:cache] @config.except!(*ALLOWED_YAML_KEYS) # anything that doesn't have script is considered as unknown @@ -78,7 +79,8 @@ module Ci options: { image: job[:image] || @image, services: job[:services] || @services, - artifacts: job[:artifacts] + artifacts: job[:artifacts], + cache: job[:cache] || @cache, }.compact } end @@ -112,6 +114,16 @@ module Ci raise ValidationError, "variables should be a map of key-valued strings" end + if @cache + if @cache[:untracked] && !validate_boolean(@cache[:untracked]) + raise ValidationError, "cache:untracked parameter should be an boolean" + end + + if @cache[:paths] && !validate_array_of_strings(@cache[:paths]) + raise ValidationError, "cache:paths parameter should be an array of strings" + end + end + @jobs.each do |name, job| validate_job!(name, job) end @@ -160,6 +172,16 @@ module Ci raise ValidationError, "#{name} job: except parameter should be an array of strings" end + if job[:cache] + if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked]) + raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean" + end + + if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths]) + raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings" + end + end + if job[:artifacts] if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked]) raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean" diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 440ef5a3cb3..0d156047ff0 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -33,6 +33,9 @@ module Grack auth! + lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call + return lfs_response unless lfs_response.nil? + if project && authorized_request? # Tell gitlab-workhorse the request is OK, and what the GL_ID is render_grack_auth_ok @@ -72,7 +75,7 @@ module Grack matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login) if project && matched_login.present? && git_cmd == 'git-upload-pack' - underscored_service = matched_login['s'].underscore + underscored_service = matched_login['s'].underscore if Service.available_services_names.include?(underscored_service) service_method = "#{underscored_service}_service" diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 01b8bda05c6..87ac30b5ffe 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -1,6 +1,6 @@ module Gitlab class Shell - class AccessDenied < StandardError; end + class Error < StandardError; end class KeyAdder < Struct.new(:io) def add_key(id, key) @@ -36,8 +36,9 @@ module Gitlab # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") # def import_repository(name, url) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'import-project', - "#{name}.git", url, '240']) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '240']) + raise Error, output unless status.zero? + true end # Move repository diff --git a/lib/gitlab/compare_result.rb b/lib/gitlab/compare_result.rb index d72391dade5..0d696a1ee28 100644 --- a/lib/gitlab/compare_result.rb +++ b/lib/gitlab/compare_result.rb @@ -2,8 +2,8 @@ module Gitlab class CompareResult attr_reader :commits, :diffs - def initialize(compare) - @commits, @diffs = compare.commits, compare.diffs + def initialize(compare, diff_options = {}) + @commits, @diffs = compare.commits, compare.diffs(nil, diff_options) end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 2d3e32d9539..46a4ef0e31f 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -25,7 +25,7 @@ module Gitlab session_expire_delay: Settings.gitlab['session_expire_delay'], import_sources: Settings.gitlab['import_sources'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], - max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'], + max_artifacts_size: Settings.artifacts['max_size'], ) end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index dd393fe09d2..07b856ca64c 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -16,6 +16,17 @@ module Gitlab def trigger(gl_id, oldrev, newrev, ref) return true unless exists? + case name + when "pre-receive", "post-receive" + call_receive_hook(gl_id, oldrev, newrev, ref) + when "update" + call_update_hook(gl_id, oldrev, newrev, ref) + end + end + + private + + def call_receive_hook(gl_id, oldrev, newrev, ref) changes = [oldrev, newrev, ref].join(" ") # function will return true if succesful @@ -54,6 +65,12 @@ module Gitlab exit_status end + + def call_update_hook(gl_id, oldrev, newrev, ref) + Dir.chdir(repo_path) do + system({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) + end + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index c90184d31cf..3ed1eec517c 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -13,7 +13,7 @@ module Gitlab def user return @user if defined?(@user) - @user = + @user = case actor when User actor @@ -125,7 +125,7 @@ module Gitlab def change_access_check(change) oldrev, newrev, ref = change.split(' ') - action = + action = if project.protected_branch?(branch_name(ref)) protected_branch_action(oldrev, newrev, branch_name(ref)) elsif protected_tag?(tag_name(ref)) @@ -148,7 +148,7 @@ module Gitlab build_status_object(false, "You are not allowed to change existing tags on this project.") else # :push_code build_status_object(false, "You are not allowed to push code to this project.") - end + end return status end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 270cbcd9ccd..74d1529e1ff 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -46,7 +46,7 @@ module Gitlab end def github_options - OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + OmniAuth::Strategies::GitHub.default_options[:client_options].to_h.symbolize_keys end end end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index bd7340a80f1..b5720f6e2cb 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -19,7 +19,7 @@ module Gitlab if issue.pull_request.nil? body = @formatter.author_line(issue.user.login) - body += issue.body + body += issue.body || "" if issue.comments > 0 body += @formatter.comments_header diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index 9c00896c913..86fb6c51765 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -75,7 +75,7 @@ module Gitlab end def gitlab_options - OmniAuth::Strategies::GitLab.default_options[:client_options].symbolize_keys + OmniAuth::Strategies::GitLab.default_options[:client_options].to_h.symbolize_keys end end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 03c410726a5..87fee28dc01 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -30,7 +30,7 @@ module Gitlab def user_map @user_map ||= begin - user_map = Hash.new do |hash, user| + user_map = Hash.new do |hash, user| # Replace ... by \.\.\., so `johnsm...@gmail.com` isn't autolinked. Client.mask_email(user).sub("...", "\\.\\.\\.") end @@ -76,18 +76,7 @@ module Gitlab attachments = format_attachments(raw_issue["id"], 0, issue_comment["attachments"]) body = format_issue_body(author, date, content, attachments) - - labels = [] - raw_issue["labels"].each do |label| - name = nice_label_name(label) - labels << name - - unless @known_labels.include?(name) - create_label(name) - @known_labels << name - end - end - labels << nice_status_name(raw_issue["status"]) + labels = import_issue_labels(raw_issue) assignee_id = nil if raw_issue.has_key?("owner") @@ -110,6 +99,7 @@ module Gitlab assignee_id: assignee_id, state: raw_issue["state"] == "closed" ? "closed" : "opened" ) + issue.add_labels_by_names(labels) if issue.iid != raw_issue["id"] @@ -120,6 +110,23 @@ module Gitlab end end + def import_issue_labels(raw_issue) + labels = [] + + raw_issue["labels"].each do |label| + name = nice_label_name(label) + labels << name + + unless @known_labels.include?(name) + create_label(name) + @known_labels << name + end + end + + labels << nice_status_name(raw_issue["status"]) + labels + end + def import_issue_comments(issue, comments) Note.transaction do while raw_comment = comments.shift @@ -172,7 +179,7 @@ module Gitlab "#5cb85c" when "Status: Started" "#8e44ad" - + when "Priority: Critical" "#ffcfcf" when "Priority: High" @@ -181,7 +188,7 @@ module Gitlab "#fff5cc" when "Priority: Low" "#cfe9ff" - + when "Type: Defect" "#d9534f" when "Type: Enhancement" @@ -249,8 +256,8 @@ module Gitlab end if raw_updates.has_key?("cc") - cc = raw_updates["cc"].map do |l| - deleted = l.start_with?("-") + cc = raw_updates["cc"].map do |l| + deleted = l.start_with?("-") l = l[1..-1] if deleted l = user_map[l] l = "~~#{l}~~" if deleted @@ -261,8 +268,8 @@ module Gitlab end if raw_updates.has_key?("labels") - labels = raw_updates["labels"].map do |l| - deleted = l.start_with?("-") + labels = raw_updates["labels"].map do |l| + deleted = l.start_with?("-") l = l[1..-1] if deleted l = nice_label_name(l) l = "~~#{l}~~" if deleted @@ -278,45 +285,39 @@ module Gitlab if raw_updates.has_key?("blockedOn") blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on| - name, id = raw_blocked_on.split(":", 2) - - deleted = name.start_with?("-") - name = name[1..-1] if deleted - - text = - if name == project.import_source - "##{id}" - else - "#{project.namespace.path}/#{name}##{id}" - end - text = "~~#{text}~~" if deleted - text + format_blocking_updates(raw_blocked_on) end + updates << "*Blocked on: #{blocked_ons.join(", ")}*" end if raw_updates.has_key?("blocking") blockings = raw_updates["blocking"].map do |raw_blocked_on| - name, id = raw_blocked_on.split(":", 2) - - deleted = name.start_with?("-") - name = name[1..-1] if deleted - - text = - if name == project.import_source - "##{id}" - else - "#{project.namespace.path}/#{name}##{id}" - end - text = "~~#{text}~~" if deleted - text + format_blocking_updates(raw_blocked_on) end + updates << "*Blocking: #{blockings.join(", ")}*" end updates end + def format_blocking_updates(raw_blocked_on) + name, id = raw_blocked_on.split(":", 2) + + deleted = name.start_with?("-") + name = name[1..-1] if deleted + + text = + if name == project.import_source + "##{id}" + else + "#{project.namespace.path}/#{name}##{id}" + end + text = "~~#{text}~~" if deleted + text + end + def format_attachments(issue_id, comment_id, raw_attachments) return [] unless raw_attachments @@ -325,7 +326,7 @@ module Gitlab filename = attachment["fileName"] link = "https://storage.googleapis.com/google-code-attachments/#{@repo.name}/issue-#{issue_id}/comment-#{comment_id}/#{filename}" - + text = "[#{filename}](#{link})" text = "!#{text}" if filename =~ /\.(png|jpg|jpeg|gif|bmp|tiff)\z/i text diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb index 99e7b529ba9..44507bde25d 100644 --- a/lib/gitlab/inline_diff.rb +++ b/lib/gitlab/inline_diff.rb @@ -11,48 +11,71 @@ module Gitlab indexes.each do |index| first_line = diff_arr[index+1] second_line = diff_arr[index+2] - max_length = [first_line.size, second_line.size].max # Skip inline diff if empty line was replaced with content next if first_line == "-\n" - first_the_same_symbols = 0 - (0..max_length + 1).each do |i| - first_the_same_symbols = i - 1 - if first_line[i] != second_line[i] && i > 0 - break - end - end + first_token = find_first_token(first_line, second_line) + apply_first_token(diff_arr, index, first_token) + + last_token = find_last_token(first_line, second_line, first_token) + apply_last_token(diff_arr, index, last_token) + end + + diff_arr + end + + def apply_first_token(diff_arr, index, first_token) + start = first_token + START + + if first_token.empty? + # In case if we remove string of spaces in commit + diff_arr[index+1].sub!("-", "-" => "-#{START}") + diff_arr[index+2].sub!("+", "+" => "+#{START}") + else + diff_arr[index+1].sub!(first_token, first_token => start) + diff_arr[index+2].sub!(first_token, first_token => start) + end + end - first_token = first_line[0..first_the_same_symbols][1..-1] - start = first_token + START + def apply_last_token(diff_arr, index, last_token) + # This is tricky: escape backslashes so that `sub` doesn't interpret them + # as backreferences. Regexp.escape does NOT do the right thing. + replace_token = FINISH + last_token.gsub(/\\/, '\&\&') + diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) + diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) + end + + def find_first_token(first_line, second_line) + max_length = [first_line.size, second_line.size].max + first_the_same_symbols = 0 + + (0..max_length + 1).each do |i| + first_the_same_symbols = i - 1 - if first_token.empty? - # In case if we remove string of spaces in commit - diff_arr[index+1].sub!("-", "-" => "-#{START}") - diff_arr[index+2].sub!("+", "+" => "+#{START}") - else - diff_arr[index+1].sub!(first_token, first_token => start) - diff_arr[index+2].sub!(first_token, first_token => start) + if first_line[i] != second_line[i] && i > 0 + break end + end + + first_line[0..first_the_same_symbols][1..-1] + end + + def find_last_token(first_line, second_line, first_token) + max_length = [first_line.size, second_line.size].max + last_the_same_symbols = 0 + + (1..max_length + 1).each do |i| + last_the_same_symbols = -i + shortest_line = second_line.size > first_line.size ? first_line : second_line - last_the_same_symbols = 0 - (1..max_length + 1).each do |i| - last_the_same_symbols = -i - shortest_line = second_line.size > first_line.size ? first_line : second_line - if ( first_line[-i] != second_line[-i] ) || "#{first_token}#{START}".size == shortest_line[1..-i].size - break - end + if (first_line[-i] != second_line[-i]) || "#{first_token}#{START}".size == shortest_line[1..-i].size + break end - last_the_same_symbols += 1 - last_token = first_line[last_the_same_symbols..-1] - # This is tricky: escape backslashes so that `sub` doesn't interpret them - # as backreferences. Regexp.escape does NOT do the right thing. - replace_token = FINISH + last_token.gsub(/\\/, '\&\&') - diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) end - diff_arr + + last_the_same_symbols += 1 + first_line[last_the_same_symbols..-1] end def _indexes_of_changed_lines(diff_arr) diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb new file mode 100644 index 00000000000..9be9a65671b --- /dev/null +++ b/lib/gitlab/lfs/response.rb @@ -0,0 +1,327 @@ +module Gitlab + module Lfs + class Response + + def initialize(project, user, request) + @origin_project = project + @project = storage_project(project) + @user = user + @env = request.env + @request = request + end + + def render_download_object_response(oid) + render_response_to_download do + if check_download_sendfile_header? + render_lfs_sendfile(oid) + else + render_not_found + end + end + end + + def render_batch_operation_response + request_body = JSON.parse(@request.body.read) + case request_body["operation"] + when "download" + render_batch_download(request_body) + when "upload" + render_batch_upload(request_body) + else + render_not_found + end + end + + def render_storage_upload_authorize_response(oid, size) + render_response_to_push do + [ + 200, + { "Content-Type" => "application/json; charset=utf-8" }, + [JSON.dump({ + 'StoreLFSPath' => "#{Gitlab.config.lfs.storage_path}/tmp/upload", + 'LfsOid' => oid, + 'LfsSize' => size + })] + ] + end + end + + def render_storage_upload_store_response(oid, size, tmp_file_name) + render_response_to_push do + render_lfs_upload_ok(oid, size, tmp_file_name) + end + end + + def render_unsupported_deprecated_api + [ + 501, + { "Content-Type" => "application/json; charset=utf-8" }, + [JSON.dump({ + 'message' => 'Server supports batch API only, please update your Git LFS client to version 1.0.1 and up.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + private + + def render_not_enabled + [ + 501, + { + "Content-Type" => "application/json; charset=utf-8", + }, + [JSON.dump({ + 'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_unauthorized + [ + 401, + { + 'Content-Type' => 'text/plain' + }, + ['Unauthorized'] + ] + end + + def render_not_found + [ + 404, + { + "Content-Type" => "application/vnd.git-lfs+json" + }, + [JSON.dump({ + 'message' => 'Not found.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_forbidden + [ + 403, + { + "Content-Type" => "application/vnd.git-lfs+json" + }, + [JSON.dump({ + 'message' => 'Access forbidden. Check your access level.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_lfs_sendfile(oid) + return render_not_found unless oid.present? + + lfs_object = object_for_download(oid) + + if lfs_object && lfs_object.file.exists? + [ + 200, + { + # GitLab-workhorse will forward Content-Type header + "Content-Type" => "application/octet-stream", + "X-Sendfile" => lfs_object.file.path + }, + [] + ] + else + render_not_found + end + end + + def render_batch_upload(body) + return render_not_found if body.empty? || body['objects'].nil? + + render_response_to_push do + response = build_upload_batch_response(body['objects']) + [ + 200, + { + "Content-Type" => "application/json; charset=utf-8", + "Cache-Control" => "private", + }, + [JSON.dump(response)] + ] + end + end + + def render_batch_download(body) + return render_not_found if body.empty? || body['objects'].nil? + + render_response_to_download do + response = build_download_batch_response(body['objects']) + [ + 200, + { + "Content-Type" => "application/json; charset=utf-8", + "Cache-Control" => "private", + }, + [JSON.dump(response)] + ] + end + end + + def render_lfs_upload_ok(oid, size, tmp_file) + if store_file(oid, size, tmp_file) + [ + 200, + { + 'Content-Type' => 'text/plain', + 'Content-Length' => 0 + }, + [] + ] + else + [ + 422, + { 'Content-Type' => 'text/plain' }, + ["Unprocessable entity"] + ] + end + end + + def render_response_to_download + return render_not_enabled unless Gitlab.config.lfs.enabled + + unless @project.public? + return render_unauthorized unless @user + return render_forbidden unless user_can_fetch? + end + + yield + end + + def render_response_to_push + return render_not_enabled unless Gitlab.config.lfs.enabled + return render_unauthorized unless @user + return render_forbidden unless user_can_push? + + yield + end + + def check_download_sendfile_header? + @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile" + end + + def user_can_fetch? + # Check user access against the project they used to initiate the pull + @user.can?(:download_code, @origin_project) + end + + def user_can_push? + # Check user access against the project they used to initiate the push + @user.can?(:push_code, @origin_project) + end + + def storage_project(project) + if project.forked? + project.forked_from_project + else + project + end + end + + def store_file(oid, size, tmp_file) + tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file) + + object = LfsObject.find_or_create_by(oid: oid, size: size) + if object.file.exists? + success = true + else + success = move_tmp_file_to_storage(object, tmp_file_path) + end + + if success + success = link_to_project(object) + end + + success + ensure + # Ensure that the tmp file is removed + FileUtils.rm_f(tmp_file_path) + end + + def object_for_download(oid) + @project.lfs_objects.find_by(oid: oid) + end + + def move_tmp_file_to_storage(object, path) + File.open(path) do |f| + object.file = f + end + + object.file.store! + object.save + end + + def link_to_project(object) + if object && !object.projects.exists?(@project.id) + object.projects << @project + object.save + end + end + + def select_existing_objects(objects) + objects_oids = objects.map { |o| o['oid'] } + @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set + end + + def build_upload_batch_response(objects) + selected_objects = select_existing_objects(objects) + + upload_hypermedia_links(objects, selected_objects) + end + + def build_download_batch_response(objects) + selected_objects = select_existing_objects(objects) + + download_hypermedia_links(objects, selected_objects) + end + + def download_hypermedia_links(all_objects, existing_objects) + all_objects.each do |object| + if existing_objects.include?(object['oid']) + object['actions'] = { + 'download' => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}", + 'header' => { + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } + } + else + object['error'] = { + 'code' => 404, + 'message' => "Object does not exist on the server or you don't have permissions to access it", + } + end + end + + { 'objects' => all_objects } + end + + def upload_hypermedia_links(all_objects, existing_objects) + all_objects.each do |object| + # generate actions only for non-existing objects + next if existing_objects.include?(object['oid']) + + object['actions'] = { + 'upload' => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", + 'header' => { + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } + } + end + + { 'objects' => all_objects } + end + end + end +end diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb new file mode 100644 index 00000000000..78d02891102 --- /dev/null +++ b/lib/gitlab/lfs/router.rb @@ -0,0 +1,97 @@ +module Gitlab + module Lfs + class Router + def initialize(project, user, request) + @project = project + @user = user + @env = request.env + @request = request + end + + def try_call + return unless @request && @request.path.present? + + case @request.request_method + when 'GET' + get_response + when 'POST' + post_response + when 'PUT' + put_response + else + nil + end + end + + private + + def get_response + path_match = @request.path.match(/\/(info\/lfs|gitlab-lfs)\/objects\/([0-9a-f]{64})$/) + return nil unless path_match + + oid = path_match[2] + return nil unless oid + + case path_match[1] + when "info/lfs" + lfs.render_unsupported_deprecated_api + when "gitlab-lfs" + lfs.render_download_object_response(oid) + else + nil + end + end + + def post_response + post_path = @request.path.match(/\/info\/lfs\/objects(\/batch)?$/) + return nil unless post_path + + # Check for Batch API + if post_path[0].ends_with?("/info/lfs/objects/batch") + lfs.render_batch_operation_response + elsif post_path[0].ends_with?("/info/lfs/objects") + lfs.render_unsupported_deprecated_api + else + nil + end + end + + def put_response + object_match = @request.path.match(/\/gitlab-lfs\/objects\/([0-9a-f]{64})\/([0-9]+)(|\/authorize){1}$/) + return nil if object_match.nil? + + oid = object_match[1] + size = object_match[2].try(:to_i) + return nil if oid.nil? || size.nil? + + # GitLab-workhorse requests + # 1. Try to authorize the request + # 2. send a request with a header containing the name of the temporary file + if object_match[3] && object_match[3] == '/authorize' + lfs.render_storage_upload_authorize_response(oid, size) + else + tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP']) + return nil unless tmp_file_name + + lfs.render_storage_upload_store_response(oid, size, tmp_file_name) + end + end + + def lfs + return unless @project + + Gitlab::Lfs::Response.new(@project, @user, @request) + end + + def sanitize_tmp_filename(name) + if name.present? + name.gsub!(/^.*(\\|\/)/, '') + name = name.match(/[0-9a-f]{73}/) + name[0] if name + else + nil + end + end + end + end +end diff --git a/lib/gitlab/markdown/abstract_reference_filter.rb b/lib/gitlab/markdown/abstract_reference_filter.rb new file mode 100644 index 00000000000..fd5b7eb9332 --- /dev/null +++ b/lib/gitlab/markdown/abstract_reference_filter.rb @@ -0,0 +1,100 @@ +require 'gitlab/markdown' + +module Gitlab + module Markdown + # Issues, Snippets and Merge Requests shares similar functionality in refernce filtering. + # All this functionality moved to this class + class AbstractReferenceFilter < ReferenceFilter + include CrossProjectReference + + def self.object_class + # Implement in child class + # Example: MergeRequest + end + + def self.object_name + object_class.name.underscore + end + + def self.object_sym + object_name.to_sym + end + + def self.data_reference + "data-#{object_name.dasherize}" + end + + # Public: Find references in text (like `!123` for merge requests) + # + # AnyReferenceFilter.references_in(text) do |match, object| + # "<a href=...>PREFIX#{object}</a>" + # end + # + # PREFIX - symbol that detects reference (like ! for merge requests) + # object - reference object (snippet, merget request etc) + # text - String text to search. + # + # Yields the String match, the Integer referenced object ID, and an optional String + # of the external project reference. + # + # Returns a String replaced with the return of the block. + def self.references_in(text) + text.gsub(object_class.reference_pattern) do |match| + yield match, $~[object_sym].to_i, $~[:project] + end + end + + def self.referenced_by(node) + { object_sym => LazyReference.new(object_class, node.attr(data_reference)) } + end + + delegate :object_class, :object_sym, :references_in, to: :class + + def find_object(project, id) + # Implement in child class + # Example: project.merge_requests.find + end + + def url_for_object(object, project) + # Implement in child class + # Example: project_merge_request_url + end + + def call + replace_text_nodes_matching(object_class.reference_pattern) do |content| + object_link_filter(content) + end + end + + # Replace references (like `!123` for merge requests) in text with links + # to the referenced object's details page. + # + # text - String text to replace references in. + # + # Returns a String with references replaced with links. All links + # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. + def object_link_filter(text) + references_in(text) do |match, id, project_ref| + project = project_from_ref(project_ref) + + if project && object = find_object(project, id) + title = escape_once("#{object_title}: #{object.title}") + klass = reference_class(object_sym) + data = data_attribute(project: project.id, object_sym => object.id) + url = url_for_object(object, project) + + %(<a href="#{url}" #{data} + title="#{title}" + class="#{klass}">#{match}</a>) + else + match + end + end + end + + def object_title + object_class.name.titleize + end + end + end +end diff --git a/lib/gitlab/markdown/issue_reference_filter.rb b/lib/gitlab/markdown/issue_reference_filter.rb index 481d282f7b1..1ed69e2f431 100644 --- a/lib/gitlab/markdown/issue_reference_filter.rb +++ b/lib/gitlab/markdown/issue_reference_filter.rb @@ -6,66 +6,17 @@ module Gitlab # issues that do not exist are ignored. # # This filter supports cross-project references. - class IssueReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find `#123` issue references in text - # - # IssueReferenceFilter.references_in(text) do |match, issue, project_ref| - # "<a href=...>##{issue}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the Integer issue ID, and an optional String of - # the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Issue.reference_pattern) do |match| - yield match, $~[:issue].to_i, $~[:project] - end - end - - def self.referenced_by(node) - { issue: LazyReference.new(Issue, node.attr("data-issue")) } - end - - def call - replace_text_nodes_matching(Issue.reference_pattern) do |content| - issue_link_filter(content) - end + class IssueReferenceFilter < AbstractReferenceFilter + def self.object_class + Issue end - # Replace `#123` issue references in text with links to the referenced - # issue's details page. - # - # text - String text to replace references in. - # - # Returns a String with `#123` references replaced with links. All links - # have `gfm` and `gfm-issue` class names attached for styling. - def issue_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if project && issue = project.get_issue(id) - url = url_for_issue(id, project, only_path: context[:only_path]) - - title = escape_once("Issue: #{issue.title}") - klass = reference_class(:issue) - data = data_attribute(project: project.id, issue: issue.id) - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{match}</a>) - else - match - end - end + def find_object(project, id) + project.get_issue(id) end - def url_for_issue(*args) - IssuesHelper.url_for_issue(*args) + def url_for_object(issue, project) + IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path]) end end end diff --git a/lib/gitlab/markdown/label_reference_filter.rb b/lib/gitlab/markdown/label_reference_filter.rb index 618acb7a578..13581b8fb13 100644 --- a/lib/gitlab/markdown/label_reference_filter.rb +++ b/lib/gitlab/markdown/label_reference_filter.rb @@ -60,8 +60,7 @@ module Gitlab def url_for_label(project, label) h = Gitlab::Application.routes.url_helpers h.namespace_project_issues_path(project.namespace, project, - label_name: label.name, - only_path: context[:only_path]) + label_name: label.name) end def render_colored_label(label) diff --git a/lib/gitlab/markdown/merge_request_reference_filter.rb b/lib/gitlab/markdown/merge_request_reference_filter.rb index 5bc63269808..1f47f03c94e 100644 --- a/lib/gitlab/markdown/merge_request_reference_filter.rb +++ b/lib/gitlab/markdown/merge_request_reference_filter.rb @@ -6,65 +6,16 @@ module Gitlab # to merge requests that do not exist are ignored. # # This filter supports cross-project references. - class MergeRequestReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find `!123` merge request references in text - # - # MergeRequestReferenceFilter.references_in(text) do |match, merge_request, project_ref| - # "<a href=...>##{merge_request}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the Integer merge request ID, and an optional - # String of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(MergeRequest.reference_pattern) do |match| - yield match, $~[:merge_request].to_i, $~[:project] - end - end - - def self.referenced_by(node) - { merge_request: LazyReference.new(MergeRequest, node.attr("data-merge-request")) } - end - - def call - replace_text_nodes_matching(MergeRequest.reference_pattern) do |content| - merge_request_link_filter(content) - end + class MergeRequestReferenceFilter < AbstractReferenceFilter + def self.object_class + MergeRequest end - # Replace `!123` merge request references in text with links to the - # referenced merge request's details page. - # - # text - String text to replace references in. - # - # Returns a String with `!123` references replaced with links. All links - # have `gfm` and `gfm-merge_request` class names attached for styling. - def merge_request_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if project && merge_request = project.merge_requests.find_by(iid: id) - title = escape_once("Merge Request: #{merge_request.title}") - klass = reference_class(:merge_request) - data = data_attribute(project: project.id, merge_request: merge_request.id) - - url = url_for_merge_request(merge_request, project) - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{match}</a>) - else - match - end - end + def find_object(project, id) + project.merge_requests.find_by(iid: id) end - def url_for_merge_request(mr, project) + def url_for_object(mr, project) h = Gitlab::Application.routes.url_helpers h.namespace_project_merge_request_url(project.namespace, project, mr, only_path: context[:only_path]) diff --git a/lib/gitlab/markdown/relative_link_filter.rb b/lib/gitlab/markdown/relative_link_filter.rb index 6ee3d1ce039..632be4d7542 100644 --- a/lib/gitlab/markdown/relative_link_filter.rb +++ b/lib/gitlab/markdown/relative_link_filter.rb @@ -51,7 +51,7 @@ module Gitlab relative_url_root, context[:project].path_with_namespace, path_type(file_path), - ref || 'master', # assume that if no ref exists we can point to master + ref || context[:project].default_branch, # if no ref exists, point to the default branch file_path ].compact.join('/').squeeze('/').chomp('/') diff --git a/lib/gitlab/markdown/snippet_reference_filter.rb b/lib/gitlab/markdown/snippet_reference_filter.rb index f783f951711..f7bd07c2a34 100644 --- a/lib/gitlab/markdown/snippet_reference_filter.rb +++ b/lib/gitlab/markdown/snippet_reference_filter.rb @@ -6,65 +6,16 @@ module Gitlab # snippets that do not exist are ignored. # # This filter supports cross-project references. - class SnippetReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find `$123` snippet references in text - # - # SnippetReferenceFilter.references_in(text) do |match, snippet| - # "<a href=...>$#{snippet}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the Integer snippet ID, and an optional String - # of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Snippet.reference_pattern) do |match| - yield match, $~[:snippet].to_i, $~[:project] - end - end - - def self.referenced_by(node) - { snippet: LazyReference.new(Snippet, node.attr("data-snippet")) } - end - - def call - replace_text_nodes_matching(Snippet.reference_pattern) do |content| - snippet_link_filter(content) - end + class SnippetReferenceFilter < AbstractReferenceFilter + def self.object_class + Snippet end - # Replace `$123` snippet references in text with links to the referenced - # snippets's details page. - # - # text - String text to replace references in. - # - # Returns a String with `$123` references replaced with links. All links - # have `gfm` and `gfm-snippet` class names attached for styling. - def snippet_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if project && snippet = project.snippets.find_by(id: id) - title = escape_once("Snippet: #{snippet.title}") - klass = reference_class(:snippet) - data = data_attribute(project: project.id, snippet: snippet.id) - - url = url_for_snippet(snippet, project) - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{match}</a>) - else - match - end - end + def find_object(project, id) + project.snippets.find_by(id: id) end - def url_for_snippet(snippet, project) + def url_for_object(snippet, project) h = Gitlab::Application.routes.url_helpers h.namespace_project_snippet_url(project.namespace, project, snippet, only_path: context[:only_path]) diff --git a/lib/gitlab/markdown/user_reference_filter.rb b/lib/gitlab/markdown/user_reference_filter.rb index 2a594e1662e..ab5e1f6fe9e 100644 --- a/lib/gitlab/markdown/user_reference_filter.rb +++ b/lib/gitlab/markdown/user_reference_filter.rb @@ -85,13 +85,12 @@ module Gitlab def link_to_all project = context[:project] - url = urls.namespace_project_url(project.namespace, project, only_path: context[:only_path]) data = data_attribute(project: project.id) - text = User.reference_prefix + 'all' - %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) + + link_tag(url, data, text) end def link_to_namespace(namespace) @@ -105,16 +104,20 @@ module Gitlab def link_to_group(group, namespace) url = urls.group_url(group, only_path: context[:only_path]) data = data_attribute(group: namespace.id) - text = Group.reference_prefix + group - %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) + + link_tag(url, data, text) end def link_to_user(user, namespace) url = urls.user_url(user, only_path: context[:only_path]) data = data_attribute(user: namespace.owner_id) - text = User.reference_prefix + user + + link_tag(url, data, text) + end + + def link_tag(url, data, text) %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) end end diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 31aa3528c4c..2ef0e982256 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -14,7 +14,7 @@ module Gitlab def self.mute_mailer code = <<-eos -def Notify.delay +def Notify.deliver_later self end eos diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb index d87a4c9bb4a..3489fb251b6 100644 --- a/lib/gitlab/sherlock/transaction.rb +++ b/lib/gitlab/sherlock/transaction.rb @@ -36,6 +36,11 @@ module Gitlab @duration ||= started_at && finished_at ? finished_at - started_at : 0 end + # Returns the total query duration in seconds. + def query_duration + @query_duration ||= @queries.map { |q| q.duration }.inject(:+) / 1000.0 + end + def to_param @id end diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb new file mode 100644 index 00000000000..1cd89b3a9c4 --- /dev/null +++ b/lib/gitlab/sql/union.rb @@ -0,0 +1,34 @@ +module Gitlab + module SQL + # Class for building SQL UNION statements. + # + # ORDER BYs are dropped from the relations as the final sort order is not + # guaranteed any way. + # + # Example usage: + # + # union = Gitlab::SQL::Union.new(user.personal_projects, user.projects) + # sql = union.to_sql + # + # Project.where("id IN (#{sql})") + class Union + def initialize(relations) + @relations = relations + end + + def to_sql + # Some relations may include placeholders for prepared statements, these + # aren't incremented properly when joining relations together this way. + # By using "unprepared_statements" we remove the usage of placeholders + # (thus fixing this problem), at a slight performance cost. + fragments = ActiveRecord::Base.connection.unprepared_statement do + @relations.map do |rel| + rel.reorder(nil).to_sql + end + end + + fragments.join("\nUNION\n") + end + end + end +end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 0a7a4118077..2a79fbdcf93 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -44,7 +44,7 @@ upstream gitlab-workhorse { ## Normal HTTP host server { - ## Either remove "default_server" from the listen line below, + ## Either remove "default_server" from the listen line below, ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab ## to be served if you visit any address that your server responds to, eg. ## the ip address of the server (http://x.x.x.x/)n 0.0.0.0:80 default_server; @@ -67,7 +67,7 @@ server { location / { ## Serve static files from defined root folder. ## @gitlab is a named location for the upstream fallback, see below. - try_files $uri $uri/index.html $uri.html @gitlab; + try_files $uri /index.html $uri.html @gitlab; } ## We route uploads through GitLab to prevent XSS and enforce access control. @@ -113,19 +113,29 @@ server { proxy_pass http://gitlab; } + location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { + client_max_body_size 0; # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; return 418; } location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { + client_max_body_size 0; # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; return 418; } location ~ ^/api/v3/projects/.*/repository/archive { + client_max_body_size 0; # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; return 418; @@ -133,21 +143,22 @@ server { # Build artifacts should be submitted to this location location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; } # Build artifacts should be submitted to this location location ~ /ci/api/v1/builds/[0-9]+/artifacts { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; } location @gitlab-workhorse { + client_max_body_size 0; ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. # gzip off; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index b463d5b6aa9..016f7a536fb 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -48,7 +48,7 @@ upstream gitlab-workhorse { ## Redirects all HTTP traffic to the HTTPS host server { - ## Either remove "default_server" from the listen line below, + ## Either remove "default_server" from the listen line below, ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab ## to be served if you visit any address that your server responds to, eg. ## the ip address of the server (http://x.x.x.x/) @@ -112,7 +112,7 @@ server { location / { ## Serve static files from defined root folder. ## @gitlab is a named location for the upstream fallback, see below. - try_files $uri $uri/index.html $uri.html @gitlab; + try_files $uri /index.html $uri.html @gitlab; } ## We route uploads through GitLab to prevent XSS and enforce access control. @@ -160,19 +160,29 @@ server { proxy_pass http://gitlab; } + location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { + client_max_body_size 0; # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; return 418; } location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { + client_max_body_size 0; # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; return 418; } location ~ ^/api/v3/projects/.*/repository/archive { + client_max_body_size 0; # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; return 418; @@ -180,21 +190,22 @@ server { # Build artifacts should be submitted to this location location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; } # Build artifacts should be submitted to this location location ~ /ci/api/v1/builds/[0-9]+/artifacts { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; } location @gitlab-workhorse { + client_max_body_size 0; ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. gzip off; diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake index 5efffc2cdac..e9587595fef 100644 --- a/lib/tasks/flay.rake +++ b/lib/tasks/flay.rake @@ -1,6 +1,6 @@ desc 'Code duplication analyze via flay' task :flay do - output = %x(bundle exec flay app/ lib/gitlab/) + output = %x(bundle exec flay --mass 35 app/ lib/gitlab/) if output.include? "Similar code found" puts output diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 3c46bcea40e..cb4abe13799 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -13,6 +13,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:uploads:create"].invoke Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:artifacts:create"].invoke + Rake::Task["gitlab:backup:lfs:create"].invoke backup = Backup::Manager.new backup.pack @@ -34,6 +35,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts") + Rake::Task["gitlab:backup:lfs:restore"].invoke unless backup.skipped?("lfs") Rake::Task["gitlab:shell:setup"].invoke backup.cleanup @@ -134,6 +136,25 @@ namespace :gitlab do end end + namespace :lfs do + task create: :environment do + $progress.puts "Dumping lfs objects ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("lfs") + $progress.puts "[SKIPPED]".cyan + else + Backup::Lfs.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring lfs objects ... ".blue + Backup::Lfs.new.restore + $progress.puts "done".green + end + end + def configure_cron_mode if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index c95b6540ebc..efb863a8764 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -2,16 +2,6 @@ module Gitlab class TaskAbortedByUserError < StandardError; end end -unless STDOUT.isatty - module Colored - extend self - - def colorize(string, options={}) - string - end - end -end - namespace :gitlab do # Ask if the user wants to continue @@ -103,7 +93,7 @@ namespace :gitlab do gitlab_user = Gitlab.config.gitlab.user current_user = run(%W(whoami)).chomp unless current_user == gitlab_user - puts "#{Colored.color(:black)+Colored.color(:on_yellow)} Warning #{Colored.extra(:clear)}" + puts " Warning ".colorize(:black).on_yellow puts " You are running as user #{current_user.magenta}, we hope you know what you are doing." puts " Things may work\/fail for the wrong reasons." puts " For correct results you should run this as user #{gitlab_user.magenta}." diff --git a/lib/tasks/grape.rake b/lib/tasks/grape.rake new file mode 100644 index 00000000000..9980e0b7984 --- /dev/null +++ b/lib/tasks/grape.rake @@ -0,0 +1,8 @@ +namespace :grape do + desc 'Print compiled grape routes' + task routes: :environment do + API::API.routes.each do |route| + puts route + end + end +end |