diff options
Diffstat (limited to 'lib')
118 files changed, 3078 insertions, 1275 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index c09488d3547..fe1bf8a4816 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -25,7 +25,7 @@ module API format :json content_type :txt, "text/plain" - helpers APIHelpers + helpers Helpers mount Groups mount GroupMembers @@ -46,10 +46,12 @@ module API mount Services mount Files mount Commits + mount CommitStatus mount Namespaces mount Branches mount Labels mount Settings mount Keys + mount Tags end end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb new file mode 100644 index 00000000000..2c0596c9dfb --- /dev/null +++ b/lib/api/commit_statuses.rb @@ -0,0 +1,80 @@ +require 'mime/types' + +module API + # Project commit statuses API + class CommitStatus < Grape::API + resource :projects do + before { authenticate! } + + # Get a commit's statuses + # + # Parameters: + # id (required) - The ID of a project + # sha (required) - The commit hash + # ref (optional) - The ref + # stage (optional) - The stage + # name (optional) - The name + # all (optional) - Show all statuses, default: false + # Examples: + # GET /projects/:id/repository/commits/:sha/statuses + get ':id/repository/commits/:sha/statuses' do + authorize! :read_commit_statuses, user_project + sha = params[:sha] + ci_commit = user_project.ci_commit(sha) + not_found! 'Commit' unless ci_commit + statuses = ci_commit.statuses + statuses = statuses.latest unless parse_boolean(params[:all]) + statuses = statuses.where(ref: params[:ref]) if params[:ref].present? + statuses = statuses.where(stage: params[:stage]) if params[:stage].present? + statuses = statuses.where(name: params[:name]) if params[:name].present? + present paginate(statuses), with: Entities::CommitStatus + end + + # Post status to commit + # + # Parameters: + # id (required) - The ID of a project + # sha (required) - The commit hash + # ref (optional) - The ref + # state (required) - The state of the status. Can be: pending, running, success, error or failure + # target_url (optional) - The target URL to associate with this status + # description (optional) - A short description of the status + # name or context (optional) - A string label to differentiate this status from the status of other systems. Default: "default" + # Examples: + # POST /projects/:id/statuses/:sha + post ':id/statuses/:sha' do + authorize! :create_commit_status, user_project + required_attributes! [:state] + attrs = attributes_for_keys [:ref, :target_url, :description, :context, :name] + commit = @project.commit(params[:sha]) + not_found! 'Commit' unless commit + + ci_commit = @project.ensure_ci_commit(commit.sha) + + name = params[:name] || params[:context] + status = GenericCommitStatus.running_or_pending.find_by(commit: ci_commit, name: name, ref: params[:ref]) + status ||= GenericCommitStatus.new(commit: ci_commit, user: current_user) + status.update(attrs) + + case params[:state].to_s + when 'running' + status.run + when 'success' + status.success + when 'failed' + status.drop + when 'canceled' + status.cancel + else + status.status = params[:state].to_s + end + + if status.save + present status, with: Entities::CommitStatus + else + render_validation_error!(status) + end + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9620d36ac41..96b73df6af9 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -62,7 +62,7 @@ module API expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace - expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :snippets_enabled, :created_at, :last_activity_at + expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :created_at, :last_activity_at expose :creator_id expose :namespace expose :forked_from_project, using: Entities::ForkedFromProject, if: lambda{ | project, options | project.forked? } @@ -95,25 +95,6 @@ module API end end - class RepoTag < Grape::Entity - expose :name - expose :message do |repo_obj, _options| - if repo_obj.respond_to?(:message) - repo_obj.message - else - nil - end - end - - expose :commit do |repo_obj, options| - if repo_obj.respond_to?(:commit) - repo_obj.commit - elsif options[:project] - options[:project].repository.commit(repo_obj.target) - end - end - end - class RepoObject < Grape::Entity expose :name @@ -149,6 +130,7 @@ module API class RepoCommitDetail < RepoCommit expose :parent_ids, :committed_date, :authored_date + expose :status end class ProjectSnippet < Grape::Entity @@ -180,7 +162,9 @@ module API end class MergeRequest < ProjectEntity - expose :target_branch, :source_branch, :upvotes, :downvotes + expose :target_branch, :source_branch + # deprecated, always returns 0 + expose :upvotes, :downvotes expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id expose :label_names, as: :labels @@ -210,6 +194,8 @@ module API expose :author, using: Entities::UserBasic expose :created_at expose :system?, as: :system + expose :noteable_id, :noteable_type + # upvote? and downvote? are deprecated, always return false expose :upvote?, as: :upvote expose :downvote?, as: :downvote end @@ -228,11 +214,19 @@ module API expose :created_at end + class CommitStatus < Grape::Entity + expose :id, :sha, :ref, :status, :name, :target_url, :description, + :created_at, :started_at, :finished_at, :allow_failure + expose :author, using: Entities::UserBasic + end + class Event < Grape::Entity expose :title, :project_id, :action_name expose :target_id, :target_type, :author_id expose :data, :target_title expose :created_at + expose :note, using: Entities::Note, if: ->(event, options) { event.note? } + expose :author, using: Entities::UserBasic, if: ->(event, options) { event.author } expose :author_username do |event, options| if event.author @@ -255,6 +249,18 @@ module API expose :notification_level end + class ProjectService < Grape::Entity + expose :id, :title, :created_at, :updated_at, :active + expose :push_events, :issues_events, :merge_requests_events, :tag_push_events, :note_events + # Expose serialized properties + expose :properties do |service, options| + field_names = service.fields. + select { |field| options[:include_passwords] || field[:type] != 'password' }. + map { |field| field[:name] } + service.properties.slice(*field_names) + end + end + class ProjectWithAccess < Project expose :permissions do expose :project_access, using: Entities::ProjectAccess do |project, options| @@ -322,5 +328,35 @@ module API expose :user_oauth_applications expose :after_sign_out_path end + + class Release < Grape::Entity + expose :tag, as: :tag_name + expose :description + end + + class RepoTag < Grape::Entity + expose :name + expose :message do |repo_obj, _options| + if repo_obj.respond_to?(:message) + repo_obj.message + else + nil + end + end + + expose :commit do |repo_obj, options| + if repo_obj.respond_to?(:commit) + repo_obj.commit + elsif options[:project] + options[:project].repository.commit(repo_obj.target) + end + end + + expose :release, using: Entities::Release do |repo_obj, options| + if options[:project] + options[:project].releases.find_by(tag: repo_obj.name) + end + end + end end end diff --git a/lib/api/files.rb b/lib/api/files.rb index 308c84dd135..a7a768f8895 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -43,7 +43,8 @@ module API # "content": "IyA9PSBTY2hlbWEgSW5mb3...", # "ref": "master", # "blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83", - # "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50" + # "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50", + # "last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d", # } # get ":id/repository/files" do @@ -71,6 +72,7 @@ module API ref: ref, blob_id: blob.id, commit_id: commit.id, + last_commit_id: user_project.repository.last_commit_for_path(commit.sha, file_path).id } else not_found! 'File' diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 7fada98fcdc..92540ccf2b1 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -1,5 +1,5 @@ module API - module APIHelpers + module Helpers PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" PRIVATE_TOKEN_PARAM = :private_token SUDO_HEADER ="HTTP_SUDO" @@ -63,11 +63,11 @@ module API user_project.build_missing_services service_method = "#{underscored_service}_service" - + send_service(service_method) end end - + @project_service || not_found!("Service") end @@ -133,6 +133,12 @@ module API authorize! :admin_project, user_project end + def require_gitlab_workhorse! + unless env['HTTP_GITLAB_WORKHORSE'].present? + forbidden!('Request should be executed via GitLab Workhorse') + end + end + def can?(object, action, subject) abilities.allowed?(object, action, subject) end @@ -149,7 +155,6 @@ module API end def attributes_for_keys(keys, custom_params = nil) - params_hash = custom_params || params attrs = {} keys.each do |key| if params[key].present? or (params.has_key?(key) and params[key] == false) @@ -235,6 +240,10 @@ module API render_api_error!(message || '409 Conflict', 409) end + def file_to_large! + render_api_error!('413 Request Entity Too Large', 413) + end + def render_validation_error!(model) if model.errors.any? render_api_error!(model.errors.messages || '400 Bad Request', 400) @@ -283,6 +292,44 @@ module API end end + # file helpers + + def uploaded_file!(field, uploads_path) + if params[field] + bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) + return params[field] + end + + # sanitize file paths + # this requires all paths to exist + required_attributes! %W(#{field}.path) + uploads_path = File.realpath(uploads_path) + file_path = File.realpath(params["#{field}.path"]) + bad_request!('Bad file path') unless file_path.start_with?(uploads_path) + + UploadedFile.new( + file_path, + params["#{field}.name"], + params["#{field}.type"] || 'application/octet-stream', + ) + end + + def present_file!(path, filename, content_type = 'application/octet-stream') + filename ||= File.basename(path) + header['Content-Disposition'] = "attachment; filename=#{filename}" + header['Content-Transfer-Encoding'] = 'binary' + content_type content_type + + # Support download acceleration + case headers['X-Sendfile-Type'] + when 'X-Sendfile' + header['X-Sendfile'] = path + body + else + file FileStreamer.new(path) + end + end + private def add_pagination_headers(paginated, per_page) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 63ea2f05438..e7c5f808aea 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -76,6 +76,22 @@ module API present merge_request, with: Entities::MergeRequest end + # Show MR commits + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of MR + # + # Example: + # GET /projects/:id/merge_request/:merge_request_id/commits + # + get ':id/merge_request/:merge_request_id/commits' do + merge_request = user_project.merge_requests. + find(params[:merge_request_id]) + authorize! :read_merge_request, merge_request + present merge_request.commits, with: Entities::RepoCommit + end + # Show MR changes # # Parameters: @@ -99,7 +115,7 @@ module API # id (required) - The ID of a project - this will be the source of the merge request # source_branch (required) - The source branch # target_branch (required) - The target branch - # target_project - The target project of the merge request defaults to the :id of the project + # target_project_id - The target project of the merge request defaults to the :id of the project # assignee_id - Assignee user ID # title (required) - Title of MR # description - Description of MR @@ -249,8 +265,16 @@ module API required_attributes! [:note] merge_request = user_project.merge_requests.find(params[:merge_request_id]) - note = merge_request.notes.new(note: params[:note], project_id: user_project.id) - note.author = current_user + + authorize! :create_note, merge_request + + opts = { + note: params[:note], + noteable_type: 'MergeRequest', + noteable_id: merge_request.id + } + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute if note.save present note, with: Entities::MRNote diff --git a/lib/api/projects.rb b/lib/api/projects.rb index c2fb36b4143..6928fe0eb9d 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -7,8 +7,12 @@ module API helpers do def map_public_to_visibility_level(attrs) publik = attrs.delete(:public) - publik = parse_boolean(publik) - attrs[:visibility_level] = Gitlab::VisibilityLevel::PUBLIC if !attrs[:visibility_level].present? && publik == true + if publik.present? && !attrs[:visibility_level].present? + publik = parse_boolean(publik) + # Since setting the public attribute to private could mean either + # private or internal, use the more conservative option, private. + attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE + end attrs end end @@ -75,6 +79,7 @@ module API # description (optional) - short project description # issues_enabled (optional) # merge_requests_enabled (optional) + # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) # namespace_id (optional) - defaults to user namespace @@ -90,6 +95,7 @@ module API :description, :issues_enabled, :merge_requests_enabled, + :builds_enabled, :wiki_enabled, :snippets_enabled, :namespace_id, @@ -117,6 +123,7 @@ module API # default_branch (optional) - 'master' by default # issues_enabled (optional) # merge_requests_enabled (optional) + # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 @@ -132,6 +139,7 @@ module API :default_branch, :issues_enabled, :merge_requests_enabled, + :builds_enabled, :wiki_enabled, :snippets_enabled, :public, @@ -172,6 +180,7 @@ module API # description (optional) - short project description # issues_enabled (optional) # merge_requests_enabled (optional) + # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 @@ -185,6 +194,7 @@ module API :default_branch, :issues_enabled, :merge_requests_enabled, + :builds_enabled, :wiki_enabled, :snippets_enabled, :public, @@ -246,8 +256,8 @@ module API # Example Request: # DELETE /projects/:id/fork delete ":id/fork" do - authenticated_as_admin! - unless user_project.forked_project_link.nil? + authorize! :remove_fork_project, user_project + if user_project.forked? user_project.forked_project_link.destroy end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 2d96c9666d2..d7c48639eba 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -16,41 +16,6 @@ module API end end - # Get a project repository tags - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/repository/tags - get ":id/repository/tags" do - present user_project.repo.tags.sort_by(&:name).reverse, - with: Entities::RepoTag, project: user_project - end - - # Create tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # ref (required) - Create tag from commit sha or branch - # message (optional) - Specifying a message creates an annotated tag. - # Example Request: - # POST /projects/:id/repository/tags - post ':id/repository/tags' do - authorize_push_project - message = params[:message] || nil - result = CreateTagService.new(user_project, current_user). - execute(params[:tag_name], params[:ref], message) - - if result[:status] == :success - present result[:tag], - with: Entities::RepoTag, - project: user_project - else - render_api_error!(result[:message], 400) - end - end - # Get a project repository tree # # Parameters: @@ -133,7 +98,7 @@ module API authorize! :download_code, user_project begin - file_path = ArchiveRepositoryService.new( + ArchiveRepositoryService.new( user_project, params[:sha], params[:format] @@ -141,17 +106,6 @@ module API rescue not_found!('File') end - - if file_path && File.exists?(file_path) - data = File.open(file_path, 'rb').read - basename = File.basename(file_path) - header['Content-Disposition'] = "attachment; filename=\"#{basename}\"" - content_type MIME::Types.type_for(file_path).first.content_type - env['api.format'] = :binary - present data - else - redirect request.fullpath - end end # Compare two branches, tags or commits diff --git a/lib/api/services.rb b/lib/api/services.rb index 6727e80ac1e..203f04a6259 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -57,7 +57,7 @@ module API # GET /project/:id/services/gitlab-ci # get ':id/services/:service_slug' do - present project_service + present project_service, with: Entities::ProjectService, include_passwords: current_user.is_admin? end end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb new file mode 100644 index 00000000000..47621f443e6 --- /dev/null +++ b/lib/api/tags.rb @@ -0,0 +1,86 @@ +module API + # Git Tags API + class Tags < Grape::API + before { authenticate! } + before { authorize! :download_code, user_project } + + resource :projects do + # Get a project repository tags + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # GET /projects/:id/repository/tags + get ":id/repository/tags" do + present user_project.repo.tags.sort_by(&:name).reverse, + with: Entities::RepoTag, project: user_project + end + + # Create tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # ref (required) - Create tag from commit sha or branch + # message (optional) - Specifying a message creates an annotated tag. + # Example Request: + # POST /projects/:id/repository/tags + post ':id/repository/tags' do + authorize_push_project + message = params[:message] || nil + result = CreateTagService.new(user_project, current_user). + execute(params[:tag_name], params[:ref], message, params[:release_description]) + + if result[:status] == :success + present result[:tag], + with: Entities::RepoTag, + project: user_project + else + render_api_error!(result[:message], 400) + end + end + + # Add release notes to tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # description (required) - Release notes with markdown support + # Example Request: + # POST /projects/:id/repository/tags/:tag_name/release + post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.*/ } do + authorize_push_project + required_attributes! [:description] + result = CreateReleaseService.new(user_project, current_user). + execute(params[:tag_name], params[:description]) + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + + # Updates a release notes of a tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # description (required) - Release notes with markdown support + # Example Request: + # PUT /projects/:id/repository/tags/:tag_name/release + put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.*/ } do + authorize_push_project + required_attributes! [:description] + result = UpdateReleaseService.new(user_project, current_user). + execute(params[:tag_name], params[:description]) + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + end + end +end diff --git a/lib/award_emoji.rb b/lib/award_emoji.rb new file mode 100644 index 00000000000..d58a196c4ef --- /dev/null +++ b/lib/award_emoji.rb @@ -0,0 +1,12 @@ +class AwardEmoji + EMOJI_LIST = [ + "+1", "-1", "100", "blush", "heart", "smile", "rage", + "beers", "disappointed", "ok_hand", + "helicopter", "shit", "airplane", "alarm_clock", + "ambulance", "anguished", "two_hearts", "wink" + ] + + def self.path_to_emoji_image(name) + "emoji/#{Emoji.emoji_filename(name)}.png" + end +end diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb new file mode 100644 index 00000000000..51fa3867e67 --- /dev/null +++ b/lib/backup/artifacts.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Artifacts < Files + def initialize + super('artifacts', ArtifactUploader.artifacts_path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index 6f56f680bb9..635967f4bd4 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -1,34 +1,13 @@ -module Backup - class Builds - attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir +require 'backup/files' +module Backup + class Builds < Files def initialize - @app_builds_dir = Settings.gitlab_ci.builds_path - @backup_dir = Gitlab.config.backup.path - @backup_builds_dir = File.join(Gitlab.config.backup.path, 'builds') - end - - # Copy builds from builds directory to backup/builds - def dump - FileUtils.rm_rf(backup_builds_dir) - # Ensure the parent dir of backup_builds_dir exists - FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create backup_builds_dir before us - FileUtils.mkdir(backup_builds_dir, mode: 0700) - FileUtils.cp_r(app_builds_dir, backup_dir) - end - - def restore - backup_existing_builds_dir - - FileUtils.cp_r(backup_builds_dir, app_builds_dir) + super('builds', Settings.gitlab_ci.builds_path) end - def backup_existing_builds_dir - timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") - if File.exists?(app_builds_dir) - FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) - end + def create_files_dir + Dir.mkdir(app_files_dir, 0700) end end end diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 959ac4b7868..67b2a64bd10 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -2,26 +2,26 @@ require 'yaml' module Backup class Database - attr_reader :config, :db_dir + attr_reader :config, :db_file_name def initialize @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env] - @db_dir = File.join(Gitlab.config.backup.path, 'db') + @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end def dump - FileUtils.rm_rf(@db_dir) - # Ensure the parent dir of @db_dir exists - FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create @db_dir before us - FileUtils.mkdir(@db_dir, mode: 0700) + FileUtils.mkdir_p(File.dirname(db_file_name)) + FileUtils.rm_f(db_file_name) + compress_rd, compress_wr = IO.pipe + compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600]) + compress_rd.close - success = case config["adapter"] + dump_pid = case config["adapter"] when /^mysql/ then $progress.print "Dumping MySQL database #{config['database']} ... " # Workaround warnings from MySQL 5.6 about passwords on cmd line ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - system('mysqldump', *mysql_args, config['database'], out: db_file_name) + spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) when "postgresql" then $progress.print "Dumping PostgreSQL database #{config['database']} ... " pg_env @@ -30,48 +30,42 @@ module Backup pgsql_args << "-n" pgsql_args << Gitlab.config.backup.pg_schema end - system('pg_dump', *pgsql_args, config['database'], out: db_file_name) + spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr) end - report_success(success) - abort 'Backup failed' unless success + compress_wr.close + + success = [compress_pid, dump_pid].all? { |pid| Process.waitpid(pid); $?.success? } - $progress.print 'Compressing database ... ' - success = system('gzip', db_file_name) report_success(success) - abort 'Backup failed: compress error' unless success + abort 'Backup failed' unless success end def restore - $progress.print 'Decompressing database ... ' - success = system('gzip', '-d', db_file_name_gz) - report_success(success) - abort 'Restore failed: decompress error' unless success + decompress_rd, decompress_wr = IO.pipe + decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name) + decompress_wr.close - success = case config["adapter"] + restore_pid = case config["adapter"] when /^mysql/ then $progress.print "Restoring MySQL database #{config['database']} ... " # Workaround warnings from MySQL 5.6 about passwords on cmd line ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - system('mysql', *mysql_args, config['database'], in: db_file_name) + spawn('mysql', *mysql_args, config['database'], in: decompress_rd) when "postgresql" then $progress.print "Restoring PostgreSQL database #{config['database']} ... " pg_env - system('psql', config['database'], '-f', db_file_name) + spawn('psql', config['database'], in: decompress_rd) end + decompress_rd.close + + success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? } + report_success(success) abort 'Restore failed' unless success end protected - def db_file_name - File.join(db_dir, 'database.sql') - end - - def db_file_name_gz - File.join(db_dir, 'database.sql.gz') - end - def mysql_args args = { 'host' => '--host', diff --git a/lib/backup/files.rb b/lib/backup/files.rb new file mode 100644 index 00000000000..654b4d1c896 --- /dev/null +++ b/lib/backup/files.rb @@ -0,0 +1,40 @@ +require 'open3' + +module Backup + class Files + attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir + + def initialize(name, app_files_dir) + @name = name + @app_files_dir = File.realpath(app_files_dir) + @files_parent_dir = File.realpath(File.join(@app_files_dir, '..')) + @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz') + end + + # Copy files from public/files to backup/files + def dump + FileUtils.mkdir_p(Gitlab.config.backup.path) + FileUtils.rm_f(backup_tarball) + run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + end + + def restore + backup_existing_files_dir + create_files_dir + + run_pipeline!([%W(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball) + end + + def backup_existing_files_dir + timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}") + if File.exists?(app_files_dir) + FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path)) + end + end + + def run_pipeline!(cmd_list, options={}) + status_list = Open3.pipeline(*cmd_list, options) + abort 'Backup failed' unless status_list.compact.all?(&:success?) + end + end +end diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb new file mode 100644 index 00000000000..4153467fbee --- /dev/null +++ b/lib/backup/lfs.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Lfs < Files + def initialize + super('lfs', Settings.lfs.storage_path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index ac63f89c6ec..099062eeb8b 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -45,7 +45,8 @@ module Backup directory = connection.directories.get(remote_directory) if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, - multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size) + multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, + encryption: Gitlab.config.backup.upload.encryption) $progress.puts "done".green else puts "uploading backup to #{remote_directory} failed".red @@ -55,7 +56,7 @@ module Backup def cleanup $progress.print "Deleting tmp directories ... " - + backup_contents.each do |dir| next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) @@ -75,7 +76,7 @@ module Backup if keep_time > 0 removed = 0 - + Dir.chdir(Gitlab.config.backup.path) do file_list = Dir.glob('*_gitlab_backup.tar') file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ } @@ -149,17 +150,15 @@ module Backup private def backup_contents - folders_to_backup + ["backup_information.yml"] + folders_to_backup + archives_to_backup + ["backup_information.yml"] end - def folders_to_backup - folders = %w{repositories db uploads builds} - - if ENV["SKIP"] - return folders.reject{ |folder| ENV["SKIP"].include?(folder) } - end + def archives_to_backup + %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + end - folders + def folders_to_backup + %w{repositories db}.reject{ |name| skipped?(name) } end def settings diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 4d70f7883dd..a82a7e1f7bf 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -35,7 +35,7 @@ module Backup if wiki.repository.empty? $progress.puts " [SKIPPED]".cyan else - cmd = %W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) output, status = Gitlab::Popen.popen(cmd) if status.zero? $progress.puts " [DONE]".green @@ -67,7 +67,7 @@ module Backup FileUtils.mkdir_p(path_to_repo(project)) cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)}) else - cmd = %W(git init --bare #{path_to_repo(project)}) + cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_repo(project)}) end if system(*cmd, silent) @@ -87,7 +87,7 @@ module Backup # that was initialized with ProjectWiki.new() and then # try to restore with 'git clone --bare'. FileUtils.rm_rf(path_to_repo(wiki)) - cmd = %W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) if system(*cmd, silent) $progress.puts " [DONE]".green diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 1f9626644e6..9261f77f3c9 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -1,34 +1,14 @@ +require 'backup/files' + module Backup - class Uploads - attr_reader :app_uploads_dir, :backup_uploads_dir, :backup_dir + class Uploads < Files def initialize - @app_uploads_dir = File.realpath(Rails.root.join('public', 'uploads')) - @backup_dir = Gitlab.config.backup.path - @backup_uploads_dir = File.join(Gitlab.config.backup.path, 'uploads') - end - - # Copy uploads from public/uploads to backup/uploads - def dump - FileUtils.rm_rf(backup_uploads_dir) - # Ensure the parent dir of backup_uploads_dir exists - FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create backup_uploads_dir before us - FileUtils.mkdir(backup_uploads_dir, mode: 0700) - FileUtils.cp_r(app_uploads_dir, backup_dir) - end - - def restore - backup_existing_uploads_dir - - FileUtils.cp_r(backup_uploads_dir, app_uploads_dir) + super('uploads', Rails.root.join('public/uploads')) end - def backup_existing_uploads_dir - timestamped_uploads_path = File.join(app_uploads_dir, '..', "uploads.#{Time.now.to_i}") - if File.exists?(app_uploads_dir) - FileUtils.mv(app_uploads_dir, File.expand_path(timestamped_uploads_path)) - end + def create_files_dir + Dir.mkdir(app_files_dir) end end end diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 5109c84e0ea..07e68216d7f 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -23,14 +23,11 @@ module Ci rack_response({ 'message' => '500 Internal Server Error' }, 500) end - before do - check_enable_flag! - end - format :json helpers Helpers - helpers ::API::APIHelpers + helpers ::API::Helpers + helpers Gitlab::CurrentSettings mount Builds mount Commits diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 83ca1e6481c..15faa6edd84 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -47,6 +47,108 @@ module Ci build.drop end end + + # Authorize artifacts uploading for build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # filesize (optional) - the size of uploaded file + # Example Request: + # POST /builds/:id/artifacts/authorize + post ":id/artifacts/authorize" do + require_gitlab_workhorse! + not_allowed! unless Gitlab.config.artifacts.enabled + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + forbidden!('build is not running') unless build.running? + + if params[:filesize] + file_size = params[:filesize].to_i + file_to_large! unless file_size < max_artifacts_size + end + + status 200 + { TempPath: ArtifactUploader.artifacts_upload_path } + end + + # Upload artifacts to build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # file (required) - The uploaded file + # Parameters (accelerated by GitLab Workhorse): + # file.path - path to locally stored body (generated by Workhorse) + # file.name - real filename as send in Content-Disposition + # file.type - real content type as send in Content-Type + # Headers: + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Body: + # The file content + # + # Example Request: + # POST /builds/:id/artifacts + post ":id/artifacts" do + require_gitlab_workhorse! + not_allowed! unless Gitlab.config.artifacts.enabled + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + forbidden!('build is not running') unless build.running? + + file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path) + file_to_large! unless file.size < max_artifacts_size + + if build.update_attributes(artifacts_file: file) + present build, with: Entities::Build + else + render_validation_error!(build) + end + end + + # Download the artifacts file from build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Headers: + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Example Request: + # GET /builds/:id/artifacts + get ":id/artifacts" do + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + artifacts_file = build.artifacts_file + + unless artifacts_file.file_storage? + return redirect_to build.artifacts_file.url + end + + unless artifacts_file.exists? + not_found! + end + + present_file!(artifacts_file.path, artifacts_file.filename) + end + + # Remove the artifacts file from build + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Headers: + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Example Request: + # DELETE /builds/:id/artifacts + delete ":id/artifacts" do + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + build.remove_artifacts_file! + end end end end diff --git a/lib/ci/api/commits.rb b/lib/ci/api/commits.rb index bac463a5909..a60769d8305 100644 --- a/lib/ci/api/commits.rb +++ b/lib/ci/api/commits.rb @@ -51,7 +51,7 @@ module Ci required_attributes! [:project_id, :data, :project_token] project = Ci::Project.find(params[:project_id]) authenticate_project_token!(project) - commit = Ci::CreateCommitService.new.execute(project, params[:data]) + commit = Ci::CreateCommitService.new.execute(project, current_user, params[:data]) if commit.persisted? present commit, with: Entities::CommitWithBuilds diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index f47bc1236b8..750f421872d 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -2,7 +2,7 @@ module Ci module API module Entities class Commit < Grape::Entity - expose :id, :ref, :sha, :project_id, :before_sha, :created_at + expose :id, :sha, :project_id, :created_at expose :status, :finished_at, :duration expose :git_commit_message, :git_author_name, :git_author_email end @@ -11,10 +11,16 @@ module Ci expose :builds end + class ArtifactFile < Grape::Entity + expose :filename, :size + end + class Build < Grape::Entity - expose :id, :commands, :ref, :sha, :project_id, :repo_url, + expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url, :before_sha, :allow_git_fetch, :project_name + expose :name, :token, :stage + expose :options do |model| model.options end @@ -24,6 +30,7 @@ module Ci end expose :variables + expose :artifacts_file, using: ArtifactFile end class Runner < Grape::Entity diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 8e893aa5cc6..02502333756 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -1,14 +1,10 @@ module Ci module API module Helpers + BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN" + BUILD_TOKEN_PARAM = :token UPDATE_RUNNER_EVERY = 60 - def check_enable_flag! - unless current_application_settings.ci_enabled - render_api_error!('400 (Bad request) CI is disabled', 400) - end - end - def authenticate_runners! forbidden! unless params[:token] == GitlabCi::REGISTRATION_TOKEN end @@ -21,8 +17,15 @@ module Ci forbidden! unless project.valid_token?(params[:project_token]) end + def authenticate_build_token!(build) + token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s + forbidden! unless token && build.valid_token?(token) + end + def update_runner_last_contact - if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= UPDATE_RUNNER_EVERY + # Use a random threshold to prevent beating DB updates + contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) + if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age current_runner.update_attributes(contacted_at: Time.now) end end @@ -36,6 +39,10 @@ module Ci info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) current_runner.update(info) end + + def max_artifacts_size + current_application_settings.max_artifacts_size.megabytes.to_i + end end end end diff --git a/lib/ci/api/projects.rb b/lib/ci/api/projects.rb index 66bcf65e8c4..d719ad9e8d5 100644 --- a/lib/ci/api/projects.rb +++ b/lib/ci/api/projects.rb @@ -75,23 +75,17 @@ module Ci # Create Gitlab CI project using Gitlab project info # # Parameters: - # name (required) - The name of the project # gitlab_id (required) - The gitlab id of the project - # path (required) - The gitlab project path, ex. randx/six - # ssh_url_to_repo (required) - The gitlab ssh url to the repo # default_ref - The branch to run against (defaults to `master`) # Example Request: # POST /projects post do - required_attributes! [:name, :gitlab_id, :ssh_url_to_repo] + required_attributes! [:gitlab_id] filtered_params = { - name: params[:name], gitlab_id: params[:gitlab_id], # we accept gitlab_url for backward compatibility for a while (added to 7.11) - path: params[:path] || params[:gitlab_url].sub(/.*\/(.*\/.*)$/, '\1'), - default_ref: params[:default_ref] || 'master', - ssh_url_to_repo: params[:ssh_url_to_repo] + default_ref: params[:default_ref] || 'master' } project = Ci::Project.new(filtered_params) @@ -109,11 +103,7 @@ module Ci # # Parameters: # id (required) - The ID of a project - # name - The name of the project - # gitlab_id - The gitlab id of the project - # path - The gitlab project path, ex. randx/six - # ssh_url_to_repo - The gitlab ssh url to the repo - # default_ref - The branch to run against (defaults to `master`) + # default_ref - The branch to run against (defaults to `master`) # Example Request: # PUT /projects/:id put ":id" do @@ -121,12 +111,7 @@ module Ci unauthorized! unless can?(current_user, :admin_project, project.gl_project) - attrs = attributes_for_keys [:name, :gitlab_id, :path, :gitlab_url, :default_ref, :ssh_url_to_repo] - - # we accept gitlab_url for backward compatibility for a while (added to 7.11) - if attrs[:gitlab_url] && !attrs[:path] - attrs[:path] = attrs[:gitlab_url].sub(/.*\/(.*\/.*)$/, '\1') - end + attrs = attributes_for_keys [:default_ref] if project.update_attributes(attrs) present project, with: Entities::Project diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index 915a4f526a6..5ff7407c6fe 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -60,7 +60,8 @@ module Ci class BuildTime < Chart def collect - commits = project.commits.joins(:builds).where("#{Ci::Build.table_name}.finished_at is NOT NULL AND #{Ci::Build.table_name}.started_at is NOT NULL").last(30) + commits = project.commits.last(30) + commits.each do |commit| @labels << commit.short_sha @build_times << (commit.duration / 60) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index e625e790df8..3beafcad117 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -4,13 +4,14 @@ module Ci DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' - ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables] - ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage] + ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables, :cache] + ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache] - attr_reader :before_script, :image, :services, :variables + attr_reader :before_script, :image, :services, :variables, :path, :cache - def initialize(config) + def initialize(config, path = nil) @config = YAML.load(config) + @path = path unless @config.is_a? Hash raise ValidationError, "YAML should be a hash" @@ -45,6 +46,7 @@ module Ci @services = @config[:services] @stages = @config[:stages] || @config[:types] @variables = @config[:variables] || {} + @cache = @config[:cache] @config.except!(*ALLOWED_YAML_KEYS) # anything that doesn't have script is considered as unknown @@ -63,50 +65,26 @@ module Ci end end - def process?(only_params, except_params, ref, tag) - return true if only_params.nil? && except_params.nil? - - if only_params - return true if tag && only_params.include?("tags") - return true if !tag && only_params.include?("branches") - - only_params.find do |pattern| - match_ref?(pattern, ref) - end - else - return false if tag && except_params.include?("tags") - return false if !tag && except_params.include?("branches") - - except_params.each do |pattern| - return false if match_ref?(pattern, ref) - end - end - end - def build_job(name, job) { + stage_idx: stages.index(job[:stage]), stage: job[:stage], - script: "#{@before_script.join("\n")}\n#{normalize_script(job[:script])}", - tags: job[:tags] || [], + commands: "#{@before_script.join("\n")}\n#{normalize_script(job[:script])}", + tag_list: job[:tags] || [], name: name, only: job[:only], except: job[:except], allow_failure: job[:allow_failure] || false, + when: job[:when] || 'on_success', options: { image: job[:image] || @image, - services: job[:services] || @services + services: job[:services] || @services, + artifacts: job[:artifacts], + cache: job[:cache] || @cache, }.compact } end - def match_ref?(pattern, ref) - if pattern.first == "/" && pattern.last == "/" - Regexp.new(pattern[1...-1]) =~ ref - else - pattern == ref - end - end - def normalize_script(script) if script.is_a? Array script.join("\n") @@ -136,63 +114,140 @@ module Ci raise ValidationError, "variables should be a map of key-valued strings" end + if @cache + if @cache[:untracked] && !validate_boolean(@cache[:untracked]) + raise ValidationError, "cache:untracked parameter should be an boolean" + end + + if @cache[:paths] && !validate_array_of_strings(@cache[:paths]) + raise ValidationError, "cache:paths parameter should be an array of strings" + end + end + @jobs.each do |name, job| - validate_job!("#{name} job", job) + validate_job!(name, job) end true end def validate_job!(name, job) + if name.blank? || !validate_string(name) + raise ValidationError, "job name should be non-empty string" + end + job.keys.each do |key| unless ALLOWED_JOB_KEYS.include? key - raise ValidationError, "#{name}: unknown parameter #{key}" + raise ValidationError, "#{name} job: unknown parameter #{key}" end end - if !job[:script].is_a?(String) && !validate_array_of_strings(job[:script]) - raise ValidationError, "#{name}: script should be a string or an array of a strings" + if !validate_string(job[:script]) && !validate_array_of_strings(job[:script]) + raise ValidationError, "#{name} job: script should be a string or an array of a strings" end if job[:stage] unless job[:stage].is_a?(String) && job[:stage].in?(stages) - raise ValidationError, "#{name}: stage parameter should be #{stages.join(", ")}" + raise ValidationError, "#{name} job: stage parameter should be #{stages.join(", ")}" end end - if job[:image] && !job[:image].is_a?(String) - raise ValidationError, "#{name}: image should be a string" + if job[:image] && !validate_string(job[:image]) + raise ValidationError, "#{name} job: image should be a string" end if job[:services] && !validate_array_of_strings(job[:services]) - raise ValidationError, "#{name}: services should be an array of strings" + raise ValidationError, "#{name} job: services should be an array of strings" end if job[:tags] && !validate_array_of_strings(job[:tags]) - raise ValidationError, "#{name}: tags parameter should be an array of strings" + raise ValidationError, "#{name} job: tags parameter should be an array of strings" end if job[:only] && !validate_array_of_strings(job[:only]) - raise ValidationError, "#{name}: only parameter should be an array of strings" + raise ValidationError, "#{name} job: only parameter should be an array of strings" end if job[:except] && !validate_array_of_strings(job[:except]) - raise ValidationError, "#{name}: except parameter should be an array of strings" + raise ValidationError, "#{name} job: except parameter should be an array of strings" end - if job[:allow_failure] && !job[:allow_failure].in?([true, false]) - raise ValidationError, "#{name}: allow_failure parameter should be an boolean" + if job[:cache] + if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked]) + raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean" + end + + if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths]) + raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings" + end + end + + if job[:artifacts] + if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked]) + raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean" + end + + if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths]) + raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings" + end + end + + if job[:allow_failure] && !validate_boolean(job[:allow_failure]) + raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" + end + + if job[:when] && !job[:when].in?(%w(on_success on_failure always)) + raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" end end private def validate_array_of_strings(values) - values.is_a?(Array) && values.all? {|tag| tag.is_a?(String)} + values.is_a?(Array) && values.all? { |value| validate_string(value) } end def validate_variables(variables) - variables.is_a?(Hash) && variables.all? {|key, value| key.is_a?(Symbol) && value.is_a?(String)} + variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) } + end + + def validate_string(value) + value.is_a?(String) || value.is_a?(Symbol) + end + + def validate_boolean(value) + value.in?([true, false]) + end + + def process?(only_params, except_params, ref, tag) + if only_params.present? + return false unless matching?(only_params, ref, tag) + end + + if except_params.present? + return false if matching?(except_params, ref, tag) + end + + true + end + + def matching?(patterns, ref, tag) + patterns.any? do |pattern| + match_ref?(pattern, ref, tag) + end + end + + def match_ref?(pattern, ref, tag) + pattern, path = pattern.split('@', 2) + return false if path && path != self.path + return true if tag && pattern == 'tags' + return true if !tag && pattern == 'branches' + + if pattern.first == "/" && pattern.last == "/" + Regexp.new(pattern[1...-1]) =~ ref + else + pattern == ref + end end end end diff --git a/lib/ci/migrate/builds.rb b/lib/ci/migrate/builds.rb deleted file mode 100644 index c4f62e55295..00000000000 --- a/lib/ci/migrate/builds.rb +++ /dev/null @@ -1,29 +0,0 @@ -module Ci - module Migrate - class Builds - attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir - - def initialize - @app_builds_dir = Settings.gitlab_ci.builds_path - @backup_dir = Gitlab.config.backup.path - @backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz') - end - - def restore - backup_existing_builds_dir - - FileUtils.mkdir_p(app_builds_dir, mode: 0700) - unless system('tar', '-C', app_builds_dir, '-zxf', backup_builds_tarball) - abort 'Restore failed'.red - end - end - - def backup_existing_builds_dir - timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") - if File.exists?(app_builds_dir) - FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) - end - end - end - end -end diff --git a/lib/ci/migrate/database.rb b/lib/ci/migrate/database.rb deleted file mode 100644 index bf9b80f1f62..00000000000 --- a/lib/ci/migrate/database.rb +++ /dev/null @@ -1,67 +0,0 @@ -require 'yaml' - -module Ci - module Migrate - class Database - attr_reader :config - - def initialize - @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] - end - - def restore - decompress_rd, decompress_wr = IO.pipe - decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name) - decompress_wr.close - - restore_pid = case config["adapter"] - when /^mysql/ then - $progress.print "Restoring MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysql', *mysql_args, config['database'], in: decompress_rd) - when "postgresql" then - $progress.print "Restoring PostgreSQL database #{config['database']} ... " - pg_env - spawn('psql', config['database'], in: decompress_rd) - end - decompress_rd.close - - success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? } - abort 'Restore failed' unless success - end - - protected - - def db_file_name - File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') - end - - def mysql_args - args = { - 'host' => '--host', - 'port' => '--port', - 'socket' => '--socket', - 'username' => '--user', - 'encoding' => '--default-character-set' - } - args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact - end - - def pg_env - ENV['PGUSER'] = config["username"] if config["username"] - ENV['PGHOST'] = config["host"] if config["host"] - ENV['PGPORT'] = config["port"].to_s if config["port"] - ENV['PGPASSWORD'] = config["password"].to_s if config["password"] - end - - def report_success(success) - if success - puts '[DONE]'.green - else - puts '[FAILED]'.red - end - end - end - end -end diff --git a/lib/ci/migrate/manager.rb b/lib/ci/migrate/manager.rb deleted file mode 100644 index e5e4fb784eb..00000000000 --- a/lib/ci/migrate/manager.rb +++ /dev/null @@ -1,72 +0,0 @@ -module Ci - module Migrate - class Manager - CI_IMPORT_PREFIX = '8.0' # Only allow imports from CI 8.0.x - - def cleanup - $progress.print "Deleting tmp directories ... " - - backup_contents.each do |dir| - next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) - - if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) - $progress.puts "done".green - else - puts "deleting tmp directory '#{dir}' failed".red - abort 'Backup failed' - end - end - end - - def unpack - Dir.chdir(Gitlab.config.backup.path) - - # check for existing backups in the backup dir - file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i } - puts "no backups found" if file_list.count == 0 - - if file_list.count > 1 && ENV["BACKUP"].nil? - puts "Found more than one backup, please specify which one you want to restore:" - puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" - exit 1 - end - - tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar") - - unless File.exists?(tar_file) - puts "The specified CI backup doesn't exist!" - exit 1 - end - - $progress.print "Unpacking backup ... " - - unless Kernel.system(*%W(tar -xf #{tar_file})) - puts "unpacking backup failed".red - exit 1 - else - $progress.puts "done".green - end - - ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 - - # restoring mismatching backups can lead to unexpected problems - if !settings[:gitlab_version].start_with?(CI_IMPORT_PREFIX) - puts "GitLab CI version mismatch:".red - puts " Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red - exit 1 - end - end - - private - - def backup_contents - ["db", "builds", "backup_information.yml"] - end - - def settings - @settings ||= YAML.load_file("backup_information.yml") - end - end - end -end - diff --git a/lib/ci/migrate/tags.rb b/lib/ci/migrate/tags.rb deleted file mode 100644 index 97e043ece27..00000000000 --- a/lib/ci/migrate/tags.rb +++ /dev/null @@ -1,42 +0,0 @@ -require 'yaml' - -module Ci - module Migrate - class Tags - def restore - puts 'Inserting tags...' - connection.select_all('SELECT ci_tags.name FROM ci_tags').each do |tag| - begin - connection.execute("INSERT INTO tags (name) VALUES(#{ActiveRecord::Base::sanitize(tag['name'])})") - rescue ActiveRecord::RecordNotUnique - end - end - - ActiveRecord::Base.transaction do - puts 'Deleting old taggings...' - connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'" - - puts 'Inserting taggings...' - connection.execute( - 'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' + - "SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " + - 'JOIN ci_tags ON ci_tags.id = ci_taggings.tag_id ' + - 'JOIN tags ON tags.name = ci_tags.name ' - ) - - puts 'Resetting counters... ' - connection.execute( - 'UPDATE tags SET ' + - 'taggings_count = (SELECT COUNT(*) FROM taggings WHERE tags.id = taggings.tag_id)' - ) - end - end - - protected - - def connection - ActiveRecord::Base.connection - end - end - end -end diff --git a/lib/ci/status.rb b/lib/ci/status.rb new file mode 100644 index 00000000000..c02b3b8f3e4 --- /dev/null +++ b/lib/ci/status.rb @@ -0,0 +1,21 @@ +module Ci + class Status + def self.get_status(statuses) + statuses.reject! { |status| status.try(&:allow_failure?) } + + if statuses.none? + 'skipped' + elsif statuses.all?(&:success?) + 'success' + elsif statuses.all?(&:pending?) + 'pending' + elsif statuses.any?(&:running?) || statuses.any?(&:pending?) + 'running' + elsif statuses.all?(&:canceled?) + 'canceled' + else + 'failed' + end + end + end +end diff --git a/lib/email_validator.rb b/lib/email_validator.rb deleted file mode 100644 index f509f0a5843..00000000000 --- a/lib/email_validator.rb +++ /dev/null @@ -1,21 +0,0 @@ -# Based on https://github.com/balexand/email_validator -# -# Extended to use only strict mode with following allowed characters: -# ' - apostrophe -# -# See http://www.remote.org/jochen/mail/info/chars.html -# -class EmailValidator < ActiveModel::EachValidator - @@default_options = {} - - def self.default_options - @@default_options - end - - def validate_each(record, attribute, value) - options = @@default_options.merge(self.options) - unless value =~ /\A\s*([-a-z0-9+._']{1,64})@((?:[-a-z0-9]+\.)+[a-z]{2,})\s*\z/i - record.errors.add(attribute, options[:message] || :invalid) - end - end -end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 163937c02cf..f15b2cfd231 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -47,7 +47,7 @@ class EventFilter actions << Event::COMMENTED if filter.include? 'comments' - events = events.where(action: actions) + events.where(action: actions) end def options(key) diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index 322aed5e27c..51e46da82cc 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -110,7 +110,7 @@ module ExtractsPath @project, @ref, @path) rescue RuntimeError, NoMethodError, InvalidPathError - not_found! + render_404 end def tree diff --git a/lib/file_streamer.rb b/lib/file_streamer.rb new file mode 100644 index 00000000000..4e3c6d3c773 --- /dev/null +++ b/lib/file_streamer.rb @@ -0,0 +1,16 @@ +class FileStreamer #:nodoc: + attr_reader :to_path + + def initialize(path) + @to_path = path + end + + # Stream the file's contents if Rack::Sendfile isn't present. + def each + File.open(to_path, 'rb') do |file| + while chunk = file.read(16384) + yield chunk + end + end + end +end diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 0353b3b7ed3..0d156047ff0 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -33,8 +33,11 @@ module Grack auth! + lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call + return lfs_response unless lfs_response.nil? + if project && authorized_request? - # Tell gitlab-git-http-server the request is OK, and what the GL_ID is + # Tell gitlab-workhorse the request is OK, and what the GL_ID is render_grack_auth_ok elsif @user.nil? && !@ci unauthorized @@ -72,7 +75,7 @@ module Grack matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login) if project && matched_login.present? && git_cmd == 'git-upload-pack' - underscored_service = matched_login['s'].underscore + underscored_service = matched_login['s'].underscore if Service.available_services_names.include?(underscored_service) service_method = "#{underscored_service}_service" @@ -193,7 +196,21 @@ module Grack end def render_grack_auth_ok - [200, { "Content-Type" => "application/json" }, [JSON.dump({ 'GL_ID' => Gitlab::ShellEnv.gl_id(@user) })]] + repo_path = + if @request.path_info =~ /^([\w\.\/-]+)\.wiki\.git/ + ProjectWiki.new(project).repository.path_to_repo + else + project.repository.path_to_repo + end + + [ + 200, + { "Content-Type" => "application/json" }, + [JSON.dump({ + 'GL_ID' => Gitlab::ShellEnv.gl_id(@user), + 'RepoPath' => repo_path, + })] + ] end def render_not_found diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 14ee4701e7b..87ac30b5ffe 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -1,10 +1,11 @@ module Gitlab class Shell - class AccessDenied < StandardError; end + class Error < StandardError; end class KeyAdder < Struct.new(:io) def add_key(id, key) - io.puts("#{id}\t#{key.strip}") + key.gsub!(/[[:space:]]+/, ' ').strip! + io.puts("#{id}\t#{key}") end end @@ -35,8 +36,9 @@ module Gitlab # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") # def import_repository(name, url) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'import-project', - "#{name}.git", url, '240']) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '240']) + raise Error, output unless status.zero? + true end # Move repository diff --git a/lib/gitlab/blacklist.rb b/lib/gitlab/blacklist.rb deleted file mode 100644 index 43145e0ee1b..00000000000 --- a/lib/gitlab/blacklist.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Gitlab - module Blacklist - extend self - - def path - %w( - admin - dashboard - files - groups - help - profile - projects - search - public - assets - u - s - teams - merge_requests - issues - users - snippets - services - repository - hooks - notes - unsubscribes - all - ci - ) - end - end -end diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index aeec595782c..9bef9037ad6 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -1,6 +1,12 @@ module Gitlab class ClosingIssueExtractor - ISSUE_CLOSING_REGEX = Regexp.new(Gitlab.config.gitlab.issue_closing_pattern) + ISSUE_CLOSING_REGEX = begin + link_pattern = URI.regexp(%w(http https)) + + pattern = Gitlab.config.gitlab.issue_closing_pattern + pattern = pattern.sub('%{issue_ref}', "(?:(?:#{link_pattern})|(?:#{Issue.reference_pattern}))") + Regexp.new(pattern).freeze + end def initialize(project, current_user = nil) @extractor = Gitlab::ReferenceExtractor.new(project, current_user) @@ -9,10 +15,12 @@ module Gitlab def closed_by_message(message) return [] if message.nil? - closing_statements = message.scan(ISSUE_CLOSING_REGEX). - map { |ref| ref[0] }.join(" ") + closing_statements = [] + message.scan(ISSUE_CLOSING_REGEX) do + closing_statements << Regexp.last_match[0] + end - @extractor.analyze(closing_statements) + @extractor.analyze(closing_statements.join(" ")) @extractor.issues end diff --git a/lib/gitlab/compare_result.rb b/lib/gitlab/compare_result.rb index d72391dade5..0d696a1ee28 100644 --- a/lib/gitlab/compare_result.rb +++ b/lib/gitlab/compare_result.rb @@ -2,8 +2,8 @@ module Gitlab class CompareResult attr_reader :commits, :diffs - def initialize(compare) - @commits, @diffs = compare.commits, compare.diffs + def initialize(compare, diff_options = {}) + @commits, @diffs = compare.commits, compare.diffs(nil, diff_options) end end end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 45bb904ed7a..8a7f8dc5003 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -12,7 +12,6 @@ module Gitlab @timestamps = {} date_from = 1.year.ago - date_to = Date.today events = Event.reorder(nil).contributions.where(author_id: user.id). where("created_at > ?", date_from).where(project_id: projects). diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 0ea1b6a2f6f..46a4ef0e31f 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -23,7 +23,9 @@ module Gitlab restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], max_attachment_size: Settings.gitlab['max_attachment_size'], session_expire_delay: Settings.gitlab['session_expire_delay'], - import_sources: Settings.gitlab['import_sources'] + import_sources: Settings.gitlab['import_sources'], + shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], + max_artifacts_size: Settings.artifacts['max_size'], ) end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb new file mode 100644 index 00000000000..71f37f1fef8 --- /dev/null +++ b/lib/gitlab/database.rb @@ -0,0 +1,11 @@ +module Gitlab + module Database + def self.mysql? + ActiveRecord::Base.connection.adapter_name.downcase == 'mysql2' + end + + def self.postgresql? + ActiveRecord::Base.connection.adapter_name.downcase == 'postgresql' + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 4daf65331e8..142058aa69d 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -44,6 +44,14 @@ module Gitlab diff.old_path end end + + def added_lines + diff_lines.select(&:added?).size + end + + def removed_lines + diff_lines.select(&:removed?).size + end end end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 8ac1b15e88a..0072194606e 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -7,6 +7,14 @@ module Gitlab @text, @type, @index = text, type, index @old_pos, @new_pos = old_pos, new_pos end + + def added? + type == 'new' + end + + def removed? + type == 'old' + end end end end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index c1d9520ddf1..7015fe36c3d 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -14,8 +14,6 @@ module Gitlab lines_arr = ::Gitlab::InlineDiff.processing lines lines_arr.each do |line| - raw_line = line.dup - next if filename?(line) full_line = html_escape(line.gsub(/\n/, '')) diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 61e08b23543..496256700b8 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -154,7 +154,7 @@ module Gitlab while comment = comments.shift verb = comment['sVerb'] - next if verb == 'Opened' || verb === 'Closed' + next if verb == 'Opened' content = format_content(comment['s']) attachments = format_attachments(comment['rgAttachments']) diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index f02ea43910f..8b1b6f48ed5 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -23,7 +23,7 @@ module Gitlab import_url: Project::UNKNOWN_IMPORT_URL ).execute - import_data = project.create_import_data( + project.create_import_data( data: { 'repo' => repo.raw_data, 'user_map' => user_map, diff --git a/lib/gitlab/force_push_check.rb b/lib/gitlab/force_push_check.rb index fdb6a35c78d..93c6a5bb7f5 100644 --- a/lib/gitlab/force_push_check.rb +++ b/lib/gitlab/force_push_check.rb @@ -7,7 +7,7 @@ module Gitlab if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) false else - missed_refs, _ = Gitlab::Popen.popen(%W(git --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})) + missed_refs, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})) missed_refs.split("\n").size > 0 end end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index dd393fe09d2..07b856ca64c 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -16,6 +16,17 @@ module Gitlab def trigger(gl_id, oldrev, newrev, ref) return true unless exists? + case name + when "pre-receive", "post-receive" + call_receive_hook(gl_id, oldrev, newrev, ref) + when "update" + call_update_hook(gl_id, oldrev, newrev, ref) + end + end + + private + + def call_receive_hook(gl_id, oldrev, newrev, ref) changes = [oldrev, newrev, ref].join(" ") # function will return true if succesful @@ -54,6 +65,12 @@ module Gitlab exit_status end + + def call_update_hook(gl_id, oldrev, newrev, ref) + Dir.chdir(repo_path) do + system({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) + end + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index c90184d31cf..3ed1eec517c 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -13,7 +13,7 @@ module Gitlab def user return @user if defined?(@user) - @user = + @user = case actor when User actor @@ -125,7 +125,7 @@ module Gitlab def change_access_check(change) oldrev, newrev, ref = change.split(' ') - action = + action = if project.protected_branch?(branch_name(ref)) protected_branch_action(oldrev, newrev, branch_name(ref)) elsif protected_tag?(tag_name(ref)) @@ -148,7 +148,7 @@ module Gitlab build_status_object(false, "You are not allowed to change existing tags on this project.") else # :push_code build_status_object(false, "You are not allowed to push code to this project.") - end + end return status end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 39d17def930..4d83d8e72a8 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -6,7 +6,7 @@ module Gitlab # Returns true for a valid reference name, false otherwise def validate(ref_name) Gitlab::Utils.system_silent( - %W(git check-ref-format refs/#{ref_name})) + %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name})) end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 270cbcd9ccd..74d1529e1ff 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -46,7 +46,7 @@ module Gitlab end def github_options - OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + OmniAuth::Strategies::GitHub.default_options[:client_options].to_h.symbolize_keys end end end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index bd7340a80f1..b5720f6e2cb 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -19,7 +19,7 @@ module Gitlab if issue.pull_request.nil? body = @formatter.author_line(issue.user.login) - body += issue.body + body += issue.body || "" if issue.comments > 0 body += @formatter.comments_header diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index 9c00896c913..86fb6c51765 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -75,7 +75,7 @@ module Gitlab end def gitlab_options - OmniAuth::Strategies::GitLab.default_options[:client_options].symbolize_keys + OmniAuth::Strategies::GitLab.default_options[:client_options].to_h.symbolize_keys end end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 03c410726a5..87fee28dc01 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -30,7 +30,7 @@ module Gitlab def user_map @user_map ||= begin - user_map = Hash.new do |hash, user| + user_map = Hash.new do |hash, user| # Replace ... by \.\.\., so `johnsm...@gmail.com` isn't autolinked. Client.mask_email(user).sub("...", "\\.\\.\\.") end @@ -76,18 +76,7 @@ module Gitlab attachments = format_attachments(raw_issue["id"], 0, issue_comment["attachments"]) body = format_issue_body(author, date, content, attachments) - - labels = [] - raw_issue["labels"].each do |label| - name = nice_label_name(label) - labels << name - - unless @known_labels.include?(name) - create_label(name) - @known_labels << name - end - end - labels << nice_status_name(raw_issue["status"]) + labels = import_issue_labels(raw_issue) assignee_id = nil if raw_issue.has_key?("owner") @@ -110,6 +99,7 @@ module Gitlab assignee_id: assignee_id, state: raw_issue["state"] == "closed" ? "closed" : "opened" ) + issue.add_labels_by_names(labels) if issue.iid != raw_issue["id"] @@ -120,6 +110,23 @@ module Gitlab end end + def import_issue_labels(raw_issue) + labels = [] + + raw_issue["labels"].each do |label| + name = nice_label_name(label) + labels << name + + unless @known_labels.include?(name) + create_label(name) + @known_labels << name + end + end + + labels << nice_status_name(raw_issue["status"]) + labels + end + def import_issue_comments(issue, comments) Note.transaction do while raw_comment = comments.shift @@ -172,7 +179,7 @@ module Gitlab "#5cb85c" when "Status: Started" "#8e44ad" - + when "Priority: Critical" "#ffcfcf" when "Priority: High" @@ -181,7 +188,7 @@ module Gitlab "#fff5cc" when "Priority: Low" "#cfe9ff" - + when "Type: Defect" "#d9534f" when "Type: Enhancement" @@ -249,8 +256,8 @@ module Gitlab end if raw_updates.has_key?("cc") - cc = raw_updates["cc"].map do |l| - deleted = l.start_with?("-") + cc = raw_updates["cc"].map do |l| + deleted = l.start_with?("-") l = l[1..-1] if deleted l = user_map[l] l = "~~#{l}~~" if deleted @@ -261,8 +268,8 @@ module Gitlab end if raw_updates.has_key?("labels") - labels = raw_updates["labels"].map do |l| - deleted = l.start_with?("-") + labels = raw_updates["labels"].map do |l| + deleted = l.start_with?("-") l = l[1..-1] if deleted l = nice_label_name(l) l = "~~#{l}~~" if deleted @@ -278,45 +285,39 @@ module Gitlab if raw_updates.has_key?("blockedOn") blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on| - name, id = raw_blocked_on.split(":", 2) - - deleted = name.start_with?("-") - name = name[1..-1] if deleted - - text = - if name == project.import_source - "##{id}" - else - "#{project.namespace.path}/#{name}##{id}" - end - text = "~~#{text}~~" if deleted - text + format_blocking_updates(raw_blocked_on) end + updates << "*Blocked on: #{blocked_ons.join(", ")}*" end if raw_updates.has_key?("blocking") blockings = raw_updates["blocking"].map do |raw_blocked_on| - name, id = raw_blocked_on.split(":", 2) - - deleted = name.start_with?("-") - name = name[1..-1] if deleted - - text = - if name == project.import_source - "##{id}" - else - "#{project.namespace.path}/#{name}##{id}" - end - text = "~~#{text}~~" if deleted - text + format_blocking_updates(raw_blocked_on) end + updates << "*Blocking: #{blockings.join(", ")}*" end updates end + def format_blocking_updates(raw_blocked_on) + name, id = raw_blocked_on.split(":", 2) + + deleted = name.start_with?("-") + name = name[1..-1] if deleted + + text = + if name == project.import_source + "##{id}" + else + "#{project.namespace.path}/#{name}##{id}" + end + text = "~~#{text}~~" if deleted + text + end + def format_attachments(issue_id, comment_id, raw_attachments) return [] unless raw_attachments @@ -325,7 +326,7 @@ module Gitlab filename = attachment["fileName"] link = "https://storage.googleapis.com/google-code-attachments/#{@repo.name}/issue-#{issue_id}/comment-#{comment_id}/#{filename}" - + text = "[#{filename}](#{link})" text = "!#{text}" if filename =~ /\.(png|jpg|jpeg|gif|bmp|tiff)\z/i text diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb index 0cfeaf9d61c..1cb7d16aeb3 100644 --- a/lib/gitlab/google_code_import/project_creator.rb +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -23,7 +23,7 @@ module Gitlab import_url: repo.import_url ).execute - import_data = project.create_import_data( + project.create_import_data( data: { "repo" => repo.raw_data, "user_map" => user_map diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 856ccc71084..9068d79c95e 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -24,12 +24,12 @@ module Gitlab match[1] end - private - def config Gitlab.config.incoming_email end + private + def address_regex wildcard_address = config.address return nil unless wildcard_address diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb index 99e7b529ba9..44507bde25d 100644 --- a/lib/gitlab/inline_diff.rb +++ b/lib/gitlab/inline_diff.rb @@ -11,48 +11,71 @@ module Gitlab indexes.each do |index| first_line = diff_arr[index+1] second_line = diff_arr[index+2] - max_length = [first_line.size, second_line.size].max # Skip inline diff if empty line was replaced with content next if first_line == "-\n" - first_the_same_symbols = 0 - (0..max_length + 1).each do |i| - first_the_same_symbols = i - 1 - if first_line[i] != second_line[i] && i > 0 - break - end - end + first_token = find_first_token(first_line, second_line) + apply_first_token(diff_arr, index, first_token) + + last_token = find_last_token(first_line, second_line, first_token) + apply_last_token(diff_arr, index, last_token) + end + + diff_arr + end + + def apply_first_token(diff_arr, index, first_token) + start = first_token + START + + if first_token.empty? + # In case if we remove string of spaces in commit + diff_arr[index+1].sub!("-", "-" => "-#{START}") + diff_arr[index+2].sub!("+", "+" => "+#{START}") + else + diff_arr[index+1].sub!(first_token, first_token => start) + diff_arr[index+2].sub!(first_token, first_token => start) + end + end - first_token = first_line[0..first_the_same_symbols][1..-1] - start = first_token + START + def apply_last_token(diff_arr, index, last_token) + # This is tricky: escape backslashes so that `sub` doesn't interpret them + # as backreferences. Regexp.escape does NOT do the right thing. + replace_token = FINISH + last_token.gsub(/\\/, '\&\&') + diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) + diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) + end + + def find_first_token(first_line, second_line) + max_length = [first_line.size, second_line.size].max + first_the_same_symbols = 0 + + (0..max_length + 1).each do |i| + first_the_same_symbols = i - 1 - if first_token.empty? - # In case if we remove string of spaces in commit - diff_arr[index+1].sub!("-", "-" => "-#{START}") - diff_arr[index+2].sub!("+", "+" => "+#{START}") - else - diff_arr[index+1].sub!(first_token, first_token => start) - diff_arr[index+2].sub!(first_token, first_token => start) + if first_line[i] != second_line[i] && i > 0 + break end + end + + first_line[0..first_the_same_symbols][1..-1] + end + + def find_last_token(first_line, second_line, first_token) + max_length = [first_line.size, second_line.size].max + last_the_same_symbols = 0 + + (1..max_length + 1).each do |i| + last_the_same_symbols = -i + shortest_line = second_line.size > first_line.size ? first_line : second_line - last_the_same_symbols = 0 - (1..max_length + 1).each do |i| - last_the_same_symbols = -i - shortest_line = second_line.size > first_line.size ? first_line : second_line - if ( first_line[-i] != second_line[-i] ) || "#{first_token}#{START}".size == shortest_line[1..-i].size - break - end + if (first_line[-i] != second_line[-i]) || "#{first_token}#{START}".size == shortest_line[1..-i].size + break end - last_the_same_symbols += 1 - last_token = first_line[last_the_same_symbols..-1] - # This is tricky: escape backslashes so that `sub` doesn't interpret them - # as backreferences. Regexp.escape does NOT do the right thing. - replace_token = FINISH + last_token.gsub(/\\/, '\&\&') - diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) end - diff_arr + + last_the_same_symbols += 1 + first_line[last_the_same_symbols..-1] end def _indexes_of_changed_lines(diff_arr) diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index cb66fd500fe..4be99dd88c2 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -14,7 +14,7 @@ module Gitlab # LDAP distinguished name is case-insensitive identity = ::Identity. where(provider: provider). - where('lower(extern_uid) = ?', uid.downcase).last + where('lower(extern_uid) = ?', uid.mb_chars.downcase.to_s).last identity && identity.user end end @@ -35,7 +35,7 @@ module Gitlab end def find_by_email - ::User.find_by(email: auth_hash.email) + ::User.find_by(email: auth_hash.email.downcase) end def update_user_attributes diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb new file mode 100644 index 00000000000..9be9a65671b --- /dev/null +++ b/lib/gitlab/lfs/response.rb @@ -0,0 +1,327 @@ +module Gitlab + module Lfs + class Response + + def initialize(project, user, request) + @origin_project = project + @project = storage_project(project) + @user = user + @env = request.env + @request = request + end + + def render_download_object_response(oid) + render_response_to_download do + if check_download_sendfile_header? + render_lfs_sendfile(oid) + else + render_not_found + end + end + end + + def render_batch_operation_response + request_body = JSON.parse(@request.body.read) + case request_body["operation"] + when "download" + render_batch_download(request_body) + when "upload" + render_batch_upload(request_body) + else + render_not_found + end + end + + def render_storage_upload_authorize_response(oid, size) + render_response_to_push do + [ + 200, + { "Content-Type" => "application/json; charset=utf-8" }, + [JSON.dump({ + 'StoreLFSPath' => "#{Gitlab.config.lfs.storage_path}/tmp/upload", + 'LfsOid' => oid, + 'LfsSize' => size + })] + ] + end + end + + def render_storage_upload_store_response(oid, size, tmp_file_name) + render_response_to_push do + render_lfs_upload_ok(oid, size, tmp_file_name) + end + end + + def render_unsupported_deprecated_api + [ + 501, + { "Content-Type" => "application/json; charset=utf-8" }, + [JSON.dump({ + 'message' => 'Server supports batch API only, please update your Git LFS client to version 1.0.1 and up.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + private + + def render_not_enabled + [ + 501, + { + "Content-Type" => "application/json; charset=utf-8", + }, + [JSON.dump({ + 'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_unauthorized + [ + 401, + { + 'Content-Type' => 'text/plain' + }, + ['Unauthorized'] + ] + end + + def render_not_found + [ + 404, + { + "Content-Type" => "application/vnd.git-lfs+json" + }, + [JSON.dump({ + 'message' => 'Not found.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_forbidden + [ + 403, + { + "Content-Type" => "application/vnd.git-lfs+json" + }, + [JSON.dump({ + 'message' => 'Access forbidden. Check your access level.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_lfs_sendfile(oid) + return render_not_found unless oid.present? + + lfs_object = object_for_download(oid) + + if lfs_object && lfs_object.file.exists? + [ + 200, + { + # GitLab-workhorse will forward Content-Type header + "Content-Type" => "application/octet-stream", + "X-Sendfile" => lfs_object.file.path + }, + [] + ] + else + render_not_found + end + end + + def render_batch_upload(body) + return render_not_found if body.empty? || body['objects'].nil? + + render_response_to_push do + response = build_upload_batch_response(body['objects']) + [ + 200, + { + "Content-Type" => "application/json; charset=utf-8", + "Cache-Control" => "private", + }, + [JSON.dump(response)] + ] + end + end + + def render_batch_download(body) + return render_not_found if body.empty? || body['objects'].nil? + + render_response_to_download do + response = build_download_batch_response(body['objects']) + [ + 200, + { + "Content-Type" => "application/json; charset=utf-8", + "Cache-Control" => "private", + }, + [JSON.dump(response)] + ] + end + end + + def render_lfs_upload_ok(oid, size, tmp_file) + if store_file(oid, size, tmp_file) + [ + 200, + { + 'Content-Type' => 'text/plain', + 'Content-Length' => 0 + }, + [] + ] + else + [ + 422, + { 'Content-Type' => 'text/plain' }, + ["Unprocessable entity"] + ] + end + end + + def render_response_to_download + return render_not_enabled unless Gitlab.config.lfs.enabled + + unless @project.public? + return render_unauthorized unless @user + return render_forbidden unless user_can_fetch? + end + + yield + end + + def render_response_to_push + return render_not_enabled unless Gitlab.config.lfs.enabled + return render_unauthorized unless @user + return render_forbidden unless user_can_push? + + yield + end + + def check_download_sendfile_header? + @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile" + end + + def user_can_fetch? + # Check user access against the project they used to initiate the pull + @user.can?(:download_code, @origin_project) + end + + def user_can_push? + # Check user access against the project they used to initiate the push + @user.can?(:push_code, @origin_project) + end + + def storage_project(project) + if project.forked? + project.forked_from_project + else + project + end + end + + def store_file(oid, size, tmp_file) + tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file) + + object = LfsObject.find_or_create_by(oid: oid, size: size) + if object.file.exists? + success = true + else + success = move_tmp_file_to_storage(object, tmp_file_path) + end + + if success + success = link_to_project(object) + end + + success + ensure + # Ensure that the tmp file is removed + FileUtils.rm_f(tmp_file_path) + end + + def object_for_download(oid) + @project.lfs_objects.find_by(oid: oid) + end + + def move_tmp_file_to_storage(object, path) + File.open(path) do |f| + object.file = f + end + + object.file.store! + object.save + end + + def link_to_project(object) + if object && !object.projects.exists?(@project.id) + object.projects << @project + object.save + end + end + + def select_existing_objects(objects) + objects_oids = objects.map { |o| o['oid'] } + @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set + end + + def build_upload_batch_response(objects) + selected_objects = select_existing_objects(objects) + + upload_hypermedia_links(objects, selected_objects) + end + + def build_download_batch_response(objects) + selected_objects = select_existing_objects(objects) + + download_hypermedia_links(objects, selected_objects) + end + + def download_hypermedia_links(all_objects, existing_objects) + all_objects.each do |object| + if existing_objects.include?(object['oid']) + object['actions'] = { + 'download' => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}", + 'header' => { + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } + } + else + object['error'] = { + 'code' => 404, + 'message' => "Object does not exist on the server or you don't have permissions to access it", + } + end + end + + { 'objects' => all_objects } + end + + def upload_hypermedia_links(all_objects, existing_objects) + all_objects.each do |object| + # generate actions only for non-existing objects + next if existing_objects.include?(object['oid']) + + object['actions'] = { + 'upload' => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", + 'header' => { + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } + } + end + + { 'objects' => all_objects } + end + end + end +end diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb new file mode 100644 index 00000000000..78d02891102 --- /dev/null +++ b/lib/gitlab/lfs/router.rb @@ -0,0 +1,97 @@ +module Gitlab + module Lfs + class Router + def initialize(project, user, request) + @project = project + @user = user + @env = request.env + @request = request + end + + def try_call + return unless @request && @request.path.present? + + case @request.request_method + when 'GET' + get_response + when 'POST' + post_response + when 'PUT' + put_response + else + nil + end + end + + private + + def get_response + path_match = @request.path.match(/\/(info\/lfs|gitlab-lfs)\/objects\/([0-9a-f]{64})$/) + return nil unless path_match + + oid = path_match[2] + return nil unless oid + + case path_match[1] + when "info/lfs" + lfs.render_unsupported_deprecated_api + when "gitlab-lfs" + lfs.render_download_object_response(oid) + else + nil + end + end + + def post_response + post_path = @request.path.match(/\/info\/lfs\/objects(\/batch)?$/) + return nil unless post_path + + # Check for Batch API + if post_path[0].ends_with?("/info/lfs/objects/batch") + lfs.render_batch_operation_response + elsif post_path[0].ends_with?("/info/lfs/objects") + lfs.render_unsupported_deprecated_api + else + nil + end + end + + def put_response + object_match = @request.path.match(/\/gitlab-lfs\/objects\/([0-9a-f]{64})\/([0-9]+)(|\/authorize){1}$/) + return nil if object_match.nil? + + oid = object_match[1] + size = object_match[2].try(:to_i) + return nil if oid.nil? || size.nil? + + # GitLab-workhorse requests + # 1. Try to authorize the request + # 2. send a request with a header containing the name of the temporary file + if object_match[3] && object_match[3] == '/authorize' + lfs.render_storage_upload_authorize_response(oid, size) + else + tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP']) + return nil unless tmp_file_name + + lfs.render_storage_upload_store_response(oid, size, tmp_file_name) + end + end + + def lfs + return unless @project + + Gitlab::Lfs::Response.new(@project, @user, @request) + end + + def sanitize_tmp_filename(name) + if name.present? + name.gsub!(/^.*(\\|\/)/, '') + name = name.match(/[0-9a-f]{73}/) + name[0] if name + else + nil + end + end + end + end +end diff --git a/lib/gitlab/markdown.rb b/lib/gitlab/markdown.rb index ae5f2544691..886a09f52af 100644 --- a/lib/gitlab/markdown.rb +++ b/lib/gitlab/markdown.rb @@ -7,6 +7,14 @@ module Gitlab module Markdown # Convert a Markdown String into an HTML-safe String of HTML # + # Note that while the returned HTML will have been sanitized of dangerous + # HTML, it may post a risk of information leakage if it's not also passed + # through `post_process`. + # + # Also note that the returned String is always HTML, not XHTML. Views + # requiring XHTML, such as Atom feeds, need to call `post_process` on the + # result, providing the appropriate `pipeline` option. + # # markdown - Markdown String # context - Hash of context options passed to our HTML Pipeline # @@ -31,6 +39,33 @@ module Gitlab renderer.render(markdown) end + # Perform post-processing on an HTML String + # + # This method is used to perform state-dependent changes to a String of + # HTML, such as removing references that the current user doesn't have + # permission to make (`RedactorFilter`). + # + # html - String to process + # options - Hash of options to customize output + # :pipeline - Symbol pipeline type + # :project - Project + # :user - User object + # + # Returns an HTML-safe String + def self.post_process(html, options) + context = { + project: options[:project], + current_user: options[:user] + } + doc = post_processor.to_document(html, context) + + if options[:pipeline] == :atom + doc.to_html(save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML) + else + doc.to_html + end.html_safe + end + # Provide autoload paths for filters to prevent a circular dependency error autoload :AutolinkFilter, 'gitlab/markdown/autolink_filter' autoload :CommitRangeReferenceFilter, 'gitlab/markdown/commit_range_reference_filter' @@ -41,6 +76,7 @@ module Gitlab autoload :IssueReferenceFilter, 'gitlab/markdown/issue_reference_filter' autoload :LabelReferenceFilter, 'gitlab/markdown/label_reference_filter' autoload :MergeRequestReferenceFilter, 'gitlab/markdown/merge_request_reference_filter' + autoload :RedactorFilter, 'gitlab/markdown/redactor_filter' autoload :RelativeLinkFilter, 'gitlab/markdown/relative_link_filter' autoload :SanitizationFilter, 'gitlab/markdown/sanitization_filter' autoload :SnippetReferenceFilter, 'gitlab/markdown/snippet_reference_filter' @@ -48,27 +84,22 @@ module Gitlab autoload :TableOfContentsFilter, 'gitlab/markdown/table_of_contents_filter' autoload :TaskListFilter, 'gitlab/markdown/task_list_filter' autoload :UserReferenceFilter, 'gitlab/markdown/user_reference_filter' + autoload :UploadLinkFilter, 'gitlab/markdown/upload_link_filter' - # Public: Parse the provided text with GitLab-Flavored Markdown + # Public: Parse the provided HTML with GitLab-Flavored Markdown + # + # html - HTML String + # options - A Hash of options used to customize output (default: {}) + # :no_header_anchors - Disable header anchors in TableOfContentsFilter + # :path - Current path String + # :pipeline - Symbol pipeline type + # :project - Current Project object + # :project_wiki - Current ProjectWiki object + # :ref - Current ref String # - # text - the source text - # options - A Hash of options used to customize output (default: {}): - # :xhtml - output XHTML instead of HTML - # :reference_only_path - Use relative path for reference links - def self.gfm(text, options = {}) - return text if text.nil? - - # Duplicate the string so we don't alter the original, then call to_str - # to cast it back to a String instead of a SafeBuffer. This is required - # for gsub calls to work as we need them to. - text = text.dup.to_str - - options.reverse_merge!( - xhtml: false, - reference_only_path: true, - project: options[:project], - current_user: options[:current_user] - ) + # Returns an HTML-safe String + def self.gfm(html, options = {}) + return '' unless html.present? @pipeline ||= HTML::Pipeline.new(filters) @@ -77,41 +108,36 @@ module Gitlab pipeline: options[:pipeline], # EmojiFilter - asset_root: Gitlab.config.gitlab.base_url, asset_host: Gitlab::Application.config.asset_host, - - # TableOfContentsFilter - no_header_anchors: options[:no_header_anchors], + asset_root: Gitlab.config.gitlab.base_url, # ReferenceFilter - current_user: options[:current_user], - only_path: options[:reference_only_path], - project: options[:project], + only_path: only_path_pipeline?(options[:pipeline]), + project: options[:project], # RelativeLinkFilter + project_wiki: options[:project_wiki], ref: options[:ref], requested_path: options[:path], - project_wiki: options[:project_wiki] - } - - result = @pipeline.call(text, context) - - save_options = 0 - if options[:xhtml] - save_options |= Nokogiri::XML::Node::SaveOptions::AS_XHTML - end - text = result[:output].to_html(save_with: save_options) + # TableOfContentsFilter + no_header_anchors: options[:no_header_anchors] + } - text.html_safe + @pipeline.to_html(html, context).html_safe end private - def self.renderer - @markdown ||= begin - renderer = Redcarpet::Render::HTML.new - Redcarpet::Markdown.new(renderer, redcarpet_options) + # Check if a pipeline enables the `only_path` context option + # + # Returns Boolean + def self.only_path_pipeline?(pipeline) + case pipeline + when :atom, :email + false + else + true end end @@ -129,6 +155,17 @@ module Gitlab }.freeze end + def self.renderer + @markdown ||= begin + renderer = Redcarpet::Render::HTML.new + Redcarpet::Markdown.new(renderer, redcarpet_options) + end + end + + def self.post_processor + @post_processor ||= HTML::Pipeline.new([Gitlab::Markdown::RedactorFilter]) + end + # Filters used in our pipeline # # SanitizationFilter should come first so that all generated reference HTML @@ -140,7 +177,7 @@ module Gitlab Gitlab::Markdown::SyntaxHighlightFilter, Gitlab::Markdown::SanitizationFilter, - Gitlab::Markdown::RelativeLinkFilter, + Gitlab::Markdown::UploadLinkFilter, Gitlab::Markdown::EmojiFilter, Gitlab::Markdown::TableOfContentsFilter, Gitlab::Markdown::AutolinkFilter, @@ -155,6 +192,8 @@ module Gitlab Gitlab::Markdown::CommitReferenceFilter, Gitlab::Markdown::LabelReferenceFilter, + Gitlab::Markdown::RelativeLinkFilter, + Gitlab::Markdown::TaskListFilter ] end diff --git a/lib/gitlab/markdown/abstract_reference_filter.rb b/lib/gitlab/markdown/abstract_reference_filter.rb new file mode 100644 index 00000000000..9488e980c08 --- /dev/null +++ b/lib/gitlab/markdown/abstract_reference_filter.rb @@ -0,0 +1,145 @@ +require 'gitlab/markdown' + +module Gitlab + module Markdown + # Issues, Merge Requests, Snippets, Commits and Commit Ranges share + # similar functionality in reference filtering. + class AbstractReferenceFilter < ReferenceFilter + include CrossProjectReference + + def self.object_class + # Implement in child class + # Example: MergeRequest + end + + def self.object_name + object_class.name.underscore + end + + def self.object_sym + object_name.to_sym + end + + def self.data_reference + "data-#{object_name.dasherize}" + end + + # Public: Find references in text (like `!123` for merge requests) + # + # AnyReferenceFilter.references_in(text) do |match, id, project_ref, matches| + # object = find_object(project_ref, id) + # "<a href=...>#{object.to_reference}</a>" + # end + # + # text - String text to search. + # + # Yields the String match, the Integer referenced object ID, an optional String + # of the external project reference, and all of the matchdata. + # + # Returns a String replaced with the return of the block. + def self.references_in(text, pattern = object_class.reference_pattern) + text.gsub(pattern) do |match| + yield match, $~[object_sym].to_i, $~[:project], $~ + end + end + + def self.referenced_by(node) + { object_sym => LazyReference.new(object_class, node.attr(data_reference)) } + end + + delegate :object_class, :object_sym, :references_in, to: :class + + def find_object(project, id) + # Implement in child class + # Example: project.merge_requests.find + end + + def url_for_object(object, project) + # Implement in child class + # Example: project_merge_request_url + end + + def call + # `#123` + replace_text_nodes_matching(object_class.reference_pattern) do |content| + object_link_filter(content, object_class.reference_pattern) + end + + # `[Issue](#123)`, which is turned into + # `<a href="#123">Issue</a>` + replace_link_nodes_with_href(object_class.reference_pattern) do |link, text| + object_link_filter(link, object_class.reference_pattern, link_text: text) + end + + # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into + # `<a href="http://gitlab.example.com/namespace/project/issues/123">http://gitlab.example.com/namespace/project/issues/123</a>` + replace_link_nodes_with_text(object_class.link_reference_pattern) do |text| + object_link_filter(text, object_class.link_reference_pattern) + end + + # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into + # `<a href="http://gitlab.example.com/namespace/project/issues/123">Issue</a>` + replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text| + object_link_filter(link, object_class.link_reference_pattern, link_text: text) + end + end + + # Replace references (like `!123` for merge requests) in text with links + # to the referenced object's details page. + # + # text - String text to replace references in. + # pattern - Reference pattern to match against. + # link_text - Original content of the link being replaced. + # + # Returns a String with references replaced with links. All links + # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. + def object_link_filter(text, pattern, link_text: nil) + references_in(text, pattern) do |match, id, project_ref, matches| + project = project_from_ref(project_ref) + + if project && object = find_object(project, id) + title = escape_once(object_link_title(object)) + klass = reference_class(object_sym) + + data = data_attribute( + original: link_text || match, + project: project.id, + object_sym => object.id + ) + + url = matches[:url] if matches.names.include?("url") + url ||= url_for_object(object, project) + + text = link_text + unless text + text = object.reference_link_text(context[:project]) + + extras = object_link_text_extras(object, matches) + text += " (#{extras.join(", ")})" if extras.any? + end + + %(<a href="#{url}" #{data} + title="#{title}" + class="#{klass}">#{text}</a>) + else + match + end + end + end + + def object_link_text_extras(object, matches) + extras = [] + + if matches.names.include?("anchor") && matches[:anchor] && matches[:anchor] =~ /\A\#note_(\d+)\z/ + extras << "comment #{$1}" + end + + extras + end + + def object_link_title(object) + "#{object_class.name.titleize}: #{object.title}" + end + end + end +end diff --git a/lib/gitlab/markdown/commit_range_reference_filter.rb b/lib/gitlab/markdown/commit_range_reference_filter.rb index bb496135d92..36b3258ef76 100644 --- a/lib/gitlab/markdown/commit_range_reference_filter.rb +++ b/lib/gitlab/markdown/commit_range_reference_filter.rb @@ -5,78 +5,54 @@ module Gitlab # HTML filter that replaces commit range references with links. # # This filter supports cross-project references. - class CommitRangeReferenceFilter < ReferenceFilter - include CrossProjectReference + class CommitRangeReferenceFilter < AbstractReferenceFilter + def self.object_class + CommitRange + end - # Public: Find commit range references in text - # - # CommitRangeReferenceFilter.references_in(text) do |match, commit_range, project_ref| - # "<a href=...>#{commit_range}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the String commit range, and an optional String - # of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(CommitRange.reference_pattern) do |match| - yield match, $~[:commit_range], $~[:project] + def self.references_in(text, pattern = CommitRange.reference_pattern) + text.gsub(pattern) do |match| + yield match, $~[:commit_range], $~[:project], $~ end end - def initialize(*args) - super + def self.referenced_by(node) + project = Project.find(node.attr("data-project")) rescue nil + return unless project - @commit_map = {} - end + id = node.attr("data-commit-range") + range = find_object(project, id) - def call - replace_text_nodes_matching(CommitRange.reference_pattern) do |content| - commit_range_link_filter(content) - end - end + return unless range - # Replace commit range references in text with links to compare the commit - # ranges. - # - # text - String text to replace references in. - # - # Returns a String with commit range references replaced with links. All - # links have `gfm` and `gfm-commit_range` class names attached for - # styling. - def commit_range_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - range = CommitRange.new(id, project) + { commit_range: range } + end - if range.valid_commits? - push_result(:commit_range, range) + def initialize(*args) + super - url = url_for_commit_range(project, range) + @commit_map = {} + end - title = range.reference_title - klass = reference_class(:commit_range) - data = data_attribute(project.id) + def self.find_object(project, id) + range = CommitRange.new(id, project) - project_ref += '@' if project_ref + range.valid_commits? ? range : nil + end - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{project_ref}#{range}</a>) - else - match - end - end + def find_object(*args) + self.class.find_object(*args) end - def url_for_commit_range(project, range) + def url_for_object(range, project) h = Gitlab::Application.routes.url_helpers h.namespace_project_compare_url(project.namespace, project, range.to_param.merge(only_path: context[:only_path])) end + + def object_link_title(range) + range.reference_title + end end end end diff --git a/lib/gitlab/markdown/commit_reference_filter.rb b/lib/gitlab/markdown/commit_reference_filter.rb index fcbb2e936a5..b4036578e60 100644 --- a/lib/gitlab/markdown/commit_reference_filter.rb +++ b/lib/gitlab/markdown/commit_reference_filter.rb @@ -5,74 +5,48 @@ module Gitlab # HTML filter that replaces commit references with links. # # This filter supports cross-project references. - class CommitReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find commit references in text - # - # CommitReferenceFilter.references_in(text) do |match, commit, project_ref| - # "<a href=...>#{commit}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the String commit identifier, and an optional - # String of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Commit.reference_pattern) do |match| - yield match, $~[:commit], $~[:project] - end + class CommitReferenceFilter < AbstractReferenceFilter + def self.object_class + Commit end - def call - replace_text_nodes_matching(Commit.reference_pattern) do |content| - commit_link_filter(content) + def self.references_in(text, pattern = Commit.reference_pattern) + text.gsub(pattern) do |match| + yield match, $~[:commit], $~[:project], $~ end end - # Replace commit references in text with links to the commit specified. - # - # text - String text to replace references in. - # - # Returns a String with commit references replaced with links. All links - # have `gfm` and `gfm-commit` class names attached for styling. - def commit_link_filter(text) - self.class.references_in(text) do |match, commit_ref, project_ref| - project = self.project_from_ref(project_ref) - - if commit = commit_from_ref(project, commit_ref) - push_result(:commit, commit) + def self.referenced_by(node) + project = Project.find(node.attr("data-project")) rescue nil + return unless project - url = url_for_commit(project, commit) + id = node.attr("data-commit") + commit = find_object(project, id) - title = escape_once(commit.link_title) - klass = reference_class(:commit) - data = data_attribute(project.id) + return unless commit - project_ref += '@' if project_ref - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{project_ref}#{commit.short_id}</a>) - else - match - end - end + { commit: commit } end - def commit_from_ref(project, commit_ref) + def self.find_object(project, id) if project && project.valid_repo? - project.commit(commit_ref) + project.commit(id) end end - def url_for_commit(project, commit) + def find_object(*args) + self.class.find_object(*args) + end + + def url_for_object(commit, project) h = Gitlab::Application.routes.url_helpers h.namespace_project_commit_url(project.namespace, project, commit, only_path: context[:only_path]) end + + def object_link_title(commit) + commit.link_title + end end end end diff --git a/lib/gitlab/markdown/cross_project_reference.rb b/lib/gitlab/markdown/cross_project_reference.rb index 855748fdccc..6ab04a584b0 100644 --- a/lib/gitlab/markdown/cross_project_reference.rb +++ b/lib/gitlab/markdown/cross_project_reference.rb @@ -13,18 +13,11 @@ module Gitlab # # ref - String reference. # - # Returns a Project, or nil if the reference can't be accessed + # Returns a Project, or nil if the reference can't be found def project_from_ref(ref) return context[:project] unless ref - other = Project.find_with_namespace(ref) - return nil unless other && user_can_reference_project?(other) - - other - end - - def user_can_reference_project?(project, user = context[:current_user]) - Ability.abilities.allowed?(user, :read_project, project) + Project.find_with_namespace(ref) end end end diff --git a/lib/gitlab/markdown/external_issue_reference_filter.rb b/lib/gitlab/markdown/external_issue_reference_filter.rb index f7c43e1ca89..14bdf5521fc 100644 --- a/lib/gitlab/markdown/external_issue_reference_filter.rb +++ b/lib/gitlab/markdown/external_issue_reference_filter.rb @@ -30,6 +30,10 @@ module Gitlab replace_text_nodes_matching(ExternalIssue.reference_pattern) do |content| issue_link_filter(content) end + + replace_link_nodes_with_href(ExternalIssue.reference_pattern) do |link, text| + issue_link_filter(link, link_text: text) + end end # Replace `JIRA-123` issue references in text with links to the referenced @@ -39,7 +43,7 @@ module Gitlab # # Returns a String with `JIRA-123` references replaced with links. All # links have `gfm` and `gfm-issue` class names attached for styling. - def issue_link_filter(text) + def issue_link_filter(text, link_text: nil) project = context[:project] self.class.references_in(text) do |match, issue| @@ -47,10 +51,13 @@ module Gitlab title = escape_once("Issue in #{project.external_issue_tracker.title}") klass = reference_class(:issue) + data = data_attribute(project: project.id) + + text = link_text || match - %(<a href="#{url}" + %(<a href="#{url}" #{data} title="#{title}" - class="#{klass}">#{match}</a>) + class="#{klass}">#{text}</a>) end end diff --git a/lib/gitlab/markdown/external_link_filter.rb b/lib/gitlab/markdown/external_link_filter.rb index 29e51b6ade6..e09dfcb83c8 100644 --- a/lib/gitlab/markdown/external_link_filter.rb +++ b/lib/gitlab/markdown/external_link_filter.rb @@ -8,9 +8,9 @@ module Gitlab class ExternalLinkFilter < HTML::Pipeline::Filter def call doc.search('a').each do |node| - next unless node.has_attribute?('href') + link = node.attr('href') - link = node.attribute('href').value + next unless link # Skip non-HTTP(S) links next unless link.start_with?('http') diff --git a/lib/gitlab/markdown/issue_reference_filter.rb b/lib/gitlab/markdown/issue_reference_filter.rb index 01320f80796..1ed69e2f431 100644 --- a/lib/gitlab/markdown/issue_reference_filter.rb +++ b/lib/gitlab/markdown/issue_reference_filter.rb @@ -6,64 +6,17 @@ module Gitlab # issues that do not exist are ignored. # # This filter supports cross-project references. - class IssueReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find `#123` issue references in text - # - # IssueReferenceFilter.references_in(text) do |match, issue, project_ref| - # "<a href=...>##{issue}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the Integer issue ID, and an optional String of - # the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Issue.reference_pattern) do |match| - yield match, $~[:issue].to_i, $~[:project] - end - end - - def call - replace_text_nodes_matching(Issue.reference_pattern) do |content| - issue_link_filter(content) - end + class IssueReferenceFilter < AbstractReferenceFilter + def self.object_class + Issue end - # Replace `#123` issue references in text with links to the referenced - # issue's details page. - # - # text - String text to replace references in. - # - # Returns a String with `#123` references replaced with links. All links - # have `gfm` and `gfm-issue` class names attached for styling. - def issue_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if project && issue = project.get_issue(id) - push_result(:issue, issue) - - url = url_for_issue(id, project, only_path: context[:only_path]) - - title = escape_once("Issue: #{issue.title}") - klass = reference_class(:issue) - data = data_attribute(project.id) - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{match}</a>) - else - match - end - end + def find_object(project, id) + project.get_issue(id) end - def url_for_issue(*args) - IssuesHelper.url_for_issue(*args) + def url_for_object(issue, project) + IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path]) end end end diff --git a/lib/gitlab/markdown/label_reference_filter.rb b/lib/gitlab/markdown/label_reference_filter.rb index 1e5cb12071e..a2026eecaeb 100644 --- a/lib/gitlab/markdown/label_reference_filter.rb +++ b/lib/gitlab/markdown/label_reference_filter.rb @@ -22,10 +22,18 @@ module Gitlab end end + def self.referenced_by(node) + { label: LazyReference.new(Label, node.attr("data-label")) } + end + def call replace_text_nodes_matching(Label.reference_pattern) do |content| label_link_filter(content) end + + replace_link_nodes_with_href(Label.reference_pattern) do |link, text| + label_link_filter(link, link_text: text) + end end # Replace label references in text with links to the label specified. @@ -34,21 +42,25 @@ module Gitlab # # Returns a String with label references replaced with links. All links # have `gfm` and `gfm-label` class names attached for styling. - def label_link_filter(text) + def label_link_filter(text, link_text: nil) project = context[:project] self.class.references_in(text) do |match, id, name| params = label_params(id, name) if label = project.labels.find_by(params) - push_result(:label, label) - url = url_for_label(project, label) klass = reference_class(:label) - data = data_attribute(project.id) + data = data_attribute( + original: link_text || match, + project: project.id, + label: label.id + ) + + text = link_text || render_colored_label(label) %(<a href="#{url}" #{data} - class="#{klass}">#{render_colored_label(label)}</a>) + class="#{klass}">#{text}</a>) else match end @@ -57,9 +69,8 @@ module Gitlab def url_for_label(project, label) h = Gitlab::Application.routes.url_helpers - h.namespace_project_issues_path(project.namespace, project, - label_name: label.name, - only_path: context[:only_path]) + h.namespace_project_issues_url( project.namespace, project, label_name: label.name, + only_path: context[:only_path]) end def render_colored_label(label) diff --git a/lib/gitlab/markdown/merge_request_reference_filter.rb b/lib/gitlab/markdown/merge_request_reference_filter.rb index ecbd263d0e0..de71fc76a9b 100644 --- a/lib/gitlab/markdown/merge_request_reference_filter.rb +++ b/lib/gitlab/markdown/merge_request_reference_filter.rb @@ -6,66 +6,29 @@ module Gitlab # to merge requests that do not exist are ignored. # # This filter supports cross-project references. - class MergeRequestReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find `!123` merge request references in text - # - # MergeRequestReferenceFilter.references_in(text) do |match, merge_request, project_ref| - # "<a href=...>##{merge_request}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the Integer merge request ID, and an optional - # String of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(MergeRequest.reference_pattern) do |match| - yield match, $~[:merge_request].to_i, $~[:project] - end + class MergeRequestReferenceFilter < AbstractReferenceFilter + def self.object_class + MergeRequest end - def call - replace_text_nodes_matching(MergeRequest.reference_pattern) do |content| - merge_request_link_filter(content) - end + def find_object(project, id) + project.merge_requests.find_by(iid: id) end - # Replace `!123` merge request references in text with links to the - # referenced merge request's details page. - # - # text - String text to replace references in. - # - # Returns a String with `!123` references replaced with links. All links - # have `gfm` and `gfm-merge_request` class names attached for styling. - def merge_request_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if project && merge_request = project.merge_requests.find_by(iid: id) - push_result(:merge_request, merge_request) - - title = escape_once("Merge Request: #{merge_request.title}") - klass = reference_class(:merge_request) - data = data_attribute(project.id) + def url_for_object(mr, project) + h = Gitlab::Application.routes.url_helpers + h.namespace_project_merge_request_url(project.namespace, project, mr, + only_path: context[:only_path]) + end - url = url_for_merge_request(merge_request, project) + def object_link_text_extras(object, matches) + extras = super - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{match}</a>) - else - match - end + if matches.names.include?("path") && matches[:path] && matches[:path] == '/diffs' + extras.unshift "diffs" end - end - def url_for_merge_request(mr, project) - h = Gitlab::Application.routes.url_helpers - h.namespace_project_merge_request_url(project.namespace, project, mr, - only_path: context[:only_path]) + extras end end end diff --git a/lib/gitlab/markdown/redactor_filter.rb b/lib/gitlab/markdown/redactor_filter.rb new file mode 100644 index 00000000000..bea714a01e7 --- /dev/null +++ b/lib/gitlab/markdown/redactor_filter.rb @@ -0,0 +1,43 @@ +require 'gitlab/markdown' +require 'html/pipeline/filter' + +module Gitlab + module Markdown + # HTML filter that removes references to records that the current user does + # not have permission to view. + # + # Expected to be run in its own post-processing pipeline. + # + class RedactorFilter < HTML::Pipeline::Filter + def call + doc.css('a.gfm').each do |node| + unless user_can_reference?(node) + # The reference should be replaced by the original text, + # which is not always the same as the rendered text. + text = node.attr('data-original') || node.text + node.replace(text) + end + end + + doc + end + + private + + def user_can_reference?(node) + if node.has_attribute?('data-reference-filter') + reference_type = node.attr('data-reference-filter') + reference_filter = reference_type.constantize + + reference_filter.user_can_reference?(current_user, node, context) + else + true + end + end + + def current_user + context[:current_user] + end + end + end +end diff --git a/lib/gitlab/markdown/reference_filter.rb b/lib/gitlab/markdown/reference_filter.rb index 9b293c957d6..b6d93e05ec7 100644 --- a/lib/gitlab/markdown/reference_filter.rb +++ b/lib/gitlab/markdown/reference_filter.rb @@ -11,30 +11,57 @@ module Gitlab # Context options: # :project (required) - Current project, ignored if reference is cross-project. # :only_path - Generate path-only links. - # - # Results: - # :references - A Hash of references that were found and replaced. class ReferenceFilter < HTML::Pipeline::Filter - def initialize(*args) - super + LazyReference = Struct.new(:klass, :ids) do + def self.load(refs) + lazy_references, values = refs.partition { |ref| ref.is_a?(self) } + + lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs| + ids = refs.flat_map(&:ids) + klass.where(id: ids) + end + + values + lazy_values + end + + def load + self.klass.where(id: self.ids) + end + end + + def self.user_can_reference?(user, node, context) + if node.has_attribute?('data-project') + project_id = node.attr('data-project').to_i + return true if project_id == context[:project].try(:id) - result[:references] = Hash.new { |hash, type| hash[type] = [] } + project = Project.find(project_id) rescue nil + Ability.abilities.allowed?(user, :read_project, project) + else + true + end + end + + def self.referenced_by(node) + raise NotImplementedError, "#{self} does not implement #{__method__}" end # Returns a data attribute String to attach to a reference link # - # id - Object ID - # type - Object type (default: :project) + # attributes - Hash, where the key becomes the data attribute name and the + # value is the data attribute value # # Examples: # - # data_attribute(1) # => "data-project-id=\"1\"" - # data_attribute(2, :user) # => "data-user-id=\"2\"" - # data_attribute(3, :group) # => "data-group-id=\"3\"" + # data_attribute(project: 1, issue: 2) + # # => "data-reference-filter=\"Gitlab::Markdown::SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" + # + # data_attribute(project: 3, merge_request: 4) + # # => "data-reference-filter=\"Gitlab::Markdown::SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" # # Returns a String - def data_attribute(id, type = :project) - %Q(data-#{type}-id="#{id}") + def data_attribute(attributes = {}) + attributes[:reference_filter] = self.class.name + attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{value}") }.join(" ") end def escape_once(html) @@ -59,16 +86,6 @@ module Gitlab context[:project] end - # Add a reference to the pipeline's result Hash - # - # type - Singular Symbol reference type (e.g., :issue, :user, etc.) - # values - One or more Objects to add - def push_result(type, *values) - return if values.empty? - - result[:references][type].push(*values) - end - def reference_class(type) "gfm gfm-#{type}" end @@ -85,15 +102,15 @@ module Gitlab # Yields the current node's String contents. The result of the block will # replace the node's existing content and update the current document. # - # Returns the updated Nokogiri::XML::Document object. + # Returns the updated Nokogiri::HTML::DocumentFragment object. def replace_text_nodes_matching(pattern) return doc if project.nil? search_text_nodes(doc).each do |node| - content = node.to_html - - next unless content.match(pattern) next if ignored_ancestry?(node) + next unless node.text =~ pattern + + content = node.to_html html = yield content @@ -105,6 +122,80 @@ module Gitlab doc end + # Iterate through the document's link nodes, yielding the current node's + # content if: + # + # * The `project` context value is present AND + # * The node's content matches `pattern` + # + # pattern - Regex pattern against which to match the node's content + # + # Yields the current node's String contents. The result of the block will + # replace the node and update the current document. + # + # Returns the updated Nokogiri::HTML::DocumentFragment object. + def replace_link_nodes_with_text(pattern) + return doc if project.nil? + + doc.search('a').each do |node| + klass = node.attr('class') + next if klass && klass.include?('gfm') + + link = node.attr('href') + text = node.text + + next unless link && text + + link = URI.decode(link) + # Ignore ending punctionation like periods or commas + next unless link == text && text =~ /\A#{pattern}/ + + html = yield text + + next if html == text + + node.replace(html) + end + + doc + end + + # Iterate through the document's link nodes, yielding the current node's + # content if: + # + # * The `project` context value is present AND + # * The node's HREF matches `pattern` + # + # pattern - Regex pattern against which to match the node's HREF + # + # Yields the current node's String HREF and String content. + # The result of the block will replace the node and update the current document. + # + # Returns the updated Nokogiri::HTML::DocumentFragment object. + def replace_link_nodes_with_href(pattern) + return doc if project.nil? + + doc.search('a').each do |node| + klass = node.attr('class') + next if klass && klass.include?('gfm') + + link = node.attr('href') + text = node.text + + next unless link && text + link = URI.decode(link) + next unless link && link =~ /\A#{pattern}\z/ + + html = yield link, text + + next if html == link + + node.replace(html) + end + + doc + end + # Ensure that a :project key exists in context # # Note that while the key might exist, its value could be nil! diff --git a/lib/gitlab/markdown/reference_gatherer_filter.rb b/lib/gitlab/markdown/reference_gatherer_filter.rb new file mode 100644 index 00000000000..00f983675e6 --- /dev/null +++ b/lib/gitlab/markdown/reference_gatherer_filter.rb @@ -0,0 +1,63 @@ +require 'gitlab/markdown' +require 'html/pipeline/filter' + +module Gitlab + module Markdown + # HTML filter that gathers all referenced records that the current user has + # permission to view. + # + # Expected to be run in its own post-processing pipeline. + # + class ReferenceGathererFilter < HTML::Pipeline::Filter + def initialize(*) + super + + result[:references] ||= Hash.new { |hash, type| hash[type] = [] } + end + + def call + doc.css('a.gfm').each do |node| + gather_references(node) + end + + load_lazy_references unless context[:load_lazy_references] == false + + doc + end + + private + + def gather_references(node) + return unless node.has_attribute?('data-reference-filter') + + reference_type = node.attr('data-reference-filter') + reference_filter = reference_type.constantize + + return if context[:reference_filter] && reference_filter != context[:reference_filter] + + return unless reference_filter.user_can_reference?(current_user, node, context) + + references = reference_filter.referenced_by(node) + return unless references + + references.each do |type, values| + Array.wrap(values).each do |value| + result[:references][type] << value + end + end + end + + # Will load all references of one type using one query. + def load_lazy_references + refs = result[:references] + refs.each do |type, values| + refs[type] = ReferenceFilter::LazyReference.load(values) + end + end + + def current_user + context[:current_user] + end + end + end +end diff --git a/lib/gitlab/markdown/relative_link_filter.rb b/lib/gitlab/markdown/relative_link_filter.rb index 6ee3d1ce039..692c51fd324 100644 --- a/lib/gitlab/markdown/relative_link_filter.rb +++ b/lib/gitlab/markdown/relative_link_filter.rb @@ -17,6 +17,9 @@ module Gitlab return doc unless linkable_files? doc.search('a').each do |el| + klass = el.attr('class') + next if klass && klass.include?('gfm') + process_link_attr el.attribute('href') end @@ -51,7 +54,7 @@ module Gitlab relative_url_root, context[:project].path_with_namespace, path_type(file_path), - ref || 'master', # assume that if no ref exists we can point to master + ref || context[:project].default_branch, # if no ref exists, point to the default branch file_path ].compact.join('/').squeeze('/').chomp('/') diff --git a/lib/gitlab/markdown/sanitization_filter.rb b/lib/gitlab/markdown/sanitization_filter.rb index e368de7d848..ffb9dc33b64 100644 --- a/lib/gitlab/markdown/sanitization_filter.rb +++ b/lib/gitlab/markdown/sanitization_filter.rb @@ -48,6 +48,12 @@ module Gitlab # Allow span elements whitelist[:elements].push('span') + # Allow any protocol in `a` elements... + whitelist[:protocols].delete('a') + + # ...but then remove links with the `javascript` protocol + whitelist[:transformers].push(remove_javascript_links) + # Remove `rel` attribute from `a` elements whitelist[:transformers].push(remove_rel) @@ -57,6 +63,19 @@ module Gitlab whitelist end + def remove_javascript_links + lambda do |env| + node = env[:node] + + return unless node.name == 'a' + return unless node.has_attribute?('href') + + if node['href'].start_with?('javascript', ':javascript') + node.remove_attribute('href') + end + end + end + def remove_rel lambda do |env| if env[:node_name] == 'a' diff --git a/lib/gitlab/markdown/snippet_reference_filter.rb b/lib/gitlab/markdown/snippet_reference_filter.rb index e2cf89cb1d8..f7bd07c2a34 100644 --- a/lib/gitlab/markdown/snippet_reference_filter.rb +++ b/lib/gitlab/markdown/snippet_reference_filter.rb @@ -6,63 +6,16 @@ module Gitlab # snippets that do not exist are ignored. # # This filter supports cross-project references. - class SnippetReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find `$123` snippet references in text - # - # SnippetReferenceFilter.references_in(text) do |match, snippet| - # "<a href=...>$#{snippet}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the Integer snippet ID, and an optional String - # of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Snippet.reference_pattern) do |match| - yield match, $~[:snippet].to_i, $~[:project] - end - end - - def call - replace_text_nodes_matching(Snippet.reference_pattern) do |content| - snippet_link_filter(content) - end + class SnippetReferenceFilter < AbstractReferenceFilter + def self.object_class + Snippet end - # Replace `$123` snippet references in text with links to the referenced - # snippets's details page. - # - # text - String text to replace references in. - # - # Returns a String with `$123` references replaced with links. All links - # have `gfm` and `gfm-snippet` class names attached for styling. - def snippet_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if project && snippet = project.snippets.find_by(id: id) - push_result(:snippet, snippet) - - title = escape_once("Snippet: #{snippet.title}") - klass = reference_class(:snippet) - data = data_attribute(project.id) - - url = url_for_snippet(snippet, project) - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{match}</a>) - else - match - end - end + def find_object(project, id) + project.snippets.find_by(id: id) end - def url_for_snippet(snippet, project) + def url_for_object(snippet, project) h = Gitlab::Application.routes.url_helpers h.namespace_project_snippet_url(project.namespace, project, snippet, only_path: context[:only_path]) diff --git a/lib/gitlab/markdown/upload_link_filter.rb b/lib/gitlab/markdown/upload_link_filter.rb new file mode 100644 index 00000000000..fbada73ab86 --- /dev/null +++ b/lib/gitlab/markdown/upload_link_filter.rb @@ -0,0 +1,47 @@ +require 'gitlab/markdown' +require 'html/pipeline/filter' +require 'uri' + +module Gitlab + module Markdown + # HTML filter that "fixes" relative upload links to files. + # Context options: + # :project (required) - Current project + # + class UploadLinkFilter < HTML::Pipeline::Filter + def call + doc.search('a').each do |el| + process_link_attr el.attribute('href') + end + + doc.search('img').each do |el| + process_link_attr el.attribute('src') + end + + doc + end + + protected + + def process_link_attr(html_attr) + return if html_attr.blank? + + uri = html_attr.value + if uri.starts_with?("/uploads/") + html_attr.value = build_url(uri).to_s + end + end + + def build_url(uri) + File.join(Gitlab.config.gitlab.url, context[:project].path_with_namespace, uri) + end + + # Ensure that a :project key exists in context + # + # Note that while the key might exist, its value could be nil! + def validate + needs :project + end + end + end +end diff --git a/lib/gitlab/markdown/user_reference_filter.rb b/lib/gitlab/markdown/user_reference_filter.rb index 6f436ea7167..0a20d9c0347 100644 --- a/lib/gitlab/markdown/user_reference_filter.rb +++ b/lib/gitlab/markdown/user_reference_filter.rb @@ -23,10 +23,39 @@ module Gitlab end end + def self.referenced_by(node) + if node.has_attribute?('data-group') + group = Group.find(node.attr('data-group')) rescue nil + return unless group + + { user: group.users } + elsif node.has_attribute?('data-user') + { user: LazyReference.new(User, node.attr('data-user')) } + elsif node.has_attribute?('data-project') + project = Project.find(node.attr('data-project')) rescue nil + return unless project + + { user: project.team.members.flatten } + end + end + + def self.user_can_reference?(user, node, context) + if node.has_attribute?('data-group') + group = Group.find(node.attr('data-group')) rescue nil + Ability.abilities.allowed?(user, :read_group, group) + else + super + end + end + def call replace_text_nodes_matching(User.reference_pattern) do |content| user_link_filter(content) end + + replace_link_nodes_with_href(User.reference_pattern) do |link, text| + user_link_filter(link, link_text: text) + end end # Replace `@user` user references in text with links to the referenced @@ -36,12 +65,12 @@ module Gitlab # # Returns a String with `@user` references replaced with links. All links # have `gfm` and `gfm-project_member` class names attached for styling. - def user_link_filter(text) + def user_link_filter(text, link_text: nil) self.class.references_in(text) do |match, username| if username == 'all' - link_to_all + link_to_all(link_text: link_text) elsif namespace = Namespace.find_by(path: username) - link_to_namespace(namespace) || match + link_to_namespace(namespace, link_text: link_text) || match else match end @@ -58,51 +87,42 @@ module Gitlab reference_class(:project_member) end - def link_to_all + def link_to_all(link_text: nil) project = context[:project] - - # FIXME (rspeicher): Law of Demeter - push_result(:user, *project.team.members.flatten) - url = urls.namespace_project_url(project.namespace, project, only_path: context[:only_path]) + data = data_attribute(project: project.id) + text = link_text || User.reference_prefix + 'all' - text = User.reference_prefix + 'all' - %(<a href="#{url}" class="#{link_class}">#{text}</a>) + link_tag(url, data, text) end - def link_to_namespace(namespace) + def link_to_namespace(namespace, link_text: nil) if namespace.is_a?(Group) - link_to_group(namespace.path, namespace) + link_to_group(namespace.path, namespace, link_text: link_text) else - link_to_user(namespace.path, namespace) + link_to_user(namespace.path, namespace, link_text: link_text) end end - def link_to_group(group, namespace) - return unless user_can_reference_group?(namespace) - - push_result(:user, *namespace.users) - + def link_to_group(group, namespace, link_text: nil) url = urls.group_url(group, only_path: context[:only_path]) - data = data_attribute(namespace.id, :group) + data = data_attribute(group: namespace.id) + text = link_text || Group.reference_prefix + group - text = Group.reference_prefix + group - %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) + link_tag(url, data, text) end - def link_to_user(user, namespace) - push_result(:user, namespace.owner) - + def link_to_user(user, namespace, link_text: nil) url = urls.user_url(user, only_path: context[:only_path]) - data = data_attribute(namespace.owner_id, :user) + data = data_attribute(user: namespace.owner_id) + text = link_text || User.reference_prefix + user - text = User.reference_prefix + user - %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) + link_tag(url, data, text) end - def user_can_reference_group?(group) - Ability.abilities.allowed?(context[:current_user], :read_group, group) + def link_tag(url, data, text) + %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) end end end diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb index 90c3fe8da33..9ad7a38d505 100644 --- a/lib/gitlab/o_auth/provider.rb +++ b/lib/gitlab/o_auth/provider.rb @@ -1,6 +1,12 @@ module Gitlab module OAuth class Provider + LABELS = { + "github" => "GitHub", + "gitlab" => "GitLab.com", + "google_oauth2" => "Google" + }.freeze + def self.providers Devise.omniauth_providers end @@ -23,8 +29,9 @@ module Gitlab end def self.label_for(name) + name = name.to_s config = config_for(name) - (config && config['label']) || name.to_s.titleize + (config && config['label']) || LABELS[name] || name.titleize end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 0dab7bcfa4d..70de6a74e76 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -9,7 +9,7 @@ module Gitlab else nil end - @query = Shellwords.shellescape(query) if query.present? + @query = query end def objects(scope, page = nil) @@ -20,6 +20,8 @@ module Gitlab Kaminari.paginate_array(blobs).page(page).per(per_page) when 'wiki_blobs' Kaminari.paginate_array(wiki_blobs).page(page).per(per_page) + when 'commits' + Kaminari.paginate_array(commits).page(page).per(per_page) else super end @@ -27,7 +29,7 @@ module Gitlab def total_count @total_count ||= issues_count + merge_requests_count + blobs_count + - notes_count + wiki_blobs_count + notes_count + wiki_blobs_count + commits_count end def blobs_count @@ -42,6 +44,10 @@ module Gitlab @wiki_blobs_count ||= wiki_blobs.count end + def commits_count + @commits_count ||= commits.count + end + private def blobs @@ -70,6 +76,14 @@ module Gitlab Note.where(project_id: limit_project_ids).user.search(query).order('updated_at DESC') end + def commits + if project.empty_repo? || query.blank? + [] + else + project.repository.find_commits_by_message(query).compact + end + end + def limit_project_ids [project.id] end diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb index d010ade704e..4f9cdef3869 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/push_data_builder.rb @@ -30,9 +30,12 @@ module Gitlab # For performance purposes maximum 20 latest commits # will be passed as post receive hook data. - commit_attrs = commits_limited.map(&:hook_attrs) + commit_attrs = commits_limited.map do |commit| + commit.hook_attrs(with_changed_files: true) + end type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push" + # Hash to be passed as post_receive_data data = { object_kind: type, diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 0961bd80421..3c3478a1271 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -3,11 +3,12 @@ require 'gitlab/markdown' module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor - attr_accessor :project, :current_user + attr_accessor :project, :current_user, :load_lazy_references - def initialize(project, current_user = nil) + def initialize(project, current_user = nil, load_lazy_references: true) @project = project @current_user = current_user + @load_lazy_references = load_lazy_references end def analyze(text) @@ -26,9 +27,9 @@ module Gitlab def references @references ||= Hash.new do |references, type| type = type.to_sym - return references[type] if references.has_key?(type) + next references[type] if references.has_key?(type) - references[type] = pipeline_result(type).uniq + references[type] = pipeline_result(type) end end @@ -39,21 +40,42 @@ module Gitlab # # Returns the results Array for the requested filter type def pipeline_result(filter_type) - klass = filter_type.to_s.camelize + 'ReferenceFilter' + return [] if @text.blank? + + klass = "#{filter_type.to_s.camelize}ReferenceFilter" filter = Gitlab::Markdown.const_get(klass) context = { project: project, current_user: current_user, + # We don't actually care about the links generated only_path: true, - ignore_blockquotes: true + ignore_blockquotes: true, + + # ReferenceGathererFilter + load_lazy_references: false, + reference_filter: filter } - pipeline = HTML::Pipeline.new([filter], context) + # We need to autolink first to finds links to referables, and to prevent + # numeric anchors to be parsed as issue references. + filters = [ + Gitlab::Markdown::AutolinkFilter, + filter, + Gitlab::Markdown::ReferenceGathererFilter + ] + + pipeline = HTML::Pipeline.new(filters, context) result = pipeline.call(@text) - result[:references][filter_type] + values = result[:references][filter_type].uniq + + if @load_lazy_references + values = Gitlab::Markdown::ReferenceFilter::LazyReference.load(values).uniq + end + + values end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 9f1adc860d1..53ab2686b43 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -51,6 +51,23 @@ module Gitlab "can contain only letters, digits, '_', '-' and '.'. " end + def file_path_regex + @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/]*\z/.freeze + end + + def file_path_regex_message + "can contain only letters, digits, '_', '-' and '.'. Separate directories with a '/'. " + end + + + def directory_traversal_regex + @directory_traversal_regex ||= /\.{2}/.freeze + end + + def directory_traversal_regex_message + "cannot include directory traversal. " + end + def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 31aa3528c4c..2ef0e982256 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -14,7 +14,7 @@ module Gitlab def self.mute_mailer code = <<-eos -def Notify.delay +def Notify.deliver_later self end eos diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb new file mode 100644 index 00000000000..6360527a7aa --- /dev/null +++ b/lib/gitlab/sherlock.rb @@ -0,0 +1,19 @@ +require 'securerandom' + +module Gitlab + module Sherlock + @collection = Collection.new + + class << self + attr_reader :collection + end + + def self.enabled? + Rails.env.development? && !!ENV['ENABLE_SHERLOCK'] + end + + def self.enable_line_profiler? + RUBY_ENGINE == 'ruby' + end + end +end diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb new file mode 100644 index 00000000000..66bd6258521 --- /dev/null +++ b/lib/gitlab/sherlock/collection.rb @@ -0,0 +1,49 @@ +module Gitlab + module Sherlock + # A collection of transactions recorded by Sherlock. + # + # Method calls for this class are synchronized using a mutex to allow + # sharing of a single Collection instance between threads (e.g. when using + # Puma as a webserver). + class Collection + include Enumerable + + def initialize + @transactions = [] + @mutex = Mutex.new + end + + def add(transaction) + synchronize { @transactions << transaction } + end + + alias_method :<<, :add + + def each(&block) + synchronize { @transactions.each(&block) } + end + + def clear + synchronize { @transactions.clear } + end + + def empty? + synchronize { @transactions.empty? } + end + + def find_transaction(id) + find { |trans| trans.id == id } + end + + def newest_first + sort { |a, b| b.finished_at <=> a.finished_at } + end + + private + + def synchronize(&block) + @mutex.synchronize(&block) + end + end + end +end diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb new file mode 100644 index 00000000000..8a3e1a5e5bf --- /dev/null +++ b/lib/gitlab/sherlock/file_sample.rb @@ -0,0 +1,31 @@ +module Gitlab + module Sherlock + class FileSample + attr_reader :id, :file, :line_samples, :events, :duration + + # file - The full path to the file this sample belongs to. + # line_samples - An array of LineSample objects. + # duration - The total execution time in milliseconds. + # events - The total amount of events. + def initialize(file, line_samples, duration, events) + @id = SecureRandom.uuid + @file = file + @line_samples = line_samples + @duration = duration + @events = events + end + + def relative_path + @relative_path ||= @file.gsub(/^#{Rails.root.to_s}\/?/, '') + end + + def to_param + @id + end + + def source + @source ||= File.read(@file) + end + end + end +end diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb new file mode 100644 index 00000000000..aa1468bff6b --- /dev/null +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -0,0 +1,98 @@ +module Gitlab + module Sherlock + # Class for profiling code on a per line basis. + # + # The LineProfiler class can be used to profile code on per line basis + # without littering your code with Ruby implementation specific profiling + # methods. + # + # This profiler only includes samples taking longer than a given threshold + # and those that occur in the actual application (e.g. files from Gems are + # ignored). + class LineProfiler + # The minimum amount of time that has to be spent in a file for it to be + # included in a list of samples. + MINIMUM_DURATION = 10.0 + + # Profiles the given block. + # + # Example: + # + # profiler = LineProfiler.new + # + # retval, samples = profiler.profile do + # "cats are amazing" + # end + # + # retval # => "cats are amazing" + # samples # => [#<Gitlab::Sherlock::FileSample ...>, ...] + # + # Returns an Array containing the block's return value and an Array of + # FileSample objects. + def profile(&block) + if mri? + profile_mri(&block) + else + raise NotImplementedError, + 'Line profiling is not supported on this platform' + end + end + + # Profiles the given block using rblineprof (MRI only). + def profile_mri + require 'rblineprof' + + retval = nil + samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield } + + file_samples = aggregate_rblineprof(samples) + + [retval, file_samples] + end + + # Returns an Array of file samples based on the output of rblineprof. + # + # lineprof_stats - A Hash containing rblineprof statistics on a per file + # basis. + # + # Returns an Array of FileSample objects. + def aggregate_rblineprof(lineprof_stats) + samples = [] + + lineprof_stats.each do |(file, stats)| + source_lines = File.read(file).each_line.to_a + line_samples = [] + + total_duration = microsec_to_millisec(stats[0][0]) + total_events = stats[0][2] + + next if total_duration <= MINIMUM_DURATION + + stats[1..-1].each_with_index do |data, index| + next unless source_lines[index] + + duration = microsec_to_millisec(data[0]) + events = data[2] + + line_samples << LineSample.new(duration, events) + end + + samples << FileSample. + new(file, line_samples, total_duration, total_events) + end + + samples + end + + private + + def microsec_to_millisec(microsec) + microsec / 1000.0 + end + + def mri? + RUBY_ENGINE == 'ruby' + end + end + end +end diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb new file mode 100644 index 00000000000..eb1948eb6d6 --- /dev/null +++ b/lib/gitlab/sherlock/line_sample.rb @@ -0,0 +1,36 @@ +module Gitlab + module Sherlock + class LineSample + attr_reader :duration, :events + + # duration - The execution time in milliseconds. + # events - The amount of events. + def initialize(duration, events) + @duration = duration + @events = events + end + + # Returns the sample duration percentage relative to the given duration. + # + # Example: + # + # sample.duration # => 150 + # sample.percentage_of(1500) # => 10.0 + # + # total_duration - The total duration to compare with. + # + # Returns a float + def percentage_of(total_duration) + (duration.to_f / total_duration) * 100.0 + end + + # Returns true if the current sample takes up the majority of the given + # duration. + # + # total_duration - The total duration to compare with. + def majority_of?(total_duration) + percentage_of(total_duration) >= 30 + end + end + end +end diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb new file mode 100644 index 00000000000..5ac265618ad --- /dev/null +++ b/lib/gitlab/sherlock/location.rb @@ -0,0 +1,26 @@ +module Gitlab + module Sherlock + class Location + attr_reader :path, :line + + SHERLOCK_DIR = File.dirname(__FILE__) + + # Creates a new Location from a `Thread::Backtrace::Location`. + def self.from_ruby_location(location) + new(location.path, location.lineno) + end + + # path - The full path of the frame as a String. + # line - The line number of the frame as a Fixnum. + def initialize(path, line) + @path = path + @line = line + end + + # Returns true if the current frame originated from the application. + def application? + @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR) + end + end + end +end diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb new file mode 100644 index 00000000000..687332fc5fc --- /dev/null +++ b/lib/gitlab/sherlock/middleware.rb @@ -0,0 +1,41 @@ +module Gitlab + module Sherlock + # Rack middleware used for tracking request metrics. + class Middleware + CONTENT_TYPES = /text\/html|application\/json/i + + IGNORE_PATHS = %r{^/sherlock} + + def initialize(app) + @app = app + end + + # env - A Hash containing Rack environment details. + def call(env) + if instrument?(env) + call_with_instrumentation(env) + else + @app.call(env) + end + end + + def call_with_instrumentation(env) + trans = transaction_from_env(env) + retval = trans.run { @app.call(env) } + + Sherlock.collection.add(trans) + + retval + end + + def instrument?(env) + !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES && + env['REQUEST_URI'] !~ IGNORE_PATHS) + end + + def transaction_from_env(env) + Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI']) + end + end + end +end diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb new file mode 100644 index 00000000000..4917c4ae2ac --- /dev/null +++ b/lib/gitlab/sherlock/query.rb @@ -0,0 +1,114 @@ +module Gitlab + module Sherlock + class Query + attr_reader :id, :query, :started_at, :finished_at, :backtrace + + # SQL identifiers that should be prefixed with newlines. + PREFIX_NEWLINE = / + \s+(FROM + |(LEFT|RIGHT)?INNER\s+JOIN + |(LEFT|RIGHT)?OUTER\s+JOIN + |WHERE + |AND + |GROUP\s+BY + |ORDER\s+BY + |LIMIT + |OFFSET)\s+/ix # Vim indent breaks when this is on a newline :< + + # Creates a new Query using a String and a separate Array of bindings. + # + # query - A String containing a SQL query, optionally with numeric + # placeholders (`$1`, `$2`, etc). + # + # bindings - An Array of ActiveRecord columns and their values. + # started_at - The start time of the query as a Time-like object. + # finished_at - The completion time of the query as a Time-like object. + # + # Returns a new Query object. + def self.new_with_bindings(query, bindings, started_at, finished_at) + bindings.each_with_index do |(_, value), index| + quoted_value = ActiveRecord::Base.connection.quote(value) + + query = query.gsub("$#{index + 1}", quoted_value) + end + + new(query, started_at, finished_at) + end + + # query - The SQL query as a String (without placeholders). + # started_at - The start time of the query as a Time-like object. + # finished_at - The completion time of the query as a Time-like object. + def initialize(query, started_at, finished_at) + @id = SecureRandom.uuid + @query = query + @started_at = started_at + @finished_at = finished_at + @backtrace = caller_locations.map do |loc| + Location.from_ruby_location(loc) + end + + unless @query.end_with?(';') + @query += ';' + end + end + + # Returns the query duration in milliseconds. + def duration + @duration ||= (@finished_at - @started_at) * 1000.0 + end + + def to_param + @id + end + + # Returns a human readable version of the query. + def formatted_query + @formatted_query ||= format_sql(@query) + end + + # Returns the last application frame of the backtrace. + def last_application_frame + @last_application_frame ||= @backtrace.find(&:application?) + end + + # Returns an Array of application frames (excluding Gems and the likes). + def application_backtrace + @application_backtrace ||= @backtrace.select(&:application?) + end + + # Returns the query plan as a String. + def explain + unless @explain + ActiveRecord::Base.connection.transaction do + @explain = raw_explain(@query).values.flatten.join("\n") + + # Roll back any queries that mutate data so we don't mess up + # anything when running explain on an INSERT, UPDATE, DELETE, etc. + raise ActiveRecord::Rollback + end + end + + @explain + end + + private + + def raw_explain(query) + if Gitlab::Database.postgresql? + explain = "EXPLAIN ANALYZE #{query};" + else + explain = "EXPLAIN #{query};" + end + + ActiveRecord::Base.connection.execute(explain) + end + + def format_sql(query) + query.each_line. + map { |line| line.strip }. + join("\n"). + gsub(PREFIX_NEWLINE) { "\n#{$1} " } + end + end + end +end diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb new file mode 100644 index 00000000000..3489fb251b6 --- /dev/null +++ b/lib/gitlab/sherlock/transaction.rb @@ -0,0 +1,136 @@ +module Gitlab + module Sherlock + class Transaction + attr_reader :id, :type, :path, :queries, :file_samples, :started_at, + :finished_at, :view_counts + + # type - The type of transaction (e.g. "GET", "POST", etc) + # path - The path of the transaction (e.g. the HTTP request path) + def initialize(type, path) + @id = SecureRandom.uuid + @type = type + @path = path + @queries = [] + @file_samples = [] + @started_at = nil + @finished_at = nil + @thread = Thread.current + @view_counts = Hash.new(0) + end + + # Runs the transaction and returns the block's return value. + def run + @started_at = Time.now + + retval = with_subscriptions do + profile_lines { yield } + end + + @finished_at = Time.now + + retval + end + + # Returns the duration in seconds. + def duration + @duration ||= started_at && finished_at ? finished_at - started_at : 0 + end + + # Returns the total query duration in seconds. + def query_duration + @query_duration ||= @queries.map { |q| q.duration }.inject(:+) / 1000.0 + end + + def to_param + @id + end + + # Returns the queries sorted in descending order by their durations. + def sorted_queries + @queries.sort { |a, b| b.duration <=> a.duration } + end + + # Returns the file samples sorted in descending order by their durations. + def sorted_file_samples + @file_samples.sort { |a, b| b.duration <=> a.duration } + end + + # Finds a query by the given ID. + # + # id - The query ID as a String. + # + # Returns a Query object if one could be found, nil otherwise. + def find_query(id) + @queries.find { |query| query.id == id } + end + + # Finds a file sample by the given ID. + # + # id - The query ID as a String. + # + # Returns a FileSample object if one could be found, nil otherwise. + def find_file_sample(id) + @file_samples.find { |sample| sample.id == id } + end + + def profile_lines + retval = nil + + if Sherlock.enable_line_profiler? + retval, @file_samples = LineProfiler.new.profile { yield } + else + retval = yield + end + + retval + end + + def subscribe_to_active_record + ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data| + next unless same_thread? + + track_query(data[:sql].strip, data[:binds], start, finish) + end + end + + def subscribe_to_action_view + regex = /render_(template|partial)\.action_view/ + + ActiveSupport::Notifications.subscribe(regex) do |_, start, finish, _, data| + next unless same_thread? + + track_view(data[:identifier]) + end + end + + private + + def track_query(query, bindings, start, finish) + @queries << Query.new_with_bindings(query, bindings, start, finish) + end + + def track_view(path) + @view_counts[path] += 1 + end + + def with_subscriptions + ar_subscriber = subscribe_to_active_record + av_subscriber = subscribe_to_action_view + + retval = yield + + ActiveSupport::Notifications.unsubscribe(ar_subscriber) + ActiveSupport::Notifications.unsubscribe(av_subscriber) + + retval + end + + # In case somebody uses a multi-threaded server locally (e.g. Puma) we + # _only_ want to track notifications that originate from the transaction + # thread. + def same_thread? + Thread.current == @thread + end + end + end +end diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb new file mode 100644 index 00000000000..1cd89b3a9c4 --- /dev/null +++ b/lib/gitlab/sql/union.rb @@ -0,0 +1,34 @@ +module Gitlab + module SQL + # Class for building SQL UNION statements. + # + # ORDER BYs are dropped from the relations as the final sort order is not + # guaranteed any way. + # + # Example usage: + # + # union = Gitlab::SQL::Union.new(user.personal_projects, user.projects) + # sql = union.to_sql + # + # Project.where("id IN (#{sql})") + class Union + def initialize(relations) + @relations = relations + end + + def to_sql + # Some relations may include placeholders for prepared statements, these + # aren't incremented properly when joining relations together this way. + # By using "unprepared_statements" we remove the usage of placeholders + # (thus fixing this problem), at a slight performance cost. + fragments = ActiveRecord::Base.connection.unprepared_statement do + @relations.map do |rel| + rel.reorder(nil).to_sql + end + end + + fragments.join("\nUNION\n") + end + end + end +end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index cf040971c6e..f3567f3ef85 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -50,15 +50,15 @@ module Gitlab end def fetch_git_tags - remote_tags, _ = Gitlab::Popen.popen(%W(git ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) + remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) remote_tags.split("\n").grep(/tags\/v#{current_version.major}/) end def update_commands { - "Stash changed files" => %W(git stash), - "Get latest code" => %W(git fetch), - "Switch to new version" => %W(git checkout v#{latest_version}), + "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash), + "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch), + "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}), "Install gems" => %W(bundle), "Migrate DB" => %W(bundle exec rake db:migrate), "Recompile assets" => %W(bundle exec rake assets:clean assets:precompile), diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb new file mode 100644 index 00000000000..be8fcc7b2d2 --- /dev/null +++ b/lib/gitlab/uploads_transfer.rb @@ -0,0 +1,35 @@ +module Gitlab + class UploadsTransfer + def move_project(project_path, namespace_path_was, namespace_path) + new_namespace_folder = File.join(root_dir, namespace_path) + FileUtils.mkdir_p(new_namespace_folder) unless Dir.exist?(new_namespace_folder) + from = File.join(root_dir, namespace_path_was, project_path) + to = File.join(root_dir, namespace_path, project_path) + move(from, to, "") + end + + def rename_project(path_was, path, namespace_path) + base_dir = File.join(root_dir, namespace_path) + move(path_was, path, base_dir) + end + + def rename_namespace(path_was, path) + move(path_was, path) + end + + private + + def move(path_was, path, base_dir = nil) + base_dir = root_dir unless base_dir + from = File.join(base_dir, path_was) + to = File.join(base_dir, path) + FileUtils.mv(from, to) + rescue Errno::ENOENT + false + end + + def root_dir + File.join(Rails.root, "public", "uploads") + end + end +end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index a80e7e77430..43fda6fa92e 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -37,10 +37,9 @@ web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" mail_room_enabled=false mail_room_pid_path="$pid_path/mail_room.pid" -gitlab_git_http_server_pid_path="$pid_path/gitlab-git-http-server.pid" -gitlab_git_http_server_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-git-http-server.socket -authBackend http://127.0.0.1:8080" -gitlab_git_http_server_repo_root='/home/git/repositories' -gitlab_git_http_server_log="$app_root/log/gitlab-git-http-server.log" +gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080" +gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" shell_path="/bin/bash" # Read configuration variable file if it is present @@ -76,8 +75,8 @@ check_pids(){ else spid=0 fi - if [ -f "$gitlab_git_http_server_pid_path" ]; then - hpid=$(cat "$gitlab_git_http_server_pid_path") + if [ -f "$gitlab_workhorse_pid_path" ]; then + hpid=$(cat "$gitlab_workhorse_pid_path") else hpid=0 fi @@ -94,7 +93,7 @@ check_pids(){ wait_for_pids(){ # We are sleeping a bit here mostly because sidekiq is slow at writing it's pid i=0; - while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_git_http_server_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do + while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do sleep 0.1; i=$((i+1)) if [ $((i%10)) = 0 ]; then @@ -131,9 +130,9 @@ check_status(){ fi if [ $hpid -ne 0 ]; then kill -0 "$hpid" 2>/dev/null - gitlab_git_http_server_status="$?" + gitlab_workhorse_status="$?" else - gitlab_git_http_server_status="-1" + gitlab_workhorse_status="-1" fi if [ "$mail_room_enabled" = true ]; then if [ $mpid -ne 0 ]; then @@ -143,7 +142,7 @@ check_status(){ mail_room_status="-1" fi fi - if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_git_http_server_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then + if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then gitlab_status=0 else # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html @@ -171,9 +170,9 @@ check_stale_pids(){ exit 1 fi fi - if [ "$hpid" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ]; then - echo "Removing stale gitlab-git-http-server pid. This is most likely caused by gitlab-git-http-server crashing the last time it ran." - if ! rm "$gitlab_git_http_server_pid_path"; then + if [ "$hpid" != "0" ] && [ "$gitlab_workhorse_status" != "0" ]; then + echo "Removing stale gitlab-workhorse pid. This is most likely caused by gitlab-workhorse crashing the last time it ran." + if ! rm "$gitlab_workhorse_pid_path"; then echo "Unable to remove stale pid, exiting" exit 1 fi @@ -190,7 +189,7 @@ check_stale_pids(){ ## If no parts of the service is running, bail out. exit_if_not_running(){ check_stale_pids - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then echo "GitLab is not running." exit fi @@ -206,8 +205,8 @@ start_gitlab() { if [ "$sidekiq_status" != "0" ]; then echo "Starting GitLab Sidekiq" fi - if [ "$gitlab_git_http_server_status" != "0" ]; then - echo "Starting gitlab-git-http-server" + if [ "$gitlab_workhorse_status" != "0" ]; then + echo "Starting gitlab-workhorse" fi if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" != "0" ]; then echo "Starting GitLab MailRoom" @@ -230,15 +229,14 @@ start_gitlab() { RAILS_ENV=$RAILS_ENV bin/background_jobs start & fi - if [ "$gitlab_git_http_server_status" = "0" ]; then - echo "The gitlab-git-http-server is already running with pid $spid, not restarting" + if [ "$gitlab_workhorse_status" = "0" ]; then + echo "The gitlab-workhorse is already running with pid $spid, not restarting" else - # No need to remove a socket, gitlab-git-http-server does this itself - $app_root/bin/daemon_with_pidfile $gitlab_git_http_server_pid_path \ - $app_root/../gitlab-git-http-server/gitlab-git-http-server \ - $gitlab_git_http_server_options \ - $gitlab_git_http_server_repo_root \ - >> $gitlab_git_http_server_log 2>&1 & + # No need to remove a socket, gitlab-workhorse does this itself + $app_root/bin/daemon_with_pidfile $gitlab_workhorse_pid_path \ + $app_root/../gitlab-workhorse/gitlab-workhorse \ + $gitlab_workhorse_options \ + >> $gitlab_workhorse_log 2>&1 & fi if [ "$mail_room_enabled" = true ]; then @@ -268,9 +266,9 @@ stop_gitlab() { echo "Shutting down GitLab Sidekiq" RAILS_ENV=$RAILS_ENV bin/background_jobs stop fi - if [ "$gitlab_git_http_server_status" = "0" ]; then - echo "Shutting down gitlab-git-http-server" - kill -- $(cat $gitlab_git_http_server_pid_path) + if [ "$gitlab_workhorse_status" = "0" ]; then + echo "Shutting down gitlab-workhorse" + kill -- $(cat $gitlab_workhorse_pid_path) fi if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; then echo "Shutting down GitLab MailRoom" @@ -278,11 +276,11 @@ stop_gitlab() { fi # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script. - while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_git_http_server_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do + while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do sleep 1 check_status printf "." - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then printf "\n" break fi @@ -292,7 +290,7 @@ stop_gitlab() { # Cleaning up unused pids rm "$web_server_pid_path" 2>/dev/null # rm "$sidekiq_pid_path" 2>/dev/null # Sidekiq seems to be cleaning up it's own pid. - rm -f "$gitlab_git_http_server_pid_path" + rm -f "$gitlab_workhorse_pid_path" if [ "$mail_room_enabled" = true ]; then rm "$mail_room_pid_path" 2>/dev/null fi @@ -303,7 +301,7 @@ stop_gitlab() { ## Prints the status of GitLab and it's components. print_status() { check_status - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then echo "GitLab is not running." return fi @@ -317,10 +315,10 @@ print_status() { else printf "The GitLab Sidekiq job dispatcher is \033[31mnot running\033[0m.\n" fi - if [ "$gitlab_git_http_server_status" = "0" ]; then - echo "The gitlab-git-http-server with pid $hpid is running." + if [ "$gitlab_workhorse_status" = "0" ]; then + echo "The gitlab-workhorse with pid $hpid is running." else - printf "The gitlab-git-http-server is \033[31mnot running\033[0m.\n" + printf "The gitlab-workhorse is \033[31mnot running\033[0m.\n" fi if [ "$mail_room_enabled" = true ]; then if [ "$mail_room_status" = "0" ]; then @@ -329,7 +327,7 @@ print_status() { printf "The GitLab MailRoom email processor is \033[31mnot running\033[0m.\n" fi fi - if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; }; then + if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; }; then printf "GitLab and all its components are \033[32mup and running\033[0m.\n" fi } @@ -360,7 +358,7 @@ reload_gitlab(){ ## Restarts Sidekiq and Unicorn. restart_gitlab(){ check_status - if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_git_http_server" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then + if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then stop_gitlab fi start_gitlab diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index aab5acaa72c..79ae8e0ae55 100755 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -30,15 +30,14 @@ web_server_pid_path="$pid_path/unicorn.pid" # The default is "$pid_path/sidekiq.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" -gitlab_git_http_server_pid_path="$pid_path/gitlab-git-http-server.pid" -# The -listenXxx settings determine where gitlab-git-http-server +gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" +# The -listenXxx settings determine where gitlab-workhorse # listens for connections from NGINX. To listen on localhost:8181, write # '-listenNetwork tcp -listenAddr localhost:8181'. -# The -authBackend setting tells gitlab-git-http-server where it can reach +# The -authBackend setting tells gitlab-workhorse where it can reach # Unicorn. -gitlab_git_http_server_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-git-http-server.socket -authBackend http://127.0.0.1:8080" -gitlab_git_http_server_repo_root="/home/git/repositories" -gitlab_git_http_server_log="$app_root/log/gitlab-git-http-server.log" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080" +gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" # mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled. # This is required for the Reply by email feature. diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 7218a4d2f20..2a79fbdcf93 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -38,13 +38,13 @@ upstream gitlab { server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; } -upstream gitlab-git-http-server { - server unix:/home/git/gitlab/tmp/sockets/gitlab-git-http-server.socket fail_timeout=0; +upstream gitlab-workhorse { + server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } ## Normal HTTP host server { - ## Either remove "default_server" from the listen line below, + ## Either remove "default_server" from the listen line below, ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab ## to be served if you visit any address that your server responds to, eg. ## the ip address of the server (http://x.x.x.x/)n 0.0.0.0:80 default_server; @@ -67,7 +67,7 @@ server { location / { ## Serve static files from defined root folder. ## @gitlab is a named location for the upstream fallback, see below. - try_files $uri $uri/index.html $uri.html @gitlab; + try_files $uri /index.html $uri.html @gitlab; } ## We route uploads through GitLab to prevent XSS and enforce access control. @@ -113,7 +113,52 @@ server { proxy_pass http://gitlab; } - location ~ [-\/\w\.]+\.git\/ { + location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location ~ ^/api/v3/projects/.*/repository/archive { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + # Build artifacts should be submitted to this location + location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + # Build artifacts should be submitted to this location + location ~ /ci/api/v1/builds/[0-9]+/artifacts { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location @gitlab-workhorse { + client_max_body_size 0; ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. # gzip off; @@ -129,7 +174,7 @@ server { # The following settings only work with NGINX 1.7.11 or newer # - # # Pass chunked request bodies to gitlab-git-http-server as-is + # # Pass chunked request bodies to gitlab-workhorse as-is # proxy_request_buffering off; # proxy_http_version 1.1; @@ -138,7 +183,7 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://gitlab-git-http-server; + proxy_pass http://gitlab-workhorse; } ## Enable gzip compression as per rails guide: diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 7dabfba87e2..79fe1474821 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -42,13 +42,13 @@ upstream gitlab { server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; } -upstream gitlab-git-http-server { - server unix:/home/git/gitlab/tmp/sockets/gitlab-git-http-server.socket fail_timeout=0; +upstream gitlab-workhorse { + server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } ## Redirects all HTTP traffic to the HTTPS host server { - ## Either remove "default_server" from the listen line below, + ## Either remove "default_server" from the listen line below, ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab ## to be served if you visit any address that your server responds to, eg. ## the ip address of the server (http://x.x.x.x/) @@ -56,7 +56,7 @@ server { listen [::]:80 ipv6only=on default_server; server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com server_tokens off; ## Don't show the nginx version number, a security best practice - return 301 https://$server_name$request_uri; + return 301 https://$http_host$request_uri; access_log /var/log/nginx/gitlab_access.log; error_log /var/log/nginx/gitlab_error.log; } @@ -112,7 +112,7 @@ server { location / { ## Serve static files from defined root folder. ## @gitlab is a named location for the upstream fallback, see below. - try_files $uri $uri/index.html $uri.html @gitlab; + try_files $uri /index.html $uri.html @gitlab; } ## We route uploads through GitLab to prevent XSS and enforce access control. @@ -160,7 +160,52 @@ server { proxy_pass http://gitlab; } - location ~ [-\/\w\.]+\.git\/ { + location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location ~ ^/api/v3/projects/.*/repository/archive { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + # Build artifacts should be submitted to this location + location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + # Build artifacts should be submitted to this location + location ~ /ci/api/v1/builds/[0-9]+/artifacts { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location @gitlab-workhorse { + client_max_body_size 0; ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. gzip off; @@ -176,7 +221,7 @@ server { # The following settings only work with NGINX 1.7.11 or newer # - # # Pass chunked request bodies to gitlab-git-http-server as-is + # # Pass chunked request bodies to gitlab-workhorse as-is # proxy_request_buffering off; # proxy_http_version 1.1; @@ -185,7 +230,7 @@ server { proxy_set_header X-Forwarded-Ssl on; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://gitlab-git-http-server; + proxy_pass http://gitlab-workhorse; } ## Enable gzip compression as per rails guide: diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake deleted file mode 100644 index 1de664c85e1..00000000000 --- a/lib/tasks/ci/migrate.rake +++ /dev/null @@ -1,87 +0,0 @@ -namespace :ci do - desc 'GitLab | Import and migrate CI database' - task migrate: :environment do - warn_user_is_not_gitlab - configure_cron_mode - - unless ENV['force'] == 'yes' - puts 'This will remove all CI related data and restore it from the provided backup.' - ask_to_continue - puts '' - end - - # disable CI for time of migration - enable_ci(false) - - # unpack archives - migrate = Ci::Migrate::Manager.new - migrate.unpack - - Rake::Task['ci:migrate:db'].invoke - Rake::Task['ci:migrate:builds'].invoke - Rake::Task['ci:migrate:tags'].invoke - Rake::Task['ci:migrate:services'].invoke - - # enable CI for time of migration - enable_ci(true) - - migrate.cleanup - end - - namespace :migrate do - desc 'GitLab | Import CI database' - task db: :environment do - configure_cron_mode - $progress.puts 'Restoring database ... '.blue - Ci::Migrate::Database.new.restore - $progress.puts 'done'.green - end - - desc 'GitLab | Import CI builds' - task builds: :environment do - configure_cron_mode - $progress.puts 'Restoring builds ... '.blue - Ci::Migrate::Builds.new.restore - $progress.puts 'done'.green - end - - desc 'GitLab | Migrate CI tags' - task tags: :environment do - configure_cron_mode - $progress.puts 'Migrating tags ... '.blue - ::Ci::Migrate::Tags.new.restore - $progress.puts 'done'.green - end - - desc 'GitLab | Migrate CI auto-increments' - task autoincrements: :environment do - c = ActiveRecord::Base.connection - c.tables.select { |t| t.start_with?('ci_') }.each do |table| - result = c.select_one("SELECT id FROM #{table} ORDER BY id DESC LIMIT 1") - if result - ai_val = result['id'].to_i + 1 - puts "Resetting auto increment ID for #{table} to #{ai_val}" - if c.adapter_name == 'PostgreSQL' - c.execute("ALTER SEQUENCE #{table}_id_seq RESTART WITH #{ai_val}") - else - c.execute("ALTER TABLE #{table} AUTO_INCREMENT = #{ai_val}") - end - end - end - end - - desc 'GitLab | Migrate CI services' - task services: :environment do - $progress.puts 'Migrating services ... '.blue - c = ActiveRecord::Base.connection - c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'") - $progress.puts 'done'.green - end - end - - def enable_ci(enabled) - settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults - settings.ci_enabled = enabled - settings.save! - end -end diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake new file mode 100644 index 00000000000..e9587595fef --- /dev/null +++ b/lib/tasks/flay.rake @@ -0,0 +1,9 @@ +desc 'Code duplication analyze via flay' +task :flay do + output = %x(bundle exec flay --mass 35 app/ lib/gitlab/) + + if output.include? "Similar code found" + puts output + exit 1 + end +end diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake new file mode 100644 index 00000000000..3bfe999ae74 --- /dev/null +++ b/lib/tasks/flog.rake @@ -0,0 +1,25 @@ +desc 'Code complexity analyze via flog' +task :flog do + output = %x(bundle exec flog -m app/ lib/gitlab) + exit_code = 0 + minimum_score = 70 + output = output.lines + + # Skip total complexity score + output.shift + + # Skip some trash info + output.shift + + output.each do |line| + score, method = line.split(" ") + score = score.to_i + + if score > minimum_score + exit_code = 1 + puts "High complexity in #{method}. Score: #{score}" + end + end + + exit exit_code +end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index f20c7f71ba5..cb4abe13799 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -12,6 +12,8 @@ namespace :gitlab do Rake::Task["gitlab:backup:repo:create"].invoke Rake::Task["gitlab:backup:uploads:create"].invoke Rake::Task["gitlab:backup:builds:create"].invoke + Rake::Task["gitlab:backup:artifacts:create"].invoke + Rake::Task["gitlab:backup:lfs:create"].invoke backup = Backup::Manager.new backup.pack @@ -32,6 +34,8 @@ namespace :gitlab do Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories") Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") + Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts") + Rake::Task["gitlab:backup:lfs:restore"].invoke unless backup.skipped?("lfs") Rake::Task["gitlab:shell:setup"].invoke backup.cleanup @@ -113,6 +117,44 @@ namespace :gitlab do end end + namespace :artifacts do + task create: :environment do + $progress.puts "Dumping artifacts ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") + $progress.puts "[SKIPPED]".cyan + else + Backup::Artifacts.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring artifacts ... ".blue + Backup::Artifacts.new.restore + $progress.puts "done".green + end + end + + namespace :lfs do + task create: :environment do + $progress.puts "Dumping lfs objects ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("lfs") + $progress.puts "[SKIPPED]".cyan + else + Backup::Lfs.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring lfs objects ... ".blue + Backup::Lfs.new.restore + $progress.puts "done".green + end + end + def configure_cron_mode if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 66f1ecf385f..a25fac62cfc 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -335,7 +335,7 @@ namespace :gitlab do print "Redis version >= #{min_redis_version}? ... " redis_version = run(%W(redis-cli --version)) - redis_version = redis_version.try(:match, /redis-cli (.*)/) + redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) if redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) puts "yes".green @@ -642,7 +642,6 @@ namespace :gitlab do if Gitlab.config.incoming_email.enabled check_address_formatted_correctly - check_mail_room_config_exists check_imap_authentication if Rails.env.production? @@ -744,42 +743,16 @@ namespace :gitlab do end end - def check_mail_room_config_exists - print "MailRoom config exists? ... " - - mail_room_config_file = Rails.root.join("config", "mail_room.yml") - - if File.exists?(mail_room_config_file) - puts "yes".green - else - puts "no".red - try_fixing_it( - "Copy config/mail_room.yml.example to config/mail_room.yml", - "Check that the information in config/mail_room.yml is correct" - ) - for_more_information( - "doc/incoming_email/README.md" - ) - fix_and_rerun - end - end - def check_imap_authentication print "IMAP server credentials are correct? ... " - mail_room_config_file = Rails.root.join("config", "mail_room.yml") - - unless File.exists?(mail_room_config_file) - puts "can't check because of previous errors".magenta - return - end - - config = YAML.load_file(mail_room_config_file)[:mailboxes].first rescue nil + config = Gitlab.config.incoming_email if config begin - imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) - imap.login(config[:email], config[:password]) + imap = Net::IMAP.new(config.host, port: config.port, ssl: config.ssl) + imap.starttls if config.start_tls + imap.login(config.user, config.password) connected = true rescue connected = false @@ -791,7 +764,7 @@ namespace :gitlab do else puts "no".red try_fixing_it( - "Check that the information in config/mail_room.yml is correct" + "Check that the information in config/gitlab.yml is correct" ) for_more_information( "doc/incoming_email/README.md" @@ -851,7 +824,7 @@ namespace :gitlab do repo_dirs = Dir.glob(File.join(namespace_dir, '*')) repo_dirs.each do |dir| puts "\nChecking repo at #{dir}" - system(*%w(git fsck), chdir: dir) + system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: dir) end end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 6b1e3716147..9f5852ac613 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -46,43 +46,24 @@ namespace :gitlab do desc "GitLab | Cleanup | Clean repositories" task repos: :environment do warn_user_is_not_gitlab - remove_flag = ENV['REMOVE'] - - git_base_path = Gitlab.config.gitlab_shell.repos_path - all_dirs = Dir.glob(git_base_path + '/*') - - global_projects = Project.in_namespace(nil).pluck(:path) - - puts git_base_path.yellow - puts "Looking for global repos to remove... " - - # skip non git repo - all_dirs.select! do |dir| - dir =~ /.git$/ - end - - # skip existing repos - all_dirs.reject! do |dir| - repo_name = File.basename dir - path = repo_name.gsub(/\.git$/, "") - global_projects.include?(path) - end - all_dirs.each do |dir_path| - if remove_flag - if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".red - else - puts "Cannot remove #{dir_path}".red - end - else - puts "Can be removed: #{dir_path}".red + move_suffix = "+orphaned+#{Time.now.to_i}" + repo_root = Gitlab.config.gitlab_shell.repos_path + # Look for global repos (legacy, depth 1) and normal repos (depth 2) + IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| + find.each_line do |path| + path.chomp! + repo_with_namespace = path. + sub(repo_root, ''). + sub(%r{^/*}, ''). + chomp('.git'). + chomp('.wiki') + next if Project.find_with_namespace(repo_with_namespace) + new_path = path + move_suffix + puts path.inspect + ' -> ' + new_path.inspect + File.rename(path, new_path) end end - - unless remove_flag - puts "To cleanup this directories run this command with REMOVE=true".yellow - end end desc "GitLab | Cleanup | Block users that have been removed in LDAP" diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 0ac4b0fa8a3..4cbccf2ca89 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -16,6 +16,7 @@ namespace :gitlab do Rake::Task["db:setup"].invoke Rake::Task["add_limits_mysql"].invoke + Rake::Task["setup_postgresql"].invoke Rake::Task["db:seed_fu"].invoke rescue Gitlab::TaskAbortedByUserError puts "Quitting...".red diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 3c0cc763d17..dd61632e557 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -17,7 +17,7 @@ namespace :gitlab do # Clone if needed unless File.directory?(target_dir) - system(*%W(git clone -- #{args.repo} #{target_dir})) + system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir})) end # Make sure we're on the right tag @@ -27,7 +27,7 @@ namespace :gitlab do reseted = reset_to_commit(args) unless reseted - system(*%W(git fetch origin)) + system(*%W(#{Gitlab.config.git.bin_path} fetch origin)) reset_to_commit(args) end @@ -128,14 +128,14 @@ namespace :gitlab do end def reset_to_commit(args) - tag, status = Gitlab::Popen.popen(%W(git describe -- #{args.tag})) + tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- #{args.tag})) unless status.zero? - tag, status = Gitlab::Popen.popen(%W(git describe -- origin/#{args.tag})) + tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- origin/#{args.tag})) end tag = tag.strip - system(*%W(git reset --hard #{tag})) + system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) end end diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index c95b6540ebc..efb863a8764 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -2,16 +2,6 @@ module Gitlab class TaskAbortedByUserError < StandardError; end end -unless STDOUT.isatty - module Colored - extend self - - def colorize(string, options={}) - string - end - end -end - namespace :gitlab do # Ask if the user wants to continue @@ -103,7 +93,7 @@ namespace :gitlab do gitlab_user = Gitlab.config.gitlab.user current_user = run(%W(whoami)).chomp unless current_user == gitlab_user - puts "#{Colored.color(:black)+Colored.color(:on_yellow)} Warning #{Colored.extra(:clear)}" + puts " Warning ".colorize(:black).on_yellow puts " You are running as user #{current_user.magenta}, we hope you know what you are doing." puts " Things may work\/fail for the wrong reasons." puts " For correct results you should run this as user #{gitlab_user.magenta}." diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake new file mode 100644 index 00000000000..9196677a017 --- /dev/null +++ b/lib/tasks/gitlab/two_factor.rake @@ -0,0 +1,23 @@ +namespace :gitlab do + namespace :two_factor do + desc "GitLab | Disable Two-factor authentication (2FA) for all users" + task disable_for_all_users: :environment do + scope = User.with_two_factor + count = scope.count + + if count > 0 + puts "This will disable 2FA for #{count.to_s.red} users..." + + begin + ask_to_continue + scope.find_each(&:disable_two_factor!) + puts "Successfully disabled 2FA for #{count} users.".green + rescue Gitlab::TaskAbortedByUserError + puts "Quitting...".red + end + else + puts "There are currently no users with 2FA enabled.".yellow + end + end + end +end diff --git a/lib/tasks/grape.rake b/lib/tasks/grape.rake new file mode 100644 index 00000000000..9980e0b7984 --- /dev/null +++ b/lib/tasks/grape.rake @@ -0,0 +1,8 @@ +namespace :grape do + desc 'Print compiled grape routes' + task routes: :environment do + API::API.routes.each do |route| + puts route + end + end +end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake new file mode 100644 index 00000000000..141a0b74ec0 --- /dev/null +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -0,0 +1,8 @@ +require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') +require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') + +desc 'GitLab | Sets up PostgreSQL' +task setup_postgresql: :environment do + NamespacesProjectsPathLowerIndexes.new.up + AddUsersLowerUsernameEmailIndexes.new.up +end diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 831746815d7..365ff2defd4 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -19,11 +19,20 @@ namespace :spec do run_commands(cmds) end + desc 'GitLab | Rspec | Run benchmark specs' + task :benchmark do + cmds = [ + %W(rake gitlab:setup), + %W(rspec spec --tag @benchmark) + ] + run_commands(cmds) + end + desc 'GitLab | Rspec | Run other specs' task :other do cmds = [ %W(rake gitlab:setup), - %W(rspec spec --tag ~@api --tag ~@feature) + %W(rspec spec --tag ~@api --tag ~@feature --tag ~@benchmark) ] run_commands(cmds) end @@ -33,7 +42,7 @@ desc "GitLab | Run specs" task :spec do cmds = [ %W(rake gitlab:setup), - %W(rspec spec), + %W(rspec spec --tag ~@benchmark), ] run_commands(cmds) end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index c8881be0954..d5a96fd38f4 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -5,7 +5,7 @@ namespace :spinach do task :project do cmds = [ %W(rake gitlab:setup), - %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets), + %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@commits), ] run_commands(cmds) end @@ -14,7 +14,7 @@ namespace :spinach do task :other do cmds = [ %W(rake gitlab:setup), - %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets), + %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets,@commits), ] run_commands(cmds) end @@ -33,4 +33,4 @@ def run_commands(cmds) cmds.each do |cmd| system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!") end -end +end
\ No newline at end of file diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb new file mode 100644 index 00000000000..d4291f012d3 --- /dev/null +++ b/lib/uploaded_file.rb @@ -0,0 +1,37 @@ +require "tempfile" +require "fileutils" + +# Taken from: Rack::Test::UploadedFile +class UploadedFile + + # The filename, *not* including the path, of the "uploaded" file + attr_reader :original_filename + + # The tempfile + attr_reader :tempfile + + # The content type of the "uploaded" file + attr_accessor :content_type + + def initialize(path, filename, content_type = "text/plain") + raise "#{path} file does not exist" unless ::File.exist?(path) + + @content_type = content_type + @original_filename = filename || ::File.basename(path) + @tempfile = File.new(path, 'rb') + end + + def path + @tempfile.path + end + + alias_method :local_path, :path + + def method_missing(method_name, *args, &block) #:nodoc: + @tempfile.__send__(method_name, *args, &block) + end + + def respond_to?(method_name, include_private = false) #:nodoc: + @tempfile.respond_to?(method_name, include_private) || super + end +end |