diff options
| author | Zeger-Jan van de Weg <mail@zjvandeweg.nl> | 2015-11-11 13:22:00 +0100 |
|---|---|---|
| committer | Zeger-Jan van de Weg <mail@zjvandeweg.nl> | 2015-11-11 13:22:00 +0100 |
| commit | 9f548dffc33158adb02ef1ec9a6e98b3d090f22b (patch) | |
| tree | 39f5619db8f33c70dcdf52dbc8d07b3cb8486e59 /lib | |
| parent | 8db063b579322238af43f6d04b5968d9c6ea935d (diff) | |
| parent | dfa09789061bbd3891c17ebbcc23f7bd6b95336c (diff) | |
| download | gitlab-ce-9f548dffc33158adb02ef1ec9a6e98b3d090f22b.tar.gz | |
Merge branch 'master' into rake-tasks-git
Diffstat (limited to 'lib')
68 files changed, 1366 insertions, 687 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index afc0402f9e1..40671e2517c 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -25,7 +25,7 @@ module API format :json content_type :txt, "text/plain" - helpers APIHelpers + helpers Helpers mount Groups mount GroupMembers diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 883a5e14b17..20cadae2291 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -231,7 +231,7 @@ module API class CommitStatus < Grape::Entity expose :id, :sha, :ref, :status, :name, :target_url, :description, - :created_at, :started_at, :finished_at + :created_at, :started_at, :finished_at, :allow_failure expose :author, using: Entities::UserBasic end diff --git a/lib/api/files.rb b/lib/api/files.rb index 308c84dd135..a7a768f8895 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -43,7 +43,8 @@ module API # "content": "IyA9PSBTY2hlbWEgSW5mb3...", # "ref": "master", # "blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83", - # "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50" + # "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50", + # "last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d", # } # get ":id/repository/files" do @@ -71,6 +72,7 @@ module API ref: ref, blob_id: blob.id, commit_id: commit.id, + last_commit_id: user_project.repository.last_commit_for_path(commit.sha, file_path).id } else not_found! 'File' diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 549b1f9e9a7..652bdf9b278 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -1,5 +1,5 @@ module API - module APIHelpers + module Helpers PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" PRIVATE_TOKEN_PARAM = :private_token SUDO_HEADER ="HTTP_SUDO" diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index f3a59fadf24..6eb84baf9cb 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -249,8 +249,16 @@ module API required_attributes! [:note] merge_request = user_project.merge_requests.find(params[:merge_request_id]) - note = merge_request.notes.new(note: params[:note], project_id: user_project.id) - note.author = current_user + + authorize! :create_note, merge_request + + opts = { + note: params[:note], + noteable_type: 'MergeRequest', + noteable_id: merge_request.id + } + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute if note.save present note, with: Entities::MRNote diff --git a/lib/api/projects.rb b/lib/api/projects.rb index c2fb36b4143..67ee66a2058 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -246,8 +246,8 @@ module API # Example Request: # DELETE /projects/:id/fork delete ":id/fork" do - authenticated_as_admin! - unless user_project.forked_project_link.nil? + authorize! :remove_fork_project, user_project + if user_project.forked? user_project.forked_project_link.destroy end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 2d96c9666d2..20d568cf462 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -133,7 +133,7 @@ module API authorize! :download_code, user_project begin - file_path = ArchiveRepositoryService.new( + ArchiveRepositoryService.new( user_project, params[:sha], params[:format] @@ -141,17 +141,6 @@ module API rescue not_found!('File') end - - if file_path && File.exists?(file_path) - data = File.open(file_path, 'rb').read - basename = File.basename(file_path) - header['Content-Disposition'] = "attachment; filename=\"#{basename}\"" - content_type MIME::Types.type_for(file_path).first.content_type - env['api.format'] = :binary - present data - else - redirect request.fullpath - end end # Compare two branches, tags or commits diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index 6f56f680bb9..635967f4bd4 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -1,34 +1,13 @@ -module Backup - class Builds - attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir +require 'backup/files' +module Backup + class Builds < Files def initialize - @app_builds_dir = Settings.gitlab_ci.builds_path - @backup_dir = Gitlab.config.backup.path - @backup_builds_dir = File.join(Gitlab.config.backup.path, 'builds') - end - - # Copy builds from builds directory to backup/builds - def dump - FileUtils.rm_rf(backup_builds_dir) - # Ensure the parent dir of backup_builds_dir exists - FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create backup_builds_dir before us - FileUtils.mkdir(backup_builds_dir, mode: 0700) - FileUtils.cp_r(app_builds_dir, backup_dir) - end - - def restore - backup_existing_builds_dir - - FileUtils.cp_r(backup_builds_dir, app_builds_dir) + super('builds', Settings.gitlab_ci.builds_path) end - def backup_existing_builds_dir - timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") - if File.exists?(app_builds_dir) - FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) - end + def create_files_dir + Dir.mkdir(app_files_dir, 0700) end end end diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 959ac4b7868..67b2a64bd10 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -2,26 +2,26 @@ require 'yaml' module Backup class Database - attr_reader :config, :db_dir + attr_reader :config, :db_file_name def initialize @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env] - @db_dir = File.join(Gitlab.config.backup.path, 'db') + @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end def dump - FileUtils.rm_rf(@db_dir) - # Ensure the parent dir of @db_dir exists - FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create @db_dir before us - FileUtils.mkdir(@db_dir, mode: 0700) + FileUtils.mkdir_p(File.dirname(db_file_name)) + FileUtils.rm_f(db_file_name) + compress_rd, compress_wr = IO.pipe + compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600]) + compress_rd.close - success = case config["adapter"] + dump_pid = case config["adapter"] when /^mysql/ then $progress.print "Dumping MySQL database #{config['database']} ... " # Workaround warnings from MySQL 5.6 about passwords on cmd line ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - system('mysqldump', *mysql_args, config['database'], out: db_file_name) + spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) when "postgresql" then $progress.print "Dumping PostgreSQL database #{config['database']} ... " pg_env @@ -30,48 +30,42 @@ module Backup pgsql_args << "-n" pgsql_args << Gitlab.config.backup.pg_schema end - system('pg_dump', *pgsql_args, config['database'], out: db_file_name) + spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr) end - report_success(success) - abort 'Backup failed' unless success + compress_wr.close + + success = [compress_pid, dump_pid].all? { |pid| Process.waitpid(pid); $?.success? } - $progress.print 'Compressing database ... ' - success = system('gzip', db_file_name) report_success(success) - abort 'Backup failed: compress error' unless success + abort 'Backup failed' unless success end def restore - $progress.print 'Decompressing database ... ' - success = system('gzip', '-d', db_file_name_gz) - report_success(success) - abort 'Restore failed: decompress error' unless success + decompress_rd, decompress_wr = IO.pipe + decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name) + decompress_wr.close - success = case config["adapter"] + restore_pid = case config["adapter"] when /^mysql/ then $progress.print "Restoring MySQL database #{config['database']} ... " # Workaround warnings from MySQL 5.6 about passwords on cmd line ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - system('mysql', *mysql_args, config['database'], in: db_file_name) + spawn('mysql', *mysql_args, config['database'], in: decompress_rd) when "postgresql" then $progress.print "Restoring PostgreSQL database #{config['database']} ... " pg_env - system('psql', config['database'], '-f', db_file_name) + spawn('psql', config['database'], in: decompress_rd) end + decompress_rd.close + + success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? } + report_success(success) abort 'Restore failed' unless success end protected - def db_file_name - File.join(db_dir, 'database.sql') - end - - def db_file_name_gz - File.join(db_dir, 'database.sql.gz') - end - def mysql_args args = { 'host' => '--host', diff --git a/lib/backup/files.rb b/lib/backup/files.rb new file mode 100644 index 00000000000..654b4d1c896 --- /dev/null +++ b/lib/backup/files.rb @@ -0,0 +1,40 @@ +require 'open3' + +module Backup + class Files + attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir + + def initialize(name, app_files_dir) + @name = name + @app_files_dir = File.realpath(app_files_dir) + @files_parent_dir = File.realpath(File.join(@app_files_dir, '..')) + @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz') + end + + # Copy files from public/files to backup/files + def dump + FileUtils.mkdir_p(Gitlab.config.backup.path) + FileUtils.rm_f(backup_tarball) + run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + end + + def restore + backup_existing_files_dir + create_files_dir + + run_pipeline!([%W(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball) + end + + def backup_existing_files_dir + timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}") + if File.exists?(app_files_dir) + FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path)) + end + end + + def run_pipeline!(cmd_list, options={}) + status_list = Open3.pipeline(*cmd_list, options) + abort 'Backup failed' unless status_list.compact.all?(&:success?) + end + end +end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 5c42f25f4a2..f011fd03de0 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -150,11 +150,11 @@ module Backup private def backup_contents - folders_to_backup + ["backup_information.yml"] + folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] end def folders_to_backup - folders = %w{repositories db uploads builds} + folders = %w{repositories db} if ENV["SKIP"] return folders.reject{ |folder| ENV["SKIP"].include?(folder) } diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 4d70f7883dd..a82a7e1f7bf 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -35,7 +35,7 @@ module Backup if wiki.repository.empty? $progress.puts " [SKIPPED]".cyan else - cmd = %W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) output, status = Gitlab::Popen.popen(cmd) if status.zero? $progress.puts " [DONE]".green @@ -67,7 +67,7 @@ module Backup FileUtils.mkdir_p(path_to_repo(project)) cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)}) else - cmd = %W(git init --bare #{path_to_repo(project)}) + cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_repo(project)}) end if system(*cmd, silent) @@ -87,7 +87,7 @@ module Backup # that was initialized with ProjectWiki.new() and then # try to restore with 'git clone --bare'. FileUtils.rm_rf(path_to_repo(wiki)) - cmd = %W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) if system(*cmd, silent) $progress.puts " [DONE]".green diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 1f9626644e6..9261f77f3c9 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -1,34 +1,14 @@ +require 'backup/files' + module Backup - class Uploads - attr_reader :app_uploads_dir, :backup_uploads_dir, :backup_dir + class Uploads < Files def initialize - @app_uploads_dir = File.realpath(Rails.root.join('public', 'uploads')) - @backup_dir = Gitlab.config.backup.path - @backup_uploads_dir = File.join(Gitlab.config.backup.path, 'uploads') - end - - # Copy uploads from public/uploads to backup/uploads - def dump - FileUtils.rm_rf(backup_uploads_dir) - # Ensure the parent dir of backup_uploads_dir exists - FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create backup_uploads_dir before us - FileUtils.mkdir(backup_uploads_dir, mode: 0700) - FileUtils.cp_r(app_uploads_dir, backup_dir) - end - - def restore - backup_existing_uploads_dir - - FileUtils.cp_r(backup_uploads_dir, app_uploads_dir) + super('uploads', Rails.root.join('public/uploads')) end - def backup_existing_uploads_dir - timestamped_uploads_path = File.join(app_uploads_dir, '..', "uploads.#{Time.now.to_i}") - if File.exists?(app_uploads_dir) - FileUtils.mv(app_uploads_dir, File.expand_path(timestamped_uploads_path)) - end + def create_files_dir + Dir.mkdir(app_files_dir) end end end diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 218d8c3adcc..0a4cbf69b63 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -26,7 +26,7 @@ module Ci format :json helpers Helpers - helpers ::API::APIHelpers + helpers ::API::Helpers mount Builds mount Commits diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index e602cda81d6..7e4986b6af3 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -16,7 +16,9 @@ module Ci end def update_runner_last_contact - if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= UPDATE_RUNNER_EVERY + # Use a random threshold to prevent beating DB updates + contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) + if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age current_runner.update_attributes(contacted_at: Time.now) end end diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index 915a4f526a6..5ff7407c6fe 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -60,7 +60,8 @@ module Ci class BuildTime < Chart def collect - commits = project.commits.joins(:builds).where("#{Ci::Build.table_name}.finished_at is NOT NULL AND #{Ci::Build.table_name}.started_at is NOT NULL").last(30) + commits = project.commits.last(30) + commits.each do |commit| @labels << commit.short_sha @build_times << (commit.duration / 60) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index c47951bc5d1..0f57a4f53ab 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -5,12 +5,13 @@ module Ci DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables] - ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage] + ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when] - attr_reader :before_script, :image, :services, :variables + attr_reader :before_script, :image, :services, :variables, :path - def initialize(config) + def initialize(config, path = nil) @config = YAML.load(config) + @path = path unless @config.is_a? Hash raise ValidationError, "YAML should be a hash" @@ -63,26 +64,6 @@ module Ci end end - def process?(only_params, except_params, ref, tag) - return true if only_params.nil? && except_params.nil? - - if only_params - return true if tag && only_params.include?("tags") - return true if !tag && only_params.include?("branches") - - only_params.find do |pattern| - match_ref?(pattern, ref) - end - else - return false if tag && except_params.include?("tags") - return false if !tag && except_params.include?("branches") - - except_params.each do |pattern| - return false if match_ref?(pattern, ref) - end - end - end - def build_job(name, job) { stage_idx: stages.index(job[:stage]), @@ -93,6 +74,7 @@ module Ci only: job[:only], except: job[:except], allow_failure: job[:allow_failure] || false, + when: job[:when] || 'on_success', options: { image: job[:image] || @image, services: job[:services] || @services @@ -100,14 +82,6 @@ module Ci } end - def match_ref?(pattern, ref) - if pattern.first == "/" && pattern.last == "/" - Regexp.new(pattern[1...-1]) =~ ref - else - pattern == ref - end - end - def normalize_script(script) if script.is_a? Array script.join("\n") @@ -138,62 +112,105 @@ module Ci end @jobs.each do |name, job| - validate_job!("#{name} job", job) + validate_job!(name, job) end true end def validate_job!(name, job) + if name.blank? || !validate_string(name) + raise ValidationError, "job name should be non-empty string" + end + job.keys.each do |key| unless ALLOWED_JOB_KEYS.include? key - raise ValidationError, "#{name}: unknown parameter #{key}" + raise ValidationError, "#{name} job: unknown parameter #{key}" end end - if !job[:script].is_a?(String) && !validate_array_of_strings(job[:script]) - raise ValidationError, "#{name}: script should be a string or an array of a strings" + if !validate_string(job[:script]) && !validate_array_of_strings(job[:script]) + raise ValidationError, "#{name} job: script should be a string or an array of a strings" end if job[:stage] unless job[:stage].is_a?(String) && job[:stage].in?(stages) - raise ValidationError, "#{name}: stage parameter should be #{stages.join(", ")}" + raise ValidationError, "#{name} job: stage parameter should be #{stages.join(", ")}" end end - if job[:image] && !job[:image].is_a?(String) - raise ValidationError, "#{name}: image should be a string" + if job[:image] && !validate_string(job[:image]) + raise ValidationError, "#{name} job: image should be a string" end if job[:services] && !validate_array_of_strings(job[:services]) - raise ValidationError, "#{name}: services should be an array of strings" + raise ValidationError, "#{name} job: services should be an array of strings" end if job[:tags] && !validate_array_of_strings(job[:tags]) - raise ValidationError, "#{name}: tags parameter should be an array of strings" + raise ValidationError, "#{name} job: tags parameter should be an array of strings" end if job[:only] && !validate_array_of_strings(job[:only]) - raise ValidationError, "#{name}: only parameter should be an array of strings" + raise ValidationError, "#{name} job: only parameter should be an array of strings" end if job[:except] && !validate_array_of_strings(job[:except]) - raise ValidationError, "#{name}: except parameter should be an array of strings" + raise ValidationError, "#{name} job: except parameter should be an array of strings" end if job[:allow_failure] && !job[:allow_failure].in?([true, false]) - raise ValidationError, "#{name}: allow_failure parameter should be an boolean" + raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" + end + + if job[:when] && !job[:when].in?(%w(on_success on_failure always)) + raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" end end private def validate_array_of_strings(values) - values.is_a?(Array) && values.all? {|tag| tag.is_a?(String)} + values.is_a?(Array) && values.all? { |value| validate_string(value) } end def validate_variables(variables) - variables.is_a?(Hash) && variables.all? {|key, value| key.is_a?(Symbol) && value.is_a?(String)} + variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) } + end + + def validate_string(value) + value.is_a?(String) || value.is_a?(Symbol) + end + + def process?(only_params, except_params, ref, tag) + if only_params.present? + return false unless matching?(only_params, ref, tag) + end + + if except_params.present? + return false if matching?(except_params, ref, tag) + end + + true + end + + def matching?(patterns, ref, tag) + patterns.any? do |pattern| + match_ref?(pattern, ref, tag) + end + end + + def match_ref?(pattern, ref, tag) + pattern, path = pattern.split('@', 2) + return false if path && path != self.path + return true if tag && pattern == 'tags' + return true if !tag && pattern == 'branches' + + if pattern.first == "/" && pattern.last == "/" + Regexp.new(pattern[1...-1]) =~ ref + else + pattern == ref + end end end end diff --git a/lib/ci/migrate/builds.rb b/lib/ci/migrate/builds.rb deleted file mode 100644 index c4f62e55295..00000000000 --- a/lib/ci/migrate/builds.rb +++ /dev/null @@ -1,29 +0,0 @@ -module Ci - module Migrate - class Builds - attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir - - def initialize - @app_builds_dir = Settings.gitlab_ci.builds_path - @backup_dir = Gitlab.config.backup.path - @backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz') - end - - def restore - backup_existing_builds_dir - - FileUtils.mkdir_p(app_builds_dir, mode: 0700) - unless system('tar', '-C', app_builds_dir, '-zxf', backup_builds_tarball) - abort 'Restore failed'.red - end - end - - def backup_existing_builds_dir - timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") - if File.exists?(app_builds_dir) - FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) - end - end - end - end -end diff --git a/lib/ci/migrate/database.rb b/lib/ci/migrate/database.rb deleted file mode 100644 index bf9b80f1f62..00000000000 --- a/lib/ci/migrate/database.rb +++ /dev/null @@ -1,67 +0,0 @@ -require 'yaml' - -module Ci - module Migrate - class Database - attr_reader :config - - def initialize - @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] - end - - def restore - decompress_rd, decompress_wr = IO.pipe - decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name) - decompress_wr.close - - restore_pid = case config["adapter"] - when /^mysql/ then - $progress.print "Restoring MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysql', *mysql_args, config['database'], in: decompress_rd) - when "postgresql" then - $progress.print "Restoring PostgreSQL database #{config['database']} ... " - pg_env - spawn('psql', config['database'], in: decompress_rd) - end - decompress_rd.close - - success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? } - abort 'Restore failed' unless success - end - - protected - - def db_file_name - File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') - end - - def mysql_args - args = { - 'host' => '--host', - 'port' => '--port', - 'socket' => '--socket', - 'username' => '--user', - 'encoding' => '--default-character-set' - } - args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact - end - - def pg_env - ENV['PGUSER'] = config["username"] if config["username"] - ENV['PGHOST'] = config["host"] if config["host"] - ENV['PGPORT'] = config["port"].to_s if config["port"] - ENV['PGPASSWORD'] = config["password"].to_s if config["password"] - end - - def report_success(success) - if success - puts '[DONE]'.green - else - puts '[FAILED]'.red - end - end - end - end -end diff --git a/lib/ci/migrate/manager.rb b/lib/ci/migrate/manager.rb deleted file mode 100644 index e5e4fb784eb..00000000000 --- a/lib/ci/migrate/manager.rb +++ /dev/null @@ -1,72 +0,0 @@ -module Ci - module Migrate - class Manager - CI_IMPORT_PREFIX = '8.0' # Only allow imports from CI 8.0.x - - def cleanup - $progress.print "Deleting tmp directories ... " - - backup_contents.each do |dir| - next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) - - if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) - $progress.puts "done".green - else - puts "deleting tmp directory '#{dir}' failed".red - abort 'Backup failed' - end - end - end - - def unpack - Dir.chdir(Gitlab.config.backup.path) - - # check for existing backups in the backup dir - file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i } - puts "no backups found" if file_list.count == 0 - - if file_list.count > 1 && ENV["BACKUP"].nil? - puts "Found more than one backup, please specify which one you want to restore:" - puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" - exit 1 - end - - tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar") - - unless File.exists?(tar_file) - puts "The specified CI backup doesn't exist!" - exit 1 - end - - $progress.print "Unpacking backup ... " - - unless Kernel.system(*%W(tar -xf #{tar_file})) - puts "unpacking backup failed".red - exit 1 - else - $progress.puts "done".green - end - - ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 - - # restoring mismatching backups can lead to unexpected problems - if !settings[:gitlab_version].start_with?(CI_IMPORT_PREFIX) - puts "GitLab CI version mismatch:".red - puts " Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red - exit 1 - end - end - - private - - def backup_contents - ["db", "builds", "backup_information.yml"] - end - - def settings - @settings ||= YAML.load_file("backup_information.yml") - end - end - end -end - diff --git a/lib/ci/migrate/tags.rb b/lib/ci/migrate/tags.rb deleted file mode 100644 index 97e043ece27..00000000000 --- a/lib/ci/migrate/tags.rb +++ /dev/null @@ -1,42 +0,0 @@ -require 'yaml' - -module Ci - module Migrate - class Tags - def restore - puts 'Inserting tags...' - connection.select_all('SELECT ci_tags.name FROM ci_tags').each do |tag| - begin - connection.execute("INSERT INTO tags (name) VALUES(#{ActiveRecord::Base::sanitize(tag['name'])})") - rescue ActiveRecord::RecordNotUnique - end - end - - ActiveRecord::Base.transaction do - puts 'Deleting old taggings...' - connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'" - - puts 'Inserting taggings...' - connection.execute( - 'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' + - "SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " + - 'JOIN ci_tags ON ci_tags.id = ci_taggings.tag_id ' + - 'JOIN tags ON tags.name = ci_tags.name ' - ) - - puts 'Resetting counters... ' - connection.execute( - 'UPDATE tags SET ' + - 'taggings_count = (SELECT COUNT(*) FROM taggings WHERE tags.id = taggings.tag_id)' - ) - end - end - - protected - - def connection - ActiveRecord::Base.connection - end - end - end -end diff --git a/lib/ci/status.rb b/lib/ci/status.rb new file mode 100644 index 00000000000..c02b3b8f3e4 --- /dev/null +++ b/lib/ci/status.rb @@ -0,0 +1,21 @@ +module Ci + class Status + def self.get_status(statuses) + statuses.reject! { |status| status.try(&:allow_failure?) } + + if statuses.none? + 'skipped' + elsif statuses.all?(&:success?) + 'success' + elsif statuses.all?(&:pending?) + 'pending' + elsif statuses.any?(&:running?) || statuses.any?(&:pending?) + 'running' + elsif statuses.all?(&:canceled?) + 'canceled' + else + 'failed' + end + end + end +end diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 0353b3b7ed3..440ef5a3cb3 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -34,7 +34,7 @@ module Grack auth! if project && authorized_request? - # Tell gitlab-git-http-server the request is OK, and what the GL_ID is + # Tell gitlab-workhorse the request is OK, and what the GL_ID is render_grack_auth_ok elsif @user.nil? && !@ci unauthorized @@ -193,7 +193,21 @@ module Grack end def render_grack_auth_ok - [200, { "Content-Type" => "application/json" }, [JSON.dump({ 'GL_ID' => Gitlab::ShellEnv.gl_id(@user) })]] + repo_path = + if @request.path_info =~ /^([\w\.\/-]+)\.wiki\.git/ + ProjectWiki.new(project).repository.path_to_repo + else + project.repository.path_to_repo + end + + [ + 200, + { "Content-Type" => "application/json" }, + [JSON.dump({ + 'GL_ID' => Gitlab::ShellEnv.gl_id(@user), + 'RepoPath' => repo_path, + })] + ] end def render_not_found diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 0ea1b6a2f6f..cd84afa31d5 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -23,7 +23,8 @@ module Gitlab restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], max_attachment_size: Settings.gitlab['max_attachment_size'], session_expire_delay: Settings.gitlab['session_expire_delay'], - import_sources: Settings.gitlab['import_sources'] + import_sources: Settings.gitlab['import_sources'], + shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], ) end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 741a52714ac..71f37f1fef8 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -1,7 +1,7 @@ module Gitlab module Database def self.mysql? - ActiveRecord::Base.connection.adapter_name.downcase == 'mysql' + ActiveRecord::Base.connection.adapter_name.downcase == 'mysql2' end def self.postgresql? diff --git a/lib/gitlab/force_push_check.rb b/lib/gitlab/force_push_check.rb index fdb6a35c78d..93c6a5bb7f5 100644 --- a/lib/gitlab/force_push_check.rb +++ b/lib/gitlab/force_push_check.rb @@ -7,7 +7,7 @@ module Gitlab if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) false else - missed_refs, _ = Gitlab::Popen.popen(%W(git --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})) + missed_refs, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})) missed_refs.split("\n").size > 0 end end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 39d17def930..4d83d8e72a8 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -6,7 +6,7 @@ module Gitlab # Returns true for a valid reference name, false otherwise def validate(ref_name) Gitlab::Utils.system_silent( - %W(git check-ref-format refs/#{ref_name})) + %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name})) end end end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 856ccc71084..9068d79c95e 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -24,12 +24,12 @@ module Gitlab match[1] end - private - def config Gitlab.config.incoming_email end + private + def address_regex wildcard_address = config.address return nil unless wildcard_address diff --git a/lib/gitlab/markdown.rb b/lib/gitlab/markdown.rb index ae5f2544691..b082bfc434b 100644 --- a/lib/gitlab/markdown.rb +++ b/lib/gitlab/markdown.rb @@ -7,6 +7,14 @@ module Gitlab module Markdown # Convert a Markdown String into an HTML-safe String of HTML # + # Note that while the returned HTML will have been sanitized of dangerous + # HTML, it may post a risk of information leakage if it's not also passed + # through `post_process`. + # + # Also note that the returned String is always HTML, not XHTML. Views + # requiring XHTML, such as Atom feeds, need to call `post_process` on the + # result, providing the appropriate `pipeline` option. + # # markdown - Markdown String # context - Hash of context options passed to our HTML Pipeline # @@ -31,6 +39,33 @@ module Gitlab renderer.render(markdown) end + # Perform post-processing on an HTML String + # + # This method is used to perform state-dependent changes to a String of + # HTML, such as removing references that the current user doesn't have + # permission to make (`RedactorFilter`). + # + # html - String to process + # options - Hash of options to customize output + # :pipeline - Symbol pipeline type + # :project - Project + # :user - User object + # + # Returns an HTML-safe String + def self.post_process(html, options) + context = { + project: options[:project], + current_user: options[:user] + } + doc = post_processor.to_document(html, context) + + if options[:pipeline] == :atom + doc.to_html(save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML) + else + doc.to_html + end.html_safe + end + # Provide autoload paths for filters to prevent a circular dependency error autoload :AutolinkFilter, 'gitlab/markdown/autolink_filter' autoload :CommitRangeReferenceFilter, 'gitlab/markdown/commit_range_reference_filter' @@ -41,6 +76,7 @@ module Gitlab autoload :IssueReferenceFilter, 'gitlab/markdown/issue_reference_filter' autoload :LabelReferenceFilter, 'gitlab/markdown/label_reference_filter' autoload :MergeRequestReferenceFilter, 'gitlab/markdown/merge_request_reference_filter' + autoload :RedactorFilter, 'gitlab/markdown/redactor_filter' autoload :RelativeLinkFilter, 'gitlab/markdown/relative_link_filter' autoload :SanitizationFilter, 'gitlab/markdown/sanitization_filter' autoload :SnippetReferenceFilter, 'gitlab/markdown/snippet_reference_filter' @@ -48,27 +84,22 @@ module Gitlab autoload :TableOfContentsFilter, 'gitlab/markdown/table_of_contents_filter' autoload :TaskListFilter, 'gitlab/markdown/task_list_filter' autoload :UserReferenceFilter, 'gitlab/markdown/user_reference_filter' + autoload :UploadLinkFilter, 'gitlab/markdown/upload_link_filter' - # Public: Parse the provided text with GitLab-Flavored Markdown + # Public: Parse the provided HTML with GitLab-Flavored Markdown + # + # html - HTML String + # options - A Hash of options used to customize output (default: {}) + # :no_header_anchors - Disable header anchors in TableOfContentsFilter + # :path - Current path String + # :pipeline - Symbol pipeline type + # :project - Current Project object + # :project_wiki - Current ProjectWiki object + # :ref - Current ref String # - # text - the source text - # options - A Hash of options used to customize output (default: {}): - # :xhtml - output XHTML instead of HTML - # :reference_only_path - Use relative path for reference links - def self.gfm(text, options = {}) - return text if text.nil? - - # Duplicate the string so we don't alter the original, then call to_str - # to cast it back to a String instead of a SafeBuffer. This is required - # for gsub calls to work as we need them to. - text = text.dup.to_str - - options.reverse_merge!( - xhtml: false, - reference_only_path: true, - project: options[:project], - current_user: options[:current_user] - ) + # Returns an HTML-safe String + def self.gfm(html, options = {}) + return '' unless html.present? @pipeline ||= HTML::Pipeline.new(filters) @@ -77,41 +108,36 @@ module Gitlab pipeline: options[:pipeline], # EmojiFilter - asset_root: Gitlab.config.gitlab.base_url, asset_host: Gitlab::Application.config.asset_host, - - # TableOfContentsFilter - no_header_anchors: options[:no_header_anchors], + asset_root: Gitlab.config.gitlab.base_url, # ReferenceFilter - current_user: options[:current_user], - only_path: options[:reference_only_path], - project: options[:project], + only_path: only_path_pipeline?(options[:pipeline]), + project: options[:project], # RelativeLinkFilter + project_wiki: options[:project_wiki], ref: options[:ref], requested_path: options[:path], - project_wiki: options[:project_wiki] - } - - result = @pipeline.call(text, context) - save_options = 0 - if options[:xhtml] - save_options |= Nokogiri::XML::Node::SaveOptions::AS_XHTML - end - - text = result[:output].to_html(save_with: save_options) + # TableOfContentsFilter + no_header_anchors: options[:no_header_anchors] + } - text.html_safe + @pipeline.to_html(html, context).html_safe end private - def self.renderer - @markdown ||= begin - renderer = Redcarpet::Render::HTML.new - Redcarpet::Markdown.new(renderer, redcarpet_options) + # Check if a pipeline enables the `only_path` context option + # + # Returns Boolean + def self.only_path_pipeline?(pipeline) + case pipeline + when :atom, :email + false + else + true end end @@ -129,6 +155,17 @@ module Gitlab }.freeze end + def self.renderer + @markdown ||= begin + renderer = Redcarpet::Render::HTML.new + Redcarpet::Markdown.new(renderer, redcarpet_options) + end + end + + def self.post_processor + @post_processor ||= HTML::Pipeline.new([Gitlab::Markdown::RedactorFilter]) + end + # Filters used in our pipeline # # SanitizationFilter should come first so that all generated reference HTML @@ -140,6 +177,7 @@ module Gitlab Gitlab::Markdown::SyntaxHighlightFilter, Gitlab::Markdown::SanitizationFilter, + Gitlab::Markdown::UploadLinkFilter, Gitlab::Markdown::RelativeLinkFilter, Gitlab::Markdown::EmojiFilter, Gitlab::Markdown::TableOfContentsFilter, diff --git a/lib/gitlab/markdown/commit_range_reference_filter.rb b/lib/gitlab/markdown/commit_range_reference_filter.rb index bb496135d92..e070edae0a4 100644 --- a/lib/gitlab/markdown/commit_range_reference_filter.rb +++ b/lib/gitlab/markdown/commit_range_reference_filter.rb @@ -26,6 +26,18 @@ module Gitlab end end + def self.referenced_by(node) + project = Project.find(node.attr("data-project")) rescue nil + return unless project + + id = node.attr("data-commit-range") + range = CommitRange.new(id, project) + + return unless range.valid_commits? + + { commit_range: range } + end + def initialize(*args) super @@ -53,13 +65,11 @@ module Gitlab range = CommitRange.new(id, project) if range.valid_commits? - push_result(:commit_range, range) - url = url_for_commit_range(project, range) title = range.reference_title klass = reference_class(:commit_range) - data = data_attribute(project.id) + data = data_attribute(project: project.id, commit_range: id) project_ref += '@' if project_ref diff --git a/lib/gitlab/markdown/commit_reference_filter.rb b/lib/gitlab/markdown/commit_reference_filter.rb index fcbb2e936a5..8cdbeb1f9cf 100644 --- a/lib/gitlab/markdown/commit_reference_filter.rb +++ b/lib/gitlab/markdown/commit_reference_filter.rb @@ -26,6 +26,18 @@ module Gitlab end end + def self.referenced_by(node) + project = Project.find(node.attr("data-project")) rescue nil + return unless project + + id = node.attr("data-commit") + commit = commit_from_ref(project, id) + + return unless commit + + { commit: commit } + end + def call replace_text_nodes_matching(Commit.reference_pattern) do |content| commit_link_filter(content) @@ -39,17 +51,15 @@ module Gitlab # Returns a String with commit references replaced with links. All links # have `gfm` and `gfm-commit` class names attached for styling. def commit_link_filter(text) - self.class.references_in(text) do |match, commit_ref, project_ref| + self.class.references_in(text) do |match, id, project_ref| project = self.project_from_ref(project_ref) - if commit = commit_from_ref(project, commit_ref) - push_result(:commit, commit) - + if commit = self.class.commit_from_ref(project, id) url = url_for_commit(project, commit) title = escape_once(commit.link_title) klass = reference_class(:commit) - data = data_attribute(project.id) + data = data_attribute(project: project.id, commit: id) project_ref += '@' if project_ref @@ -62,9 +72,9 @@ module Gitlab end end - def commit_from_ref(project, commit_ref) + def self.commit_from_ref(project, id) if project && project.valid_repo? - project.commit(commit_ref) + project.commit(id) end end diff --git a/lib/gitlab/markdown/cross_project_reference.rb b/lib/gitlab/markdown/cross_project_reference.rb index 855748fdccc..6ab04a584b0 100644 --- a/lib/gitlab/markdown/cross_project_reference.rb +++ b/lib/gitlab/markdown/cross_project_reference.rb @@ -13,18 +13,11 @@ module Gitlab # # ref - String reference. # - # Returns a Project, or nil if the reference can't be accessed + # Returns a Project, or nil if the reference can't be found def project_from_ref(ref) return context[:project] unless ref - other = Project.find_with_namespace(ref) - return nil unless other && user_can_reference_project?(other) - - other - end - - def user_can_reference_project?(project, user = context[:current_user]) - Ability.abilities.allowed?(user, :read_project, project) + Project.find_with_namespace(ref) end end end diff --git a/lib/gitlab/markdown/external_issue_reference_filter.rb b/lib/gitlab/markdown/external_issue_reference_filter.rb index f7c43e1ca89..8f86f13976a 100644 --- a/lib/gitlab/markdown/external_issue_reference_filter.rb +++ b/lib/gitlab/markdown/external_issue_reference_filter.rb @@ -47,8 +47,9 @@ module Gitlab title = escape_once("Issue in #{project.external_issue_tracker.title}") klass = reference_class(:issue) + data = data_attribute(project: project.id) - %(<a href="#{url}" + %(<a href="#{url}" #{data} title="#{title}" class="#{klass}">#{match}</a>) end diff --git a/lib/gitlab/markdown/issue_reference_filter.rb b/lib/gitlab/markdown/issue_reference_filter.rb index 01320f80796..481d282f7b1 100644 --- a/lib/gitlab/markdown/issue_reference_filter.rb +++ b/lib/gitlab/markdown/issue_reference_filter.rb @@ -27,6 +27,10 @@ module Gitlab end end + def self.referenced_by(node) + { issue: LazyReference.new(Issue, node.attr("data-issue")) } + end + def call replace_text_nodes_matching(Issue.reference_pattern) do |content| issue_link_filter(content) @@ -45,13 +49,11 @@ module Gitlab project = self.project_from_ref(project_ref) if project && issue = project.get_issue(id) - push_result(:issue, issue) - url = url_for_issue(id, project, only_path: context[:only_path]) title = escape_once("Issue: #{issue.title}") klass = reference_class(:issue) - data = data_attribute(project.id) + data = data_attribute(project: project.id, issue: issue.id) %(<a href="#{url}" #{data} title="#{title}" diff --git a/lib/gitlab/markdown/label_reference_filter.rb b/lib/gitlab/markdown/label_reference_filter.rb index 1e5cb12071e..618acb7a578 100644 --- a/lib/gitlab/markdown/label_reference_filter.rb +++ b/lib/gitlab/markdown/label_reference_filter.rb @@ -22,6 +22,10 @@ module Gitlab end end + def self.referenced_by(node) + { label: LazyReference.new(Label, node.attr("data-label")) } + end + def call replace_text_nodes_matching(Label.reference_pattern) do |content| label_link_filter(content) @@ -41,11 +45,9 @@ module Gitlab params = label_params(id, name) if label = project.labels.find_by(params) - push_result(:label, label) - url = url_for_label(project, label) klass = reference_class(:label) - data = data_attribute(project.id) + data = data_attribute(project: project.id, label: label.id) %(<a href="#{url}" #{data} class="#{klass}">#{render_colored_label(label)}</a>) diff --git a/lib/gitlab/markdown/merge_request_reference_filter.rb b/lib/gitlab/markdown/merge_request_reference_filter.rb index ecbd263d0e0..5bc63269808 100644 --- a/lib/gitlab/markdown/merge_request_reference_filter.rb +++ b/lib/gitlab/markdown/merge_request_reference_filter.rb @@ -27,6 +27,10 @@ module Gitlab end end + def self.referenced_by(node) + { merge_request: LazyReference.new(MergeRequest, node.attr("data-merge-request")) } + end + def call replace_text_nodes_matching(MergeRequest.reference_pattern) do |content| merge_request_link_filter(content) @@ -45,11 +49,9 @@ module Gitlab project = self.project_from_ref(project_ref) if project && merge_request = project.merge_requests.find_by(iid: id) - push_result(:merge_request, merge_request) - title = escape_once("Merge Request: #{merge_request.title}") klass = reference_class(:merge_request) - data = data_attribute(project.id) + data = data_attribute(project: project.id, merge_request: merge_request.id) url = url_for_merge_request(merge_request, project) diff --git a/lib/gitlab/markdown/redactor_filter.rb b/lib/gitlab/markdown/redactor_filter.rb new file mode 100644 index 00000000000..a1f3a8a8ebf --- /dev/null +++ b/lib/gitlab/markdown/redactor_filter.rb @@ -0,0 +1,40 @@ +require 'gitlab/markdown' +require 'html/pipeline/filter' + +module Gitlab + module Markdown + # HTML filter that removes references to records that the current user does + # not have permission to view. + # + # Expected to be run in its own post-processing pipeline. + # + class RedactorFilter < HTML::Pipeline::Filter + def call + doc.css('a.gfm').each do |node| + unless user_can_reference?(node) + node.replace(node.text) + end + end + + doc + end + + private + + def user_can_reference?(node) + if node.has_attribute?('data-reference-filter') + reference_type = node.attr('data-reference-filter') + reference_filter = reference_type.constantize + + reference_filter.user_can_reference?(current_user, node, context) + else + true + end + end + + def current_user + context[:current_user] + end + end + end +end diff --git a/lib/gitlab/markdown/reference_filter.rb b/lib/gitlab/markdown/reference_filter.rb index 9b293c957d6..a4c560f578c 100644 --- a/lib/gitlab/markdown/reference_filter.rb +++ b/lib/gitlab/markdown/reference_filter.rb @@ -11,30 +11,57 @@ module Gitlab # Context options: # :project (required) - Current project, ignored if reference is cross-project. # :only_path - Generate path-only links. - # - # Results: - # :references - A Hash of references that were found and replaced. class ReferenceFilter < HTML::Pipeline::Filter - def initialize(*args) - super + LazyReference = Struct.new(:klass, :ids) do + def self.load(refs) + lazy_references, values = refs.partition { |ref| ref.is_a?(self) } + + lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs| + ids = refs.flat_map(&:ids) + klass.where(id: ids) + end + + values + lazy_values + end + + def load + self.klass.where(id: self.ids) + end + end + + def self.user_can_reference?(user, node, context) + if node.has_attribute?('data-project') + project_id = node.attr('data-project').to_i + return true if project_id == context[:project].try(:id) + + project = Project.find(project_id) rescue nil + Ability.abilities.allowed?(user, :read_project, project) + else + true + end + end - result[:references] = Hash.new { |hash, type| hash[type] = [] } + def self.referenced_by(node) + raise NotImplementedError, "#{self} does not implement #{__method__}" end # Returns a data attribute String to attach to a reference link # - # id - Object ID - # type - Object type (default: :project) + # attributes - Hash, where the key becomes the data attribute name and the + # value is the data attribute value # # Examples: # - # data_attribute(1) # => "data-project-id=\"1\"" - # data_attribute(2, :user) # => "data-user-id=\"2\"" - # data_attribute(3, :group) # => "data-group-id=\"3\"" + # data_attribute(project: 1, issue: 2) + # # => "data-reference-filter=\"Gitlab::Markdown::SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" + # + # data_attribute(project: 3, merge_request: 4) + # # => "data-reference-filter=\"Gitlab::Markdown::SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" # # Returns a String - def data_attribute(id, type = :project) - %Q(data-#{type}-id="#{id}") + def data_attribute(attributes = {}) + attributes[:reference_filter] = self.class.name + attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{value}") }.join(" ") end def escape_once(html) @@ -59,16 +86,6 @@ module Gitlab context[:project] end - # Add a reference to the pipeline's result Hash - # - # type - Singular Symbol reference type (e.g., :issue, :user, etc.) - # values - One or more Objects to add - def push_result(type, *values) - return if values.empty? - - result[:references][type].push(*values) - end - def reference_class(type) "gfm gfm-#{type}" end @@ -85,15 +102,15 @@ module Gitlab # Yields the current node's String contents. The result of the block will # replace the node's existing content and update the current document. # - # Returns the updated Nokogiri::XML::Document object. + # Returns the updated Nokogiri::HTML::DocumentFragment object. def replace_text_nodes_matching(pattern) return doc if project.nil? search_text_nodes(doc).each do |node| - content = node.to_html - - next unless content.match(pattern) next if ignored_ancestry?(node) + next unless node.text =~ pattern + + content = node.to_html html = yield content diff --git a/lib/gitlab/markdown/reference_gatherer_filter.rb b/lib/gitlab/markdown/reference_gatherer_filter.rb new file mode 100644 index 00000000000..00f983675e6 --- /dev/null +++ b/lib/gitlab/markdown/reference_gatherer_filter.rb @@ -0,0 +1,63 @@ +require 'gitlab/markdown' +require 'html/pipeline/filter' + +module Gitlab + module Markdown + # HTML filter that gathers all referenced records that the current user has + # permission to view. + # + # Expected to be run in its own post-processing pipeline. + # + class ReferenceGathererFilter < HTML::Pipeline::Filter + def initialize(*) + super + + result[:references] ||= Hash.new { |hash, type| hash[type] = [] } + end + + def call + doc.css('a.gfm').each do |node| + gather_references(node) + end + + load_lazy_references unless context[:load_lazy_references] == false + + doc + end + + private + + def gather_references(node) + return unless node.has_attribute?('data-reference-filter') + + reference_type = node.attr('data-reference-filter') + reference_filter = reference_type.constantize + + return if context[:reference_filter] && reference_filter != context[:reference_filter] + + return unless reference_filter.user_can_reference?(current_user, node, context) + + references = reference_filter.referenced_by(node) + return unless references + + references.each do |type, values| + Array.wrap(values).each do |value| + result[:references][type] << value + end + end + end + + # Will load all references of one type using one query. + def load_lazy_references + refs = result[:references] + refs.each do |type, values| + refs[type] = ReferenceFilter::LazyReference.load(values) + end + end + + def current_user + context[:current_user] + end + end + end +end diff --git a/lib/gitlab/markdown/sanitization_filter.rb b/lib/gitlab/markdown/sanitization_filter.rb index e368de7d848..ffb9dc33b64 100644 --- a/lib/gitlab/markdown/sanitization_filter.rb +++ b/lib/gitlab/markdown/sanitization_filter.rb @@ -48,6 +48,12 @@ module Gitlab # Allow span elements whitelist[:elements].push('span') + # Allow any protocol in `a` elements... + whitelist[:protocols].delete('a') + + # ...but then remove links with the `javascript` protocol + whitelist[:transformers].push(remove_javascript_links) + # Remove `rel` attribute from `a` elements whitelist[:transformers].push(remove_rel) @@ -57,6 +63,19 @@ module Gitlab whitelist end + def remove_javascript_links + lambda do |env| + node = env[:node] + + return unless node.name == 'a' + return unless node.has_attribute?('href') + + if node['href'].start_with?('javascript', ':javascript') + node.remove_attribute('href') + end + end + end + def remove_rel lambda do |env| if env[:node_name] == 'a' diff --git a/lib/gitlab/markdown/snippet_reference_filter.rb b/lib/gitlab/markdown/snippet_reference_filter.rb index e2cf89cb1d8..f783f951711 100644 --- a/lib/gitlab/markdown/snippet_reference_filter.rb +++ b/lib/gitlab/markdown/snippet_reference_filter.rb @@ -27,6 +27,10 @@ module Gitlab end end + def self.referenced_by(node) + { snippet: LazyReference.new(Snippet, node.attr("data-snippet")) } + end + def call replace_text_nodes_matching(Snippet.reference_pattern) do |content| snippet_link_filter(content) @@ -45,11 +49,9 @@ module Gitlab project = self.project_from_ref(project_ref) if project && snippet = project.snippets.find_by(id: id) - push_result(:snippet, snippet) - title = escape_once("Snippet: #{snippet.title}") klass = reference_class(:snippet) - data = data_attribute(project.id) + data = data_attribute(project: project.id, snippet: snippet.id) url = url_for_snippet(snippet, project) diff --git a/lib/gitlab/markdown/upload_link_filter.rb b/lib/gitlab/markdown/upload_link_filter.rb new file mode 100644 index 00000000000..fbada73ab86 --- /dev/null +++ b/lib/gitlab/markdown/upload_link_filter.rb @@ -0,0 +1,47 @@ +require 'gitlab/markdown' +require 'html/pipeline/filter' +require 'uri' + +module Gitlab + module Markdown + # HTML filter that "fixes" relative upload links to files. + # Context options: + # :project (required) - Current project + # + class UploadLinkFilter < HTML::Pipeline::Filter + def call + doc.search('a').each do |el| + process_link_attr el.attribute('href') + end + + doc.search('img').each do |el| + process_link_attr el.attribute('src') + end + + doc + end + + protected + + def process_link_attr(html_attr) + return if html_attr.blank? + + uri = html_attr.value + if uri.starts_with?("/uploads/") + html_attr.value = build_url(uri).to_s + end + end + + def build_url(uri) + File.join(Gitlab.config.gitlab.url, context[:project].path_with_namespace, uri) + end + + # Ensure that a :project key exists in context + # + # Note that while the key might exist, its value could be nil! + def validate + needs :project + end + end + end +end diff --git a/lib/gitlab/markdown/user_reference_filter.rb b/lib/gitlab/markdown/user_reference_filter.rb index 6f436ea7167..2a594e1662e 100644 --- a/lib/gitlab/markdown/user_reference_filter.rb +++ b/lib/gitlab/markdown/user_reference_filter.rb @@ -23,6 +23,31 @@ module Gitlab end end + def self.referenced_by(node) + if node.has_attribute?('data-group') + group = Group.find(node.attr('data-group')) rescue nil + return unless group + + { user: group.users } + elsif node.has_attribute?('data-user') + { user: LazyReference.new(User, node.attr('data-user')) } + elsif node.has_attribute?('data-project') + project = Project.find(node.attr('data-project')) rescue nil + return unless project + + { user: project.team.members.flatten } + end + end + + def self.user_can_reference?(user, node, context) + if node.has_attribute?('data-group') + group = Group.find(node.attr('data-group')) rescue nil + Ability.abilities.allowed?(user, :read_group, group) + else + super + end + end + def call replace_text_nodes_matching(User.reference_pattern) do |content| user_link_filter(content) @@ -61,14 +86,12 @@ module Gitlab def link_to_all project = context[:project] - # FIXME (rspeicher): Law of Demeter - push_result(:user, *project.team.members.flatten) - url = urls.namespace_project_url(project.namespace, project, only_path: context[:only_path]) + data = data_attribute(project: project.id) text = User.reference_prefix + 'all' - %(<a href="#{url}" class="#{link_class}">#{text}</a>) + %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) end def link_to_namespace(namespace) @@ -80,30 +103,20 @@ module Gitlab end def link_to_group(group, namespace) - return unless user_can_reference_group?(namespace) - - push_result(:user, *namespace.users) - url = urls.group_url(group, only_path: context[:only_path]) - data = data_attribute(namespace.id, :group) + data = data_attribute(group: namespace.id) text = Group.reference_prefix + group %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) end def link_to_user(user, namespace) - push_result(:user, namespace.owner) - url = urls.user_url(user, only_path: context[:only_path]) - data = data_attribute(namespace.owner_id, :user) + data = data_attribute(user: namespace.owner_id) text = User.reference_prefix + user %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) end - - def user_can_reference_group?(group) - Ability.abilities.allowed?(context[:current_user], :read_group, group) - end end end end diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb index 90c3fe8da33..9ad7a38d505 100644 --- a/lib/gitlab/o_auth/provider.rb +++ b/lib/gitlab/o_auth/provider.rb @@ -1,6 +1,12 @@ module Gitlab module OAuth class Provider + LABELS = { + "github" => "GitHub", + "gitlab" => "GitLab.com", + "google_oauth2" => "Google" + }.freeze + def self.providers Devise.omniauth_providers end @@ -23,8 +29,9 @@ module Gitlab end def self.label_for(name) + name = name.to_s config = config_for(name) - (config && config['label']) || name.to_s.titleize + (config && config['label']) || LABELS[name] || name.titleize end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 0dab7bcfa4d..70de6a74e76 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -9,7 +9,7 @@ module Gitlab else nil end - @query = Shellwords.shellescape(query) if query.present? + @query = query end def objects(scope, page = nil) @@ -20,6 +20,8 @@ module Gitlab Kaminari.paginate_array(blobs).page(page).per(per_page) when 'wiki_blobs' Kaminari.paginate_array(wiki_blobs).page(page).per(per_page) + when 'commits' + Kaminari.paginate_array(commits).page(page).per(per_page) else super end @@ -27,7 +29,7 @@ module Gitlab def total_count @total_count ||= issues_count + merge_requests_count + blobs_count + - notes_count + wiki_blobs_count + notes_count + wiki_blobs_count + commits_count end def blobs_count @@ -42,6 +44,10 @@ module Gitlab @wiki_blobs_count ||= wiki_blobs.count end + def commits_count + @commits_count ||= commits.count + end + private def blobs @@ -70,6 +76,14 @@ module Gitlab Note.where(project_id: limit_project_ids).user.search(query).order('updated_at DESC') end + def commits + if project.empty_repo? || query.blank? + [] + else + project.repository.find_commits_by_message(query).compact + end + end + def limit_project_ids [project.id] end diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb index d010ade704e..fa068d50763 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/push_data_builder.rb @@ -18,7 +18,10 @@ module Gitlab # homepage: String, # }, # commits: Array, - # total_commits_count: Fixnum + # total_commits_count: Fixnum, + # added: ["CHANGELOG"], + # modified: [], + # removed: ["tmp/file.txt"] # } # def build(project, user, oldrev, newrev, ref, commits = [], message = nil) @@ -33,6 +36,8 @@ module Gitlab commit_attrs = commits_limited.map(&:hook_attrs) type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push" + + repo_changes = repo_changes(project, newrev, oldrev) # Hash to be passed as post_receive_data data = { object_kind: type, @@ -55,7 +60,10 @@ module Gitlab visibility_level: project.visibility_level }, commits: commit_attrs, - total_commits_count: commits_count + total_commits_count: commits_count, + added: repo_changes[:added], + modified: repo_changes[:modified], + removed: repo_changes[:removed] } data @@ -86,6 +94,27 @@ module Gitlab newrev end end + + def repo_changes(project, newrev, oldrev) + changes = { added: [], modified: [], removed: [] } + compare_result = CompareService.new. + execute(project, newrev, project, oldrev) + + if compare_result + compare_result.diffs.each do |diff| + case true + when diff.deleted_file + changes[:removed] << diff.old_path + when diff.renamed_file, diff.new_file + changes[:added] << diff.new_path + else + changes[:modified] << diff.new_path + end + end + end + + changes + end end end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 0961bd80421..da8df8a3025 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -3,11 +3,12 @@ require 'gitlab/markdown' module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor - attr_accessor :project, :current_user + attr_accessor :project, :current_user, :load_lazy_references - def initialize(project, current_user = nil) + def initialize(project, current_user = nil, load_lazy_references: true) @project = project @current_user = current_user + @load_lazy_references = load_lazy_references end def analyze(text) @@ -26,9 +27,9 @@ module Gitlab def references @references ||= Hash.new do |references, type| type = type.to_sym - return references[type] if references.has_key?(type) + next references[type] if references.has_key?(type) - references[type] = pipeline_result(type).uniq + references[type] = pipeline_result(type) end end @@ -39,21 +40,34 @@ module Gitlab # # Returns the results Array for the requested filter type def pipeline_result(filter_type) - klass = filter_type.to_s.camelize + 'ReferenceFilter' + return [] if @text.blank? + + klass = "#{filter_type.to_s.camelize}ReferenceFilter" filter = Gitlab::Markdown.const_get(klass) context = { project: project, current_user: current_user, + # We don't actually care about the links generated only_path: true, - ignore_blockquotes: true + ignore_blockquotes: true, + + # ReferenceGathererFilter + load_lazy_references: false, + reference_filter: filter } - pipeline = HTML::Pipeline.new([filter], context) + pipeline = HTML::Pipeline.new([filter, Gitlab::Markdown::ReferenceGathererFilter], context) result = pipeline.call(@text) - result[:references][filter_type] + values = result[:references][filter_type].uniq + + if @load_lazy_references + values = Gitlab::Markdown::ReferenceFilter::LazyReference.load(values).uniq + end + + values end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 9f1adc860d1..53ab2686b43 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -51,6 +51,23 @@ module Gitlab "can contain only letters, digits, '_', '-' and '.'. " end + def file_path_regex + @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/]*\z/.freeze + end + + def file_path_regex_message + "can contain only letters, digits, '_', '-' and '.'. Separate directories with a '/'. " + end + + + def directory_traversal_regex + @directory_traversal_regex ||= /\.{2}/.freeze + end + + def directory_traversal_regex_message + "cannot include directory traversal. " + end + def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb new file mode 100644 index 00000000000..6360527a7aa --- /dev/null +++ b/lib/gitlab/sherlock.rb @@ -0,0 +1,19 @@ +require 'securerandom' + +module Gitlab + module Sherlock + @collection = Collection.new + + class << self + attr_reader :collection + end + + def self.enabled? + Rails.env.development? && !!ENV['ENABLE_SHERLOCK'] + end + + def self.enable_line_profiler? + RUBY_ENGINE == 'ruby' + end + end +end diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb new file mode 100644 index 00000000000..66bd6258521 --- /dev/null +++ b/lib/gitlab/sherlock/collection.rb @@ -0,0 +1,49 @@ +module Gitlab + module Sherlock + # A collection of transactions recorded by Sherlock. + # + # Method calls for this class are synchronized using a mutex to allow + # sharing of a single Collection instance between threads (e.g. when using + # Puma as a webserver). + class Collection + include Enumerable + + def initialize + @transactions = [] + @mutex = Mutex.new + end + + def add(transaction) + synchronize { @transactions << transaction } + end + + alias_method :<<, :add + + def each(&block) + synchronize { @transactions.each(&block) } + end + + def clear + synchronize { @transactions.clear } + end + + def empty? + synchronize { @transactions.empty? } + end + + def find_transaction(id) + find { |trans| trans.id == id } + end + + def newest_first + sort { |a, b| b.finished_at <=> a.finished_at } + end + + private + + def synchronize(&block) + @mutex.synchronize(&block) + end + end + end +end diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb new file mode 100644 index 00000000000..8a3e1a5e5bf --- /dev/null +++ b/lib/gitlab/sherlock/file_sample.rb @@ -0,0 +1,31 @@ +module Gitlab + module Sherlock + class FileSample + attr_reader :id, :file, :line_samples, :events, :duration + + # file - The full path to the file this sample belongs to. + # line_samples - An array of LineSample objects. + # duration - The total execution time in milliseconds. + # events - The total amount of events. + def initialize(file, line_samples, duration, events) + @id = SecureRandom.uuid + @file = file + @line_samples = line_samples + @duration = duration + @events = events + end + + def relative_path + @relative_path ||= @file.gsub(/^#{Rails.root.to_s}\/?/, '') + end + + def to_param + @id + end + + def source + @source ||= File.read(@file) + end + end + end +end diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb new file mode 100644 index 00000000000..aa1468bff6b --- /dev/null +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -0,0 +1,98 @@ +module Gitlab + module Sherlock + # Class for profiling code on a per line basis. + # + # The LineProfiler class can be used to profile code on per line basis + # without littering your code with Ruby implementation specific profiling + # methods. + # + # This profiler only includes samples taking longer than a given threshold + # and those that occur in the actual application (e.g. files from Gems are + # ignored). + class LineProfiler + # The minimum amount of time that has to be spent in a file for it to be + # included in a list of samples. + MINIMUM_DURATION = 10.0 + + # Profiles the given block. + # + # Example: + # + # profiler = LineProfiler.new + # + # retval, samples = profiler.profile do + # "cats are amazing" + # end + # + # retval # => "cats are amazing" + # samples # => [#<Gitlab::Sherlock::FileSample ...>, ...] + # + # Returns an Array containing the block's return value and an Array of + # FileSample objects. + def profile(&block) + if mri? + profile_mri(&block) + else + raise NotImplementedError, + 'Line profiling is not supported on this platform' + end + end + + # Profiles the given block using rblineprof (MRI only). + def profile_mri + require 'rblineprof' + + retval = nil + samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield } + + file_samples = aggregate_rblineprof(samples) + + [retval, file_samples] + end + + # Returns an Array of file samples based on the output of rblineprof. + # + # lineprof_stats - A Hash containing rblineprof statistics on a per file + # basis. + # + # Returns an Array of FileSample objects. + def aggregate_rblineprof(lineprof_stats) + samples = [] + + lineprof_stats.each do |(file, stats)| + source_lines = File.read(file).each_line.to_a + line_samples = [] + + total_duration = microsec_to_millisec(stats[0][0]) + total_events = stats[0][2] + + next if total_duration <= MINIMUM_DURATION + + stats[1..-1].each_with_index do |data, index| + next unless source_lines[index] + + duration = microsec_to_millisec(data[0]) + events = data[2] + + line_samples << LineSample.new(duration, events) + end + + samples << FileSample. + new(file, line_samples, total_duration, total_events) + end + + samples + end + + private + + def microsec_to_millisec(microsec) + microsec / 1000.0 + end + + def mri? + RUBY_ENGINE == 'ruby' + end + end + end +end diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb new file mode 100644 index 00000000000..eb1948eb6d6 --- /dev/null +++ b/lib/gitlab/sherlock/line_sample.rb @@ -0,0 +1,36 @@ +module Gitlab + module Sherlock + class LineSample + attr_reader :duration, :events + + # duration - The execution time in milliseconds. + # events - The amount of events. + def initialize(duration, events) + @duration = duration + @events = events + end + + # Returns the sample duration percentage relative to the given duration. + # + # Example: + # + # sample.duration # => 150 + # sample.percentage_of(1500) # => 10.0 + # + # total_duration - The total duration to compare with. + # + # Returns a float + def percentage_of(total_duration) + (duration.to_f / total_duration) * 100.0 + end + + # Returns true if the current sample takes up the majority of the given + # duration. + # + # total_duration - The total duration to compare with. + def majority_of?(total_duration) + percentage_of(total_duration) >= 30 + end + end + end +end diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb new file mode 100644 index 00000000000..5ac265618ad --- /dev/null +++ b/lib/gitlab/sherlock/location.rb @@ -0,0 +1,26 @@ +module Gitlab + module Sherlock + class Location + attr_reader :path, :line + + SHERLOCK_DIR = File.dirname(__FILE__) + + # Creates a new Location from a `Thread::Backtrace::Location`. + def self.from_ruby_location(location) + new(location.path, location.lineno) + end + + # path - The full path of the frame as a String. + # line - The line number of the frame as a Fixnum. + def initialize(path, line) + @path = path + @line = line + end + + # Returns true if the current frame originated from the application. + def application? + @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR) + end + end + end +end diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb new file mode 100644 index 00000000000..687332fc5fc --- /dev/null +++ b/lib/gitlab/sherlock/middleware.rb @@ -0,0 +1,41 @@ +module Gitlab + module Sherlock + # Rack middleware used for tracking request metrics. + class Middleware + CONTENT_TYPES = /text\/html|application\/json/i + + IGNORE_PATHS = %r{^/sherlock} + + def initialize(app) + @app = app + end + + # env - A Hash containing Rack environment details. + def call(env) + if instrument?(env) + call_with_instrumentation(env) + else + @app.call(env) + end + end + + def call_with_instrumentation(env) + trans = transaction_from_env(env) + retval = trans.run { @app.call(env) } + + Sherlock.collection.add(trans) + + retval + end + + def instrument?(env) + !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES && + env['REQUEST_URI'] !~ IGNORE_PATHS) + end + + def transaction_from_env(env) + Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI']) + end + end + end +end diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb new file mode 100644 index 00000000000..4917c4ae2ac --- /dev/null +++ b/lib/gitlab/sherlock/query.rb @@ -0,0 +1,114 @@ +module Gitlab + module Sherlock + class Query + attr_reader :id, :query, :started_at, :finished_at, :backtrace + + # SQL identifiers that should be prefixed with newlines. + PREFIX_NEWLINE = / + \s+(FROM + |(LEFT|RIGHT)?INNER\s+JOIN + |(LEFT|RIGHT)?OUTER\s+JOIN + |WHERE + |AND + |GROUP\s+BY + |ORDER\s+BY + |LIMIT + |OFFSET)\s+/ix # Vim indent breaks when this is on a newline :< + + # Creates a new Query using a String and a separate Array of bindings. + # + # query - A String containing a SQL query, optionally with numeric + # placeholders (`$1`, `$2`, etc). + # + # bindings - An Array of ActiveRecord columns and their values. + # started_at - The start time of the query as a Time-like object. + # finished_at - The completion time of the query as a Time-like object. + # + # Returns a new Query object. + def self.new_with_bindings(query, bindings, started_at, finished_at) + bindings.each_with_index do |(_, value), index| + quoted_value = ActiveRecord::Base.connection.quote(value) + + query = query.gsub("$#{index + 1}", quoted_value) + end + + new(query, started_at, finished_at) + end + + # query - The SQL query as a String (without placeholders). + # started_at - The start time of the query as a Time-like object. + # finished_at - The completion time of the query as a Time-like object. + def initialize(query, started_at, finished_at) + @id = SecureRandom.uuid + @query = query + @started_at = started_at + @finished_at = finished_at + @backtrace = caller_locations.map do |loc| + Location.from_ruby_location(loc) + end + + unless @query.end_with?(';') + @query += ';' + end + end + + # Returns the query duration in milliseconds. + def duration + @duration ||= (@finished_at - @started_at) * 1000.0 + end + + def to_param + @id + end + + # Returns a human readable version of the query. + def formatted_query + @formatted_query ||= format_sql(@query) + end + + # Returns the last application frame of the backtrace. + def last_application_frame + @last_application_frame ||= @backtrace.find(&:application?) + end + + # Returns an Array of application frames (excluding Gems and the likes). + def application_backtrace + @application_backtrace ||= @backtrace.select(&:application?) + end + + # Returns the query plan as a String. + def explain + unless @explain + ActiveRecord::Base.connection.transaction do + @explain = raw_explain(@query).values.flatten.join("\n") + + # Roll back any queries that mutate data so we don't mess up + # anything when running explain on an INSERT, UPDATE, DELETE, etc. + raise ActiveRecord::Rollback + end + end + + @explain + end + + private + + def raw_explain(query) + if Gitlab::Database.postgresql? + explain = "EXPLAIN ANALYZE #{query};" + else + explain = "EXPLAIN #{query};" + end + + ActiveRecord::Base.connection.execute(explain) + end + + def format_sql(query) + query.each_line. + map { |line| line.strip }. + join("\n"). + gsub(PREFIX_NEWLINE) { "\n#{$1} " } + end + end + end +end diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb new file mode 100644 index 00000000000..d87a4c9bb4a --- /dev/null +++ b/lib/gitlab/sherlock/transaction.rb @@ -0,0 +1,131 @@ +module Gitlab + module Sherlock + class Transaction + attr_reader :id, :type, :path, :queries, :file_samples, :started_at, + :finished_at, :view_counts + + # type - The type of transaction (e.g. "GET", "POST", etc) + # path - The path of the transaction (e.g. the HTTP request path) + def initialize(type, path) + @id = SecureRandom.uuid + @type = type + @path = path + @queries = [] + @file_samples = [] + @started_at = nil + @finished_at = nil + @thread = Thread.current + @view_counts = Hash.new(0) + end + + # Runs the transaction and returns the block's return value. + def run + @started_at = Time.now + + retval = with_subscriptions do + profile_lines { yield } + end + + @finished_at = Time.now + + retval + end + + # Returns the duration in seconds. + def duration + @duration ||= started_at && finished_at ? finished_at - started_at : 0 + end + + def to_param + @id + end + + # Returns the queries sorted in descending order by their durations. + def sorted_queries + @queries.sort { |a, b| b.duration <=> a.duration } + end + + # Returns the file samples sorted in descending order by their durations. + def sorted_file_samples + @file_samples.sort { |a, b| b.duration <=> a.duration } + end + + # Finds a query by the given ID. + # + # id - The query ID as a String. + # + # Returns a Query object if one could be found, nil otherwise. + def find_query(id) + @queries.find { |query| query.id == id } + end + + # Finds a file sample by the given ID. + # + # id - The query ID as a String. + # + # Returns a FileSample object if one could be found, nil otherwise. + def find_file_sample(id) + @file_samples.find { |sample| sample.id == id } + end + + def profile_lines + retval = nil + + if Sherlock.enable_line_profiler? + retval, @file_samples = LineProfiler.new.profile { yield } + else + retval = yield + end + + retval + end + + def subscribe_to_active_record + ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data| + next unless same_thread? + + track_query(data[:sql].strip, data[:binds], start, finish) + end + end + + def subscribe_to_action_view + regex = /render_(template|partial)\.action_view/ + + ActiveSupport::Notifications.subscribe(regex) do |_, start, finish, _, data| + next unless same_thread? + + track_view(data[:identifier]) + end + end + + private + + def track_query(query, bindings, start, finish) + @queries << Query.new_with_bindings(query, bindings, start, finish) + end + + def track_view(path) + @view_counts[path] += 1 + end + + def with_subscriptions + ar_subscriber = subscribe_to_active_record + av_subscriber = subscribe_to_action_view + + retval = yield + + ActiveSupport::Notifications.unsubscribe(ar_subscriber) + ActiveSupport::Notifications.unsubscribe(av_subscriber) + + retval + end + + # In case somebody uses a multi-threaded server locally (e.g. Puma) we + # _only_ want to track notifications that originate from the transaction + # thread. + def same_thread? + Thread.current == @thread + end + end + end +end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index cf040971c6e..f3567f3ef85 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -50,15 +50,15 @@ module Gitlab end def fetch_git_tags - remote_tags, _ = Gitlab::Popen.popen(%W(git ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) + remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) remote_tags.split("\n").grep(/tags\/v#{current_version.major}/) end def update_commands { - "Stash changed files" => %W(git stash), - "Get latest code" => %W(git fetch), - "Switch to new version" => %W(git checkout v#{latest_version}), + "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash), + "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch), + "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}), "Install gems" => %W(bundle), "Migrate DB" => %W(bundle exec rake db:migrate), "Recompile assets" => %W(bundle exec rake assets:clean assets:precompile), diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb new file mode 100644 index 00000000000..be8fcc7b2d2 --- /dev/null +++ b/lib/gitlab/uploads_transfer.rb @@ -0,0 +1,35 @@ +module Gitlab + class UploadsTransfer + def move_project(project_path, namespace_path_was, namespace_path) + new_namespace_folder = File.join(root_dir, namespace_path) + FileUtils.mkdir_p(new_namespace_folder) unless Dir.exist?(new_namespace_folder) + from = File.join(root_dir, namespace_path_was, project_path) + to = File.join(root_dir, namespace_path, project_path) + move(from, to, "") + end + + def rename_project(path_was, path, namespace_path) + base_dir = File.join(root_dir, namespace_path) + move(path_was, path, base_dir) + end + + def rename_namespace(path_was, path) + move(path_was, path) + end + + private + + def move(path_was, path, base_dir = nil) + base_dir = root_dir unless base_dir + from = File.join(base_dir, path_was) + to = File.join(base_dir, path) + FileUtils.mv(from, to) + rescue Errno::ENOENT + false + end + + def root_dir + File.join(Rails.root, "public", "uploads") + end + end +end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index a80e7e77430..f0a6c2b30e9 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -37,10 +37,9 @@ web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" mail_room_enabled=false mail_room_pid_path="$pid_path/mail_room.pid" -gitlab_git_http_server_pid_path="$pid_path/gitlab-git-http-server.pid" -gitlab_git_http_server_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-git-http-server.socket -authBackend http://127.0.0.1:8080" -gitlab_git_http_server_repo_root='/home/git/repositories' -gitlab_git_http_server_log="$app_root/log/gitlab-git-http-server.log" +gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080" +gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" shell_path="/bin/bash" # Read configuration variable file if it is present @@ -76,8 +75,8 @@ check_pids(){ else spid=0 fi - if [ -f "$gitlab_git_http_server_pid_path" ]; then - hpid=$(cat "$gitlab_git_http_server_pid_path") + if [ -f "$gitlab_workhorse_pid_path" ]; then + hpid=$(cat "$gitlab_workhorse_pid_path") else hpid=0 fi @@ -94,7 +93,7 @@ check_pids(){ wait_for_pids(){ # We are sleeping a bit here mostly because sidekiq is slow at writing it's pid i=0; - while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_git_http_server_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do + while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do sleep 0.1; i=$((i+1)) if [ $((i%10)) = 0 ]; then @@ -131,9 +130,9 @@ check_status(){ fi if [ $hpid -ne 0 ]; then kill -0 "$hpid" 2>/dev/null - gitlab_git_http_server_status="$?" + gitlab_workhorse_status="$?" else - gitlab_git_http_server_status="-1" + gitlab_workhorse_status="-1" fi if [ "$mail_room_enabled" = true ]; then if [ $mpid -ne 0 ]; then @@ -143,7 +142,7 @@ check_status(){ mail_room_status="-1" fi fi - if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_git_http_server_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then + if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then gitlab_status=0 else # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html @@ -171,9 +170,9 @@ check_stale_pids(){ exit 1 fi fi - if [ "$hpid" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ]; then - echo "Removing stale gitlab-git-http-server pid. This is most likely caused by gitlab-git-http-server crashing the last time it ran." - if ! rm "$gitlab_git_http_server_pid_path"; then + if [ "$hpid" != "0" ] && [ "$gitlab_workhorse_status" != "0" ]; then + echo "Removing stale gitlab-workhorse pid. This is most likely caused by gitlab-workhorse crashing the last time it ran." + if ! rm "$gitlab_workhorse_pid_path"; then echo "Unable to remove stale pid, exiting" exit 1 fi @@ -190,7 +189,7 @@ check_stale_pids(){ ## If no parts of the service is running, bail out. exit_if_not_running(){ check_stale_pids - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then echo "GitLab is not running." exit fi @@ -206,8 +205,8 @@ start_gitlab() { if [ "$sidekiq_status" != "0" ]; then echo "Starting GitLab Sidekiq" fi - if [ "$gitlab_git_http_server_status" != "0" ]; then - echo "Starting gitlab-git-http-server" + if [ "$gitlab_workhorse_status" != "0" ]; then + echo "Starting gitlab-workhorse" fi if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" != "0" ]; then echo "Starting GitLab MailRoom" @@ -230,15 +229,14 @@ start_gitlab() { RAILS_ENV=$RAILS_ENV bin/background_jobs start & fi - if [ "$gitlab_git_http_server_status" = "0" ]; then - echo "The gitlab-git-http-server is already running with pid $spid, not restarting" + if [ "$gitlab_workhorse_status" = "0" ]; then + echo "The gitlab-workhorse is already running with pid $spid, not restarting" else - # No need to remove a socket, gitlab-git-http-server does this itself - $app_root/bin/daemon_with_pidfile $gitlab_git_http_server_pid_path \ - $app_root/../gitlab-git-http-server/gitlab-git-http-server \ - $gitlab_git_http_server_options \ - $gitlab_git_http_server_repo_root \ - >> $gitlab_git_http_server_log 2>&1 & + # No need to remove a socket, gitlab-workhorse does this itself + $app_root/bin/daemon_with_pidfile $gitlab_workhorse_pid_path \ + $app_root/../gitlab-workhorse/gitlab-workhorse \ + $gitlab_workhorse_options \ + >> $gitlab_workhorse_log 2>&1 & fi if [ "$mail_room_enabled" = true ]; then @@ -268,9 +266,9 @@ stop_gitlab() { echo "Shutting down GitLab Sidekiq" RAILS_ENV=$RAILS_ENV bin/background_jobs stop fi - if [ "$gitlab_git_http_server_status" = "0" ]; then - echo "Shutting down gitlab-git-http-server" - kill -- $(cat $gitlab_git_http_server_pid_path) + if [ "$gitlab_workhorse_status" = "0" ]; then + echo "Shutting down gitlab-workhorse" + kill -- $(cat $gitlab_workhorse_pid_path) fi if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; then echo "Shutting down GitLab MailRoom" @@ -278,11 +276,11 @@ stop_gitlab() { fi # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script. - while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_git_http_server_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do + while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do sleep 1 check_status printf "." - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then printf "\n" break fi @@ -292,7 +290,7 @@ stop_gitlab() { # Cleaning up unused pids rm "$web_server_pid_path" 2>/dev/null # rm "$sidekiq_pid_path" 2>/dev/null # Sidekiq seems to be cleaning up it's own pid. - rm -f "$gitlab_git_http_server_pid_path" + rm -f "$gitlab_workhorse_pid_path" if [ "$mail_room_enabled" = true ]; then rm "$mail_room_pid_path" 2>/dev/null fi @@ -303,7 +301,7 @@ stop_gitlab() { ## Prints the status of GitLab and it's components. print_status() { check_status - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then echo "GitLab is not running." return fi @@ -317,10 +315,10 @@ print_status() { else printf "The GitLab Sidekiq job dispatcher is \033[31mnot running\033[0m.\n" fi - if [ "$gitlab_git_http_server_status" = "0" ]; then - echo "The gitlab-git-http-server with pid $hpid is running." + if [ "$gitlab_workhorse_status" = "0" ]; then + echo "The gitlab-workhorse with pid $hpid is running." else - printf "The gitlab-git-http-server is \033[31mnot running\033[0m.\n" + printf "The gitlab-workhorse is \033[31mnot running\033[0m.\n" fi if [ "$mail_room_enabled" = true ]; then if [ "$mail_room_status" = "0" ]; then @@ -360,7 +358,7 @@ reload_gitlab(){ ## Restarts Sidekiq and Unicorn. restart_gitlab(){ check_status - if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_git_http_server" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then + if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then stop_gitlab fi start_gitlab diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index aab5acaa72c..79ae8e0ae55 100755 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -30,15 +30,14 @@ web_server_pid_path="$pid_path/unicorn.pid" # The default is "$pid_path/sidekiq.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" -gitlab_git_http_server_pid_path="$pid_path/gitlab-git-http-server.pid" -# The -listenXxx settings determine where gitlab-git-http-server +gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" +# The -listenXxx settings determine where gitlab-workhorse # listens for connections from NGINX. To listen on localhost:8181, write # '-listenNetwork tcp -listenAddr localhost:8181'. -# The -authBackend setting tells gitlab-git-http-server where it can reach +# The -authBackend setting tells gitlab-workhorse where it can reach # Unicorn. -gitlab_git_http_server_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-git-http-server.socket -authBackend http://127.0.0.1:8080" -gitlab_git_http_server_repo_root="/home/git/repositories" -gitlab_git_http_server_log="$app_root/log/gitlab-git-http-server.log" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080" +gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" # mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled. # This is required for the Reply by email feature. diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 7218a4d2f20..e767027dc29 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -38,8 +38,8 @@ upstream gitlab { server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; } -upstream gitlab-git-http-server { - server unix:/home/git/gitlab/tmp/sockets/gitlab-git-http-server.socket fail_timeout=0; +upstream gitlab-workhorse { + server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } ## Normal HTTP host @@ -113,7 +113,25 @@ server { proxy_pass http://gitlab; } - location ~ [-\/\w\.]+\.git\/ { + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location ~ ^/api/v3/projects/.*/repository/archive { + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location @gitlab-workhorse { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. # gzip off; @@ -129,7 +147,7 @@ server { # The following settings only work with NGINX 1.7.11 or newer # - # # Pass chunked request bodies to gitlab-git-http-server as-is + # # Pass chunked request bodies to gitlab-workhorse as-is # proxy_request_buffering off; # proxy_http_version 1.1; @@ -138,7 +156,7 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://gitlab-git-http-server; + proxy_pass http://gitlab-workhorse; } ## Enable gzip compression as per rails guide: diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 7dabfba87e2..4d31e31f8d5 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -42,8 +42,8 @@ upstream gitlab { server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; } -upstream gitlab-git-http-server { - server unix:/home/git/gitlab/tmp/sockets/gitlab-git-http-server.socket fail_timeout=0; +upstream gitlab-workhorse { + server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } ## Redirects all HTTP traffic to the HTTPS host @@ -160,7 +160,25 @@ server { proxy_pass http://gitlab; } - location ~ [-\/\w\.]+\.git\/ { + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location ~ ^/api/v3/projects/.*/repository/archive { + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location @gitlab-workhorse { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. gzip off; @@ -176,7 +194,7 @@ server { # The following settings only work with NGINX 1.7.11 or newer # - # # Pass chunked request bodies to gitlab-git-http-server as-is + # # Pass chunked request bodies to gitlab-workhorse as-is # proxy_request_buffering off; # proxy_http_version 1.1; @@ -185,7 +203,7 @@ server { proxy_set_header X-Forwarded-Ssl on; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://gitlab-git-http-server; + proxy_pass http://gitlab-workhorse; } ## Enable gzip compression as per rails guide: diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake deleted file mode 100644 index 1de664c85e1..00000000000 --- a/lib/tasks/ci/migrate.rake +++ /dev/null @@ -1,87 +0,0 @@ -namespace :ci do - desc 'GitLab | Import and migrate CI database' - task migrate: :environment do - warn_user_is_not_gitlab - configure_cron_mode - - unless ENV['force'] == 'yes' - puts 'This will remove all CI related data and restore it from the provided backup.' - ask_to_continue - puts '' - end - - # disable CI for time of migration - enable_ci(false) - - # unpack archives - migrate = Ci::Migrate::Manager.new - migrate.unpack - - Rake::Task['ci:migrate:db'].invoke - Rake::Task['ci:migrate:builds'].invoke - Rake::Task['ci:migrate:tags'].invoke - Rake::Task['ci:migrate:services'].invoke - - # enable CI for time of migration - enable_ci(true) - - migrate.cleanup - end - - namespace :migrate do - desc 'GitLab | Import CI database' - task db: :environment do - configure_cron_mode - $progress.puts 'Restoring database ... '.blue - Ci::Migrate::Database.new.restore - $progress.puts 'done'.green - end - - desc 'GitLab | Import CI builds' - task builds: :environment do - configure_cron_mode - $progress.puts 'Restoring builds ... '.blue - Ci::Migrate::Builds.new.restore - $progress.puts 'done'.green - end - - desc 'GitLab | Migrate CI tags' - task tags: :environment do - configure_cron_mode - $progress.puts 'Migrating tags ... '.blue - ::Ci::Migrate::Tags.new.restore - $progress.puts 'done'.green - end - - desc 'GitLab | Migrate CI auto-increments' - task autoincrements: :environment do - c = ActiveRecord::Base.connection - c.tables.select { |t| t.start_with?('ci_') }.each do |table| - result = c.select_one("SELECT id FROM #{table} ORDER BY id DESC LIMIT 1") - if result - ai_val = result['id'].to_i + 1 - puts "Resetting auto increment ID for #{table} to #{ai_val}" - if c.adapter_name == 'PostgreSQL' - c.execute("ALTER SEQUENCE #{table}_id_seq RESTART WITH #{ai_val}") - else - c.execute("ALTER TABLE #{table} AUTO_INCREMENT = #{ai_val}") - end - end - end - end - - desc 'GitLab | Migrate CI services' - task services: :environment do - $progress.puts 'Migrating services ... '.blue - c = ActiveRecord::Base.connection - c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'") - $progress.puts 'done'.green - end - end - - def enable_ci(enabled) - settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults - settings.ci_enabled = enabled - settings.save! - end -end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 66f1ecf385f..a25fac62cfc 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -335,7 +335,7 @@ namespace :gitlab do print "Redis version >= #{min_redis_version}? ... " redis_version = run(%W(redis-cli --version)) - redis_version = redis_version.try(:match, /redis-cli (.*)/) + redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) if redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) puts "yes".green @@ -642,7 +642,6 @@ namespace :gitlab do if Gitlab.config.incoming_email.enabled check_address_formatted_correctly - check_mail_room_config_exists check_imap_authentication if Rails.env.production? @@ -744,42 +743,16 @@ namespace :gitlab do end end - def check_mail_room_config_exists - print "MailRoom config exists? ... " - - mail_room_config_file = Rails.root.join("config", "mail_room.yml") - - if File.exists?(mail_room_config_file) - puts "yes".green - else - puts "no".red - try_fixing_it( - "Copy config/mail_room.yml.example to config/mail_room.yml", - "Check that the information in config/mail_room.yml is correct" - ) - for_more_information( - "doc/incoming_email/README.md" - ) - fix_and_rerun - end - end - def check_imap_authentication print "IMAP server credentials are correct? ... " - mail_room_config_file = Rails.root.join("config", "mail_room.yml") - - unless File.exists?(mail_room_config_file) - puts "can't check because of previous errors".magenta - return - end - - config = YAML.load_file(mail_room_config_file)[:mailboxes].first rescue nil + config = Gitlab.config.incoming_email if config begin - imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) - imap.login(config[:email], config[:password]) + imap = Net::IMAP.new(config.host, port: config.port, ssl: config.ssl) + imap.starttls if config.start_tls + imap.login(config.user, config.password) connected = true rescue connected = false @@ -791,7 +764,7 @@ namespace :gitlab do else puts "no".red try_fixing_it( - "Check that the information in config/mail_room.yml is correct" + "Check that the information in config/gitlab.yml is correct" ) for_more_information( "doc/incoming_email/README.md" @@ -851,7 +824,7 @@ namespace :gitlab do repo_dirs = Dir.glob(File.join(namespace_dir, '*')) repo_dirs.each do |dir| puts "\nChecking repo at #{dir}" - system(*%w(git fsck), chdir: dir) + system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: dir) end end end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 3c0cc763d17..dd61632e557 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -17,7 +17,7 @@ namespace :gitlab do # Clone if needed unless File.directory?(target_dir) - system(*%W(git clone -- #{args.repo} #{target_dir})) + system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir})) end # Make sure we're on the right tag @@ -27,7 +27,7 @@ namespace :gitlab do reseted = reset_to_commit(args) unless reseted - system(*%W(git fetch origin)) + system(*%W(#{Gitlab.config.git.bin_path} fetch origin)) reset_to_commit(args) end @@ -128,14 +128,14 @@ namespace :gitlab do end def reset_to_commit(args) - tag, status = Gitlab::Popen.popen(%W(git describe -- #{args.tag})) + tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- #{args.tag})) unless status.zero? - tag, status = Gitlab::Popen.popen(%W(git describe -- origin/#{args.tag})) + tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- origin/#{args.tag})) end tag = tag.strip - system(*%W(git reset --hard #{tag})) + system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) end end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index bf6894a8351..141a0b74ec0 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -1,6 +1,8 @@ require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') +require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do NamespacesProjectsPathLowerIndexes.new.up + AddUsersLowerUsernameEmailIndexes.new.up end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index c8881be0954..d5a96fd38f4 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -5,7 +5,7 @@ namespace :spinach do task :project do cmds = [ %W(rake gitlab:setup), - %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets), + %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@commits), ] run_commands(cmds) end @@ -14,7 +14,7 @@ namespace :spinach do task :other do cmds = [ %W(rake gitlab:setup), - %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets), + %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets,@commits), ] run_commands(cmds) end @@ -33,4 +33,4 @@ def run_commands(cmds) cmds.each do |cmd| system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!") end -end +end
\ No newline at end of file |