From 16f8ca566b8637dc8092a6b630c23a82a905b437 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 1 Oct 2015 23:40:29 -0400 Subject: Add custom protocol whitelisting to SanitizationFilter Addresses internal https://dev.gitlab.org/gitlab/gitlabhq/issues/2613 --- lib/gitlab/markdown/sanitization_filter.rb | 19 ++++ .../gitlab/markdown/sanitization_filter_spec.rb | 101 +++++++++++++++++++-- 2 files changed, 110 insertions(+), 10 deletions(-) diff --git a/lib/gitlab/markdown/sanitization_filter.rb b/lib/gitlab/markdown/sanitization_filter.rb index e368de7d848..ffb9dc33b64 100644 --- a/lib/gitlab/markdown/sanitization_filter.rb +++ b/lib/gitlab/markdown/sanitization_filter.rb @@ -48,6 +48,12 @@ module Gitlab # Allow span elements whitelist[:elements].push('span') + # Allow any protocol in `a` elements... + whitelist[:protocols].delete('a') + + # ...but then remove links with the `javascript` protocol + whitelist[:transformers].push(remove_javascript_links) + # Remove `rel` attribute from `a` elements whitelist[:transformers].push(remove_rel) @@ -57,6 +63,19 @@ module Gitlab whitelist end + def remove_javascript_links + lambda do |env| + node = env[:node] + + return unless node.name == 'a' + return unless node.has_attribute?('href') + + if node['href'].start_with?('javascript', ':javascript') + node.remove_attribute('href') + end + end + end + def remove_rel lambda do |env| if env[:node_name] == 'a' diff --git a/spec/lib/gitlab/markdown/sanitization_filter_spec.rb b/spec/lib/gitlab/markdown/sanitization_filter_spec.rb index e50c82d0b3c..27cd00e8054 100644 --- a/spec/lib/gitlab/markdown/sanitization_filter_spec.rb +++ b/spec/lib/gitlab/markdown/sanitization_filter_spec.rb @@ -44,7 +44,7 @@ module Gitlab::Markdown instance = described_class.new('Foo') 3.times { instance.whitelist } - expect(instance.whitelist[:transformers].size).to eq 4 + expect(instance.whitelist[:transformers].size).to eq 5 end it 'allows syntax highlighting' do @@ -77,19 +77,100 @@ module Gitlab::Markdown end it 'removes `rel` attribute from `a` elements' do - doc = filter(%q{Link}) + act = %q{Link} + exp = %q{Link} - expect(doc.css('a').size).to eq 1 - expect(doc.at_css('a')['href']).to eq '#' - expect(doc.at_css('a')['rel']).to be_nil + expect(filter(act).to_html).to eq exp end - it 'removes script-like `href` attribute from `a` elements' do - html = %q{Hi} - doc = filter(html) + # Adapted from the Sanitize test suite: http://git.io/vczrM + protocols = { + 'protocol-based JS injection: simple, no spaces' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: simple, spaces before' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: simple, spaces after' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: simple, spaces before and after' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: preceding colon' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: UTF-8 encoding' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: long UTF-8 encoding' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: long UTF-8 encoding without semicolons' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: hex encoding' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: long hex encoding' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: hex encoding without semicolons' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: null char' => { + input: "foo", + output: '' + }, + + 'protocol-based JS injection: spaces and entities' => { + input: 'foo', + output: 'foo' + }, + } + + protocols.each do |name, data| + it "handles #{name}" do + doc = filter(data[:input]) + + expect(doc.to_html).to eq data[:output] + end + end + + it 'allows non-standard anchor schemes' do + exp = %q{IRC} + act = filter(exp) + + expect(act.to_html).to eq exp + end + + it 'allows relative links' do + exp = %q{foo/bar.md} + act = filter(exp) - expect(doc.css('a').size).to eq 1 - expect(doc.at_css('a')['href']).to be_nil + expect(act.to_html).to eq exp end end -- cgit v1.2.1 From d3734fbd89c069d35856b440f12109af8a7ef9c9 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 14:43:19 +0200 Subject: Use tar for intermediate backup storage During the backup we create an intermediate copy of two directories: builds and uploads. Instead of creating many small files with 'cp -r', we now use tar (and fast gzip) to create single intermediate files. This saves on disk IO and disk space while creating a backup. --- lib/backup/builds.rb | 31 ++----------------------------- lib/backup/files.rb | 39 +++++++++++++++++++++++++++++++++++++++ lib/backup/manager.rb | 4 ++-- lib/backup/uploads.rb | 30 ++---------------------------- 4 files changed, 45 insertions(+), 59 deletions(-) create mode 100644 lib/backup/files.rb diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index 6f56f680bb9..d269f8e260c 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -1,34 +1,7 @@ module Backup - class Builds - attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir - + class Builds < Files def initialize - @app_builds_dir = Settings.gitlab_ci.builds_path - @backup_dir = Gitlab.config.backup.path - @backup_builds_dir = File.join(Gitlab.config.backup.path, 'builds') - end - - # Copy builds from builds directory to backup/builds - def dump - FileUtils.rm_rf(backup_builds_dir) - # Ensure the parent dir of backup_builds_dir exists - FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create backup_builds_dir before us - FileUtils.mkdir(backup_builds_dir, mode: 0700) - FileUtils.cp_r(app_builds_dir, backup_dir) - end - - def restore - backup_existing_builds_dir - - FileUtils.cp_r(backup_builds_dir, app_builds_dir) - end - - def backup_existing_builds_dir - timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") - if File.exists?(app_builds_dir) - FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) - end + super(Settings.gitlab_ci.builds_path) end end end diff --git a/lib/backup/files.rb b/lib/backup/files.rb new file mode 100644 index 00000000000..d0a6e8f27be --- /dev/null +++ b/lib/backup/files.rb @@ -0,0 +1,39 @@ +require 'open3' + +module Backup + class Files + attr_reader :name, :app_files_dir, :backup_tarball, :backup_dir, :files_parent_dir + + def initialize(app_files_dir) + @app_files_dir = File.realpath(app_files_dir) + @name = File.basename(app_files_dir) + @files_parent_dir = File.realpath(File.join(@app_files_dir, '..')) + @backup_dir = Gitlab.config.backup.path + @backup_tarball = File.join(@backup_dir, name + '.tar.gz') + end + + # Copy files from public/files to backup/files + def dump + FileUtils.mkdir_p(Gitlab.config.backup.path) + run_pipeline!([%W(tar -C #{files_parent_dir} -cf - #{name}), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + end + + def restore + backup_existing_files_dir + + run_pipeline!([%W(gzip -cd), %W(tar -C #{files_parent_dir} -xf -)], in: backup_tarball) + end + + def backup_existing_files_dir + timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}") + if File.exists?(app_files_dir) + FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path)) + end + end + + def run_pipeline!(cmd_list, options={}) + status_list = Open3.pipeline(*cmd_list, options) + abort 'Backup failed' unless status_list.compact.all?(&:success?) + end + end +end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 5c42f25f4a2..f011fd03de0 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -150,11 +150,11 @@ module Backup private def backup_contents - folders_to_backup + ["backup_information.yml"] + folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] end def folders_to_backup - folders = %w{repositories db uploads builds} + folders = %w{repositories db} if ENV["SKIP"] return folders.reject{ |folder| ENV["SKIP"].include?(folder) } diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 1f9626644e6..7c0838cc8b7 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -1,34 +1,8 @@ module Backup - class Uploads - attr_reader :app_uploads_dir, :backup_uploads_dir, :backup_dir + class Uploads < Files def initialize - @app_uploads_dir = File.realpath(Rails.root.join('public', 'uploads')) - @backup_dir = Gitlab.config.backup.path - @backup_uploads_dir = File.join(Gitlab.config.backup.path, 'uploads') - end - - # Copy uploads from public/uploads to backup/uploads - def dump - FileUtils.rm_rf(backup_uploads_dir) - # Ensure the parent dir of backup_uploads_dir exists - FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create backup_uploads_dir before us - FileUtils.mkdir(backup_uploads_dir, mode: 0700) - FileUtils.cp_r(app_uploads_dir, backup_dir) - end - - def restore - backup_existing_uploads_dir - - FileUtils.cp_r(backup_uploads_dir, app_uploads_dir) - end - - def backup_existing_uploads_dir - timestamped_uploads_path = File.join(app_uploads_dir, '..', "uploads.#{Time.now.to_i}") - if File.exists?(app_uploads_dir) - FileUtils.mv(app_uploads_dir, File.expand_path(timestamped_uploads_path)) - end + super(Rails.root.join('public/uploads')) end end end -- cgit v1.2.1 From 90ddf140b9390647002771572d0375da0bb9dfa4 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 15:06:15 +0200 Subject: Reduce disk IO during SQL backup By using light gzip compression we can save a lot of disk IO during the backup. --- lib/backup/database.rb | 53 +++++++++++++++++++++++--------------------------- lib/backup/manager.rb | 4 ++-- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 959ac4b7868..4bdf6e1c628 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -2,26 +2,27 @@ require 'yaml' module Backup class Database - attr_reader :config, :db_dir + attr_reader :config, :db_file_name def initialize @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env] - @db_dir = File.join(Gitlab.config.backup.path, 'db') + @db_file_name = File.join(Gitlab.config.backup.path, 'database.sql.gz') end def dump - FileUtils.rm_rf(@db_dir) - # Ensure the parent dir of @db_dir exists + FileUtils.rm_f(db_file_name) + compress_rd, compress_wr = IO.pipe + compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600]) + compress_rd.close + FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create @db_dir before us - FileUtils.mkdir(@db_dir, mode: 0700) - success = case config["adapter"] + dump_pid = case config["adapter"] when /^mysql/ then $progress.print "Dumping MySQL database #{config['database']} ... " # Workaround warnings from MySQL 5.6 about passwords on cmd line ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - system('mysqldump', *mysql_args, config['database'], out: db_file_name) + spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) when "postgresql" then $progress.print "Dumping PostgreSQL database #{config['database']} ... " pg_env @@ -30,48 +31,42 @@ module Backup pgsql_args << "-n" pgsql_args << Gitlab.config.backup.pg_schema end - system('pg_dump', *pgsql_args, config['database'], out: db_file_name) + spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr) end - report_success(success) - abort 'Backup failed' unless success + compress_wr.close + + success = [compress_pid, dump_pid].all? { |pid| Process.waitpid(pid); $?.success? } - $progress.print 'Compressing database ... ' - success = system('gzip', db_file_name) report_success(success) - abort 'Backup failed: compress error' unless success + abort 'Backup failed' unless success end def restore - $progress.print 'Decompressing database ... ' - success = system('gzip', '-d', db_file_name_gz) - report_success(success) - abort 'Restore failed: decompress error' unless success + decompress_rd, decompress_wr = IO.pipe + decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name) + decompress_wr.close - success = case config["adapter"] + restore_pid = case config["adapter"] when /^mysql/ then $progress.print "Restoring MySQL database #{config['database']} ... " # Workaround warnings from MySQL 5.6 about passwords on cmd line ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - system('mysql', *mysql_args, config['database'], in: db_file_name) + spawn('mysql', *mysql_args, config['database'], in: decompress_rd) when "postgresql" then $progress.print "Restoring PostgreSQL database #{config['database']} ... " pg_env - system('psql', config['database'], '-f', db_file_name) + spawn('psql', config['database'], in: decompress_rd) end + decompress_rd.close + + success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? } + report_success(success) abort 'Restore failed' unless success end protected - def db_file_name - File.join(db_dir, 'database.sql') - end - - def db_file_name_gz - File.join(db_dir, 'database.sql.gz') - end - def mysql_args args = { 'host' => '--host', diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index f011fd03de0..53e79d4d1f7 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -150,11 +150,11 @@ module Backup private def backup_contents - folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] + folders_to_backup + ["database.sql.gz", "uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] end def folders_to_backup - folders = %w{repositories db} + folders = %w{repositories} if ENV["SKIP"] return folders.reject{ |folder| ENV["SKIP"].include?(folder) } -- cgit v1.2.1 From 7b71727c562b6f6337a180ae136be94bf0f6ed31 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 15:10:13 +0200 Subject: Remove old "files" tarball explicitly --- lib/backup/files.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/backup/files.rb b/lib/backup/files.rb index d0a6e8f27be..1b08e3324d7 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -15,6 +15,7 @@ module Backup # Copy files from public/files to backup/files def dump FileUtils.mkdir_p(Gitlab.config.backup.path) + FileUtils.rm_f(backup_tarball) run_pipeline!([%W(tar -C #{files_parent_dir} -cf - #{name}), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) end -- cgit v1.2.1 From e789644783fae55f1095ffcc38b32f810f549caa Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 15:21:15 +0200 Subject: Keep old path: db/database.sql.gz Documentation elsewhere refers to this internal path, let's keep it. --- lib/backup/database.rb | 3 ++- lib/backup/manager.rb | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 4bdf6e1c628..fe0434361e8 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -6,10 +6,11 @@ module Backup def initialize @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env] - @db_file_name = File.join(Gitlab.config.backup.path, 'database.sql.gz') + @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end def dump + FileUtils.mkdir_p(File.dirname(db_file_name)) FileUtils.rm_f(db_file_name) compress_rd, compress_wr = IO.pipe compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600]) diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 53e79d4d1f7..f011fd03de0 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -150,11 +150,11 @@ module Backup private def backup_contents - folders_to_backup + ["database.sql.gz", "uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] + folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] end def folders_to_backup - folders = %w{repositories} + folders = %w{repositories db} if ENV["SKIP"] return folders.reject{ |folder| ENV["SKIP"].include?(folder) } -- cgit v1.2.1 From 7d58489fd908b2263f02e8919b1bd0b3fae1201d Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 15:22:08 +0200 Subject: Remove unused variable --- lib/backup/files.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 1b08e3324d7..5a210a0e464 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -2,14 +2,13 @@ require 'open3' module Backup class Files - attr_reader :name, :app_files_dir, :backup_tarball, :backup_dir, :files_parent_dir + attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir def initialize(app_files_dir) @app_files_dir = File.realpath(app_files_dir) @name = File.basename(app_files_dir) @files_parent_dir = File.realpath(File.join(@app_files_dir, '..')) - @backup_dir = Gitlab.config.backup.path - @backup_tarball = File.join(@backup_dir, name + '.tar.gz') + @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz') end # Copy files from public/files to backup/files -- cgit v1.2.1 From 901f5445785a754227d8b77ca535947ab8cbbfca Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 15:38:21 +0200 Subject: Remove superfluous mkdir -p --- lib/backup/database.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/backup/database.rb b/lib/backup/database.rb index fe0434361e8..67b2a64bd10 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -16,8 +16,6 @@ module Backup compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600]) compress_rd.close - FileUtils.mkdir_p(Gitlab.config.backup.path) - dump_pid = case config["adapter"] when /^mysql/ then $progress.print "Dumping MySQL database #{config['database']} ... " -- cgit v1.2.1 From 852526e07fdac8cc11bdd4dd47cab71b39ae192c Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 16:10:14 +0200 Subject: Spec fixes for new backup contents --- spec/tasks/gitlab/backup_rake_spec.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 3be7dd4e52b..9a881455ea1 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -112,14 +112,14 @@ describe 'gitlab:app namespace rake task' do it 'should set correct permissions on the tar contents' do tar_contents, exit_status = Gitlab::Popen.popen( - %W{tar -tvf #{@backup_tar} db uploads repositories builds} + %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz} ) expect(exit_status).to eq(0) expect(tar_contents).to match('db/') - expect(tar_contents).to match('uploads/') + expect(tar_contents).to match('uploads.tar.gz') expect(tar_contents).to match('repositories/') - expect(tar_contents).to match('builds/') - expect(tar_contents).not_to match(/^.{4,9}[rwx].* (db|uploads|repositories|builds)\/$/) + expect(tar_contents).to match('builds.tar.gz') + expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz)\/$/) end it 'should delete temp directories' do @@ -160,12 +160,12 @@ describe 'gitlab:app namespace rake task' do it "does not contain skipped item" do tar_contents, _exit_status = Gitlab::Popen.popen( - %W{tar -tvf #{@backup_tar} db uploads repositories builds} + %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz} ) expect(tar_contents).to match('db/') - expect(tar_contents).to match('uploads/') - expect(tar_contents).to match('builds/') + expect(tar_contents).to match('uploads.tar.gz') + expect(tar_contents).to match('builds.tar.gz') expect(tar_contents).not_to match('repositories/') end -- cgit v1.2.1 From b847baf8c405935415fcd4ad4a620d577f6526f8 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 16:11:59 +0200 Subject: One more backup spec fix Stop the 'uploads' part from actually running. --- spec/tasks/gitlab/backup_rake_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 9a881455ea1..386ac9c8372 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -55,6 +55,7 @@ describe 'gitlab:app namespace rake task' do expect(Rake::Task["gitlab:backup:db:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:repo:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke) + expect(Rake::Task["gitlab:backup:uploads:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke) expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error end -- cgit v1.2.1 From 58260a0327a953499a07e9cad8d9aaad2d25699b Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 16 Oct 2015 17:16:17 +0200 Subject: Do no rely on basename of builds, uploads --- lib/backup/builds.rb | 6 +++++- lib/backup/files.rb | 9 +++++---- lib/backup/uploads.rb | 6 +++++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index d269f8e260c..800f30c2144 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -1,7 +1,11 @@ module Backup class Builds < Files def initialize - super(Settings.gitlab_ci.builds_path) + super('builds', Settings.gitlab_ci.builds_path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) end end end diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 5a210a0e464..654b4d1c896 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -4,9 +4,9 @@ module Backup class Files attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir - def initialize(app_files_dir) + def initialize(name, app_files_dir) + @name = name @app_files_dir = File.realpath(app_files_dir) - @name = File.basename(app_files_dir) @files_parent_dir = File.realpath(File.join(@app_files_dir, '..')) @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz') end @@ -15,13 +15,14 @@ module Backup def dump FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.rm_f(backup_tarball) - run_pipeline!([%W(tar -C #{files_parent_dir} -cf - #{name}), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) end def restore backup_existing_files_dir + create_files_dir - run_pipeline!([%W(gzip -cd), %W(tar -C #{files_parent_dir} -xf -)], in: backup_tarball) + run_pipeline!([%W(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball) end def backup_existing_files_dir diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 7c0838cc8b7..0a0ec564ba4 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -2,7 +2,11 @@ module Backup class Uploads < Files def initialize - super(Rails.root.join('public/uploads')) + super('uploads', Rails.root.join('public/uploads')) + end + + def create_files_dir + Dir.mkdir(app_files_dir) end end end -- cgit v1.2.1 From f56c7d9f8e66c69de6e984e497dd529874b8d638 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 23 Oct 2015 11:13:10 -0700 Subject: Force update refs/merge-requests/X/head upon a push to the source branch of a merge request Closes #3138 --- CHANGELOG | 1 + app/models/repository.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 0d89fca9fc1..fa13343ea2b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,7 @@ Please view this file on the master branch, on stable branches it's out of date. v 8.2.0 (unreleased) + - Force update refs/merge-requests/X/head upon a push to the source branch of a merge request (Stan Hu) - Improved performance of replacing references in comments - Show last project commit to default branch on project home page - Highlight comment based on anchor in URL diff --git a/app/models/repository.rb b/app/models/repository.rb index a3ba5f4c18a..c9b36bd8170 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -528,7 +528,7 @@ class Repository end def fetch_ref(source_path, source_ref, target_ref) - args = %W(git fetch #{source_path} #{source_ref}:#{target_ref}) + args = %W(git fetch -f #{source_path} #{source_ref}:#{target_ref}) Gitlab::Popen.popen(args, path_to_repo) end -- cgit v1.2.1 From 7794cc8b9d1e1582c5046f94d5d5cea843c7e95a Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Thu, 29 Oct 2015 10:25:38 +0100 Subject: Put delete snippet btn after edit btn Signed-off-by: Dmitriy Zaporozhets --- app/views/snippets/_actions.html.haml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/views/snippets/_actions.html.haml b/app/views/snippets/_actions.html.haml index 751fafa8942..1979ae6d5bc 100644 --- a/app/views/snippets/_actions.html.haml +++ b/app/views/snippets/_actions.html.haml @@ -1,11 +1,11 @@ = link_to new_snippet_path, class: 'btn btn-grouped new-snippet-link', title: "New Snippet" do = icon('plus') New Snippet -- if can?(current_user, :admin_personal_snippet, @snippet) - = link_to snippet_path(@snippet), method: :delete, data: { confirm: "Are you sure?" }, class: "btn btn-grouped btn-remove", title: 'Delete Snippet' do - = icon('trash-o') - Delete - if can?(current_user, :update_personal_snippet, @snippet) = link_to edit_snippet_path(@snippet), class: "btn btn-grouped snippable-edit" do = icon('pencil-square-o') Edit +- if can?(current_user, :admin_personal_snippet, @snippet) + = link_to snippet_path(@snippet), method: :delete, data: { confirm: "Are you sure?" }, class: "btn btn-grouped btn-remove", title: 'Delete Snippet' do + = icon('trash-o') + Delete -- cgit v1.2.1 From e0e311a19c025435d119d379a98ec28f0704628f Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Thu, 29 Oct 2015 10:28:41 +0100 Subject: Fix bg for labels page when no labels present Signed-off-by: Dmitriy Zaporozhets --- app/views/projects/labels/index.html.haml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/projects/labels/index.html.haml b/app/views/projects/labels/index.html.haml index 97175f8232b..fb784ee5f4f 100644 --- a/app/views/projects/labels/index.html.haml +++ b/app/views/projects/labels/index.html.haml @@ -14,8 +14,8 @@ = render @labels = paginate @labels, theme: 'gitlab' - else - .light-well + .nothing-here-block - if can? current_user, :admin_label, @project - .nothing-here-block Create first label or #{link_to 'generate', generate_namespace_project_labels_path(@project.namespace, @project), method: :post} default set of labels + Create first label or #{link_to 'generate', generate_namespace_project_labels_path(@project.namespace, @project), method: :post} default set of labels - else - .nothing-here-block No labels created + No labels created -- cgit v1.2.1 From c5132e94e1c44884f8f9b399c4def20154b6c0ce Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 29 Oct 2015 14:21:24 +0100 Subject: Switch to gitlab-workhorse --- GITLAB_GIT_HTTP_SERVER_VERSION | 1 - GITLAB_WORKHORSE_VERSION | 1 + doc/update/8.1-to-8.2.md | 187 ++++++++++++++++++++++++++++++ lib/gitlab/backend/grack_auth.rb | 2 +- lib/support/init.d/gitlab | 68 ++++++----- lib/support/init.d/gitlab.default.example | 11 +- lib/support/nginx/gitlab | 22 ++-- lib/support/nginx/gitlab-ssl | 22 ++-- 8 files changed, 249 insertions(+), 65 deletions(-) delete mode 100644 GITLAB_GIT_HTTP_SERVER_VERSION create mode 100644 GITLAB_WORKHORSE_VERSION create mode 100644 doc/update/8.1-to-8.2.md diff --git a/GITLAB_GIT_HTTP_SERVER_VERSION b/GITLAB_GIT_HTTP_SERVER_VERSION deleted file mode 100644 index 0d91a54c7d4..00000000000 --- a/GITLAB_GIT_HTTP_SERVER_VERSION +++ /dev/null @@ -1 +0,0 @@ -0.3.0 diff --git a/GITLAB_WORKHORSE_VERSION b/GITLAB_WORKHORSE_VERSION new file mode 100644 index 00000000000..9e11b32fcaa --- /dev/null +++ b/GITLAB_WORKHORSE_VERSION @@ -0,0 +1 @@ +0.3.1 diff --git a/doc/update/8.1-to-8.2.md b/doc/update/8.1-to-8.2.md new file mode 100644 index 00000000000..38e5661b368 --- /dev/null +++ b/doc/update/8.1-to-8.2.md @@ -0,0 +1,187 @@ +# From 8.1 to 8.2 + +**NOTE:** GitLab 8.0 introduced several significant changes related to +installation and configuration which *are not duplicated here*. Be sure you're +already running a working version of 8.0 before proceeding with this guide. + +### 0. Double-check your Git version + +**This notice applies only to /usr/local/bin/git** + +If you compiled Git from source on your GitLab server then please double-check +that you are using a version that protects against CVE-2014-9390. For six +months after this vulnerability became known the GitLab installation guide +still contained instructions that would install the outdated, 'vulnerable' Git +version 2.1.2. + +Run the following command to get your current Git version: + +```sh +/usr/local/bin/git --version +``` + +If you see 'No such file or directory' then you did not install Git according +to the outdated instructions from the GitLab installation guide and you can go +to the next step 'Stop server' below. + +If you see a version string then it should be v1.8.5.6, v1.9.5, v2.0.5, v2.1.4, +v2.2.1 or newer. You can use the [instructions in the GitLab source +installation +guide](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/install/installation.md#1-packages-dependencies) +to install a newer version of Git. + +### 1. Stop server + + sudo service gitlab stop + +### 2. Backup + +```bash +cd /home/git/gitlab +sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production +``` + +### 3. Get latest code + +```bash +sudo -u git -H git fetch --all +sudo -u git -H git checkout -- db/schema.rb # local changes will be restored automatically +``` + +For GitLab Community Edition: + +```bash +sudo -u git -H git checkout 8-2-stable +``` + +OR + +For GitLab Enterprise Edition: + +```bash +sudo -u git -H git checkout 8-2-stable-ee +``` + +### 4. Update gitlab-shell + +```bash +cd /home/git/gitlab-shell +sudo -u git -H git fetch +sudo -u git -H git checkout v2.6.5 +``` + +### 5. Replace gitlab-git-http-server with gitlab-workhorse + +Install and compile gitlab-workhorse. This requires [Go +1.5](https://golang.org/dl) which should already be on your system +from GitLab 8.1. + +```bash +cd /home/git +sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-workhorse.git +sudo -u git -H git checkout 0.3.1 +sudo -u git -H make +``` + +Update the GitLab init script and 'default' file. + +``` +cd /home/git/gitlab +sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab +test -e /etc/default/gitlab && \ + sudo sed -i .pre-8.2 's/^\([^=]*\)gitlab_git_http_server/\1gitlab_workhorse/' /etc/default/gitlab +``` + +Make sure that you also update your **NGINX configuration** to use +the new gitlab-workhorse.socket file. + +### 6. Install libs, migrations, etc. + +```bash +cd /home/git/gitlab + +# MySQL installations (note: the line below states '--without postgres') +sudo -u git -H bundle install --without postgres development test --deployment + +# PostgreSQL installations (note: the line below states '--without mysql') +sudo -u git -H bundle install --without mysql development test --deployment + +# Run database migrations +sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production + +# Clean up assets and cache +sudo -u git -H bundle exec rake assets:clean assets:precompile cache:clear RAILS_ENV=production + +# Update init.d script +sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab +``` + +### 7. Update configuration files + +#### New configuration options for `gitlab.yml` + +There are new configuration options available for [`gitlab.yml`](config/gitlab.yml.example). View them with the command below and apply them manually to your current `gitlab.yml`: + +```sh +git diff origin/8-1-stable:config/gitlab.yml.example origin/8-2-stable:config/gitlab.yml.example +``` + +#### Nginx configuration + +View changes between the previous recommended Nginx configuration and the +current one: + +```sh +# For HTTPS configurations +git diff origin/8-1-stable:lib/support/nginx/gitlab-ssl origin/8-2-stable:lib/support/nginx/gitlab-ssl + +# For HTTP configurations +git diff origin/8-1-stable:lib/support/nginx/gitlab origin/8-2-stable:lib/support/nginx/gitlab +``` + +If you are using Apache instead of NGINX please see the updated [Apache templates]. +Also note that because Apache does not support upstreams behind Unix sockets you +will need to let gitlab-git-http-server listen on a TCP port. You can do this +via [/etc/default/gitlab]. + +[Apache templates]: https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache +[/etc/default/gitlab]: https://gitlab.com/gitlab-org/gitlab-ce/blob/8-2-stable/lib/support/init.d/gitlab.default.example#L34 + +### 8. Start application + + sudo service gitlab start + sudo service nginx restart + +### 9. Check application status + +Check if GitLab and its environment are configured correctly: + + sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production + +To make sure you didn't miss anything run a more thorough check: + + sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production + +If all items are green, then congratulations, the upgrade is complete! + +## Things went south? Revert to previous version (8.0) + +### 1. Revert the code to the previous version + +Follow the [upgrade guide from 7.14 to 8.0](7.14-to-8.0.md), except for the database migration +(The backup is already migrated to the previous version) + +### 2. Restore from the backup + +```bash +cd /home/git/gitlab +sudo -u git -H bundle exec rake gitlab:backup:restore RAILS_ENV=production +``` + +If you have more than one backup `*.tar` file(s) please add `BACKUP=timestamp_of_backup` to the command above. + +## Troubleshooting + +### "You appear to have cloned an empty repository." + +See the [7.14 to 8.0 update guide](7.14-to-8.0.md#troubleshooting). diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 85a2d1a93a7..440ef5a3cb3 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -34,7 +34,7 @@ module Grack auth! if project && authorized_request? - # Tell gitlab-git-http-server the request is OK, and what the GL_ID is + # Tell gitlab-workhorse the request is OK, and what the GL_ID is render_grack_auth_ok elsif @user.nil? && !@ci unauthorized diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index a80e7e77430..f0a6c2b30e9 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -37,10 +37,9 @@ web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" mail_room_enabled=false mail_room_pid_path="$pid_path/mail_room.pid" -gitlab_git_http_server_pid_path="$pid_path/gitlab-git-http-server.pid" -gitlab_git_http_server_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-git-http-server.socket -authBackend http://127.0.0.1:8080" -gitlab_git_http_server_repo_root='/home/git/repositories' -gitlab_git_http_server_log="$app_root/log/gitlab-git-http-server.log" +gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080" +gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" shell_path="/bin/bash" # Read configuration variable file if it is present @@ -76,8 +75,8 @@ check_pids(){ else spid=0 fi - if [ -f "$gitlab_git_http_server_pid_path" ]; then - hpid=$(cat "$gitlab_git_http_server_pid_path") + if [ -f "$gitlab_workhorse_pid_path" ]; then + hpid=$(cat "$gitlab_workhorse_pid_path") else hpid=0 fi @@ -94,7 +93,7 @@ check_pids(){ wait_for_pids(){ # We are sleeping a bit here mostly because sidekiq is slow at writing it's pid i=0; - while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_git_http_server_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do + while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do sleep 0.1; i=$((i+1)) if [ $((i%10)) = 0 ]; then @@ -131,9 +130,9 @@ check_status(){ fi if [ $hpid -ne 0 ]; then kill -0 "$hpid" 2>/dev/null - gitlab_git_http_server_status="$?" + gitlab_workhorse_status="$?" else - gitlab_git_http_server_status="-1" + gitlab_workhorse_status="-1" fi if [ "$mail_room_enabled" = true ]; then if [ $mpid -ne 0 ]; then @@ -143,7 +142,7 @@ check_status(){ mail_room_status="-1" fi fi - if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_git_http_server_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then + if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then gitlab_status=0 else # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html @@ -171,9 +170,9 @@ check_stale_pids(){ exit 1 fi fi - if [ "$hpid" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ]; then - echo "Removing stale gitlab-git-http-server pid. This is most likely caused by gitlab-git-http-server crashing the last time it ran." - if ! rm "$gitlab_git_http_server_pid_path"; then + if [ "$hpid" != "0" ] && [ "$gitlab_workhorse_status" != "0" ]; then + echo "Removing stale gitlab-workhorse pid. This is most likely caused by gitlab-workhorse crashing the last time it ran." + if ! rm "$gitlab_workhorse_pid_path"; then echo "Unable to remove stale pid, exiting" exit 1 fi @@ -190,7 +189,7 @@ check_stale_pids(){ ## If no parts of the service is running, bail out. exit_if_not_running(){ check_stale_pids - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then echo "GitLab is not running." exit fi @@ -206,8 +205,8 @@ start_gitlab() { if [ "$sidekiq_status" != "0" ]; then echo "Starting GitLab Sidekiq" fi - if [ "$gitlab_git_http_server_status" != "0" ]; then - echo "Starting gitlab-git-http-server" + if [ "$gitlab_workhorse_status" != "0" ]; then + echo "Starting gitlab-workhorse" fi if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" != "0" ]; then echo "Starting GitLab MailRoom" @@ -230,15 +229,14 @@ start_gitlab() { RAILS_ENV=$RAILS_ENV bin/background_jobs start & fi - if [ "$gitlab_git_http_server_status" = "0" ]; then - echo "The gitlab-git-http-server is already running with pid $spid, not restarting" + if [ "$gitlab_workhorse_status" = "0" ]; then + echo "The gitlab-workhorse is already running with pid $spid, not restarting" else - # No need to remove a socket, gitlab-git-http-server does this itself - $app_root/bin/daemon_with_pidfile $gitlab_git_http_server_pid_path \ - $app_root/../gitlab-git-http-server/gitlab-git-http-server \ - $gitlab_git_http_server_options \ - $gitlab_git_http_server_repo_root \ - >> $gitlab_git_http_server_log 2>&1 & + # No need to remove a socket, gitlab-workhorse does this itself + $app_root/bin/daemon_with_pidfile $gitlab_workhorse_pid_path \ + $app_root/../gitlab-workhorse/gitlab-workhorse \ + $gitlab_workhorse_options \ + >> $gitlab_workhorse_log 2>&1 & fi if [ "$mail_room_enabled" = true ]; then @@ -268,9 +266,9 @@ stop_gitlab() { echo "Shutting down GitLab Sidekiq" RAILS_ENV=$RAILS_ENV bin/background_jobs stop fi - if [ "$gitlab_git_http_server_status" = "0" ]; then - echo "Shutting down gitlab-git-http-server" - kill -- $(cat $gitlab_git_http_server_pid_path) + if [ "$gitlab_workhorse_status" = "0" ]; then + echo "Shutting down gitlab-workhorse" + kill -- $(cat $gitlab_workhorse_pid_path) fi if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; then echo "Shutting down GitLab MailRoom" @@ -278,11 +276,11 @@ stop_gitlab() { fi # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script. - while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_git_http_server_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do + while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do sleep 1 check_status printf "." - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then printf "\n" break fi @@ -292,7 +290,7 @@ stop_gitlab() { # Cleaning up unused pids rm "$web_server_pid_path" 2>/dev/null # rm "$sidekiq_pid_path" 2>/dev/null # Sidekiq seems to be cleaning up it's own pid. - rm -f "$gitlab_git_http_server_pid_path" + rm -f "$gitlab_workhorse_pid_path" if [ "$mail_room_enabled" = true ]; then rm "$mail_room_pid_path" 2>/dev/null fi @@ -303,7 +301,7 @@ stop_gitlab() { ## Prints the status of GitLab and it's components. print_status() { check_status - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then echo "GitLab is not running." return fi @@ -317,10 +315,10 @@ print_status() { else printf "The GitLab Sidekiq job dispatcher is \033[31mnot running\033[0m.\n" fi - if [ "$gitlab_git_http_server_status" = "0" ]; then - echo "The gitlab-git-http-server with pid $hpid is running." + if [ "$gitlab_workhorse_status" = "0" ]; then + echo "The gitlab-workhorse with pid $hpid is running." else - printf "The gitlab-git-http-server is \033[31mnot running\033[0m.\n" + printf "The gitlab-workhorse is \033[31mnot running\033[0m.\n" fi if [ "$mail_room_enabled" = true ]; then if [ "$mail_room_status" = "0" ]; then @@ -360,7 +358,7 @@ reload_gitlab(){ ## Restarts Sidekiq and Unicorn. restart_gitlab(){ check_status - if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_git_http_server" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then + if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then stop_gitlab fi start_gitlab diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index aab5acaa72c..79ae8e0ae55 100755 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -30,15 +30,14 @@ web_server_pid_path="$pid_path/unicorn.pid" # The default is "$pid_path/sidekiq.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" -gitlab_git_http_server_pid_path="$pid_path/gitlab-git-http-server.pid" -# The -listenXxx settings determine where gitlab-git-http-server +gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" +# The -listenXxx settings determine where gitlab-workhorse # listens for connections from NGINX. To listen on localhost:8181, write # '-listenNetwork tcp -listenAddr localhost:8181'. -# The -authBackend setting tells gitlab-git-http-server where it can reach +# The -authBackend setting tells gitlab-workhorse where it can reach # Unicorn. -gitlab_git_http_server_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-git-http-server.socket -authBackend http://127.0.0.1:8080" -gitlab_git_http_server_repo_root="/home/git/repositories" -gitlab_git_http_server_log="$app_root/log/gitlab-git-http-server.log" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080" +gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" # mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled. # This is required for the Reply by email feature. diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 1e55c5a0486..e767027dc29 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -38,8 +38,8 @@ upstream gitlab { server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; } -upstream gitlab-git-http-server { - server unix:/home/git/gitlab/tmp/sockets/gitlab-git-http-server.socket fail_timeout=0; +upstream gitlab-workhorse { + server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } ## Normal HTTP host @@ -114,24 +114,24 @@ server { } location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { - # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block - error_page 418 = @gitlab-git-http-server; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; return 418; } location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { - # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block - error_page 418 = @gitlab-git-http-server; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; return 418; } location ~ ^/api/v3/projects/.*/repository/archive { - # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block - error_page 418 = @gitlab-git-http-server; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; return 418; } - location @gitlab-git-http-server { + location @gitlab-workhorse { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. # gzip off; @@ -147,7 +147,7 @@ server { # The following settings only work with NGINX 1.7.11 or newer # - # # Pass chunked request bodies to gitlab-git-http-server as-is + # # Pass chunked request bodies to gitlab-workhorse as-is # proxy_request_buffering off; # proxy_http_version 1.1; @@ -156,7 +156,7 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://gitlab-git-http-server; + proxy_pass http://gitlab-workhorse; } ## Enable gzip compression as per rails guide: diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 08641bbcc17..4d31e31f8d5 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -42,8 +42,8 @@ upstream gitlab { server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; } -upstream gitlab-git-http-server { - server unix:/home/git/gitlab/tmp/sockets/gitlab-git-http-server.socket fail_timeout=0; +upstream gitlab-workhorse { + server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } ## Redirects all HTTP traffic to the HTTPS host @@ -161,24 +161,24 @@ server { } location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { - # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block - error_page 418 = @gitlab-git-http-server; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; return 418; } location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { - # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block - error_page 418 = @gitlab-git-http-server; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; return 418; } location ~ ^/api/v3/projects/.*/repository/archive { - # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block - error_page 418 = @gitlab-git-http-server; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; return 418; } - location @gitlab-git-http-server { + location @gitlab-workhorse { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. gzip off; @@ -194,7 +194,7 @@ server { # The following settings only work with NGINX 1.7.11 or newer # - # # Pass chunked request bodies to gitlab-git-http-server as-is + # # Pass chunked request bodies to gitlab-workhorse as-is # proxy_request_buffering off; # proxy_http_version 1.1; @@ -203,7 +203,7 @@ server { proxy_set_header X-Forwarded-Ssl on; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://gitlab-git-http-server; + proxy_pass http://gitlab-workhorse; } ## Enable gzip compression as per rails guide: -- cgit v1.2.1 From b1547021bcceb1f622c42b0967d0aaf80cb14cd1 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Thu, 29 Oct 2015 14:22:11 +0100 Subject: Fix label destroy js Signed-off-by: Dmitriy Zaporozhets --- app/views/projects/labels/destroy.js.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/projects/labels/destroy.js.haml b/app/views/projects/labels/destroy.js.haml index 1b4c83ab097..d59563b122a 100644 --- a/app/views/projects/labels/destroy.js.haml +++ b/app/views/projects/labels/destroy.js.haml @@ -1,2 +1,2 @@ - if @project.labels.size == 0 - $('.labels').load(document.URL + ' .light-well').hide().fadeIn(1000) + $('.labels').load(document.URL + ' .nothing-here-block').hide().fadeIn(1000) -- cgit v1.2.1 From 3be9d2c422b8651498abec3a2ee9bb6a3685f040 Mon Sep 17 00:00:00 2001 From: Ben Ford Date: Mon, 19 Oct 2015 14:52:46 -0700 Subject: Add ability to create directories in the editor Simply type a name with a `/` directory separator and new directories will be created. This does not do the fancy UI work that github.com does, but it will get the job done. I could not find tests for file creation, so I didn't add a test for this slight behaviour modification. I did test directory traversals though, using both absolute paths like `/tmp/foo.txt` and relative paths like `../../foo.txt`. Neither case escaped the repository, though attempting to traverse with a relative path resulted in a 500 error that did not affect application stability upon reload. --- CHANGELOG | 3 +++ app/assets/stylesheets/pages/editor.scss | 2 +- app/controllers/projects/blob_controller.rb | 2 +- app/services/files/create_dir_service.rb | 11 +++++++++++ app/services/files/create_service.rb | 11 ++++++++--- features/project/source/browse_files.feature | 10 ++++++++++ features/steps/project/source/browse_files.rb | 15 +++++++++++++++ lib/gitlab/regex.rb | 17 +++++++++++++++++ 8 files changed, 66 insertions(+), 5 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 0d89fca9fc1..200e105db4f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,8 @@ Please view this file on the master branch, on stable branches it's out of date. +development + - Adds ability to create directories using the web editor + v 8.2.0 (unreleased) - Improved performance of replacing references in comments - Show last project commit to default branch on project home page diff --git a/app/assets/stylesheets/pages/editor.scss b/app/assets/stylesheets/pages/editor.scss index 1d565477dd4..e2c521af91e 100644 --- a/app/assets/stylesheets/pages/editor.scss +++ b/app/assets/stylesheets/pages/editor.scss @@ -50,7 +50,7 @@ .editor-file-name { .new-file-name { display: inline-block; - width: 200px; + width: 450px; } .form-control { diff --git a/app/controllers/projects/blob_controller.rb b/app/controllers/projects/blob_controller.rb index 8cc2f21d887..93738aa1ee5 100644 --- a/app/controllers/projects/blob_controller.rb +++ b/app/controllers/projects/blob_controller.rb @@ -161,7 +161,7 @@ class Projects::BlobController < Projects::ApplicationController if params[:file].present? params[:file_name] = params[:file].original_filename end - File.join(@path, File.basename(params[:file_name])) + File.join(@path, params[:file_name]) else @path end diff --git a/app/services/files/create_dir_service.rb b/app/services/files/create_dir_service.rb index 71272fb5707..6107254a34e 100644 --- a/app/services/files/create_dir_service.rb +++ b/app/services/files/create_dir_service.rb @@ -5,5 +5,16 @@ module Files def commit repository.commit_dir(current_user, @file_path, @commit_message, @target_branch) end + + def validate + super + + unless @file_path =~ Gitlab::Regex.file_path_regex + raise_error( + 'Your changes could not be committed, because the file path ' + + Gitlab::Regex.file_path_regex_message + ) + end + end end end diff --git a/app/services/files/create_service.rb b/app/services/files/create_service.rb index c8e3a910bba..2348920cc58 100644 --- a/app/services/files/create_service.rb +++ b/app/services/files/create_service.rb @@ -9,12 +9,17 @@ module Files def validate super - file_name = File.basename(@file_path) + if @file_path =~ Gitlab::Regex.directory_traversal_regex + raise_error( + 'Your changes could not be committed, because the file name ' + + Gitlab::Regex.directory_traversal_regex_message + ) + end - unless file_name =~ Gitlab::Regex.file_name_regex + unless @file_path =~ Gitlab::Regex.file_path_regex raise_error( 'Your changes could not be committed, because the file name ' + - Gitlab::Regex.file_name_regex_message + Gitlab::Regex.file_path_regex_message ) end diff --git a/features/project/source/browse_files.feature b/features/project/source/browse_files.feature index 6b0484b6a38..69aa79f2d24 100644 --- a/features/project/source/browse_files.feature +++ b/features/project/source/browse_files.feature @@ -90,6 +90,16 @@ Feature: Project Source Browse Files Then I am on the new file page And I see a commit error message + @javascript + Scenario: I can create file with a directory name + Given I click on "New file" link in repo + And I fill the new file name with a new directory + And I edit code + And I fill the commit message + And I click on "Commit changes" + Then I am redirected to the new file with directory + And I should see its new content + @javascript Scenario: I can edit file Given I click on ".gitignore" file in repo diff --git a/features/steps/project/source/browse_files.rb b/features/steps/project/source/browse_files.rb index 1b27500497a..84725b9b585 100644 --- a/features/steps/project/source/browse_files.rb +++ b/features/steps/project/source/browse_files.rb @@ -78,6 +78,10 @@ class Spinach::Features::ProjectSourceBrowseFiles < Spinach::FeatureSteps fill_in :file_name, with: 'Spaces Not Allowed' end + step 'I fill the new file name with a new directory' do + fill_in :file_name, with: new_file_name_with_directory + end + step 'I fill the commit message' do fill_in :commit_message, with: 'Not yet a commit message.', visible: true end @@ -238,6 +242,11 @@ class Spinach::Features::ProjectSourceBrowseFiles < Spinach::FeatureSteps @project.namespace, @project, 'master/' + new_file_name)) end + step 'I am redirected to the new file with directory' do + expect(current_path).to eq(namespace_project_blob_path( + @project.namespace, @project, 'master/' + new_file_name_with_directory)) + end + step 'I am redirected to the new file on new branch' do expect(current_path).to eq(namespace_project_blob_path( @project.namespace, @project, 'new_branch_name/' + new_file_name)) @@ -335,6 +344,12 @@ class Spinach::Features::ProjectSourceBrowseFiles < Spinach::FeatureSteps 'not_a_file.md' end + # Constant value that is a valid filename with directory and + # not a filename present at root of the seed repository. + def new_file_name_with_directory + 'foo/bar/baz.txt' + end + # Constant value that is a valid directory and # not a directory present at root of the seed repository. def new_dir_name diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 9f1adc860d1..53ab2686b43 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -51,6 +51,23 @@ module Gitlab "can contain only letters, digits, '_', '-' and '.'. " end + def file_path_regex + @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/]*\z/.freeze + end + + def file_path_regex_message + "can contain only letters, digits, '_', '-' and '.'. Separate directories with a '/'. " + end + + + def directory_traversal_regex + @directory_traversal_regex ||= /\.{2}/.freeze + end + + def directory_traversal_regex_message + "cannot include directory traversal. " + end + def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | -- cgit v1.2.1 From 6d0337a97a9d21a27bdd96b48ae17d2458b4ad46 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 29 Oct 2015 15:48:45 +0100 Subject: Update changelog item --- CHANGELOG | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 200e105db4f..325583722f5 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,8 +1,5 @@ Please view this file on the master branch, on stable branches it's out of date. -development - - Adds ability to create directories using the web editor - v 8.2.0 (unreleased) - Improved performance of replacing references in comments - Show last project commit to default branch on project home page @@ -32,6 +29,9 @@ v 8.1.0 - Ensure MySQL CI limits DB migrations occur after the fields have been created (Stan Hu) - Fix duplicate repositories in GitHub import page (Stan Hu) - Redirect to a default path if HTTP_REFERER is not set (Stan Hu) + - Adds ability to create directories using the web editor (Ben Ford) + +v 8.1.0 (unreleased) - Send an email to admin email when a user is reported for spam (Jonathan Rochkind) - Show notifications button when user is member of group rather than project (Grzegorz Bizon) - Fix bug preventing mentioned issued from being closed when MR is merged using fast-forward merge. -- cgit v1.2.1 From 6119b872fbb3b9dc19be3b21737dbb8380cf4847 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 29 Oct 2015 16:34:33 +0100 Subject: Add missing "cd" --- doc/update/8.1-to-8.2.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/update/8.1-to-8.2.md b/doc/update/8.1-to-8.2.md index 38e5661b368..d84d13f33e0 100644 --- a/doc/update/8.1-to-8.2.md +++ b/doc/update/8.1-to-8.2.md @@ -79,6 +79,7 @@ from GitLab 8.1. ```bash cd /home/git sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-workhorse.git +cd gitlab-workhorse sudo -u git -H git checkout 0.3.1 sudo -u git -H make ``` -- cgit v1.2.1 From bc89c4f8d993028cd903217b6e1b989ca54307b9 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 29 Oct 2015 16:34:50 +0100 Subject: Make sed command GNU-compatible --- doc/update/8.1-to-8.2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/update/8.1-to-8.2.md b/doc/update/8.1-to-8.2.md index d84d13f33e0..3772f624e98 100644 --- a/doc/update/8.1-to-8.2.md +++ b/doc/update/8.1-to-8.2.md @@ -90,7 +90,7 @@ Update the GitLab init script and 'default' file. cd /home/git/gitlab sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab test -e /etc/default/gitlab && \ - sudo sed -i .pre-8.2 's/^\([^=]*\)gitlab_git_http_server/\1gitlab_workhorse/' /etc/default/gitlab + sudo sed -i.pre-8.2 's/^\([^=]*\)gitlab_git_http_server/\1gitlab_workhorse/' /etc/default/gitlab ``` Make sure that you also update your **NGINX configuration** to use -- cgit v1.2.1 From 49c081b9f38e99bbc11e7132d87773749b5b39d5 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 28 Oct 2015 14:43:27 +0100 Subject: Improve performance of User.find_by_any_email MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This query used to rely on a JOIN, effectively producing the following SQL: SELECT users.* FROM users LEFT OUTER JOIN emails ON emails.user_id = users.id WHERE (users.email = X OR emails.email = X) LIMIT 1; The use of a JOIN means having to scan over all Emails and users, join them together and then filter out the rows that don't match the criteria (though this step may be taken into account already when joining). In the new setup this query instead uses a sub-query, producing the following SQL: SELECT * FROM users WHERE id IN (select user_id FROM emails WHERE email = X) OR email = X LIMIT 1; This query has the benefit that it: 1. Doesn't have to JOIN any rows 2. Only has to operate on a relatively small set of rows from the "emails" table. Since most users will only have a handful of Emails associated (certainly not hundreds or even thousands) the size of the set returned by the sub-query is small enough that it should not become problematic. Performance of the old versus new version can be measured using the following benchmark: # Save this in ./bench.rb require 'benchmark/ips' email = 'yorick@gitlab.com' def User.find_by_any_email_old(email) user_table = arel_table email_table = Email.arel_table query = user_table. project(user_table[Arel.star]). join(email_table, Arel::Nodes::OuterJoin). on(user_table[:id].eq(email_table[:user_id])). where(user_table[:email].eq(email).or(email_table[:email].eq(email))) find_by_sql(query.to_sql).first end Benchmark.ips do |bench| bench.report 'original' do User.find_by_any_email_old(email) end bench.report 'optimized' do User.find_by_any_email(email) end bench.compare! end Running this locally using "bundle exec rails r bench.rb" produces the following output: Calculating ------------------------------------- original 1.000 i/100ms optimized 93.000 i/100ms ------------------------------------------------- original 11.103 (± 0.0%) i/s - 56.000 optimized 948.713 (± 5.3%) i/s - 4.743k Comparison: optimized: 948.7 i/s original: 11.1 i/s - 85.45x slower In other words, the new setup is 85x faster compared to the old setup, at least when running this benchmark locally. For GitLab.com these improvements result in User.find_by_any_email taking only ~170 ms to run, instead of around 800 ms. While this is "only" an improvement of about 4.5 times (instead of 85x) it's still significantly better than before. Fixes #3242 --- app/models/user.rb | 18 +++--------------- spec/benchmarks/models/user_spec.rb | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/app/models/user.rb b/app/models/user.rb index c72beacbf0f..924cb543fab 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -235,21 +235,9 @@ class User < ActiveRecord::Base # Find a User by their primary email or any associated secondary email def find_by_any_email(email) - user_table = arel_table - email_table = Email.arel_table - - # Use ARel to build a query: - query = user_table. - # SELECT "users".* FROM "users" - project(user_table[Arel.star]). - # LEFT OUTER JOIN "emails" - join(email_table, Arel::Nodes::OuterJoin). - # ON "users"."id" = "emails"."user_id" - on(user_table[:id].eq(email_table[:user_id])). - # WHERE ("user"."email" = '' OR "emails"."email" = '') - where(user_table[:email].eq(email).or(email_table[:email].eq(email))) - - find_by_sql(query.to_sql).first + User.reorder(nil). + where('id IN (SELECT user_id FROM emails WHERE email = :email) OR email = :email', email: email). + take end def filter(filter_name) diff --git a/spec/benchmarks/models/user_spec.rb b/spec/benchmarks/models/user_spec.rb index cc5c3904193..74520aaf3f0 100644 --- a/spec/benchmarks/models/user_spec.rb +++ b/spec/benchmarks/models/user_spec.rb @@ -39,4 +39,30 @@ describe User, benchmark: true do it { is_expected.to iterate_per_second(iterations) } end end + + describe '.find_by_any_email' do + let(:user) { create(:user) } + + describe 'using a user with only a single Email address' do + let(:email) { user.email } + + benchmark_subject { User.find_by_any_email(email) } + + it { is_expected.to iterate_per_second(5000) } + end + + describe 'using a user with multiple Email addresses' do + let(:email) { user.emails.first.email } + + benchmark_subject { User.find_by_any_email(email) } + + before do + 10.times do + user.emails.create(email: FFaker::Internet.email) + end + end + + it { is_expected.to iterate_per_second(5000) } + end + end end -- cgit v1.2.1 From 0a6aaf256900ab47058d03d3b74883420c4643ae Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 28 Oct 2015 14:57:52 +0100 Subject: Changelog entry for User.find_by_any_email --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index cf44a5c7c82..6bec4f606e7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ Please view this file on the master branch, on stable branches it's out of date. v 8.2.0 (unreleased) - Force update refs/merge-requests/X/head upon a push to the source branch of a merge request (Stan Hu) + - Improved performance of finding users by one of their Email addresses - Improved performance of replacing references in comments - Show last project commit to default branch on project home page - Highlight comment based on anchor in URL -- cgit v1.2.1 From 24c8f42278844cf48cdd9871b8285445379623f0 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 29 Oct 2015 12:05:47 +0100 Subject: Use a UNION for User.find_by_any_email This is significantly faster than using a sub-query, at least when run on the GitLab.com production database. The benchmarks are a lot slower now with these changes, most likely due to PostgreSQL choosing a different (and less efficient) plan based on the amount of data present in the test database. Thanks to @dlemstra for suggesting the use of a UNION. --- app/models/user.rb | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/app/models/user.rb b/app/models/user.rb index 924cb543fab..35f5ab56798 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -235,9 +235,18 @@ class User < ActiveRecord::Base # Find a User by their primary email or any associated secondary email def find_by_any_email(email) - User.reorder(nil). - where('id IN (SELECT user_id FROM emails WHERE email = :email) OR email = :email', email: email). - take + # Arel doesn't allow for chaining operations on union nodes, thus we have + # to write this query by hand. See the following issue for more info: + # https://github.com/rails/arel/issues/98. + sql = '(SELECT * FROM users WHERE email = :email + UNION + SELECT users.* + FROM emails + INNER JOIN users ON users.id = emails.user_id + WHERE emails.email = :email) + LIMIT 1;' + + User.find_by_sql([sql, { email: email }]).first end def filter(filter_name) -- cgit v1.2.1 From bba46623c20de52942c85eb2aba3649e3f4ba296 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 29 Oct 2015 13:33:20 +0100 Subject: Fixed UNION syntax for MySQL MySQL doesn't support the previous syntax. --- app/models/user.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/user.rb b/app/models/user.rb index 35f5ab56798..66db70080b7 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -238,9 +238,9 @@ class User < ActiveRecord::Base # Arel doesn't allow for chaining operations on union nodes, thus we have # to write this query by hand. See the following issue for more info: # https://github.com/rails/arel/issues/98. - sql = '(SELECT * FROM users WHERE email = :email + sql = '(SELECT * FROM users WHERE email = :email) UNION - SELECT users.* + (SELECT users.* FROM emails INNER JOIN users ON users.id = emails.user_id WHERE emails.email = :email) -- cgit v1.2.1 From a9df714764d6138bf162acd82b780ca82a21864b Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 29 Oct 2015 17:51:49 +0100 Subject: Use a subquery with IDs only for find_by_any_email This further improves performance of User.find_by_any_email and is roughly twice as fast as the previous UNION setup. Thanks again to @dlemstra for suggesting this. --- app/models/user.rb | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/app/models/user.rb b/app/models/user.rb index 66db70080b7..67fef1c1e6a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -235,15 +235,13 @@ class User < ActiveRecord::Base # Find a User by their primary email or any associated secondary email def find_by_any_email(email) - # Arel doesn't allow for chaining operations on union nodes, thus we have - # to write this query by hand. See the following issue for more info: - # https://github.com/rails/arel/issues/98. - sql = '(SELECT * FROM users WHERE email = :email) - UNION - (SELECT users.* - FROM emails - INNER JOIN users ON users.id = emails.user_id - WHERE emails.email = :email) + sql = 'SELECT * + FROM users + WHERE id IN ( + SELECT id FROM users WHERE email = :email + UNION + SELECT emails.user_id FROM emails WHERE email = :email + ) LIMIT 1;' User.find_by_sql([sql, { email: email }]).first -- cgit v1.2.1 From 6d3068bec3a926d17f4f2d0da895856489bfcb7a Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 29 Oct 2015 17:53:56 +0100 Subject: Adjusted ips/sec for find_by_any_email benchmarks While these benchmarks run at roughly 1500 i/sec setting the threshold to 1000 leaves some room for deviations (e.g. due to different DB setups). --- spec/benchmarks/models/user_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/benchmarks/models/user_spec.rb b/spec/benchmarks/models/user_spec.rb index 74520aaf3f0..4cdba66939b 100644 --- a/spec/benchmarks/models/user_spec.rb +++ b/spec/benchmarks/models/user_spec.rb @@ -48,7 +48,7 @@ describe User, benchmark: true do benchmark_subject { User.find_by_any_email(email) } - it { is_expected.to iterate_per_second(5000) } + it { is_expected.to iterate_per_second(1000) } end describe 'using a user with multiple Email addresses' do @@ -62,7 +62,7 @@ describe User, benchmark: true do end end - it { is_expected.to iterate_per_second(5000) } + it { is_expected.to iterate_per_second(1000) } end end end -- cgit v1.2.1 From 31ea88cfed2580975f621a6aa992ac58405cb275 Mon Sep 17 00:00:00 2001 From: Takuya Noguchi Date: Sat, 31 Oct 2015 02:27:44 +0900 Subject: Fix deadlink in docs for ci/examples --- doc/ci/examples/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/ci/examples/README.md b/doc/ci/examples/README.md index e0b9fa0e25d..c8122fc63b3 100644 --- a/doc/ci/examples/README.md +++ b/doc/ci/examples/README.md @@ -1,5 +1,5 @@ # Build script examples -+ [Test and deploy Ruby Application to Heroku](test-and-deploy-ruby-application-to-heroku.md) -+ [Test and deploy Python Application to Heroku](test-and-deploy-python-application-to-heroku.md) -+ [Test Clojure applications](examples/test-clojure-application.md) ++ [Test and deploy Ruby applications to Heroku](test-and-deploy-ruby-application-to-heroku.md) ++ [Test and deploy Python applications to Heroku](test-and-deploy-python-application-to-heroku.md) ++ [Test Clojure applications](test-clojure-application.md) -- cgit v1.2.1 From 3b0039f659eab1c29e735ef2a5443e26b33d5d18 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Sat, 31 Oct 2015 16:06:06 +0100 Subject: Persist blob editor's value on submit, not on click Prior, the value of the Ace editor was only being persisted if the user physically clicked the submit button, which the "quick submit" behavior doesn't do. Now the value will be properly transferred before any form is submitted. --- app/assets/javascripts/blob/edit_blob.js.coffee | 8 ++++---- app/assets/javascripts/blob/new_blob.js.coffee | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/assets/javascripts/blob/edit_blob.js.coffee b/app/assets/javascripts/blob/edit_blob.js.coffee index 050888f9c15..f6bf836f19f 100644 --- a/app/assets/javascripts/blob/edit_blob.js.coffee +++ b/app/assets/javascripts/blob/edit_blob.js.coffee @@ -11,10 +11,10 @@ class @EditBlob if ace_mode editor.getSession().setMode "ace/mode/" + ace_mode - $(".js-commit-button").click -> - $("#file-content").val editor.getValue() - $(".file-editor form").submit() - return false + # Before a form submission, move the content from the Ace editor into the + # submitted textarea + $('form').submit -> + $("#file-content").val(editor.getValue()) editModePanes = $(".js-edit-mode-pane") editModeLinks = $(".js-edit-mode a") diff --git a/app/assets/javascripts/blob/new_blob.js.coffee b/app/assets/javascripts/blob/new_blob.js.coffee index 1f36a53f191..68c5e5195e3 100644 --- a/app/assets/javascripts/blob/new_blob.js.coffee +++ b/app/assets/javascripts/blob/new_blob.js.coffee @@ -11,10 +11,10 @@ class @NewBlob if ace_mode editor.getSession().setMode "ace/mode/" + ace_mode - $(".js-commit-button").click -> - $("#file-content").val editor.getValue() - $(".file-editor form").submit() - return false + # Before a form submission, move the content from the Ace editor into the + # submitted textarea + $('form').submit -> + $("#file-content").val(editor.getValue()) editor: -> return @editor -- cgit v1.2.1 From 98008a2db85a5320d885cd515557d4093a1cba24 Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Sat, 31 Oct 2015 22:56:24 +0100 Subject: Fix typo in rake task doc --- doc/development/rake_tasks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/development/rake_tasks.md b/doc/development/rake_tasks.md index a4a980cf0e0..9f3fd69fc4e 100644 --- a/doc/development/rake_tasks.md +++ b/doc/development/rake_tasks.md @@ -9,7 +9,7 @@ bundle exec rake setup ``` The `setup` task is a alias for `gitlab:setup`. -This tasks calls `db:setup` to create the database, calls `add_limits_mysql` that adds limits to the database schema in case of a MySQL database and fianlly it calls `db:seed_fu` to seed the database. +This tasks calls `db:setup` to create the database, calls `add_limits_mysql` that adds limits to the database schema in case of a MySQL database and finally it calls `db:seed_fu` to seed the database. Note: `db:setup` calls `db:seed` but this does nothing. ## Run tests -- cgit v1.2.1 From a1d0eca88686faabd9742f92174a0caa76e36cf6 Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Sat, 31 Oct 2015 23:31:21 +0100 Subject: Gilab -> GitLab --- doc/ci/api/README.md | 2 +- doc/ci/api/projects.md | 22 +++++++++++----------- doc/ci/api/runners.md | 4 ++-- doc/install/installation.md | 2 +- doc/operations/unicorn.md | 2 +- doc/ssh/README.md | 7 +++---- .../importing/import_projects_from_gitlab_com.md | 4 ++-- 7 files changed, 21 insertions(+), 22 deletions(-) diff --git a/doc/ci/api/README.md b/doc/ci/api/README.md index 33c5b172e98..cf9710ede57 100644 --- a/doc/ci/api/README.md +++ b/doc/ci/api/README.md @@ -25,7 +25,7 @@ GitLab CI API has 4 authentication methods: Authentication is done by sending the `private-token` of a valid user and the `url` of an -authorized Gitlab instance via a query string along with the API +authorized GitLab instance via a query string along with the API request: GET http://gitlab.example.com/ci/api/v1/projects?private_token=QVy1PB7sTxfy4pqfZM1U&url=http://demo.gitlab.com/ diff --git a/doc/ci/api/projects.md b/doc/ci/api/projects.md index 5585191e826..74a4c64d000 100644 --- a/doc/ci/api/projects.md +++ b/doc/ci/api/projects.md @@ -1,7 +1,7 @@ # Projects API This API is intended to aid in the setup and configuration of -projects on Gitlab CI. +projects on GitLab CI. __Authentication is done by GitLab user token & GitLab url__ @@ -88,23 +88,23 @@ authorized. Parameters: - * `id` (required) - The ID of the Gitlab CI project + * `id` (required) - The ID of the GitLab CI project ### Create Project -Creates a Gitlab CI project using Gitlab project details. +Creates a GitLab CI project using GitLab project details. POST /ci/projects Parameters: * `name` (required) - The name of the project - * `gitlab_id` (required) - The ID of the project on the Gitlab instance + * `gitlab_id` (required) - The ID of the project on the GitLab instance * `default_ref` (optional) - The branch to run on (default to `master`) ### Update Project -Updates a Gitlab CI project using Gitlab project details that the +Updates a GitLab CI project using GitLab project details that the authenticated user has access to. PUT /ci/projects/:id @@ -116,13 +116,13 @@ Parameters: ### Remove Project -Removes a Gitlab CI project that the authenticated user has access to. +Removes a GitLab CI project that the authenticated user has access to. DELETE /ci/projects/:id Parameters: - * `id` (required) - The ID of the Gitlab CI project + * `id` (required) - The ID of the GitLab CI project ### Link Project to Runner @@ -133,8 +133,8 @@ authorized user). Parameters: - * `id` (required) - The ID of the Gitlab CI project - * `runner_id` (required) - The ID of the Gitlab CI runner + * `id` (required) - The ID of the GitLab CI project + * `runner_id` (required) - The ID of the GitLab CI runner ### Remove Project from Runner @@ -145,5 +145,5 @@ via authorized user). Parameters: - * `id` (required) - The ID of the Gitlab CI project - * `runner_id` (required) - The ID of the Gitlab CI runner \ No newline at end of file + * `id` (required) - The ID of the GitLab CI project + * `runner_id` (required) - The ID of the GitLab CI runner \ No newline at end of file diff --git a/doc/ci/api/runners.md b/doc/ci/api/runners.md index e9f88ee066e..c383dc4bcc9 100644 --- a/doc/ci/api/runners.md +++ b/doc/ci/api/runners.md @@ -6,7 +6,7 @@ __Authentication is done by GitLab user token & GitLab url__ -Used to get information about all runners registered on the Gitlab CI +Used to get information about all runners registered on the GitLab CI instance. GET /ci/runners @@ -31,7 +31,7 @@ Returns: __Authentication is done with a Shared runner registration token or a project Specific runner registration token__ -Used to make Gitlab CI aware of available runners. +Used to make GitLab CI aware of available runners. POST /ci/runners/register diff --git a/doc/install/installation.md b/doc/install/installation.md index b48acb655ee..b02e8e8e588 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -489,7 +489,7 @@ See the [omniauth integration document](../integration/omniauth.md) ### Build your projects GitLab can build your projects. To enable that feature you need GitLab Runners to do that for you. -Checkout the [Gitlab Runner section](https://about.gitlab.com/gitlab-ci/#gitlab-runner) to install it +Checkout the [GitLab Runner section](https://about.gitlab.com/gitlab-ci/#gitlab-runner) to install it ### Custom Redis Connection diff --git a/doc/operations/unicorn.md b/doc/operations/unicorn.md index 31b432cd411..3998da01f01 100644 --- a/doc/operations/unicorn.md +++ b/doc/operations/unicorn.md @@ -78,7 +78,7 @@ threshold is a random value between 200 and 250 MB. The master process (PID ``` One other thing that stands out in the log snippet above, taken from -Gitlab.com, is that 'worker 4' was serving requests for only 23 seconds. This +GitLab.com, is that 'worker 4' was serving requests for only 23 seconds. This is a normal value for our current GitLab.com setup and traffic. The high frequency of Unicorn memory restarts on some GitLab sites can be a diff --git a/doc/ssh/README.md b/doc/ssh/README.md index b6b8000af4e..0bdb4070e74 100644 --- a/doc/ssh/README.md +++ b/doc/ssh/README.md @@ -15,8 +15,7 @@ Note: It is a best practice to use a password for an SSH key, but it is not required and you can skip creating a password by pressing enter. Note that the password you choose here can't be altered or retrieved. -To generate a new SSH key, use the following command: -```bash +To generate a new SSH key, use the following commandGitLab```bash ssh-keygen -t rsa -C "$your_email" ``` This command will prompt you for a location and filename to store the key @@ -82,7 +81,7 @@ How to add your ssh key to Eclipse: http://wiki.eclipse.org/EGit/User_Guide#Ecli ## Tip: Non-default OpenSSH key file names or locations -If, for whatever reason, you decide to specify a non-default location and filename for your Gitlab SSH key pair, you must configure your SSH client to find your Gitlab SSH private key for connections to your Gitlab server (perhaps gitlab.com). For OpenSSH clients, this is handled in the `~/.ssh/config` file with a stanza similar to the following: +If, for whatever reason, you decide to specify a non-default location and filename for your GitLab SSH key pair, you must configure your SSH client to find your GitLab SSH private key for connections to your GitLab server (perhaps gitlab.com). For OpenSSH clients, this is handled in the `~/.ssh/config` file with a stanza similar to the following: ``` # @@ -97,7 +96,7 @@ User mygitlabusername Another example ``` # -# Our company's internal Gitlab server +# Our company's internal GitLab server # Host my-gitlab.company.com RSAAuthentication yes diff --git a/doc/workflow/importing/import_projects_from_gitlab_com.md b/doc/workflow/importing/import_projects_from_gitlab_com.md index f4c4e955d46..1117db98e7e 100644 --- a/doc/workflow/importing/import_projects_from_gitlab_com.md +++ b/doc/workflow/importing/import_projects_from_gitlab_com.md @@ -2,12 +2,12 @@ You can import your existing GitLab.com projects to your GitLab instance. But keep in mind that it is possible only if GitLab support is enabled on your GitLab instance. -You can read more about Gitlab support [here](http://doc.gitlab.com/ce/integration/gitlab.html) +You can read more about GitLab support [here](http://doc.gitlab.com/ce/integration/gitlab.html) To get to the importer page you need to go to "New project" page. ![New project page](gitlab_importer/new_project_page.png) -Click on the "Import projects from Gitlab.com" link and you will be redirected to GitLab.com +Click on the "Import projects from GitLab.com" link and you will be redirected to GitLab.com for permission to access your projects. After accepting, you'll be automatically redirected to the importer. -- cgit v1.2.1 From 1045a72eb7fd16ab09b7e68e1a467ff0f5c9215d Mon Sep 17 00:00:00 2001 From: zheng_b Date: Mon, 2 Nov 2015 07:59:04 +0100 Subject: Go to gitlab installation folder before initialize database --- doc/install/installation.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/install/installation.md b/doc/install/installation.md index b48acb655ee..8ed1eba560a 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -329,6 +329,10 @@ GitLab Shell is an SSH access and repository management software developed speci sudo -u git -H make ### Initialize Database and Activate Advanced Features + + # Go to Gitlab installation folder + + cd /home/git/gilab sudo -u git -H bundle exec rake gitlab:setup RAILS_ENV=production -- cgit v1.2.1 From 1b8d324762787805fabee72d585ac40e244af4d7 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Mon, 2 Nov 2015 14:14:38 +0200 Subject: Add ability to fetch the commit ID of the last commit that actually touched a file --- CHANGELOG | 1 + doc/api/repository_files.md | 3 ++- lib/api/files.rb | 4 +++- spec/requests/api/files_spec.rb | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 6bec4f606e7..588909d2578 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -16,6 +16,7 @@ v 8.2.0 (unreleased) - Fix: 500 error returned if destroy request without HTTP referer (Kazuki Shimizu) - Remove deprecated CI events from project settings page - Use issue editor as cross reference comment author when issue is edited with a new mention. + - [API] Add ability to fetch the commit ID of the last commit that actually touched a file v 8.1.1 - Fix cloning Wiki repositories via HTTP (Stan Hu) diff --git a/doc/api/repository_files.md b/doc/api/repository_files.md index 25311b07107..623063f357b 100644 --- a/doc/api/repository_files.md +++ b/doc/api/repository_files.md @@ -23,7 +23,8 @@ Example response: "content": "IyA9PSBTY2hlbWEgSW5mb3...", "ref": "master", "blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83", - "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50" + "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50", + "last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d" } ``` diff --git a/lib/api/files.rb b/lib/api/files.rb index 308c84dd135..a7a768f8895 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -43,7 +43,8 @@ module API # "content": "IyA9PSBTY2hlbWEgSW5mb3...", # "ref": "master", # "blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83", - # "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50" + # "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50", + # "last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d", # } # get ":id/repository/files" do @@ -71,6 +72,7 @@ module API ref: ref, blob_id: blob.id, commit_id: commit.id, + last_commit_id: user_project.repository.last_commit_for_path(commit.sha, file_path).id } else not_found! 'File' diff --git a/spec/requests/api/files_spec.rb b/spec/requests/api/files_spec.rb index 042e6352567..8efa09f75fd 100644 --- a/spec/requests/api/files_spec.rb +++ b/spec/requests/api/files_spec.rb @@ -19,6 +19,7 @@ describe API::API, api: true do expect(response.status).to eq(200) expect(json_response['file_path']).to eq(file_path) expect(json_response['file_name']).to eq('popen.rb') + expect(json_response['last_commit_id']).to eq('570e7b2abdd848b95f2f578043fc23bd6f6fd24d') expect(Base64.decode64(json_response['content']).lines.first).to eq("require 'fileutils'\n") end -- cgit v1.2.1