Merge branch 'master' into assign-to-issuable-opener

10 files changed, 255 insertions, 70 deletions

diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake
index 5d4e0740373..d5a402907d8 100644
--- a/lib/tasks/brakeman.rake
+++ b/lib/tasks/brakeman.rake
@@ -2,7 +2,7 @@ desc 'Security check via brakeman'
 task :brakeman do
   # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge
   # requests are welcome!
-  if system(*%W(brakeman --skip-files lib/backup/repository.rb -w3 -z))
+  if system(*%W(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z))
     puts 'Security check succeed'
   else
     puts 'Security check failed'
diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake
index 9e2fb429d57..51e746ef923 100644
--- a/lib/tasks/cache.rake
+++ b/lib/tasks/cache.rake
@@ -1,19 +1,19 @@
 namespace :cache do
-  CLEAR_BATCH_SIZE = 1000 # The more the faster, but having too many can crash Ruby
+  CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000
   REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan
 
   desc "GitLab | Clear redis cache"
   task :clear => :environment do
-    redis_store = Rails.cache.instance_variable_get(:@data)
+    redis = Redis.new(url: Gitlab::RedisConfig.url)
     cursor = REDIS_SCAN_START_STOP
     loop do
-      cursor, keys = redis_store.scan(
+      cursor, keys = redis.scan(
         cursor,
         match: "#{Gitlab::REDIS_CACHE_NAMESPACE}*", 
         count: CLEAR_BATCH_SIZE
       )
 
-      redis_store.del(*keys) if keys.any?
+      redis.del(*keys) if keys.any?
 
       break if cursor == REDIS_SCAN_START_STOP
     end
diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake
new file mode 100644
index 00000000000..cfaf4a129b1
--- /dev/null
+++ b/lib/tasks/gemojione.rake
@@ -0,0 +1,122 @@
+# This task will generate a standard and Retina sprite of all of the current
+# Gemojione Emojis, with the accompanying SCSS map.
+#
+# It will not appear in `rake -T` output, and the dependent gems are not
+# included in the Gemfile by default, because this task will only be needed
+# occasionally, such as when new Emojis are added to Gemojione.
+
+begin
+  require 'sprite_factory'
+  require 'rmagick'
+rescue LoadError
+  # noop
+end
+
+namespace :gemojione do
+  task sprite: :environment do
+    check_requirements!
+
+    SIZE   = 20
+    RETINA = SIZE * 2
+
+    Dir.mktmpdir do |tmpdir|
+      # Copy the Gemojione assets to the temporary folder for resizing
+      FileUtils.cp_r(Gemojione.index.images_path, tmpdir)
+
+      Dir.chdir(tmpdir) do
+        Dir["**/*.png"].each do |png|
+          resize!(File.join(tmpdir, png), SIZE)
+        end
+      end
+
+      style_path = Rails.root.join(*%w(app assets stylesheets pages emojis.scss))
+
+      # Combine the resized assets into a packed sprite and re-generate the SCSS
+      SpriteFactory.cssurl = "image-url('$IMAGE')"
+      SpriteFactory.run!(File.join(tmpdir, 'images'), {
+        output_style: style_path,
+        output_image: "app/assets/images/emoji.png",
+        selector:     '.emoji-',
+        style:        :scss,
+        nocomments:   true,
+        pngcrush:     true,
+        layout:       :packed
+      })
+
+      # SpriteFactory's SCSS is a bit too verbose for our purposes here, so
+      # let's simplify it
+      system(%Q(sed -i '' "s/width: #{SIZE}px; height: #{SIZE}px; background: image-url('emoji.png')/background-position:/" #{style_path}))
+      system(%Q(sed -i '' "s/ no-repeat//" #{style_path}))
+      system(%Q(sed -i '' "s/ 0px/ 0/" #{style_path}))
+
+      # Append a generic rule that applies to all Emojis
+      File.open(style_path, 'a') do |f|
+        f.puts
+        f.puts <<-CSS.strip_heredoc
+        .emoji-icon {
+          background-image: image-url('emoji.png');
+          background-repeat: no-repeat;
+          height: #{SIZE}px;
+          width: #{SIZE}px;
+
+          @media only screen and (-webkit-min-device-pixel-ratio: 2),
+                 only screen and (min--moz-device-pixel-ratio: 2),
+                 only screen and (-o-min-device-pixel-ratio: 2/1),
+                 only screen and (min-device-pixel-ratio: 2),
+                 only screen and (min-resolution: 192dpi),
+                 only screen and (min-resolution: 2dppx) {
+            background-image: image-url('emoji@2x.png');
+            background-size: 840px 820px;
+          }
+        }
+        CSS
+      end
+    end
+
+    # Now do it again but for Retina
+    Dir.mktmpdir do |tmpdir|
+      # Copy the Gemojione assets to the temporary folder for resizing
+      FileUtils.cp_r(Gemojione.index.images_path, tmpdir)
+
+      Dir.chdir(tmpdir) do
+        Dir["**/*.png"].each do |png|
+          resize!(File.join(tmpdir, png), RETINA)
+        end
+      end
+
+      # Combine the resized assets into a packed sprite and re-generate the SCSS
+      SpriteFactory.run!(File.join(tmpdir, 'images'), {
+        output_image: "app/assets/images/emoji@2x.png",
+        style:        false,
+        nocomments:   true,
+        pngcrush:     true,
+        layout:       :packed
+      })
+    end
+  end
+
+  def check_requirements!
+    return if defined?(SpriteFactory) && defined?(Magick)
+
+    puts <<-MSG.strip_heredoc
+      This task is disabled by default and should only be run when the Gemojione
+      gem is updated with new Emojis.
+
+      To enable this task, *temporarily* add the following lines to Gemfile and
+      re-bundle:
+
+      gem 'sprite-factory'
+      gem 'rmagick'
+    MSG
+
+    exit 1
+  end
+
+  def resize!(image_path, size)
+    # Resize the image in-place, save it, and free the object
+    image = Magick::Image.read(image_path).first
+    image.resize!(size, size)
+    image.write(image_path) { self.quality = 100 }
+    image.destroy!
+  end
+end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index cb4abe13799..402bb338f27 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -22,7 +22,7 @@ namespace :gitlab do
     end
 
     # Restore backup of GitLab system
-    desc "GitLab | Restore a previously created backup"
+    desc 'GitLab | Restore a previously created backup'
     task restore: :environment do
       warn_user_is_not_gitlab
       configure_cron_mode
@@ -30,13 +30,31 @@ namespace :gitlab do
       backup = Backup::Manager.new
       backup.unpack
 
-      Rake::Task["gitlab:backup:db:restore"].invoke unless backup.skipped?("db")
-      Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
-      Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
-      Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
-      Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
-      Rake::Task["gitlab:backup:lfs:restore"].invoke unless backup.skipped?("lfs")
-      Rake::Task["gitlab:shell:setup"].invoke
+      unless backup.skipped?('db')
+        unless ENV['force'] == 'yes'
+          warning = warning = <<-MSG.strip_heredoc
+            Before restoring the database we recommend removing all existing
+            tables to avoid future upgrade problems. Be aware that if you have
+            custom tables in the GitLab database these tables and all data will be
+            removed.
+          MSG
+          ask_to_continue
+          puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.yellow
+          sleep(5)
+        end
+        # Drop all tables Load the schema to ensure we don't have any newer tables
+        # hanging out from a failed upgrade
+        $progress.puts 'Cleaning the database ... '.blue
+        Rake::Task['gitlab:db:drop_tables'].invoke
+        $progress.puts 'done'.green
+        Rake::Task['gitlab:backup:db:restore'].invoke
+      end
+      Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories')
+      Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads')
+      Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds')
+      Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts')
+      Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs')
+      Rake::Task['gitlab:shell:setup'].invoke
 
       backup.cleanup
     end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 81099cb8ba9..27ed57efe55 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -266,7 +266,7 @@ namespace :gitlab do
       unless File.directory?(Rails.root.join('public/uploads'))
         puts "no".red
         try_fixing_it(
-          "sudo -u #{gitlab_user} mkdir -m 750 #{Rails.root}/public/uploads"
+          "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads"
         )
         for_more_information(
           see_installation_guide_section "GitLab"
@@ -278,21 +278,22 @@ namespace :gitlab do
       upload_path = File.realpath(Rails.root.join('public/uploads'))
       upload_path_tmp = File.join(upload_path, 'tmp')
 
-      if File.stat(upload_path).mode == 040750
+      if File.stat(upload_path).mode == 040700
         unless Dir.exists?(upload_path_tmp)
           puts 'skipped (no tmp uploads folder yet)'.magenta
           return
         end
 
-        # if tmp upload dir has incorrect permissions, assume others do as well
-        if File.stat(upload_path_tmp).mode == 040755 && File.owned?(upload_path_tmp) # verify drwxr-xr-x permissions
+        # If tmp upload dir has incorrect permissions, assume others do as well
+        # Verify drwx------ permissions
+        if File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp)
           puts "yes".green
         else
           puts "no".red
           try_fixing_it(
             "sudo chown -R #{gitlab_user} #{upload_path}",
             "sudo find #{upload_path} -type f -exec chmod 0644 {} \\;",
-            "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0755 {} \\;"
+            "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;"
           )
           for_more_information(
             see_installation_guide_section "GitLab"
@@ -302,7 +303,7 @@ namespace :gitlab do
       else
         puts "no".red
         try_fixing_it(
-          "sudo chmod 0750 #{upload_path}",
+          "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;"
         )
         for_more_information(
           see_installation_guide_section "GitLab"
@@ -727,13 +728,15 @@ namespace :gitlab do
     def check_imap_authentication
       print "IMAP server credentials are correct? ... "
 
-      config = Gitlab.config.incoming_email
+      config_path = Rails.root.join('config', 'mail_room.yml')
+      config_file = YAML.load(ERB.new(File.read(config_path)).result)
+      config = config_file[:mailboxes].first
 
       if config
         begin
-          imap = Net::IMAP.new(config.host, port: config.port, ssl: config.ssl)
-          imap.starttls if config.start_tls
-          imap.login(config.user, config.password)
+          imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl])
+          imap.starttls if config[:start_tls]
+          imap.login(config[:email], config[:password])
           connected = true
         rescue
           connected = false
@@ -910,7 +913,7 @@ namespace :gitlab do
   end
 
   def check_git_version
-    required_version = Gitlab::VersionInfo.new(1, 7, 10)
+    required_version = Gitlab::VersionInfo.new(2, 7, 3)
     current_version = Gitlab::VersionInfo.parse(run(%W(#{Gitlab.config.git.bin_path} --version)))
 
     puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\""
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
new file mode 100644
index 00000000000..4921c6e0bcf
--- /dev/null
+++ b/lib/tasks/gitlab/db.rake
@@ -0,0 +1,35 @@
+namespace :gitlab do
+  namespace :db do
+    desc 'GitLab | Manually insert schema migration version'
+    task :mark_migration_complete, [:version] => :environment do |_, args|
+      unless args[:version]
+        puts "Must specify a migration version as an argument".red
+        exit 1
+      end
+
+      version = args[:version].to_i
+      if version == 0
+        puts "Version '#{args[:version]}' must be a non-zero integer".red
+        exit 1
+      end
+
+      sql = "INSERT INTO schema_migrations (version) VALUES (#{version})"
+      begin
+        ActiveRecord::Base.connection.execute(sql)
+        puts "Successfully marked '#{version}' as complete".green
+      rescue ActiveRecord::RecordNotUnique
+        puts "Migration version '#{version}' is already marked complete".yellow
+      end
+    end
+
+    desc 'Drop all tables'
+    task :drop_tables => :environment do
+      connection = ActiveRecord::Base.connection
+      tables = connection.tables
+      tables.delete 'schema_migrations'
+      # Truncate schema_migrations to ensure migrations re-run
+      connection.execute('TRUNCATE schema_migrations')
+      tables.each { |t| connection.execute("DROP TABLE #{t}") }
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake
index 76e443e55ee..cc0f668474e 100644
--- a/lib/tasks/gitlab/web_hook.rake
+++ b/lib/tasks/gitlab/web_hook.rake
@@ -1,13 +1,13 @@
 namespace :gitlab do
   namespace :web_hook do
-    desc "GitLab | Adds a web hook to the projects"
+    desc "GitLab | Adds a webhook to the projects"
     task :add => :environment do
       web_hook_url = ENV['URL']
       namespace_path = ENV['NAMESPACE']
 
       projects = find_projects(namespace_path)
 
-      puts "Adding web hook '#{web_hook_url}' to:"
+      puts "Adding webhook '#{web_hook_url}' to:"
       projects.find_each(batch_size: 1000) do |project|
         print "- #{project.name} ... "
         web_hook = project.hooks.new(url: web_hook_url)
@@ -20,7 +20,7 @@ namespace :gitlab do
       end
     end
 
-    desc "GitLab | Remove a web hook from the projects"
+    desc "GitLab | Remove a webhook from the projects"
     task :rm => :environment do
       web_hook_url = ENV['URL']
       namespace_path = ENV['NAMESPACE']
@@ -28,12 +28,12 @@ namespace :gitlab do
       projects = find_projects(namespace_path)
       projects_ids = projects.pluck(:id)
 
-      puts "Removing web hooks with the url '#{web_hook_url}' ... "
+      puts "Removing webhooks with the url '#{web_hook_url}' ... "
       count = WebHook.where(url: web_hook_url, project_id: projects_ids, type: 'ProjectHook').delete_all
-      puts "#{count} web hooks were removed."
+      puts "#{count} webhooks were removed."
     end
 
-    desc "GitLab | List web hooks"
+    desc "GitLab | List webhooks"
     task :list => :environment do
       namespace_path = ENV['NAMESPACE']
 
@@ -43,7 +43,7 @@ namespace :gitlab do
         puts "#{hook.project.name.truncate(20).ljust(20)} -> #{hook.url}"
       end
 
-      puts "\n#{web_hooks.size} web hooks found."
+      puts "\n#{web_hooks.size} webhooks found."
     end
   end
 
diff --git a/lib/tasks/scss-lint.rake b/lib/tasks/scss-lint.rake
new file mode 100644
index 00000000000..250fd8699e4
--- /dev/null
+++ b/lib/tasks/scss-lint.rake
@@ -0,0 +1,10 @@
+unless Rails.env.production?
+  require 'scss_lint/rake_task'
+
+  SCSSLint::RakeTask.new do |t|
+    t.config = '.scss-lint.yml'
+    # See https://github.com/brigade/scss-lint/issues/726
+    # Hack, otherwise linter won't respect scss_files option in config file.
+    t.files = []
+  end
+end
diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake
index 0985ef3a669..2cf7a25a0fd 100644
--- a/lib/tasks/spec.rake
+++ b/lib/tasks/spec.rake
@@ -46,20 +46,11 @@ namespace :spec do
     run_commands(cmds)
   end
 
-  desc 'GitLab | Rspec | Run benchmark specs'
-  task :benchmark do
-    cmds = [
-      %W(rake gitlab:setup),
-      %W(rspec spec --tag @benchmark)
-    ]
-    run_commands(cmds)
-  end
-
   desc 'GitLab | Rspec | Run other specs'
   task :other do
     cmds = [
       %W(rake gitlab:setup),
-      %W(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services --tag ~@benchmark)
+      %W(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services)
     ]
     run_commands(cmds)
   end
@@ -69,7 +60,7 @@ desc "GitLab | Run specs"
 task :spec do
   cmds = [
     %W(rake gitlab:setup),
-    %W(rspec spec --tag ~@benchmark),
+    %W(rspec spec),
   ]
   run_commands(cmds)
 end
diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake
index 3acfc6e2075..01d23b89bb7 100644
--- a/lib/tasks/spinach.rake
+++ b/lib/tasks/spinach.rake
@@ -4,53 +4,59 @@ namespace :spinach do
   namespace :project do
     desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features"
     task :half do
-      cmds = [
-        %W(rake gitlab:setup),
-        %W(spinach --tags @project_commits,@project_issues,@project_merge_requests),
-      ]
-      run_commands(cmds)
+      run_spinach_tests('@project_commits,@project_issues,@project_merge_requests')
     end
 
     desc "GitLab | Spinach | Run remaining project spinach features"
     task :rest do
-      cmds = [
-        %W(rake gitlab:setup),
-        %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests),
-      ]
-      run_commands(cmds)
+      run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests')
     end
   end
 
   desc "GitLab | Spinach | Run project spinach features"
   task :project do
-    cmds = [
-      %W(rake gitlab:setup),
-      %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets),
-    ]
-    run_commands(cmds)
+    run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets')
   end
 
   desc "GitLab | Spinach | Run other spinach features"
   task :other do
-    cmds = [
-      %W(rake gitlab:setup),
-      %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets),
-    ]
-    run_commands(cmds)
+    run_spinach_tests('@admin,@dashboard,@profile,@public,@snippets')
+  end
+
+  desc "GitLab | Spinach | Run other spinach features"
+  task :builds do
+    run_spinach_tests('@builds')
   end
 end
 
 desc "GitLab | Run spinach"
 task :spinach do
-  cmds = [
-    %W(rake gitlab:setup),
-    %W(spinach),
-  ]
-  run_commands(cmds)
+  run_spinach_tests(nil)
+end
+
+def run_command(cmd)
+  system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd)
 end
 
-def run_commands(cmds)
-  cmds.each do |cmd|
-    system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!")
+def run_spinach_command(args)
+  run_command(%w(spinach -r rerun) + args)
+end
+
+def run_spinach_tests(tags)
+  #run_command(%w(rake gitlab:setup)) or raise('gitlab:setup failed!')
+
+  success = run_spinach_command(%W(--tags #{tags}))
+  3.times do |_|
+    break if success
+    break unless File.exists?('tmp/spinach-rerun.txt')
+
+    tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp)
+    puts ''
+    puts "Spinach tests for #{tags}: Retrying tests... #{tests}".red
+    puts ''
+    sleep(3)
+    success = run_spinach_command(tests)
   end
+
+  raise("spinach tests for #{tags} failed!") unless success
 end