Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ce into fix/import-url-validator

# Conflicts:
#	spec/models/project_spec.rb

11 files changed, 115 insertions, 66 deletions

diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb
index ab7b811c5d8..ad412f56cca 100644
--- a/lib/gitlab/backend/grack_auth.rb
+++ b/lib/gitlab/backend/grack_auth.rb
@@ -22,7 +22,7 @@ module Grack
       # Need this if under RELATIVE_URL_ROOT
       unless Gitlab.config.gitlab.relative_url_root.empty?
         # If website is mounted using relative_url_root need to remove it first
-        @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root,'')
+        @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root, '')
       else
         @env['PATH_INFO'] = @request.path
       end
diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb
index 3e3986d6382..34e0143a82e 100644
--- a/lib/gitlab/backend/shell.rb
+++ b/lib/gitlab/backend/shell.rb
@@ -1,3 +1,5 @@
+require 'securerandom'
+
 module Gitlab
   class Shell
     class Error < StandardError; end
@@ -18,77 +20,82 @@ module Gitlab
 
     # Init new repository
     #
+    # storage - project's storage path
     # name - project path with namespace
     #
     # Ex.
-    #   add_repository("gitlab/gitlab-ci")
+    #   add_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
-    def add_repository(name)
+    def add_repository(storage, name)
       Gitlab::Utils.system_silent([gitlab_shell_projects_path,
-                                   'add-project', "#{name}.git"])
+                                   'add-project', storage, "#{name}.git"])
     end
 
     # Import repository
     #
+    # storage - project's storage path
     # name - project path with namespace
     #
     # Ex.
-    #   import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git")
+    #   import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git")
     #
-    def import_repository(name, url)
-      output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900'])
+    def import_repository(storage, name, url)
+      output, status = Popen::popen([gitlab_shell_projects_path, 'import-project',
+                                     storage, "#{name}.git", url, '900'])
       raise Error, output unless status.zero?
       true
     end
 
     # Move repository
-    #
+    # storage - project's storage path
     # path - project path with namespace
     # new_path - new project path with namespace
     #
     # Ex.
-    #   mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new")
+    #   mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
     #
-    def mv_repository(path, new_path)
+    def mv_repository(storage, path, new_path)
       Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project',
-                                   "#{path}.git", "#{new_path}.git"])
+                                   storage, "#{path}.git", "#{new_path}.git"])
     end
 
     # Fork repository to new namespace
-    #
+    # storage - project's storage path
     # path - project path with namespace
     # fork_namespace - namespace for forked project
     #
     # Ex.
-    #  fork_repository("gitlab/gitlab-ci", "randx")
+    #  fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx")
     #
-    def fork_repository(path, fork_namespace)
+    def fork_repository(storage, path, fork_namespace)
       Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project',
-                                   "#{path}.git", fork_namespace])
+                                   storage, "#{path}.git", fork_namespace])
     end
 
     # Remove repository from file system
     #
+    # storage - project's storage path
     # name - project path with namespace
     #
     # Ex.
-    #   remove_repository("gitlab/gitlab-ci")
+    #   remove_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
-    def remove_repository(name)
+    def remove_repository(storage, name)
       Gitlab::Utils.system_silent([gitlab_shell_projects_path,
-                                   'rm-project', "#{name}.git"])
+                                   'rm-project', storage, "#{name}.git"])
     end
 
     # Gc repository
     #
+    # storage - project storage path
     # path - project path with namespace
     #
     # Ex.
-    #   gc("gitlab/gitlab-ci")
+    #   gc("/path/to/storage", "gitlab/gitlab-ci")
     #
-    def gc(path)
+    def gc(storage, path)
       Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc',
-                                   "#{path}.git"])
+                                   storage, "#{path}.git"])
     end
 
     # Add new key to gitlab-shell
@@ -133,31 +140,31 @@ module Gitlab
     # Add empty directory for storing repositories
     #
     # Ex.
-    #   add_namespace("gitlab")
+    #   add_namespace("/path/to/storage", "gitlab")
     #
-    def add_namespace(name)
-      FileUtils.mkdir(full_path(name), mode: 0770) unless exists?(name)
+    def add_namespace(storage, name)
+      FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name)
     end
 
     # Remove directory from repositories storage
     # Every repository inside this directory will be removed too
     #
     # Ex.
-    #   rm_namespace("gitlab")
+    #   rm_namespace("/path/to/storage", "gitlab")
     #
-    def rm_namespace(name)
-      FileUtils.rm_r(full_path(name), force: true)
+    def rm_namespace(storage, name)
+      FileUtils.rm_r(full_path(storage, name), force: true)
     end
 
     # Move namespace directory inside repositories storage
     #
     # Ex.
-    #   mv_namespace("gitlab", "gitlabhq")
+    #   mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
     #
-    def mv_namespace(old_name, new_name)
-      return false if exists?(new_name) || !exists?(old_name)
+    def mv_namespace(storage, old_name, new_name)
+      return false if exists?(storage, new_name) || !exists?(storage, old_name)
 
-      FileUtils.mv(full_path(old_name), full_path(new_name))
+      FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name))
     end
 
     def url_to_repo(path)
@@ -176,11 +183,26 @@ module Gitlab
     # Check if such directory exists in repositories.
     #
     # Usage:
-    #   exists?('gitlab')
-    #   exists?('gitlab/cookies.git')
+    #   exists?(storage, 'gitlab')
+    #   exists?(storage, 'gitlab/cookies.git')
     #
-    def exists?(dir_name)
-      File.exist?(full_path(dir_name))
+    def exists?(storage, dir_name)
+      File.exist?(full_path(storage, dir_name))
+    end
+
+    # Create (if necessary) and link the secret token file
+    def generate_and_link_secret_token
+      secret_file = Gitlab.config.gitlab_shell.secret_file
+      unless File.exist? secret_file
+        # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
+        token = SecureRandom.hex(16)
+        File.write(secret_file, token)
+      end
+
+      link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret')
+      if File.exist?(gitlab_shell_path) && !File.exist?(link_path)
+        FileUtils.symlink(secret_file, link_path)
+      end
     end
 
     protected
@@ -193,14 +215,10 @@ module Gitlab
       File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}")
     end
 
-    def repos_path
-      Gitlab.config.gitlab_shell.repos_path
-    end
-
-    def full_path(dir_name)
+    def full_path(storage, dir_name)
       raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?
 
-      File.join(repos_path, dir_name)
+      File.join(storage, dir_name)
     end
 
     def gitlab_shell_projects_path
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 28c34429c1f..54b46e5d23f 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -9,10 +9,14 @@ module Gitlab
     end
 
     def ensure_application_settings!
-      settings = ::ApplicationSetting.cached
+      if connect_to_db?
+        begin
+          settings = ::ApplicationSetting.current
+        # In case Redis isn't running or the Redis UNIX socket file is not available
+        rescue ::Redis::BaseError, ::Errno::ENOENT
+          settings = ::ApplicationSetting.last
+        end
 
-      if !settings && connect_to_db?
-        settings = ::ApplicationSetting.current
         settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration?
       end
 
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 2286ac8829c..730978d502b 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -167,7 +167,7 @@ module Gitlab
       def import_wiki
         unless project.wiki_enabled?
           wiki = WikiFormatter.new(project)
-          gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url)
+          gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url)
           project.update_attribute(:wiki_enabled, true)
         end
 
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index f751a3a12fd..d4f12cb1df9 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -3,7 +3,6 @@ module Gitlab
     def add_gon_variables
       gon.api_version            = API::API.version
       gon.default_avatar_url     = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s
-      gon.default_issues_tracker = Project.new.default_issue_tracker.to_param
       gon.max_file_size          = current_application_settings.max_attachment_size
       gon.relative_url_root      = Gitlab.config.gitlab.relative_url_root
       gon.shortcuts_path         = help_shortcuts_path
diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb
index faf0d9b6318..c048fe20ba7 100644
--- a/lib/gitlab/metrics/method_call.rb
+++ b/lib/gitlab/metrics/method_call.rb
@@ -18,12 +18,12 @@ module Gitlab
 
       # Measures the real and CPU execution time of the supplied block.
       def measure
-        start_real = Time.now
+        start_real = System.monotonic_time
         start_cpu = System.cpu_time
         retval = yield
 
-        @real_time += (Time.now - start_real) * 1000.0
-        @cpu_time += System.cpu_time.to_f - start_cpu
+        @real_time += System.monotonic_time - start_real
+        @cpu_time += System.cpu_time - start_cpu
         @call_count += 1
 
         retval
diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb
index 1cd1ca30f70..f23d67e1e38 100644
--- a/lib/gitlab/metrics/metric.rb
+++ b/lib/gitlab/metrics/metric.rb
@@ -4,16 +4,15 @@ module Gitlab
     class Metric
       JITTER_RANGE = 0.000001..0.001
 
-      attr_reader :series, :values, :tags, :created_at
+      attr_reader :series, :values, :tags
 
       # series - The name of the series (as a String) to store the metric in.
       # values - A Hash containing the values to store.
       # tags   - A Hash containing extra tags to add to the metrics.
       def initialize(series, values, tags = {})
-        @values     = values
-        @series     = series
-        @tags       = tags
-        @created_at = Time.now.utc
+        @values = values
+        @series = series
+        @tags   = tags
       end
 
       # Returns a Hash in a format that can be directly written to InfluxDB.
@@ -27,20 +26,20 @@ module Gitlab
         #
         # Due to the way InfluxDB is set up there's no solution to this problem,
         # all we can do is lower the amount of collisions. We do this by using
-        # Time#to_f which returns the seconds as a Float providing greater
-        # accuracy. We then add a small random value that is large enough to
-        # distinguish most timestamps but small enough to not alter the amount
-        # of seconds.
+        # System.real_time which returns the nanoseconds as a Float providing
+        # greater accuracy. We then add a small random value that is large
+        # enough to distinguish most timestamps but small enough to not alter
+        # the timestamp significantly.
         #
         # See https://gitlab.com/gitlab-com/operations/issues/175 for more
         # information.
-        time = @created_at.to_f + rand(JITTER_RANGE)
+        time = System.real_time(:nanosecond) + rand(JITTER_RANGE)
 
         {
           series:    @series,
           tags:      @tags,
           values:    @values,
-          timestamp: (time * 1_000_000_000).to_i
+          timestamp: time.to_i
         }
       end
     end
diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb
index a7d183b2f94..82c18bb108b 100644
--- a/lib/gitlab/metrics/system.rb
+++ b/lib/gitlab/metrics/system.rb
@@ -34,13 +34,29 @@ module Gitlab
       # THREAD_CPUTIME is not supported on OS X
       if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID)
         def self.cpu_time
-          Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond)
+          Process.
+            clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond).to_f
         end
       else
         def self.cpu_time
-          Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond)
+          Process.
+            clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond).to_f
         end
       end
+
+      # Returns the current real time in a given precision.
+      #
+      # Returns the time as a Float.
+      def self.real_time(precision = :millisecond)
+        Process.clock_gettime(Process::CLOCK_REALTIME, precision).to_f
+      end
+
+      # Returns the current monotonic clock time in a given precision.
+      #
+      # Returns the time as a Float.
+      def self.monotonic_time(precision = :millisecond)
+        Process.clock_gettime(Process::CLOCK_MONOTONIC, precision).to_f
+      end
     end
   end
 end
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 4bc5081aa03..bded245da43 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -30,7 +30,7 @@ module Gitlab
       end
 
       def duration
-        @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0
+        @finished_at ? (@finished_at - @started_at) : 0.0
       end
 
       def allocated_memory
@@ -41,12 +41,12 @@ module Gitlab
         Thread.current[THREAD_KEY] = self
 
         @memory_before = System.memory_usage
-        @started_at    = Time.now
+        @started_at    = System.monotonic_time
 
         yield
       ensure
         @memory_after = System.memory_usage
-        @finished_at  = Time.now
+        @finished_at  = System.monotonic_time
 
         Thread.current[THREAD_KEY] = nil
       end
diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb
index 36e5c2670bb..7d6911a1ab3 100644
--- a/lib/gitlab/o_auth/auth_hash.rb
+++ b/lib/gitlab/o_auth/auth_hash.rb
@@ -66,7 +66,7 @@ module Gitlab
       # Get the first part of the email address (before @)
       # In addtion in removes illegal characters
       def generate_username(email)
-        email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/,'').to_s
+        email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s
       end
 
       def generate_temporarily_email(username)
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 40e8299c36b..ef1241f8600 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -52,6 +52,19 @@ module Gitlab
         ]
       end
 
+      def send_git_patch(repository, from, to)
+        params = {
+          'RepoPath'  => repository.path_to_repo,
+          'ShaFrom'   => from,
+          'ShaTo'     => to
+        }
+
+        [
+          SEND_DATA_HEADER,
+          "git-format-patch:#{encode(params)}"
+        ]
+      end
+
       protected
 
       def encode(hash)