Merge remote-tracking branch 'upstream/master' into pipeline-emails

* upstream/master: (292 commits)
  Deletes extra empty line breaking the build
  Optimize the `award_user_list` helper spec
  Fix typo and add he MWBS accronym for "Merge When Build Succeeds"
  Added missing content and improved layout
  ExpireBuildArtifactsWorker query builds table without ordering enqueuing one job per build to cleanup
  Improve the contribution and MR review guide
  Updates test in order to look for link
  Make projects API docs match parameter style
  Fix Event#reset_project_activity updates
  Update user whitelist reject message
  Call ensure_secret_token! in secret token test's before block since it would be called in an initializer.
  Add a CHANGELOG for CacheMarkdownField
  Enable CacheMarkdownField for the remaining models
  Make search results use the markdown cache columns, treating them consistently
  Use CacheMarkdownField for notes
  Add markdown cache columns to the database, but don't use them yet
  Update issue board spec
  Link to Registry docs from project settings
  Truncate long labels with ellipsis in labels page
  Improve issue load time performance by avoiding ORDER BY in find_by call
  ...

16 files changed, 172 insertions, 51 deletions

diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb
index 79eac66b364..d0060fbaca1 100644
--- a/lib/gitlab/backend/shell.rb
+++ b/lib/gitlab/backend/shell.rb
@@ -17,6 +17,18 @@ module Gitlab
     end
 
     class << self
+      def secret_token
+        @secret_token ||= begin
+          File.read(Gitlab.config.gitlab_shell.secret_file).chomp
+        end
+      end
+
+      def ensure_secret_token!
+        return if File.exist?(File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret'))
+
+        generate_and_link_secret_token
+      end
+
       def version_required
         @version_required ||= File.read(Rails.root.
                                         join('GITLAB_SHELL_VERSION')).strip
@@ -25,6 +37,25 @@ module Gitlab
       def strip_key(key)
         key.split(/ /)[0, 2].join(' ')
       end
+
+      private
+
+      # Create (if necessary) and link the secret token file
+      def generate_and_link_secret_token
+        secret_file = Gitlab.config.gitlab_shell.secret_file
+        shell_path = Gitlab.config.gitlab_shell.path
+
+        unless File.size?(secret_file)
+          # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
+          token = SecureRandom.hex(16)
+          File.write(secret_file, token)
+        end
+
+        link_path = File.join(shell_path, '.gitlab_shell_secret')
+        if File.exist?(shell_path) && !File.exist?(link_path)
+          FileUtils.symlink(secret_file, link_path)
+        end
+      end
     end
 
     # Init new repository
@@ -201,21 +232,6 @@ module Gitlab
       File.exist?(full_path(storage, dir_name))
     end
 
-    # Create (if necessary) and link the secret token file
-    def generate_and_link_secret_token
-      secret_file = Gitlab.config.gitlab_shell.secret_file
-      unless File.size?(secret_file)
-        # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
-        token = SecureRandom.hex(16)
-        File.write(secret_file, token)
-      end
-
-      link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret')
-      if File.exist?(gitlab_shell_path) && !File.exist?(link_path)
-        FileUtils.symlink(secret_file, link_path)
-      end
-    end
-
     protected
 
     def gitlab_shell_path
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index e33ac61f5ae..7f424b74efb 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -102,9 +102,19 @@ module Gitlab
       def request(method, *args, &block)
         sleep rate_limit_sleep_time if rate_limit_exceed?
 
-        data = api.send(method, *args, &block)
-        yield data
+        data = api.send(method, *args)
+        return data unless data.is_a?(Array)
 
+        if block_given?
+          yield data
+          each_response_page(&block)
+        else
+          each_response_page { |page| data.concat(page) }
+          data
+        end
+      end
+
+      def each_response_page
         last_response = api.last_response
 
         while last_response.rels[:next]
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index b8321244473..4b70f33a851 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -170,10 +170,9 @@ module Gitlab
       end
 
       def import_wiki
-        unless project.wiki_enabled?
+        unless project.wiki.repository_exists?
           wiki = WikiFormatter.new(project)
           gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url)
-          project.project.update_attribute(:wiki_access_level, ProjectFeature::ENABLED)
         end
       rescue Gitlab::Shell::Error => e
         # GitHub error message when the wiki repo has not been created,
diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb
index 605abfabdab..a2410068845 100644
--- a/lib/gitlab/github_import/project_creator.rb
+++ b/lib/gitlab/github_import/project_creator.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module GithubImport
     class ProjectCreator
-      attr_reader :repo, :namespace, :current_user, :session_data
+      attr_reader :repo, :name, :namespace, :current_user, :session_data
 
       def initialize(repo, name, namespace, current_user, session_data)
         @repo = repo
@@ -12,24 +12,37 @@ module Gitlab
       end
 
       def execute
-        project = ::Projects::CreateService.new(
+        ::Projects::CreateService.new(
           current_user,
-          name: @name,
-          path: @name,
+          name: name,
+          path: name,
           description: repo.description,
           namespace_id: namespace.id,
-          visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility,
+          visibility_level: visibility_level,
           import_type: "github",
           import_source: repo.full_name,
-          import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@")
+          import_url: import_url,
+          skip_wiki: skip_wiki
         ).execute
+      end
+
+      private
 
-        # If repo has wiki we'll import it later
-        if repo.has_wiki? && project
-          project.project_feature.update_attribute(:wiki_access_level, ProjectFeature::DISABLED)
-        end
+      def import_url
+        repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@")
+      end
+
+      def visibility_level
+        repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility
+      end
 
-        project
+      #
+      # If the GitHub project repository has wiki, we should not create the
+      # default wiki. Otherwise the GitHub importer will fail because the wiki
+      # repository already exist.
+      #
+      def skip_wiki
+        repo.has_wiki?
       end
     end
   end
diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb
index 3e5d728f3bc..f8809db21aa 100644
--- a/lib/gitlab/identifier.rb
+++ b/lib/gitlab/identifier.rb
@@ -5,19 +5,61 @@ module Gitlab
     def identify(identifier, project, newrev)
       if identifier.blank?
         # Local push from gitlab
-        email = project.commit(newrev).author_email rescue nil
-        User.find_by(email: email) if email
-
+        identify_using_commit(project, newrev)
       elsif identifier =~ /\Auser-\d+\Z/
         # git push over http
-        user_id = identifier.gsub("user-", "")
-        User.find_by(id: user_id)
-
+        identify_using_user(identifier)
       elsif identifier =~ /\Akey-\d+\Z/
         # git push over ssh
-        key_id = identifier.gsub("key-", "")
-        Key.find_by(id: key_id).try(:user)
+        identify_using_ssh_key(identifier)
+      end
+    end
+
+    # Tries to identify a user based on a commit SHA.
+    def identify_using_commit(project, ref)
+      commit = project.commit(ref)
+
+      return if !commit || !commit.author_email
+
+      email = commit.author_email
+
+      identify_with_cache(:email, email) do
+        User.find_by(email: email)
       end
     end
+
+    # Tries to identify a user based on a user identifier (e.g. "user-123").
+    def identify_using_user(identifier)
+      user_id = identifier.gsub("user-", "")
+
+      identify_with_cache(:user, user_id) do
+        User.find_by(id: user_id)
+      end
+    end
+
+    # Tries to identify a user based on an SSH key identifier (e.g. "key-123").
+    def identify_using_ssh_key(identifier)
+      key_id = identifier.gsub("key-", "")
+
+      identify_with_cache(:ssh_key, key_id) do
+        User.find_by_ssh_key_id(key_id)
+      end
+    end
+
+    def identify_with_cache(category, key)
+      if identification_cache[category].key?(key)
+        identification_cache[category][key]
+      else
+        identification_cache[category][key] = yield
+      end
+    end
+
+    def identification_cache
+      @identification_cache ||= {
+        email: {},
+        user: {},
+        ssh_key: {}
+      }
+    end
   end
 end
diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb
new file mode 100644
index 00000000000..b9e4042220a
--- /dev/null
+++ b/lib/gitlab/import_export/attribute_cleaner.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  module ImportExport
+    class AttributeCleaner
+      ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES
+
+      def self.clean!(relation_hash:)
+        relation_hash.reject! do |key, _value|
+          key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index e522a0fc8f6..f00c7460e82 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -1,6 +1,8 @@
 module Gitlab
   module ImportExport
     module CommandLineUtil
+      DEFAULT_MODE = 0700
+
       def tar_czf(archive:, dir:)
         tar_with_options(archive: archive, dir: dir, options: 'czf')
       end
@@ -21,6 +23,11 @@ module Gitlab
         execute(%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args)
       end
 
+      def mkdir_p(path)
+        FileUtils.mkdir_p(path, mode: DEFAULT_MODE)
+        FileUtils.chmod(DEFAULT_MODE, path)
+      end
+
       private
 
       def tar_with_options(archive:, dir:, options:)
@@ -45,7 +52,7 @@ module Gitlab
         # if we are copying files, create the destination folder
         destination_folder = File.file?(source) ? File.dirname(destination) : destination
 
-        FileUtils.mkdir_p(destination_folder)
+        mkdir_p(destination_folder)
         FileUtils.copy_entry(source, destination)
         true
       end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index eca6e5b6d51..113895ba22c 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -15,7 +15,7 @@ module Gitlab
       end
 
       def import
-        FileUtils.mkdir_p(@shared.export_path)
+        mkdir_p(@shared.export_path)
 
         wait_for_archived_file do
           decompress_archive
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 35ff134ea19..5a109f24f9f 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -110,9 +110,10 @@ module Gitlab
       def create_relation(relation, relation_hash_list)
         relation_array = [relation_hash_list].flatten.map do |relation_hash|
           Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
-                                                       relation_hash: relation_hash.merge('project_id' => restored_project.id),
+                                                       relation_hash: relation_hash,
                                                        members_mapper: members_mapper,
-                                                       user: @user)
+                                                       user: @user,
+                                                       project_id: restored_project.id)
         end
 
         relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb
index 9153088e966..2fbf437ec26 100644
--- a/lib/gitlab/import_export/project_tree_saver.rb
+++ b/lib/gitlab/import_export/project_tree_saver.rb
@@ -1,6 +1,8 @@
 module Gitlab
   module ImportExport
     class ProjectTreeSaver
+      include Gitlab::ImportExport::CommandLineUtil
+
       attr_reader :full_path
 
       def initialize(project:, shared:)
@@ -10,7 +12,7 @@ module Gitlab
       end
 
       def save
-        FileUtils.mkdir_p(@shared.export_path)
+        mkdir_p(@shared.export_path)
 
         File.write(full_path, project_json_tree)
         true
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 354ccd64696..9300f789e1b 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -13,6 +13,8 @@ module Gitlab
 
       USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze
 
+      PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze
+
       BUILD_MODELS = %w[Ci::Build commit_status].freeze
 
       IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
@@ -25,9 +27,9 @@ module Gitlab
         new(*args).create
       end
 
-      def initialize(relation_sym:, relation_hash:, members_mapper:, user:)
+      def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:)
         @relation_name = OVERRIDES[relation_sym] || relation_sym
-        @relation_hash = relation_hash.except('id', 'noteable_id')
+        @relation_hash = relation_hash.except('id', 'noteable_id').merge('project_id' => project_id)
         @members_mapper = members_mapper
         @user = user
         @imported_object_retries = 0
@@ -153,7 +155,11 @@ module Gitlab
       end
 
       def parsed_relation_hash
-        @parsed_relation_hash ||= @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) }
+        @parsed_relation_hash ||= begin
+          Gitlab::ImportExport::AttributeCleaner.clean!(relation_hash: @relation_hash)
+
+          @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) }
+        end
       end
 
       def set_st_diffs
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index d1e33ea8678..48a9a6fa5e2 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -12,7 +12,7 @@ module Gitlab
       def restore
         return true unless File.exist?(@path_to_bundle)
 
-        FileUtils.mkdir_p(path_to_repo)
+        mkdir_p(path_to_repo)
 
         git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) && repo_restore_hooks
       rescue => e
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
index 331e14021e6..a7028a32570 100644
--- a/lib/gitlab/import_export/repo_saver.rb
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -20,7 +20,7 @@ module Gitlab
       private
 
       def bundle_to_disk
-        FileUtils.mkdir_p(@shared.export_path)
+        mkdir_p(@shared.export_path)
         git_bundle(repo_path: path_to_repo, bundle_path: @full_path)
       rescue => e
         @shared.error(e)
diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb
index 9b642d740b7..7cf88298642 100644
--- a/lib/gitlab/import_export/version_saver.rb
+++ b/lib/gitlab/import_export/version_saver.rb
@@ -1,12 +1,14 @@
 module Gitlab
   module ImportExport
     class VersionSaver
+      include Gitlab::ImportExport::CommandLineUtil
+
       def initialize(shared:)
         @shared = shared
       end
 
       def save
-        FileUtils.mkdir_p(@shared.export_path)
+        mkdir_p(@shared.export_path)
 
         File.write(version_file, Gitlab::ImportExport.version, mode: 'w')
       rescue => e
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
index 6107420e4dd..1e6722a7bba 100644
--- a/lib/gitlab/import_export/wiki_repo_saver.rb
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -9,7 +9,7 @@ module Gitlab
       end
 
       def bundle_to_disk(full_path)
-        FileUtils.mkdir_p(@shared.export_path)
+        mkdir_p(@shared.export_path)
         git_bundle(repo_path: path_to_repo, bundle_path: full_path)
       rescue => e
         @shared.error(e)
diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb
index 3faab937726..c649da8c426 100644
--- a/lib/gitlab/redis.rb
+++ b/lib/gitlab/redis.rb
@@ -24,10 +24,20 @@ module Gitlab
       end
 
       def with
-        @pool ||= ConnectionPool.new { ::Redis.new(params) }
+        @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) }
         @pool.with { |redis| yield redis }
       end
 
+      def pool_size
+        if Sidekiq.server?
+          # the pool will be used in a multi-threaded context
+          Sidekiq.options[:concurrency] + 5
+        else
+          # probably this is a Unicorn process, so single threaded
+          5
+        end
+      end
+
       def _raw_config
         return @_raw_config if defined?(@_raw_config)