Merge branch 'master' into refactor/ci-config-add-entry-error

* master: (189 commits)
  Update CHANGELOG for !4659
  Center the header logo for all Devise emails
  Add previews for all customized Devise emails
  Customize the Devise `unlock_instructions` email
  Customize the Devise `reset_password_instructions` email
  Customize the Devise `password_change` emails
  Use gitlab-git 10.2.0
  Use Git cached counters on project show page
  Fix indentation scss-lint errors
  Added title attribute to enties in tree view Closes #18353
  Banzai::Filter::ExternalLinkFilter use XPath
  Reduce queries in IssueReferenceFilter
  Use gitlab_git 10.1.4
  Fixed ordering in Project.find_with_namespace
  Fix images in emails
  Banzai::Filter::UploadLinkFilter use XPath
  Turn Group#owners into a has_many association
  Make project_id nullable
  ...

17 files changed, 155 insertions, 66 deletions

diff --git a/lib/api/builds.rb b/lib/api/builds.rb
index 645e2dda0b7..979328efe0e 100644
--- a/lib/api/builds.rb
+++ b/lib/api/builds.rb
@@ -142,7 +142,7 @@ module API
         return not_found!(build) unless build
         return forbidden!('Build is not retryable') unless build.retryable?
 
-        build = Ci::Build.retry(build)
+        build = Ci::Build.retry(build, current_user)
 
         present build, with: Entities::Build,
                        user_can_download_artifacts: can?(current_user, :read_build, user_project)
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index db95d7c908b..4815bafe238 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -103,7 +103,7 @@ module Banzai
         ref_pattern = object_class.reference_pattern
         link_pattern = object_class.link_reference_pattern
 
-        each_node do |node|
+        nodes.each do |node|
           if text_node?(node) && ref_pattern
             replace_text_when_pattern_matches(node, ref_pattern) do |content|
               object_link_filter(content, ref_pattern)
@@ -206,6 +206,55 @@ module Banzai
         text
       end
 
+      # Returns a Hash containing all object references (e.g. issue IDs) per the
+      # project they belong to.
+      def references_per_project
+        @references_per_project ||= begin
+          refs = Hash.new { |hash, key| hash[key] = Set.new }
+
+          regex = Regexp.union(object_class.reference_pattern,
+                               object_class.link_reference_pattern)
+
+          nodes.each do |node|
+            node.to_html.scan(regex) do
+              project = $~[:project] || current_project_path
+
+              refs[project] << $~[object_sym]
+            end
+          end
+
+          refs
+        end
+      end
+
+      # Returns a Hash containing referenced projects grouped per their full
+      # path.
+      def projects_per_reference
+        @projects_per_reference ||= begin
+          hash = {}
+          refs = Set.new
+
+          references_per_project.each do |project_ref, _|
+            refs << project_ref
+          end
+
+          find_projects_for_paths(refs.to_a).each do |project|
+            hash[project.path_with_namespace] = project
+          end
+
+          hash
+        end
+      end
+
+      # Returns the projects for the given paths.
+      def find_projects_for_paths(paths)
+        Project.where_paths_in(paths).includes(:namespace)
+      end
+
+      def current_project_path
+        @current_project_path ||= project.path_with_namespace
+      end
+
       private
 
       def project_refs_cache
diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb
index f73ecfc9418..0a29c547a4d 100644
--- a/lib/banzai/filter/external_link_filter.rb
+++ b/lib/banzai/filter/external_link_filter.rb
@@ -3,17 +3,8 @@ module Banzai
     # HTML Filter to modify the attributes of external links
     class ExternalLinkFilter < HTML::Pipeline::Filter
       def call
-        doc.search('a').each do |node|
-          link = node.attr('href')
-
-          next unless link
-
-          # Skip non-HTTP(S) links
-          next unless link.start_with?('http')
-
-          # Skip internal links
-          next if link.start_with?(internal_url)
-
+        # Skip non-HTTP(S) links and internal links
+        doc.xpath("descendant-or-self::a[starts-with(@href, 'http') and not(starts-with(@href, '#{internal_url}'))]").each do |node|
           node.set_attribute('rel', 'nofollow noreferrer')
           node.set_attribute('target', '_blank')
         end
diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb
index 2496e704002..2614261f9eb 100644
--- a/lib/banzai/filter/issue_reference_filter.rb
+++ b/lib/banzai/filter/issue_reference_filter.rb
@@ -11,13 +11,40 @@ module Banzai
         Issue
       end
 
-      def find_object(project, id)
-        project.get_issue(id)
+      def find_object(project, iid)
+        issues_per_project[project][iid]
       end
 
       def url_for_object(issue, project)
         IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path])
       end
+
+      def project_from_ref(ref)
+        projects_per_reference[ref || current_project_path]
+      end
+
+      # Returns a Hash containing the issues per Project instance.
+      def issues_per_project
+        @issues_per_project ||= begin
+          hash = Hash.new { |h, k| h[k] = {} }
+
+          projects_per_reference.each do |path, project|
+            issue_ids = references_per_project[path]
+
+            next unless project.default_issues_tracker?
+
+            project.issues.where(iid: issue_ids.to_a).each do |issue|
+              hash[project][issue.iid] = issue
+            end
+          end
+
+          hash
+        end
+      end
+
+      def find_projects_for_paths(paths)
+        super(paths).includes(:gitlab_issue_tracker_service)
+      end
     end
   end
 end
diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb
index c0f503c9af3..45bb66dc99f 100644
--- a/lib/banzai/filter/upload_link_filter.rb
+++ b/lib/banzai/filter/upload_link_filter.rb
@@ -10,11 +10,11 @@ module Banzai
       def call
         return doc unless project
 
-        doc.search('a').each do |el|
+        doc.xpath('descendant-or-self::a[starts-with(@href, "/uploads/")]').each do |el|
           process_link_attr el.attribute('href')
         end
 
-        doc.search('img').each do |el|
+        doc.xpath('descendant-or-self::img[starts-with(@src, "/uploads/")]').each do |el|
           process_link_attr el.attribute('src')
         end
 
@@ -24,12 +24,7 @@ module Banzai
       protected
 
       def process_link_attr(html_attr)
-        return if html_attr.blank?
-
-        uri = html_attr.value
-        if uri.starts_with?("/uploads/")
-          html_attr.value = build_url(uri).to_s
-        end
+        html_attr.value = build_url(html_attr.value).to_s
       end
 
       def build_url(uri)
diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb
index 24076e3d9ec..f306079d833 100644
--- a/lib/banzai/reference_parser/issue_parser.rb
+++ b/lib/banzai/reference_parser/issue_parser.rb
@@ -25,7 +25,21 @@ module Banzai
       def issues_for_nodes(nodes)
         @issues_for_nodes ||= grouped_objects_for_nodes(
           nodes,
-          Issue.all.includes(:author, :assignee, :project),
+          Issue.all.includes(
+            :author,
+            :assignee,
+            {
+              # These associations are primarily used for checking permissions.
+              # Eager loading these ensures we don't end up running dozens of
+              # queries in this process.
+              project: [
+                { namespace: :owner },
+                { group: [:owners, :group_members] },
+                :invited_groups,
+                :project_members
+              ]
+            }
+          ),
           self.class.data_attribute
         )
       end
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 1a5be9dbc46..a051748cf43 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -9,7 +9,8 @@ module Ci
     ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache]
     ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services,
                         :allow_failure, :type, :stage, :when, :artifacts, :cache,
-                        :dependencies, :before_script, :after_script, :variables]
+                        :dependencies, :before_script, :after_script, :variables,
+                        :environment]
     ALLOWED_CACHE_KEYS = [:key, :untracked, :paths]
     ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in]
 
@@ -29,7 +30,10 @@ module Ci
     end
 
     def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil)
-      builds.select{|build| build[:stage] == stage && process?(build[:only], build[:except], ref, tag, trigger_request)}
+      builds.select do |build|
+        build[:stage] == stage &&
+          process?(build[:only], build[:except], ref, tag, trigger_request)
+      end
     end
 
     def builds
@@ -90,6 +94,7 @@ module Ci
         except: job[:except],
         allow_failure: job[:allow_failure] || false,
         when: job[:when] || 'on_success',
+        environment: job[:environment],
         options: {
           image: job[:image] || @image,
           services: job[:services] || @services,
@@ -214,6 +219,10 @@ module Ci
       if job[:when] && !job[:when].in?(%w[on_success on_failure always])
         raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always"
       end
+
+      if job[:environment] && !validate_environment(job[:environment])
+        raise ValidationError, "#{name} job: environment parameter #{Gitlab::Regex.environment_name_regex_message}"
+      end
     end
 
     def validate_job_script!(name, job)
diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb
index 4e20dc4f875..eb5a2596177 100644
--- a/lib/container_registry/blob.rb
+++ b/lib/container_registry/blob.rb
@@ -18,7 +18,7 @@ module ContainerRegistry
     end
 
     def digest
-      config['digest']
+      config['digest'] || config['blobSum']
     end
 
     def type
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
index 4d726692f45..e0b3f14d384 100644
--- a/lib/container_registry/client.rb
+++ b/lib/container_registry/client.rb
@@ -47,7 +47,9 @@ module ContainerRegistry
       conn.request :json
       conn.headers['Accept'] = MANIFEST_VERSION
 
-      conn.response :json, content_type: /\bjson$/
+      conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws'
+      conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json'
+      conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json'
 
       if options[:user] && options[:password]
         conn.request(:basic_auth, options[:user].to_s, options[:password].to_s)
diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb
index 43f8d6dc8c2..7a0929d774e 100644
--- a/lib/container_registry/tag.rb
+++ b/lib/container_registry/tag.rb
@@ -12,6 +12,14 @@ module ContainerRegistry
       manifest.present?
     end
 
+    def v1?
+      manifest && manifest['schemaVersion'] == 1
+    end
+
+    def v2?
+      manifest && manifest['schemaVersion'] == 2
+    end
+
     def manifest
       return @manifest if defined?(@manifest)
 
@@ -57,7 +65,9 @@ module ContainerRegistry
       return @layers if defined?(@layers)
       return unless manifest
 
-      @layers = manifest['layers'].map do |layer|
+      layers = manifest['layers'] || manifest['fsLayers']
+
+      @layers = layers.map do |layer|
         repository.blob(layer)
       end
     end
@@ -65,7 +75,7 @@ module ContainerRegistry
     def total_size
       return unless layers
 
-      layers.map(&:size).sum
+      layers.map(&:size).sum if v2?
     end
 
     def delete
diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb
index adbf5941a96..7e3f5abba62 100644
--- a/lib/gitlab/backend/grack_auth.rb
+++ b/lib/gitlab/backend/grack_auth.rb
@@ -1,5 +1,3 @@
-require_relative 'shell_env'
-
 module Grack
   class AuthSpawner
     def self.call(env)
@@ -61,11 +59,6 @@ module Grack
       end
 
       @user = authenticate_user(login, password)
-
-      if @user
-        Gitlab::ShellEnv.set_env(@user)
-        @env['REMOTE_USER'] = @auth.username
-      end
     end
 
     def ci_request?(login, password)
diff --git a/lib/gitlab/backend/shell_env.rb b/lib/gitlab/backend/shell_env.rb
deleted file mode 100644
index 9f5adee594a..00000000000
--- a/lib/gitlab/backend/shell_env.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-module Gitlab
-  # This module provide 2 methods
-  # to set specific ENV variables for GitLab Shell
-  module ShellEnv
-    extend self
-
-    def set_env(user)
-      # Set GL_ID env variable
-      if user
-        ENV['GL_ID'] = gl_id(user)
-      end
-    end
-
-    def reset_env
-      # Reset GL_ID env variable
-      ENV['GL_ID'] = nil
-    end
-
-    def gl_id(user)
-      if user.present?
-        "user-#{user.id}"
-      else
-        # This empty string is used in the render_grack_auth_ok method
-        ""
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb
index 970763670c5..095d0ac1047 100644
--- a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb
+++ b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb
@@ -24,6 +24,10 @@ module Gitlab
             value.is_a?(String) || value.is_a?(Symbol)
           end
 
+          def validate_environment(value)
+            value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex
+          end
+
           def validate_boolean(value)
             value.in?([true, false])
           end
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 04fa6a3a5de..d76ecb54017 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -30,6 +30,10 @@ module Gitlab
       order
     end
 
+    def self.random
+      Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()"
+    end
+
     def true_value
       if Gitlab::Database.postgresql?
         "'t'"
diff --git a/lib/gitlab/gl_id.rb b/lib/gitlab/gl_id.rb
new file mode 100644
index 00000000000..624fd00367e
--- /dev/null
+++ b/lib/gitlab/gl_id.rb
@@ -0,0 +1,11 @@
+module Gitlab
+  module GlId
+    def self.gl_id(user)
+      if user.present?
+        "user-#{user.id}"
+      else
+        ""
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 1cbd6d945a0..c84c68f96f6 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -100,5 +100,13 @@ module Gitlab
     def container_registry_reference_regex
       git_reference_regex
     end
+
+    def environment_name_regex
+      @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze
+    end
+
+    def environment_name_regex_message
+      "can contain only letters, digits, '-' and '_'."
+    end
   end
 end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 388f84dbe0e..40e8299c36b 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -8,7 +8,7 @@ module Gitlab
     class << self
       def git_http_ok(repository, user)
         {
-          'GL_ID' => Gitlab::ShellEnv.gl_id(user),
+          'GL_ID' => Gitlab::GlId.gl_id(user),
           'RepoPath' => repository.path_to_repo,
         }
       end