Merge branch 'master' into feature/multi-level-container-registry-images

* master: (94 commits)
  Merge branch 'open-redirect-fix-continue-to' into 'security'
  Merge branch 'open-redirect-host-fix' into 'security'
  Merge branch 'path-disclosure-proj-import-export' into 'security'
  Merge branch '29364-private-projects-mr-fix'
  Merge branch '30125-markdown-security'
  Issue title realtime
  Update CHANGELOG.md for 8.16.9
  Update CHANGELOG.md for 8.17.5
  Update CHANGELOG.md for 9.0.4
  Add "search" optional param and docs for V4
  Use PDFLab to render PDFs in GitLab
  Separate Scala from Java in CI examples
  Fix broken link
  Reorganize CI examples, add more links
  Refactor CI index page
  Remove deprecated field from workhorse response
  Use gitlab-workhorse 1.4.3
  Document how ETag caching middleware handles query parameters
  Make group skip validation in the frontend
  Use NamespaceValidator::WILDCARD_ROUTES in ETag caching middleware
  ...

12 files changed, 126 insertions, 40 deletions

diff --git a/app/models/blob.rb b/app/models/blob.rb
index 95d2111a992..f82126f8e65 100644
--- a/app/models/blob.rb
+++ b/app/models/blob.rb
@@ -46,10 +46,18 @@ class Blob < SimpleDelegator
     text? && language && language.name == 'SVG'
   end
 
+  def pdf?
+    name && File.extname(name) == '.pdf'
+  end
+
   def ipython_notebook?
     text? && language&.name == 'Jupyter Notebook'
   end
 
+  def sketch?
+    binary? && extname.downcase.delete('.') == 'sketch'
+  end
+
   def size_within_svg_limits?
     size <= MAXIMUM_SVG_SIZE
   end
@@ -67,8 +75,12 @@ class Blob < SimpleDelegator
       end
     elsif image? || svg?
       'image'
+    elsif pdf?
+      'pdf'
     elsif ipython_notebook?
       'notebook'
+    elsif sketch?
+      'sketch'
     elsif text?
       'text'
     else
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index ad0be70c32a..8431c5f228c 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -540,6 +540,8 @@ module Ci
     end
 
     def dependencies
+      return [] if empty_dependencies?
+
       depended_jobs = depends_on_builds
 
       return depended_jobs unless options[:dependencies].present?
@@ -549,6 +551,10 @@ module Ci
       end
     end
 
+    def empty_dependencies?
+      options[:dependencies]&.empty?
+    end
+
     private
 
     def update_artifacts_size
diff --git a/app/models/issue.rb b/app/models/issue.rb
index 10a5d9d2a24..f9704b0d754 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -40,6 +40,8 @@ class Issue < ActiveRecord::Base
 
   scope :include_associations, -> { includes(:assignee, :labels, project: :namespace) }
 
+  after_save :expire_etag_cache
+
   attr_spammable :title, spam_title: true
   attr_spammable :description, spam_description: true
 
@@ -59,10 +61,6 @@ class Issue < ActiveRecord::Base
     before_transition any => :closed do |issue|
       issue.closed_at = Time.zone.now
     end
-
-    before_transition closed: any do |issue|
-      issue.closed_at = nil
-    end
   end
 
   def hook_attrs
@@ -256,4 +254,13 @@ class Issue < ActiveRecord::Base
   def publicly_visible?
     project.public? && !confidential?
   end
+
+  def expire_etag_cache
+    key = Gitlab::Routing.url_helpers.rendered_title_namespace_project_issue_path(
+      project.namespace,
+      project,
+      self
+    )
+    Gitlab::EtagCaching::Store.new.touch(key)
+  end
 end
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index baee00b8fcd..6ad56b842b2 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -177,6 +177,16 @@ class MergeRequestDiff < ActiveRecord::Base
     st_commits.count
   end
 
+  def utf8_st_diffs
+    return [] if st_diffs.blank?
+
+    st_diffs.map do |diff|
+      diff.each do |k, v|
+        diff[k] = encode_utf8(v) if v.respond_to?(:encoding)
+      end
+    end
+  end
+
   private
 
   # Old GitLab implementations may have generated diffs as ["--broken-diff"].
@@ -270,14 +280,6 @@ class MergeRequestDiff < ActiveRecord::Base
     project.merge_base_commit(head_commit_sha, start_commit_sha).try(:sha)
   end
 
-  def utf8_st_diffs
-    st_diffs.map do |diff|
-      diff.each do |k, v|
-        diff[k] = encode_utf8(v) if v.respond_to?(:encoding)
-      end
-    end
-  end
-
   #
   # #save or #update_attributes providing changes on serialized attributes do a lot of
   # serialization and deserialization calls resulting in bad performance.
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index 1d4b1f7d590..9bfa731785f 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -150,7 +150,7 @@ class Namespace < ActiveRecord::Base
   end
 
   def any_project_has_container_registry_tags?
-    projects.any?(&:has_container_registry_tags?)
+    all_projects.any?(&:has_container_registry_tags?)
   end
 
   def send_update_instructions
@@ -214,6 +214,12 @@ class Namespace < ActiveRecord::Base
     @old_repository_storage_paths ||= repository_storage_paths
   end
 
+  # Includes projects from this namespace and projects from all subgroups
+  # that belongs to this namespace
+  def all_projects
+    Project.inside_path(full_path)
+  end
+
   private
 
   def repository_storage_paths
@@ -221,7 +227,7 @@ class Namespace < ActiveRecord::Base
     # pending delete. Unscoping also get rids of the default order, which causes
     # problems with SELECT DISTINCT.
     Project.unscoped do
-      projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
+      all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
     end
   end
 
diff --git a/app/models/project.rb b/app/models/project.rb
index f36cc324a5e..3d3a42ba0af 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -114,6 +114,8 @@ class Project < ActiveRecord::Base
   has_one :kubernetes_service, dependent: :destroy, inverse_of: :project
   has_one :prometheus_service, dependent: :destroy, inverse_of: :project
   has_one :mock_ci_service, dependent: :destroy
+  has_one :mock_deployment_service, dependent: :destroy
+  has_one :mock_monitoring_service, dependent: :destroy
 
   has_one  :forked_project_link,  dependent: :destroy, foreign_key: "forked_to_project_id"
   has_one  :forked_from_project,  through:   :forked_project_link
diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb
index 02fbd5497fa..9c56518c991 100644
--- a/app/models/project_services/kubernetes_service.rb
+++ b/app/models/project_services/kubernetes_service.rb
@@ -22,22 +22,21 @@ class KubernetesService < DeploymentService
   with_options presence: true, if: :activated? do
     validates :api_url, url: true
     validates :token
-
-    validates :namespace,
-      format: {
-        with: Gitlab::Regex.kubernetes_namespace_regex,
-        message: Gitlab::Regex.kubernetes_namespace_regex_message,
-      },
-      length: 1..63
   end
 
+  validates :namespace,
+    allow_blank: true,
+    length: 1..63,
+    if: :activated?,
+    format: {
+      with: Gitlab::Regex.kubernetes_namespace_regex,
+      message: Gitlab::Regex.kubernetes_namespace_regex_message
+    }
+
   after_save :clear_reactive_cache!
 
   def initialize_properties
-    if properties.nil?
-      self.properties = {}
-      self.namespace = "#{project.path}-#{project.id}" if project.present?
-    end
+    self.properties = {} if properties.nil?
   end
 
   def title
@@ -62,7 +61,7 @@ class KubernetesService < DeploymentService
         { type: 'text',
           name: 'namespace',
           title: 'Kubernetes namespace',
-          placeholder: 'Kubernetes namespace' },
+          placeholder: namespace_placeholder },
         { type: 'text',
           name: 'api_url',
           title: 'API URL',
@@ -92,7 +91,7 @@ class KubernetesService < DeploymentService
     variables = [
       { key: 'KUBE_URL', value: api_url, public: true },
       { key: 'KUBE_TOKEN', value: token, public: false },
-      { key: 'KUBE_NAMESPACE', value: namespace, public: true }
+      { key: 'KUBE_NAMESPACE', value: namespace_variable, public: true }
     ]
 
     if ca_pem.present?
@@ -135,8 +134,26 @@ class KubernetesService < DeploymentService
     { pods: pods }
   end
 
+  TEMPLATE_PLACEHOLDER = 'Kubernetes namespace'.freeze
+
   private
 
+  def namespace_placeholder
+    default_namespace || TEMPLATE_PLACEHOLDER
+  end
+
+  def namespace_variable
+    if namespace.present?
+      namespace
+    else
+      default_namespace
+    end
+  end
+
+  def default_namespace
+    "#{project.path}-#{project.id}" if project.present?
+  end
+
   def build_kubeclient!(api_path: 'api', api_version: 'v1')
     raise "Incomplete settings" unless api_url && namespace && token
 
diff --git a/app/models/project_services/mock_deployment_service.rb b/app/models/project_services/mock_deployment_service.rb
new file mode 100644
index 00000000000..59a3811ce5d
--- /dev/null
+++ b/app/models/project_services/mock_deployment_service.rb
@@ -0,0 +1,18 @@
+class MockDeploymentService < DeploymentService
+  def title
+    'Mock deployment'
+  end
+
+  def description
+    'Mock deployment service'
+  end
+
+  def self.to_param
+    'mock_deployment'
+  end
+
+  # No terminals support
+  def terminals(environment)
+    []
+  end
+end
diff --git a/app/models/project_services/mock_monitoring_service.rb b/app/models/project_services/mock_monitoring_service.rb
new file mode 100644
index 00000000000..dd04e04e198
--- /dev/null
+++ b/app/models/project_services/mock_monitoring_service.rb
@@ -0,0 +1,17 @@
+class MockMonitoringService < MonitoringService
+  def title
+    'Mock monitoring'
+  end
+
+  def description
+    'Mock monitoring service'
+  end
+
+  def self.to_param
+    'mock_monitoring'
+  end
+
+  def metrics(environment)
+    JSON.parse(File.read(Rails.root + 'spec/fixtures/metrics.json'))
+  end
+end
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 6b2383b73eb..dc1c1fab880 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -59,7 +59,7 @@ class Repository
   def raw_repository
     return nil unless path_with_namespace
 
-    @raw_repository ||= Gitlab::Git::Repository.new(path_to_repo)
+    @raw_repository ||= initialize_raw_repository
   end
 
   # Return absolute path to repository
@@ -146,12 +146,7 @@ class Repository
     # may cause the branch to "disappear" erroneously or have the wrong SHA.
     #
     # See: https://github.com/libgit2/libgit2/issues/1534 and https://gitlab.com/gitlab-org/gitlab-ce/issues/15392
-    raw_repo =
-      if fresh_repo
-        Gitlab::Git::Repository.new(path_to_repo)
-      else
-        raw_repository
-      end
+    raw_repo = fresh_repo ? initialize_raw_repository : raw_repository
 
     raw_repo.find_branch(name)
   end
@@ -505,9 +500,7 @@ class Repository
     end
   end
 
-  def branch_names
-    branches.map(&:name)
-  end
+  delegate :branch_names, to: :raw_repository
   cache_method :branch_names, fallback: []
 
   delegate :tag_names, to: :raw_repository
@@ -1168,4 +1161,8 @@ class Repository
   def repository_storage_path
     @project.repository_storage_path
   end
+
+  def initialize_raw_repository
+    Gitlab::Git::Repository.new(project.repository_storage, path_with_namespace + '.git')
+  end
 end
diff --git a/app/models/service.rb b/app/models/service.rb
index 54550177744..5a0ec58d193 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -238,7 +238,9 @@ class Service < ActiveRecord::Base
       slack
       teamcity
     ]
-    service_names << 'mock_ci' if Rails.env.development?
+    if Rails.env.development?
+      service_names += %w[mock_ci mock_deployment mock_monitoring]
+    end
 
     service_names.sort_by(&:downcase)
   end
diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb
index 5cc66574941..1e6fc837a75 100644
--- a/app/models/system_note_metadata.rb
+++ b/app/models/system_note_metadata.rb
@@ -1,7 +1,7 @@
 class SystemNoteMetadata < ActiveRecord::Base
   ICON_TYPES = %w[
-    commit merge confidentiality status label assignee cross_reference
-    title time_tracking branch milestone discussion task moved
+    commit merge confidential visible label assignee cross_reference
+    title time_tracking branch milestone discussion task moved opened closed merged
   ].freeze
 
   validates :note, presence: true