Add latest changes from gitlab-org/gitlab@master

14 files changed, 213 insertions, 33 deletions

diff --git a/app/assets/javascripts/api.js b/app/assets/javascripts/api.js
index 8a6395b42b5..dc6ea148047 100644
--- a/app/assets/javascripts/api.js
+++ b/app/assets/javascripts/api.js
@@ -476,12 +476,12 @@ const Api = {
     return axios.get(url);
   },
 
-  lsifData(projectPath, commitId, path) {
+  lsifData(projectPath, commitId, paths) {
     const url = Api.buildUrl(this.lsifPath)
       .replace(':id', encodeURIComponent(projectPath))
       .replace(':commit_id', commitId);
 
-    return axios.get(url, { params: { path } });
+    return axios.get(url, { params: { paths } });
   },
 
   environments(id) {
diff --git a/app/assets/javascripts/code_navigation/store/actions.js b/app/assets/javascripts/code_navigation/store/actions.js
index 2c52074e362..5220b1215b8 100644
--- a/app/assets/javascripts/code_navigation/store/actions.js
+++ b/app/assets/javascripts/code_navigation/store/actions.js
@@ -13,9 +13,10 @@ export default {
     commit(types.REQUEST_DATA);
 
     api
-      .lsifData(state.projectPath, state.commitId, state.blobPath)
+      .lsifData(state.projectPath, state.commitId, [state.blobPath])
       .then(({ data }) => {
-        const normalizedData = data.reduce((acc, d) => {
+        const dataForPath = data[state.blobPath];
+        const normalizedData = dataForPath.reduce((acc, d) => {
           if (d.hover) {
             acc[`${d.start_line}:${d.start_char}`] = d;
             addInteractionClass(d);
diff --git a/app/controllers/concerns/cycle_analytics_params.rb b/app/controllers/concerns/cycle_analytics_params.rb
index 3e67f1f54cb..50e340dc9b1 100644
--- a/app/controllers/concerns/cycle_analytics_params.rb
+++ b/app/controllers/concerns/cycle_analytics_params.rb
@@ -38,7 +38,7 @@ module CycleAnalyticsParams
   end
 
   def to_utc_time(field)
-    date = field.is_a?(Date) ? field : Date.parse(field)
+    date = field.is_a?(Date) || field.is_a?(Time) ? field : Date.parse(field)
     date.to_time.utc
   end
 end
diff --git a/app/graphql/resolvers/projects/snippets_resolver.rb b/app/graphql/resolvers/projects/snippets_resolver.rb
index bf9aa45349f..22895a24054 100644
--- a/app/graphql/resolvers/projects/snippets_resolver.rb
+++ b/app/graphql/resolvers/projects/snippets_resolver.rb
@@ -10,6 +10,11 @@ module Resolvers
       def resolve(**args)
         return Snippet.none if project.nil?
 
+        unless project.feature_available?(:snippets, current_user)
+          raise Gitlab::Graphql::Errors::ResourceNotAvailable,
+            'Snippets are not enabled for this Project'
+        end
+
         super
       end
 
diff --git a/app/models/project.rb b/app/models/project.rb
index fdf7452d143..b0a1e378b41 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -2039,6 +2039,16 @@ class Project < ApplicationRecord
     end
   end
 
+  def change_repository_storage(new_repository_storage_key)
+    return if repository_read_only?
+    return if repository_storage == new_repository_storage_key
+
+    raise ArgumentError unless ::Gitlab.config.repositories.storages.key?(new_repository_storage_key)
+
+    run_after_commit { ProjectUpdateRepositoryStorageWorker.perform_async(id, new_repository_storage_key) }
+    self.repository_read_only = true
+  end
+
   def pushes_since_gc
     Gitlab::Redis::SharedState.with { |redis| redis.get(pushes_since_gc_redis_shared_state_key).to_i }
   end
diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb
index 4d49c96d268..ee47acc6041 100644
--- a/app/policies/project_policy.rb
+++ b/app/policies/project_policy.rb
@@ -469,6 +469,8 @@ class ProjectPolicy < BasePolicy
     prevent :create_pipeline
   end
 
+  rule { admin }.enable :change_repository_storage
+
   private
 
   def team_member?
diff --git a/app/presenters/projects/prometheus/alert_presenter.rb b/app/presenters/projects/prometheus/alert_presenter.rb
index 7416b76b65d..02e22f8f46a 100644
--- a/app/presenters/projects/prometheus/alert_presenter.rb
+++ b/app/presenters/projects/prometheus/alert_presenter.rb
@@ -53,7 +53,7 @@ module Projects
           #### Summary
 
           #{metadata_list}
-          #{alert_details}
+          #{alert_details}#{metric_embed_for_alert}
         MARKDOWN
       end
 
@@ -118,6 +118,10 @@ module Projects
       def host_links
         Array(hosts.value).join(' ')
       end
+
+      def metric_embed_for_alert; end
     end
   end
 end
+
+Projects::Prometheus::AlertPresenter.prepend_if_ee('EE::Projects::Prometheus::AlertPresenter')
diff --git a/app/presenters/snippet_blob_presenter.rb b/app/presenters/snippet_blob_presenter.rb
index 8e593d3a5fe..ed9c28bbc2c 100644
--- a/app/presenters/snippet_blob_presenter.rb
+++ b/app/presenters/snippet_blob_presenter.rb
@@ -3,18 +3,15 @@
 class SnippetBlobPresenter < BlobPresenter
   def rich_data
     return if blob.binary?
+    return unless blob.rich_viewer
 
-    if markup?
-      blob.rendered_markup
-    else
-      highlight(plain: false)
-    end
+    render_rich_partial
   end
 
   def plain_data
     return if blob.binary?
 
-    highlight(plain: !markup?)
+    highlight(plain: false)
   end
 
   def raw_path
@@ -27,10 +24,6 @@ class SnippetBlobPresenter < BlobPresenter
 
   private
 
-  def markup?
-    blob.rich_viewer&.partial_name == 'markup'
-  end
-
   def snippet
     blob.container
   end
@@ -38,4 +31,18 @@ class SnippetBlobPresenter < BlobPresenter
   def language
     nil
   end
+
+  def render_rich_partial
+    renderer.render("projects/blob/viewers/_#{blob.rich_viewer.partial_name}",
+                    locals: { viewer: blob.rich_viewer, blob: blob, blob_raw_path: raw_path },
+                    layout: false)
+  end
+
+  def renderer
+    proxy = Warden::Proxy.new({}, Warden::Manager.new({})).tap do |proxy_instance|
+      proxy_instance.set_user(current_user, scope: :user)
+    end
+
+    ApplicationController.renderer.new('warden' => proxy)
+  end
 end
diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb
index 046745d725e..6eb8f5c27d9 100644
--- a/app/services/projects/container_repository/cleanup_tags_service.rb
+++ b/app/services/projects/container_repository/cleanup_tags_service.rb
@@ -61,10 +61,15 @@ module Projects
       end
 
       def filter_by_name(tags)
-        regex = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex']}\\z")
+        # Technical Debt: https://gitlab.com/gitlab-org/gitlab/issues/207267
+        # name_regex to be removed when container_expiration_policies is updated
+        # to have both regex columns
+        regex_delete = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_delete'] || params['name_regex']}\\z")
+        regex_retain = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_keep']}\\z")
 
         tags.select do |tag|
-          regex.scan(tag.name).any?
+          # regex_retain will override any overlapping matches by regex_delete
+          regex_delete.match?(tag.name) && !regex_retain.match?(tag.name)
         end
       end
 
diff --git a/app/services/projects/lsif_data_service.rb b/app/services/projects/lsif_data_service.rb
index 971885b680e..142a5a910d4 100644
--- a/app/services/projects/lsif_data_service.rb
+++ b/app/services/projects/lsif_data_service.rb
@@ -2,20 +2,22 @@
 
 module Projects
   class LsifDataService
-    attr_reader :file, :project, :path, :commit_id,
-      :docs, :doc_ranges, :ranges, :def_refs, :hover_refs
+    attr_reader :file, :project, :commit_id, :docs,
+      :doc_ranges, :ranges, :def_refs, :hover_refs
 
     CACHE_EXPIRE_IN = 1.hour
 
-    def initialize(file, project, params)
+    def initialize(file, project, commit_id)
       @file = file
       @project = project
-      @path = params[:path]
-      @commit_id = params[:commit_id]
-    end
+      @commit_id = commit_id
 
-    def execute
       fetch_data!
+    end
+
+    def execute(path)
+      doc_id = find_doc_id(docs, path)
+      dir_absolute_path = docs[doc_id]&.delete_suffix(path)
 
       doc_ranges[doc_id]&.map do |range_id|
         location, ref_id = ranges[range_id].values_at('loc', 'ref_id')
@@ -26,7 +28,7 @@ module Projects
           end_line: line_data.last,
           start_char: column_data.first,
           end_char: column_data.last,
-          definition_url: definition_url_for(def_refs[ref_id]),
+          definition_url: definition_url_for(def_refs[ref_id], dir_absolute_path),
           hover: highlighted_hover(hover_refs[ref_id])
         }
       end
@@ -58,8 +60,8 @@ module Projects
       @hover_refs = data['hover_refs']
     end
 
-    def doc_id
-      @doc_id ||= docs.reduce(nil) do |doc_id, (id, doc_path)|
+    def find_doc_id(docs, path)
+      docs.reduce(nil) do |doc_id, (id, doc_path)|
         next doc_id unless doc_path =~ /#{path}$/
 
         if doc_id.nil? || docs[doc_id].size > doc_path.size
@@ -70,11 +72,7 @@ module Projects
       end
     end
 
-    def dir_absolute_path
-      @dir_absolute_path ||= docs[doc_id]&.delete_suffix(path)
-    end
-
-    def definition_url_for(ref_id)
+    def definition_url_for(ref_id, dir_absolute_path)
       return unless range = ranges[ref_id]
 
       def_doc_id, location = range.values_at('doc_id', 'loc')
diff --git a/app/services/projects/update_repository_storage_service.rb b/app/services/projects/update_repository_storage_service.rb
new file mode 100644
index 00000000000..30b99e85304
--- /dev/null
+++ b/app/services/projects/update_repository_storage_service.rb
@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Projects
+  class UpdateRepositoryStorageService < BaseService
+    include Gitlab::ShellAdapter
+
+    RepositoryAlreadyMoved = Class.new(StandardError)
+
+    def initialize(project)
+      @project = project
+    end
+
+    def execute(new_repository_storage_key)
+      # Raising an exception is a little heavy handed but this behavior (doing
+      # nothing if the repo is already on the right storage) prevents data
+      # loss, so it is valuable for us to be able to observe it via the
+      # exception.
+      raise RepositoryAlreadyMoved if project.repository_storage == new_repository_storage_key
+
+      if mirror_repositories(new_repository_storage_key)
+        mark_old_paths_for_archive
+
+        project.update(repository_storage: new_repository_storage_key, repository_read_only: false)
+        project.leave_pool_repository
+        project.track_project_repository
+
+        enqueue_housekeeping
+      else
+        project.update(repository_read_only: false)
+      end
+    end
+
+    private
+
+    def mirror_repositories(new_repository_storage_key)
+      result = mirror_repository(new_repository_storage_key)
+
+      if project.wiki.repository_exists?
+        result &&= mirror_repository(new_repository_storage_key, type: Gitlab::GlRepository::WIKI)
+      end
+
+      result
+    end
+
+    def mirror_repository(new_storage_key, type: Gitlab::GlRepository::PROJECT)
+      return false unless wait_for_pushes(type)
+
+      repository = type.repository_for(project)
+      full_path = repository.full_path
+      raw_repository = repository.raw
+
+      # Initialize a git repository on the target path
+      gitlab_shell.create_repository(new_storage_key, raw_repository.relative_path, full_path)
+      new_repository = Gitlab::Git::Repository.new(new_storage_key,
+                                                   raw_repository.relative_path,
+                                                   raw_repository.gl_repository,
+                                                   full_path)
+
+      new_repository.fetch_repository_as_mirror(raw_repository)
+    end
+
+    def mark_old_paths_for_archive
+      old_repository_storage = project.repository_storage
+      new_project_path = moved_path(project.disk_path)
+
+      # Notice that the block passed to `run_after_commit` will run with `project`
+      # as its context
+      project.run_after_commit do
+        GitlabShellWorker.perform_async(:mv_repository,
+                                        old_repository_storage,
+                                        disk_path,
+                                        new_project_path)
+
+        if wiki.repository_exists?
+          GitlabShellWorker.perform_async(:mv_repository,
+                                          old_repository_storage,
+                                          wiki.disk_path,
+                                          "#{new_project_path}.wiki")
+        end
+      end
+    end
+
+    def moved_path(path)
+      "#{path}+#{project.id}+moved+#{Time.now.to_i}"
+    end
+
+    # The underlying FetchInternalRemote call uses a `git fetch` to move data
+    # to the new repository, which leaves it in a less-well-packed state,
+    # lacking bitmaps and commit graphs. Housekeeping will boost performance
+    # significantly.
+    def enqueue_housekeeping
+      return unless Gitlab::CurrentSettings.housekeeping_enabled?
+      return unless Feature.enabled?(:repack_after_shard_migration, project)
+
+      Projects::HousekeepingService.new(project, :gc).execute
+    rescue Projects::HousekeepingService::LeaseTaken
+      # No action required
+    end
+
+    def wait_for_pushes(type)
+      reference_counter = project.reference_counter(type: type)
+
+      # Try for 30 seconds, polling every 10
+      3.times do
+        return true if reference_counter.value == 0
+
+        sleep 10
+      end
+
+      false
+    end
+  end
+end
+
+Projects::UpdateRepositoryStorageService.prepend_if_ee('EE::Projects::UpdateRepositoryStorageService')
diff --git a/app/services/projects/update_service.rb b/app/services/projects/update_service.rb
index aedd7252f63..e10dede632a 100644
--- a/app/services/projects/update_service.rb
+++ b/app/services/projects/update_service.rb
@@ -13,6 +13,10 @@ module Projects
 
       ensure_wiki_exists if enabling_wiki?
 
+      if changing_storage_size?
+        project.change_repository_storage(params.delete(:repository_storage))
+      end
+
       yield if block_given?
 
       validate_classification_label(project, :external_authorization_classification_label)
@@ -140,6 +144,13 @@ module Projects
     def changing_pages_https_only?
       project.previous_changes.include?(:pages_https_only)
     end
+
+    def changing_storage_size?
+      new_repository_storage = params[:repository_storage]
+
+      new_repository_storage && project.repository.exists? &&
+        can?(current_user, :change_repository_storage, project)
+    end
   end
 end
 
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 259b2efc49f..61e9f50c2dd 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -1151,6 +1151,13 @@
   :resource_boundary: :unknown
   :weight: 1
   :idempotent: 
+- :name: project_update_repository_storage
+  :feature_category: :source_code_management
+  :has_external_dependencies: 
+  :urgency: :default
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: 
 - :name: propagate_service_template
   :feature_category: :source_code_management
   :has_external_dependencies: 
diff --git a/app/workers/project_update_repository_storage_worker.rb b/app/workers/project_update_repository_storage_worker.rb
new file mode 100644
index 00000000000..2d88b532dbf
--- /dev/null
+++ b/app/workers/project_update_repository_storage_worker.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class ProjectUpdateRepositoryStorageWorker # rubocop:disable Scalability/IdempotentWorker
+  include ApplicationWorker
+
+  feature_category :source_code_management
+
+  def perform(project_id, new_repository_storage_key)
+    project = Project.find(project_id)
+
+    ::Projects::UpdateRepositoryStorageService.new(project).execute(new_repository_storage_key)
+  rescue ::Projects::UpdateRepositoryStorageService::RepositoryAlreadyMoved
+    Rails.logger.info "#{self.class}: repository already moved: #{project}" # rubocop:disable Gitlab/RailsLogger
+  end
+end