Add latest changes from gitlab-org/gitlab@master

5 files changed, 237 insertions, 111 deletions

diff --git a/app/services/concerns/projects/container_repository/gitlab/timeoutable.rb b/app/services/concerns/projects/container_repository/gitlab/timeoutable.rb
new file mode 100644
index 00000000000..095f5aa7cfa
--- /dev/null
+++ b/app/services/concerns/projects/container_repository/gitlab/timeoutable.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Projects
+  module ContainerRepository
+    module Gitlab
+      module Timeoutable
+        extend ActiveSupport::Concern
+
+        DISABLED_TIMEOUTS = [nil, 0].freeze
+
+        TimeoutError = Class.new(StandardError)
+
+        private
+
+        def timeout?(start_time)
+          return false if service_timeout.in?(DISABLED_TIMEOUTS)
+
+          (Time.zone.now - start_time) > service_timeout
+        end
+
+        def service_timeout
+          ::Gitlab::CurrentSettings.current_application_settings.container_registry_delete_tags_service_timeout
+        end
+      end
+    end
+  end
+end
diff --git a/app/services/projects/container_repository/cleanup_tags_base_service.rb b/app/services/projects/container_repository/cleanup_tags_base_service.rb
new file mode 100644
index 00000000000..ec0528af652
--- /dev/null
+++ b/app/services/projects/container_repository/cleanup_tags_base_service.rb
@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Projects
+  module ContainerRepository
+    class CleanupTagsBaseService
+      include BaseServiceUtility
+      include ::Gitlab::Utils::StrongMemoize
+
+      private
+
+      def filter_out_latest
+        @tags.reject!(&:latest?)
+      end
+
+      def filter_by_name
+        regex_delete = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_delete || name_regex}\\z")
+        regex_retain = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_keep}\\z")
+
+        @tags.select! do |tag|
+          # regex_retain will override any overlapping matches by regex_delete
+          regex_delete.match?(tag.name) && !regex_retain.match?(tag.name)
+        end
+      end
+
+      # Should return [tags_to_delete, tags_to_keep]
+      def partition_by_keep_n
+        return [@tags, []] unless keep_n
+
+        order_by_date_desc
+
+        @tags.partition.with_index { |_, index| index >= keep_n_as_integer }
+      end
+
+      # Should return [tags_to_delete, tags_to_keep]
+      def partition_by_older_than
+        return [@tags, []] unless older_than
+
+        older_than_timestamp = older_than_in_seconds.ago
+
+        @tags.partition do |tag|
+          timestamp = pushed_at(tag)
+
+          timestamp && timestamp < older_than_timestamp
+        end
+      end
+
+      def order_by_date_desc
+        now = DateTime.current
+        @tags.sort_by! { |tag| pushed_at(tag) || now }
+             .reverse!
+      end
+
+      def delete_tags
+        return success(deleted: []) unless @tags.any?
+
+        service = Projects::ContainerRepository::DeleteTagsService.new(
+          @project,
+          @current_user,
+          tags: @tags.map(&:name),
+          container_expiration_policy: container_expiration_policy
+        )
+
+        service.execute(@container_repository)
+      end
+
+      def can_destroy?
+        return true if container_expiration_policy
+
+        can?(@current_user, :destroy_container_image, @project)
+      end
+
+      def valid_regex?
+        %w[name_regex_delete name_regex name_regex_keep].each do |param_name|
+          regex = @params[param_name]
+          ::Gitlab::UntrustedRegexp.new(regex) unless regex.blank?
+        end
+        true
+      rescue RegexpError => e
+        ::Gitlab::ErrorTracking.log_exception(e, project_id: @project.id)
+        false
+      end
+
+      def older_than
+        @params['older_than']
+      end
+
+      def name_regex_delete
+        @params['name_regex_delete']
+      end
+
+      def name_regex
+        @params['name_regex']
+      end
+
+      def name_regex_keep
+        @params['name_regex_keep']
+      end
+
+      def container_expiration_policy
+        @params['container_expiration_policy']
+      end
+
+      def keep_n
+        @params['keep_n']
+      end
+
+      def keep_n_as_integer
+        keep_n.to_i
+      end
+
+      def older_than_in_seconds
+        strong_memoize(:older_than_in_seconds) do
+          ChronicDuration.parse(older_than).seconds
+        end
+      end
+    end
+  end
+end
diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb
index 0a8e8e72766..515f9aadca0 100644
--- a/app/services/projects/container_repository/cleanup_tags_service.rb
+++ b/app/services/projects/container_repository/cleanup_tags_service.rb
@@ -2,10 +2,7 @@
 
 module Projects
   module ContainerRepository
-    class CleanupTagsService
-      include BaseServiceUtility
-      include ::Gitlab::Utils::StrongMemoize
-
+    class CleanupTagsService < CleanupTagsBaseService
       def initialize(container_repository, user = nil, params = {})
         @container_repository = container_repository
         @current_user = user
@@ -43,74 +40,20 @@ module Projects
 
       private
 
-      def delete_tags
-        return success(deleted: []) unless @tags.any?
-
-        service = Projects::ContainerRepository::DeleteTagsService.new(
-          @project,
-          @current_user,
-          tags: @tags.map(&:name),
-          container_expiration_policy: container_expiration_policy
-        )
-
-        service.execute(@container_repository)
-      end
-
-      def filter_out_latest
-        @tags.reject!(&:latest?)
-      end
-
-      def order_by_date
-        now = DateTime.current
-        @tags.sort_by! { |tag| tag.created_at || now }
-             .reverse!
-      end
-
-      def filter_by_name
-        regex_delete = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_delete || name_regex}\\z")
-        regex_retain = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_keep}\\z")
-
-        @tags.select! do |tag|
-          # regex_retain will override any overlapping matches by regex_delete
-          regex_delete.match?(tag.name) && !regex_retain.match?(tag.name)
-        end
-      end
-
       def filter_keep_n
-        return unless keep_n
+        @tags, tags_to_keep = partition_by_keep_n
 
-        order_by_date
-        cache_tags(@tags.first(keep_n_as_integer))
-        @tags = @tags.drop(keep_n_as_integer)
+        cache_tags(tags_to_keep)
       end
 
       def filter_by_older_than
-        return unless older_than
-
-        older_than_timestamp = older_than_in_seconds.ago
-
-        @tags, tags_to_keep = @tags.partition do |tag|
-          tag.created_at && tag.created_at < older_than_timestamp
-        end
+        @tags, tags_to_keep = partition_by_older_than
 
         cache_tags(tags_to_keep)
       end
 
-      def can_destroy?
-        return true if container_expiration_policy
-
-        can?(@current_user, :destroy_container_image, @project)
-      end
-
-      def valid_regex?
-        %w(name_regex_delete name_regex name_regex_keep).each do |param_name|
-          regex = @params[param_name]
-          ::Gitlab::UntrustedRegexp.new(regex) unless regex.blank?
-        end
-        true
-      rescue RegexpError => e
-        ::Gitlab::ErrorTracking.log_exception(e, project_id: @project.id)
-        false
+      def pushed_at(tag)
+        tag.created_at
       end
 
       def truncate
@@ -153,40 +96,6 @@ module Projects
       def max_list_size
         ::Gitlab::CurrentSettings.current_application_settings.container_registry_cleanup_tags_service_max_list_size.to_i
       end
-
-      def keep_n
-        @params['keep_n']
-      end
-
-      def keep_n_as_integer
-        keep_n.to_i
-      end
-
-      def older_than_in_seconds
-        strong_memoize(:older_than_in_seconds) do
-          ChronicDuration.parse(older_than).seconds
-        end
-      end
-
-      def older_than
-        @params['older_than']
-      end
-
-      def name_regex_delete
-        @params['name_regex_delete']
-      end
-
-      def name_regex
-        @params['name_regex']
-      end
-
-      def name_regex_keep
-        @params['name_regex_keep']
-      end
-
-      def container_expiration_policy
-        @params['container_expiration_policy']
-      end
     end
   end
 end
diff --git a/app/services/projects/container_repository/gitlab/cleanup_tags_service.rb b/app/services/projects/container_repository/gitlab/cleanup_tags_service.rb
new file mode 100644
index 00000000000..c515061f58f
--- /dev/null
+++ b/app/services/projects/container_repository/gitlab/cleanup_tags_service.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Projects
+  module ContainerRepository
+    module Gitlab
+      class CleanupTagsService < CleanupTagsBaseService
+        include ::Projects::ContainerRepository::Gitlab::Timeoutable
+
+        TAGS_PAGE_SIZE = 1000
+
+        def initialize(container_repository, user = nil, params = {})
+          @container_repository = container_repository
+          @current_user = user
+          @params = params.dup
+
+          @project = container_repository.project
+        end
+
+        def execute
+          return error('access denied') unless can_destroy?
+          return error('invalid regex') unless valid_regex?
+
+          with_timeout do |start_time, result|
+            @container_repository.each_tags_page(page_size: TAGS_PAGE_SIZE) do |tags|
+              execute_for_tags(tags, result)
+
+              raise TimeoutError if timeout?(start_time)
+            end
+          end
+        end
+
+        private
+
+        def execute_for_tags(tags, overall_result)
+          @tags = tags
+          original_size = @tags.size
+
+          filter_out_latest
+          filter_by_name
+
+          filter_by_keep_n
+          filter_by_older_than
+
+          overall_result[:before_delete_size] += @tags.size
+          overall_result[:original_size] += original_size
+
+          result = delete_tags
+
+          overall_result[:deleted_size] += result[:deleted]&.size
+          overall_result[:deleted] += result[:deleted]
+          overall_result[:status] = result[:status] unless overall_result[:status] == :error
+        end
+
+        def with_timeout
+          result = {
+            original_size: 0,
+            before_delete_size: 0,
+            deleted_size: 0,
+            deleted: []
+          }
+
+          yield Time.zone.now, result
+
+          result
+        rescue TimeoutError
+          result[:status] = :error
+
+          result
+        end
+
+        def filter_by_keep_n
+          @tags, _ = partition_by_keep_n
+        end
+
+        def filter_by_older_than
+          @tags, _ = partition_by_older_than
+        end
+
+        def pushed_at(tag)
+          tag.updated_at || tag.created_at
+        end
+      end
+    end
+  end
+end
diff --git a/app/services/projects/container_repository/gitlab/delete_tags_service.rb b/app/services/projects/container_repository/gitlab/delete_tags_service.rb
index 81cef554dec..530cf87c338 100644
--- a/app/services/projects/container_repository/gitlab/delete_tags_service.rb
+++ b/app/services/projects/container_repository/gitlab/delete_tags_service.rb
@@ -6,10 +6,7 @@ module Projects
       class DeleteTagsService
         include BaseServiceUtility
         include ::Gitlab::Utils::StrongMemoize
-
-        DISABLED_TIMEOUTS = [nil, 0].freeze
-
-        TimeoutError = Class.new(StandardError)
+        include ::Projects::ContainerRepository::Gitlab::Timeoutable
 
         def initialize(container_repository, tag_names)
           @container_repository = container_repository
@@ -44,16 +41,6 @@ module Projects
 
           @deleted_tags.any? ? success(deleted: @deleted_tags) : error('could not delete tags')
         end
-
-        def timeout?(start_time)
-          return false if service_timeout.in?(DISABLED_TIMEOUTS)
-
-          (Time.zone.now - start_time) > service_timeout
-        end
-
-        def service_timeout
-          ::Gitlab::CurrentSettings.current_application_settings.container_registry_delete_tags_service_timeout
-        end
       end
     end
   end