Add latest changes from gitlab-org/gitlab@master

9 files changed, 137 insertions, 8 deletions

diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb
index a497d2cabe5..8a4be3139e8 100644
--- a/app/models/ci/namespace_mirror.rb
+++ b/app/models/ci/namespace_mirror.rb
@@ -4,6 +4,34 @@ module Ci
   # This model represents a record in a shadow table of the main database's namespaces table.
   # It allows us to navigate the namespace hierarchy on the ci database without resorting to a JOIN.
   class NamespaceMirror < ApplicationRecord
-    # Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517
+    belongs_to :namespace
+
+    scope :contains_namespace, -> (id) do
+      where('traversal_ids @> ARRAY[?]::int[]', id)
+    end
+
+    class << self
+      def sync!(event)
+        namespace = event.namespace
+        traversal_ids = namespace.self_and_ancestor_ids(hierarchy_order: :desc)
+
+        upsert({ namespace_id: event.namespace_id, traversal_ids: traversal_ids },
+               unique_by: :namespace_id)
+
+        # It won't be necessary once we remove `sync_traversal_ids`.
+        # More info: https://gitlab.com/gitlab-org/gitlab/-/issues/347541
+        sync_children_namespaces!(event.namespace_id, traversal_ids)
+      end
+
+      private
+
+      def sync_children_namespaces!(namespace_id, traversal_ids)
+        contains_namespace(namespace_id)
+          .where.not(namespace_id: namespace_id)
+          .update_all(
+            "traversal_ids = ARRAY[#{sanitize_sql(traversal_ids.join(','))}]::int[] || traversal_ids[array_position(traversal_ids, #{sanitize_sql(namespace_id)}) + 1:]"
+          )
+      end
+    end
   end
 end
diff --git a/app/models/ci/project_mirror.rb b/app/models/ci/project_mirror.rb
index c6e3101fb3a..d6aaa3f50c1 100644
--- a/app/models/ci/project_mirror.rb
+++ b/app/models/ci/project_mirror.rb
@@ -4,6 +4,13 @@ module Ci
   # This model represents a shadow table of the main database's projects table.
   # It allows us to navigate the project and namespace hierarchy on the ci database.
   class ProjectMirror < ApplicationRecord
-    # Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517
+    belongs_to :project
+
+    class << self
+      def sync!(event)
+        upsert({ project_id: event.project_id, namespace_id: event.project.namespace_id },
+               unique_by: :project_id)
+      end
+    end
   end
 end
diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb
index 8e130998f11..c914819f79d 100644
--- a/app/models/container_repository.rb
+++ b/app/models/container_repository.rb
@@ -145,9 +145,14 @@ class ContainerRepository < ApplicationRecord
              name: path.repository_name)
   end
 
-  def self.create_from_path!(path)
-    safe_find_or_create_by!(project: path.repository_project,
-                                 name: path.repository_name)
+  def self.find_or_create_from_path(path)
+    repository = safe_find_or_create_by(
+      project: path.repository_project,
+      name: path.repository_name
+    )
+    return repository if repository.persisted?
+
+    find_by_path!(path)
   end
 
   def self.build_root_repository(project)
diff --git a/app/models/error_tracking/error_event.rb b/app/models/error_tracking/error_event.rb
index 0b638f65768..18c1467e6f6 100644
--- a/app/models/error_tracking/error_event.rb
+++ b/app/models/error_tracking/error_event.rb
@@ -3,6 +3,9 @@
 class ErrorTracking::ErrorEvent < ApplicationRecord
   belongs_to :error, counter_cache: :events_count
 
+  # Scrub null bytes
+  attribute :payload, Gitlab::Database::Type::JsonPgSafe.new
+
   validates :payload, json_schema: { filename: 'error_tracking_event_payload' }
 
   validates :error, presence: true
diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb
index c3b3e76f67b..0fbdd2d8a5b 100644
--- a/app/models/loose_foreign_keys/deleted_record.rb
+++ b/app/models/loose_foreign_keys/deleted_record.rb
@@ -1,15 +1,45 @@
 # frozen_string_literal: true
 
 class LooseForeignKeys::DeletedRecord < ApplicationRecord
+  PARTITION_DURATION = 1.day
+
+  include PartitionedTable
+
   self.primary_key = :id
+  self.ignored_columns = %i[partition]
+
+  partitioned_by :partition, strategy: :sliding_list,
+    next_partition_if: -> (active_partition) do
+      return false if Feature.disabled?(:lfk_automatic_partition_creation, default_enabled: :yaml)
+
+      oldest_record_in_partition = LooseForeignKeys::DeletedRecord
+        .select(:id, :created_at)
+        .for_partition(active_partition)
+        .order(:id)
+        .limit(1)
+        .take
+
+      oldest_record_in_partition.present? && oldest_record_in_partition.created_at < PARTITION_DURATION.ago
+    end,
+    detach_partition_if: -> (partition) do
+      return false if Feature.disabled?(:lfk_automatic_partition_dropping, default_enabled: :yaml)
+
+      !LooseForeignKeys::DeletedRecord
+        .for_partition(partition)
+        .status_pending
+        .exists?
+    end
 
   scope :for_table, -> (table) { where(fully_qualified_table_name: table) }
+  scope :for_partition, -> (partition) { where(partition: partition) }
   scope :consume_order, -> { order(:partition, :consume_after, :id) }
 
   enum status: { pending: 1, processed: 2 }, _prefix: :status
 
   def self.load_batch_for_table(table, batch_size)
-    for_table(table)
+    # selecting partition as partition_number to workaround the sliding partitioning column ignore
+    select(arel_table[Arel.star], arel_table[:partition].as('partition_number'))
+      .for_table(table)
       .status_pending
       .consume_order
       .limit(batch_size)
@@ -20,9 +50,9 @@ class LooseForeignKeys::DeletedRecord < ApplicationRecord
     # Run a query for each partition to optimize the row lookup by primary key (partition, id)
     update_count = 0
 
-    all_records.group_by(&:partition).each do |partition, records_within_partition|
+    all_records.group_by(&:partition_number).each do |partition, records_within_partition|
       update_count += status_pending
-        .where(partition: partition)
+        .for_partition(partition)
         .where(id: records_within_partition.pluck(:id))
         .update_all(status: :processed)
     end
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index db306221318..4b1cf2fa217 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -64,6 +64,9 @@ class Namespace < ApplicationRecord
   has_one :admin_note, inverse_of: :namespace
   accepts_nested_attributes_for :admin_note, update_only: true
 
+  has_one :ci_namespace_mirror, class_name: 'Ci::NamespaceMirror'
+  has_many :sync_events, class_name: 'Namespaces::SyncEvent'
+
   validates :owner, presence: true, if: ->(n) { n.owner_required? }
   validates :name,
     presence: true,
@@ -104,6 +107,8 @@ class Namespace < ApplicationRecord
   delegate :name, to: :owner, allow_nil: true, prefix: true
   delegate :avatar_url, to: :owner, allow_nil: true
 
+  after_save :schedule_sync_event_worker, if: -> { saved_change_to_id? || saved_change_to_parent_id? }
+
   after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') }
 
   before_create :sync_share_with_group_lock_with_parent
@@ -609,6 +614,13 @@ class Namespace < ApplicationRecord
   def enforce_minimum_path_length?
     path_changed? && !project_namespace?
   end
+
+  # SyncEvents are created by PG triggers (with the function `insert_namespaces_sync_event`)
+  def schedule_sync_event_worker
+    run_after_commit do
+      Namespaces::SyncEvent.enqueue_worker
+    end
+  end
 end
 
 Namespace.prepend_mod_with('Namespace')
diff --git a/app/models/namespaces/sync_event.rb b/app/models/namespaces/sync_event.rb
new file mode 100644
index 00000000000..8534d8afb8c
--- /dev/null
+++ b/app/models/namespaces/sync_event.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# This model serves to keep track of changes to the namespaces table in the main database, and allowing to safely
+# replicate these changes to other databases.
+class Namespaces::SyncEvent < ApplicationRecord
+  self.table_name = 'namespaces_sync_events'
+
+  belongs_to :namespace
+
+  scope :preload_synced_relation, -> { preload(:namespace) }
+  scope :order_by_id_asc, -> { order(id: :asc) }
+
+  def self.enqueue_worker
+    ::Namespaces::ProcessSyncEventsWorker.perform_async # rubocop:disable CodeReuse/Worker
+  end
+end
diff --git a/app/models/project.rb b/app/models/project.rb
index 088a2f9ea27..a751e8adeb0 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -102,6 +102,8 @@ class Project < ApplicationRecord
 
   after_save :update_project_statistics, if: :saved_change_to_namespace_id?
 
+  after_save :schedule_sync_event_worker, if: -> { saved_change_to_id? || saved_change_to_namespace_id? }
+
   after_save :create_import_state, if: ->(project) { project.import? && project.import_state.nil? }
 
   after_save :save_topics
@@ -394,6 +396,9 @@ class Project < ApplicationRecord
 
   has_many :timelogs
 
+  has_one :ci_project_mirror, class_name: 'Ci::ProjectMirror'
+  has_many :sync_events, class_name: 'Projects::SyncEvent'
+
   accepts_nested_attributes_for :variables, allow_destroy: true
   accepts_nested_attributes_for :project_feature, update_only: true
   accepts_nested_attributes_for :project_setting, update_only: true
@@ -2938,6 +2943,13 @@ class Project < ApplicationRecord
     project_namespace.shared_runners_enabled = shared_runners_enabled
     project_namespace.visibility_level = visibility_level
   end
+
+  # SyncEvents are created by PG triggers (with the function `insert_projects_sync_event`)
+  def schedule_sync_event_worker
+    run_after_commit do
+      Projects::SyncEvent.enqueue_worker
+    end
+  end
 end
 
 Project.prepend_mod_with('Project')
diff --git a/app/models/projects/sync_event.rb b/app/models/projects/sync_event.rb
new file mode 100644
index 00000000000..5221b00c55f
--- /dev/null
+++ b/app/models/projects/sync_event.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# This model serves to keep track of changes to the namespaces table in the main database as they relate to projects,
+# allowing to safely replicate changes to other databases.
+class Projects::SyncEvent < ApplicationRecord
+  self.table_name = 'projects_sync_events'
+
+  belongs_to :project
+
+  scope :preload_synced_relation, -> { preload(:project) }
+  scope :order_by_id_asc, -> { order(id: :asc) }
+
+  def self.enqueue_worker
+    ::Projects::ProcessSyncEventsWorker.perform_async # rubocop:disable CodeReuse/Worker
+  end
+end