1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
# frozen_string_literal: true
class ProjectStatistics < ApplicationRecord
include AfterCommitQueue
include CounterAttribute
belongs_to :project
belongs_to :namespace
attribute :wiki_size, default: 0
attribute :snippets_size, default: 0
counter_attribute :build_artifacts_size
counter_attribute :packages_size
counter_attribute_after_commit do |project_statistics|
project_statistics.refresh_storage_size!
Namespaces::ScheduleAggregationWorker.perform_async(project_statistics.namespace_id)
end
before_save :update_storage_size
COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count, :snippets_size, :uploads_size, :container_registry_size].freeze
INCREMENTABLE_COLUMNS = [
:pipeline_artifacts_size,
:snippets_size
].freeze
NAMESPACE_RELATABLE_COLUMNS = [:repository_size, :wiki_size, :lfs_objects_size, :uploads_size, :container_registry_size].freeze
STORAGE_SIZE_COMPONENTS = [
:repository_size,
:wiki_size,
:lfs_objects_size,
:build_artifacts_size,
:packages_size,
:snippets_size,
:pipeline_artifacts_size,
:uploads_size
].freeze
scope :for_project_ids, ->(project_ids) { where(project_id: project_ids) }
scope :for_namespaces, -> (namespaces) { where(namespace: namespaces) }
def total_repository_size
repository_size + lfs_objects_size
end
def refresh!(only: [])
return if Gitlab::Database.read_only?
columns_to_update = only.empty? ? COLUMNS_TO_REFRESH : COLUMNS_TO_REFRESH & only
columns_to_update.each do |column|
public_send("update_#{column}") # rubocop:disable GitlabSecurity/PublicSend
end
if only.empty? || only.any? { |column| NAMESPACE_RELATABLE_COLUMNS.include?(column) }
schedule_namespace_aggregation_worker
end
detect_race_on_record(log_fields: { caller: __method__, attributes: columns_to_update }) do
save!
end
end
def update_commit_count
self.commit_count = project.repository.commit_count
end
def update_repository_size
self.repository_size = project.repository.size * 1.megabyte
end
def update_wiki_size
self.wiki_size = project.wiki.repository.size * 1.megabyte
end
def update_snippets_size
self.snippets_size = project.snippets.with_statistics.sum(:repository_size)
end
def update_lfs_objects_size
self.lfs_objects_size = LfsObject.joins(:lfs_objects_projects).where(lfs_objects_projects: { project_id: project.id }).sum(:size)
end
def update_uploads_size
self.uploads_size = project.uploads.sum(:size)
end
def update_container_registry_size
self.container_registry_size = project.container_repositories_size || 0
end
# `wiki_size` and `snippets_size` have no default value in the database
# and the column can be nil.
# This means that, when the columns were added, all rows had nil
# values on them.
# Therefore, any call to any of those methods will return nil instead of 0.
#
# These two methods provide consistency and avoid returning nil.
def wiki_size
super.to_i
end
def snippets_size
super.to_i
end
def update_storage_size
self.storage_size = storage_size_components.sum { |component| method(component).call }
end
def refresh_storage_size!
detect_race_on_record(log_fields: { caller: __method__, attributes: :storage_size }) do
update!(storage_size: storage_size_sum)
end
end
# Since this incremental update method does not call update_storage_size above through before_save,
# we have to update the storage_size separately.
#
# For counter attributes, storage_size will be refreshed after the counter is flushed,
# through counter_attribute_after_commit
#
# For non-counter attributes, storage_size is updated depending on key => [columns] in INCREMENTABLE_COLUMNS
def self.increment_statistic(project, key, amount)
project.statistics.try do |project_statistics|
project_statistics.increment_statistic(key, amount)
end
end
def increment_statistic(key, amount)
raise ArgumentError, "Cannot increment attribute: #{key}" unless incrementable_attribute?(key)
increment_counter(key, amount)
end
private
def incrementable_attribute?(key)
INCREMENTABLE_COLUMNS.include?(key) || counter_attribute_enabled?(key)
end
def storage_size_components
STORAGE_SIZE_COMPONENTS
end
def storage_size_sum
storage_size_components.map { |component| "COALESCE (#{component}, 0)" }.join(' + ').freeze
end
def schedule_namespace_aggregation_worker
run_after_commit do
Namespaces::ScheduleAggregationWorker.perform_async(project.namespace_id)
end
end
end
ProjectStatistics.prepend_mod_with('ProjectStatistics')
|
