1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
|
class RemoteMirror < ActiveRecord::Base
include AfterCommitQueue
PROTECTED_BACKOFF_DELAY = 1.minute
UNPROTECTED_BACKOFF_DELAY = 5.minutes
attr_encrypted :credentials,
key: Gitlab::Application.secrets.db_key_base,
marshal: true,
encode: true,
mode: :per_attribute_iv_and_salt,
insecure_mode: true,
algorithm: 'aes-256-cbc'
default_value_for :only_protected_branches, true
belongs_to :project, inverse_of: :remote_mirrors
validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true }
validates :url, addressable_url: true, if: :url_changed?
before_save :set_new_remote_name, if: :mirror_url_changed?
after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available }
after_save :refresh_remote, if: :mirror_url_changed?
after_update :reset_fields, if: :mirror_url_changed?
after_commit :remove_remote, on: :destroy
scope :enabled, -> { where(enabled: true) }
scope :started, -> { with_update_status(:started) }
scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) }
state_machine :update_status, initial: :none do
event :update_start do
transition [:none, :finished, :failed] => :started
end
event :update_finish do
transition started: :finished
end
event :update_fail do
transition started: :failed
end
state :started
state :finished
state :failed
after_transition any => :started do |remote_mirror, _|
Gitlab::Metrics.add_event(:remote_mirrors_running, path: remote_mirror.project.full_path)
remote_mirror.update(last_update_started_at: Time.now)
end
after_transition started: :finished do |remote_mirror, _|
Gitlab::Metrics.add_event(:remote_mirrors_finished, path: remote_mirror.project.full_path)
timestamp = Time.now
remote_mirror.update_attributes!(
last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil
)
end
after_transition started: :failed do |remote_mirror, _|
Gitlab::Metrics.add_event(:remote_mirrors_failed, path: remote_mirror.project.full_path)
remote_mirror.update(last_update_at: Time.now)
end
end
def remote_name
super || fallback_remote_name
end
def update_failed?
update_status == 'failed'
end
def update_in_progress?
update_status == 'started'
end
def update_repository(options)
raw.update(options)
end
def sync
return unless enabled?
return if Gitlab::Geo.secondary?
if recently_scheduled?
RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now)
else
RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now)
end
end
def enabled
return false unless project && super
return false unless project.remote_mirror_available?
return false unless project.repository_exists?
return false if project.pending_delete?
true
end
alias_method :enabled?, :enabled
def updated_since?(timestamp)
last_update_started_at && last_update_started_at > timestamp && !update_failed?
end
def mark_for_delete_if_blank_url
mark_for_destruction if url.blank?
end
def mark_as_failed(error_message)
update_fail
update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message))
end
def url=(value)
super(value) && return unless Gitlab::UrlSanitizer.valid?(value)
mirror_url = Gitlab::UrlSanitizer.new(value)
self.credentials = mirror_url.credentials
super(mirror_url.sanitized_url)
end
def url
if super
Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url
end
rescue
super
end
def safe_url
return if url.nil?
result = URI.parse(url)
result.password = '*****' if result.password
result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user
result.to_s
end
private
def raw
@raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, remote_name)
end
def fallback_remote_name
return unless id
"remote_mirror_#{id}"
end
def recently_scheduled?
return false unless self.last_update_started_at
self.last_update_started_at >= Time.now - backoff_delay
end
def backoff_delay
if self.only_protected_branches
PROTECTED_BACKOFF_DELAY
else
UNPROTECTED_BACKOFF_DELAY
end
end
def reset_fields
update_columns(
last_error: nil,
last_update_at: nil,
last_successful_update_at: nil,
update_status: 'finished'
)
end
def set_override_remote_mirror_available
enabled = read_attribute(:enabled)
project.update(remote_mirror_available_overridden: enabled)
end
def set_new_remote_name
self.remote_name = "remote_mirror_#{SecureRandom.hex}"
end
def refresh_remote
return unless project
# Before adding a new remote we have to delete the data from
# the previous remote name
prev_remote_name = remote_name_was || fallback_remote_name
run_after_commit do
project.repository.async_remove_remote(prev_remote_name)
end
project.repository.add_remote(remote_name, url)
end
def remove_remote
return unless project # could be pending to delete so don't need to touch the git repository
project.repository.async_remove_remote(remote_name)
end
def mirror_url_changed?
url_changed? || encrypted_credentials_changed?
end
end
|
