1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
|
# frozen_string_literal: true
# This service downloads and links lfs objects from a remote URL
module Projects
module LfsPointers
class LfsDownloadService < BaseService
SizeError = Class.new(StandardError)
OidError = Class.new(StandardError)
ResponseError = Class.new(StandardError)
LARGE_FILE_SIZE = 1.megabytes
attr_reader :lfs_download_object
delegate :oid, :size, :credentials, :sanitized_url, :headers, to: :lfs_download_object, prefix: :lfs
def initialize(project, lfs_download_object)
super(project)
@lfs_download_object = lfs_download_object
end
def execute
return unless project&.lfs_enabled? && lfs_download_object
return error("LFS file with oid #{lfs_oid} has invalid attributes") unless lfs_download_object.valid?
return link_existing_lfs_object! if lfs_size > LARGE_FILE_SIZE && lfs_object
wrap_download_errors do
download_lfs_file!
end
end
private
def wrap_download_errors(&block)
yield
rescue SizeError, OidError, ResponseError, StandardError => e
error("LFS file with oid #{lfs_oid} could't be downloaded from #{lfs_sanitized_url}: #{e.message}")
end
def download_lfs_file!
with_tmp_file do |tmp_file|
download_and_save_file!(tmp_file)
project.lfs_objects << find_or_create_lfs_object(tmp_file)
success
end
end
def find_or_create_lfs_object(tmp_file)
lfs_obj = LfsObject.safe_find_or_create_by!(
oid: lfs_oid,
size: lfs_size
)
lfs_obj.update!(file: tmp_file) unless lfs_obj.file.file
lfs_obj
end
def download_and_save_file!(file)
digester = Digest::SHA256.new
fetch_file do |fragment|
digester << fragment
file.write(fragment)
raise_size_error! if file.size > lfs_size
end
raise_size_error! if file.size != lfs_size
raise_oid_error! if digester.hexdigest != lfs_oid
end
def download_options
http_options = { headers: lfs_headers, stream_body: true }
return http_options if lfs_download_object.has_authorization_header?
http_options.tap do |options|
if lfs_credentials[:user].present? || lfs_credentials[:password].present?
# Using authentication headers in the request
options[:basic_auth] = { username: lfs_credentials[:user], password: lfs_credentials[:password] }
end
end
end
def fetch_file(&block)
response = Gitlab::HTTP.get(lfs_sanitized_url, download_options, &block)
raise ResponseError, "Received error code #{response.code}" unless response.success?
end
def with_tmp_file
create_tmp_storage_dir
File.open(tmp_filename, 'wb') do |file|
yield file
rescue StandardError => e
# If the lfs file is successfully downloaded it will be removed
# when it is added to the project's lfs files.
# Nevertheless if any excetion raises the file would remain
# in the file system. Here we ensure to remove it
File.unlink(file) if File.exist?(file)
raise e
end
end
def tmp_filename
File.join(tmp_storage_dir, lfs_oid)
end
def create_tmp_storage_dir
FileUtils.makedirs(tmp_storage_dir) unless Dir.exist?(tmp_storage_dir)
end
def tmp_storage_dir
@tmp_storage_dir ||= File.join(storage_dir, 'tmp', 'download')
end
def storage_dir
@storage_dir ||= Gitlab.config.lfs.storage_path
end
def raise_size_error!
raise SizeError, 'Size mistmatch'
end
def raise_oid_error!
raise OidError, 'Oid mismatch'
end
def error(message, http_status = nil)
log_error(message)
super
end
def lfs_object
@lfs_object ||= LfsObject.find_by_oid(lfs_oid)
end
def link_existing_lfs_object!
existing_file = lfs_object.file.open
buffer_size = 0
result = fetch_file do |fragment|
unless fragment == existing_file.read(fragment.size)
break error("LFS file with oid #{lfs_oid} cannot be linked with an existing LFS object")
end
buffer_size += fragment.size
break success if buffer_size > LARGE_FILE_SIZE
end
project.lfs_objects << lfs_object
result
ensure
existing_file&.close
end
end
end
end
|
