1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
|
# frozen_string_literal: true
module Gitlab
module GithubImport
module BulkImporting
attr_reader :project, :client
# project - An instance of `Project`.
# client - An instance of `Gitlab::GithubImport::Client`.
def initialize(project, client)
@project = project
@client = client
end
# Builds and returns an Array of objects to bulk insert into the
# database.
#
# enum - An Enumerable that returns the objects to turn into database
# rows.
def build_database_rows(enum)
rows = enum.each_with_object([]) do |(object, _), result|
result << build(object) unless already_imported?(object)
end
log_and_increment_counter(rows.size, :fetched)
rows
end
# Bulk inserts the given rows into the database.
def bulk_insert(model, rows, batch_size: 100)
rows.each_slice(batch_size) do |slice|
Gitlab::Database.main.bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert
log_and_increment_counter(slice.size, :imported)
end
end
def object_type
raise NotImplementedError
end
private
def log_and_increment_counter(value, operation)
Gitlab::Import::Logger.info(
import_type: :github,
project_id: project.id,
importer: self.class.name,
message: "#{value} #{object_type.to_s.pluralize} #{operation}"
)
Gitlab::GithubImport::ObjectCounter.increment(
project,
object_type,
operation,
value: value
)
end
end
end
end
|
