summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-03-17 18:09:44 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2020-03-17 18:09:44 +0000
commit2c156e3c7bbade01c36eee18327f1ced6eebea79 (patch)
tree115fa8dbf6bc05037378b380311d31acb805f54c /lib
parent8e129497b2565b8c595ef4f806d9a9595ca654e5 (diff)
downloadgitlab-ce-2c156e3c7bbade01c36eee18327f1ced6eebea79.tar.gz
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r--lib/api/entities/discussion.rb2
-rw-r--r--lib/api/entities/note_with_gitlab_employee_badge.rb10
-rw-r--r--lib/api/entities/user_with_gitlab_employee_badge.rb9
-rw-r--r--lib/gitlab/ci/config/entry/reports.rb3
-rw-r--r--lib/gitlab/ci/parsers.rb3
-rw-r--r--lib/gitlab/ci/parsers/coverage/cobertura.rb64
-rw-r--r--lib/gitlab/ci/reports/coverage_reports.rb43
-rw-r--r--lib/gitlab/import_export/group/tree_saver.rb2
-rw-r--r--lib/gitlab/import_export/json/legacy_writer.rb73
-rw-r--r--lib/gitlab/import_export/json/streaming_serializer.rb82
-rw-r--r--lib/gitlab/import_export/legacy_relation_tree_saver.rb (renamed from lib/gitlab/import_export/relation_tree_saver.rb)2
-rw-r--r--lib/gitlab/import_export/project/legacy_tree_saver.rb68
-rw-r--r--lib/gitlab/import_export/project/tree_saver.rb46
-rw-r--r--lib/tasks/gitlab/import_export/export.rake1
-rw-r--r--lib/tasks/gitlab/import_export/import.rake1
-rw-r--r--lib/tasks/sidekiq.rake2
16 files changed, 374 insertions, 37 deletions
diff --git a/lib/api/entities/discussion.rb b/lib/api/entities/discussion.rb
index dd1dd40da23..0740de97897 100644
--- a/lib/api/entities/discussion.rb
+++ b/lib/api/entities/discussion.rb
@@ -5,7 +5,7 @@ module API
class Discussion < Grape::Entity
expose :id
expose :individual_note?, as: :individual_note
- expose :notes, using: Entities::Note
+ expose :notes, using: Entities::NoteWithGitlabEmployeeBadge
end
end
end
diff --git a/lib/api/entities/note_with_gitlab_employee_badge.rb b/lib/api/entities/note_with_gitlab_employee_badge.rb
new file mode 100644
index 00000000000..2ea300ffeb6
--- /dev/null
+++ b/lib/api/entities/note_with_gitlab_employee_badge.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class NoteWithGitlabEmployeeBadge < Note
+ expose :author, using: Entities::UserWithGitlabEmployeeBadge
+ expose :resolved_by, using: Entities::UserWithGitlabEmployeeBadge, if: ->(note, options) { note.resolvable? }
+ end
+ end
+end
diff --git a/lib/api/entities/user_with_gitlab_employee_badge.rb b/lib/api/entities/user_with_gitlab_employee_badge.rb
new file mode 100644
index 00000000000..36b9f633132
--- /dev/null
+++ b/lib/api/entities/user_with_gitlab_employee_badge.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class UserWithGitlabEmployeeBadge < UserBasic
+ expose :gitlab_employee?, as: :is_gitlab_employee, if: ->(user, options) { ::Feature.enabled?(:gitlab_employee_badge) && user.gitlab_employee? }
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb
index 994d3799004..40d37f3601a 100644
--- a/lib/gitlab/ci/config/entry/reports.rb
+++ b/lib/gitlab/ci/config/entry/reports.rb
@@ -14,7 +14,7 @@ module Gitlab
ALLOWED_KEYS =
%i[junit codequality sast dependency_scanning container_scanning
dast performance license_management license_scanning metrics lsif
- dotenv].freeze
+ dotenv cobertura].freeze
attributes ALLOWED_KEYS
@@ -35,6 +35,7 @@ module Gitlab
validates :metrics, array_of_strings_or_string: true
validates :lsif, array_of_strings_or_string: true
validates :dotenv, array_of_strings_or_string: true
+ validates :cobertura, array_of_strings_or_string: true
end
end
diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb
index c76cd5ff285..a44105d53c2 100644
--- a/lib/gitlab/ci/parsers.rb
+++ b/lib/gitlab/ci/parsers.rb
@@ -9,7 +9,8 @@ module Gitlab
def self.parsers
{
- junit: ::Gitlab::Ci::Parsers::Test::Junit
+ junit: ::Gitlab::Ci::Parsers::Test::Junit,
+ cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura
}
end
diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb
new file mode 100644
index 00000000000..006d5097148
--- /dev/null
+++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Parsers
+ module Coverage
+ class Cobertura
+ CoberturaParserError = Class.new(Gitlab::Ci::Parsers::ParserError)
+
+ def parse!(xml_data, coverage_report)
+ root = Hash.from_xml(xml_data)
+
+ parse_all(root, coverage_report)
+ rescue Nokogiri::XML::SyntaxError
+ raise CoberturaParserError, "XML parsing failed"
+ rescue
+ raise CoberturaParserError, "Cobertura parsing failed"
+ end
+
+ private
+
+ def parse_all(root, coverage_report)
+ return unless root.present?
+
+ root.each do |key, value|
+ parse_node(key, value, coverage_report)
+ end
+ end
+
+ def parse_node(key, value, coverage_report)
+ if key == 'class'
+ Array.wrap(value).each do |item|
+ parse_class(item, coverage_report)
+ end
+ elsif value.is_a?(Hash)
+ parse_all(value, coverage_report)
+ elsif value.is_a?(Array)
+ value.each do |item|
+ parse_all(item, coverage_report)
+ end
+ end
+ end
+
+ def parse_class(file, coverage_report)
+ return unless file["filename"].present? && file["lines"].present?
+
+ parsed_lines = parse_lines(file["lines"])
+
+ coverage_report.add_file(file["filename"], Hash[parsed_lines])
+ end
+
+ def parse_lines(lines)
+ line_array = Array.wrap(lines["line"])
+
+ line_array.map do |line|
+ # Using `Integer()` here to raise exception on invalid values
+ [Integer(line["number"]), Integer(line["hits"])]
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/coverage_reports.rb b/lib/gitlab/ci/reports/coverage_reports.rb
new file mode 100644
index 00000000000..31afb636d2f
--- /dev/null
+++ b/lib/gitlab/ci/reports/coverage_reports.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ class CoverageReports
+ attr_reader :files
+
+ def initialize
+ @files = {}
+ end
+
+ def pick(keys)
+ coverage_files = files.select do |key|
+ keys.include?(key)
+ end
+
+ { files: coverage_files }
+ end
+
+ def add_file(name, line_coverage)
+ if files[name].present?
+ line_coverage.each { |line, hits| combine_lines(name, line, hits) }
+
+ else
+ files[name] = line_coverage
+ end
+ end
+
+ private
+
+ def combine_lines(name, line, hits)
+ if files[name][line].present?
+ files[name][line] += hits
+
+ else
+ files[name][line] = hits
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb
index 48f6925884b..fd1eb329ad2 100644
--- a/lib/gitlab/import_export/group/tree_saver.rb
+++ b/lib/gitlab/import_export/group/tree_saver.rb
@@ -49,7 +49,7 @@ module Gitlab
end
def tree_saver
- @tree_saver ||= RelationTreeSaver.new
+ @tree_saver ||= LegacyRelationTreeSaver.new
end
end
end
diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb
new file mode 100644
index 00000000000..c935e360a65
--- /dev/null
+++ b/lib/gitlab/import_export/json/legacy_writer.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module JSON
+ class LegacyWriter
+ include Gitlab::ImportExport::CommandLineUtil
+
+ attr_reader :path
+
+ def initialize(path)
+ @path = path
+ @last_array = nil
+ @keys = Set.new
+
+ mkdir_p(File.dirname(@path))
+ file.write('{}')
+ end
+
+ def close
+ @file&.close
+ @file = nil
+ end
+
+ def set(hash)
+ hash.each do |key, value|
+ write(key, value)
+ end
+ end
+
+ def write(key, value)
+ raise ArgumentError, "key '#{key}' already written" if @keys.include?(key)
+
+ # rewind by one byte, to overwrite '}'
+ file.pos = file.size - 1
+
+ file.write(',') if @keys.any?
+ file.write(key.to_json)
+ file.write(':')
+ file.write(value.to_json)
+ file.write('}')
+
+ @keys.add(key)
+ @last_array = nil
+ @last_array_count = nil
+ end
+
+ def append(key, value)
+ unless @last_array == key
+ write(key, [])
+
+ @last_array = key
+ @last_array_count = 0
+ end
+
+ # rewind by two bytes, to overwrite ']}'
+ file.pos = file.size - 2
+
+ file.write(',') if @last_array_count > 0
+ file.write(value.to_json)
+ file.write(']}')
+ @last_array_count += 1
+ end
+
+ private
+
+ def file
+ @file ||= File.open(@path, "wb")
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb
new file mode 100644
index 00000000000..d053bf16166
--- /dev/null
+++ b/lib/gitlab/import_export/json/streaming_serializer.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module JSON
+ class StreamingSerializer
+ include Gitlab::ImportExport::CommandLineUtil
+
+ BATCH_SIZE = 100
+
+ class Raw < String
+ def to_json(*_args)
+ to_s
+ end
+ end
+
+ def initialize(exportable, relations_schema, json_writer)
+ @exportable = exportable
+ @relations_schema = relations_schema
+ @json_writer = json_writer
+ end
+
+ def execute
+ serialize_root
+
+ includes.each do |relation_definition|
+ serialize_relation(relation_definition)
+ end
+ end
+
+ private
+
+ attr_reader :json_writer, :relations_schema, :exportable
+
+ def serialize_root
+ attributes = exportable.as_json(
+ relations_schema.merge(include: nil, preloads: nil))
+ json_writer.set(attributes)
+ end
+
+ def serialize_relation(definition)
+ raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash)
+ raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one?
+
+ key, options = definition.first
+
+ record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend
+ if record.is_a?(ActiveRecord::Relation)
+ serialize_many_relations(key, record, options)
+ else
+ serialize_single_relation(key, record, options)
+ end
+ end
+
+ def serialize_many_relations(key, records, options)
+ key_preloads = preloads&.dig(key)
+ records = records.preload(key_preloads) if key_preloads
+
+ records.find_each(batch_size: BATCH_SIZE) do |record|
+ json = Raw.new(record.to_json(options))
+
+ json_writer.append(key, json)
+ end
+ end
+
+ def serialize_single_relation(key, record, options)
+ json = Raw.new(record.to_json(options))
+
+ json_writer.write(key, json)
+ end
+
+ def includes
+ relations_schema[:include]
+ end
+
+ def preloads
+ relations_schema[:preload]
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb
index ed5392c13d0..fe3e64358e5 100644
--- a/lib/gitlab/import_export/relation_tree_saver.rb
+++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb
@@ -2,7 +2,7 @@
module Gitlab
module ImportExport
- class RelationTreeSaver
+ class LegacyRelationTreeSaver
include Gitlab::ImportExport::CommandLineUtil
def serialize(exportable, relations_tree)
diff --git a/lib/gitlab/import_export/project/legacy_tree_saver.rb b/lib/gitlab/import_export/project/legacy_tree_saver.rb
new file mode 100644
index 00000000000..2ed98f52c58
--- /dev/null
+++ b/lib/gitlab/import_export/project/legacy_tree_saver.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Project
+ class LegacyTreeSaver
+ attr_reader :full_path
+
+ def initialize(project:, current_user:, shared:, params: {})
+ @params = params
+ @project = project
+ @current_user = current_user
+ @shared = shared
+ @full_path = File.join(@shared.export_path, ImportExport.project_filename)
+ end
+
+ def save
+ project_tree = tree_saver.serialize(@project, reader.project_tree)
+ fix_project_tree(project_tree)
+ tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
+
+ true
+ rescue => e
+ @shared.error(e)
+ false
+ end
+
+ private
+
+ # Aware that the resulting hash needs to be pure-hash and
+ # does not include any AR objects anymore, only objects that run `.to_json`
+ def fix_project_tree(project_tree)
+ if @params[:description].present?
+ project_tree['description'] = @params[:description]
+ end
+
+ project_tree['project_members'] += group_members_array
+ end
+
+ def reader
+ @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
+ end
+
+ def group_members_array
+ group_members.as_json(reader.group_members_tree).each do |group_member|
+ group_member['source_type'] = 'Project' # Make group members project members of the future import
+ end
+ end
+
+ def group_members
+ return [] unless @current_user.can?(:admin_group, @project.group)
+
+ # We need `.where.not(user_id: nil)` here otherwise when a group has an
+ # invitee, it would make the following query return 0 rows since a NULL
+ # user_id would be present in the subquery
+ # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
+ non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
+
+ GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
+ end
+
+ def tree_saver
+ @tree_saver ||= Gitlab::ImportExport::LegacyRelationTreeSaver.new
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb
index 32b3b518ece..01000c9d6d9 100644
--- a/lib/gitlab/import_export/project/tree_saver.rb
+++ b/lib/gitlab/import_export/project/tree_saver.rb
@@ -15,52 +15,40 @@ module Gitlab
end
def save
- project_tree = tree_saver.serialize(@project, reader.project_tree)
- fix_project_tree(project_tree)
- tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
+ json_writer = ImportExport::JSON::LegacyWriter.new(@full_path)
+
+ serializer = ImportExport::JSON::StreamingSerializer.new(exportable, reader.project_tree, json_writer)
+ serializer.execute
true
rescue => e
@shared.error(e)
false
+ ensure
+ json_writer&.close
end
private
- # Aware that the resulting hash needs to be pure-hash and
- # does not include any AR objects anymore, only objects that run `.to_json`
- def fix_project_tree(project_tree)
- if @params[:description].present?
- project_tree['description'] = @params[:description]
- end
-
- project_tree['project_members'] += group_members_array
- end
-
def reader
@reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
end
- def group_members_array
- group_members.as_json(reader.group_members_tree).each do |group_member|
- group_member['source_type'] = 'Project' # Make group members project members of the future import
- end
+ def exportable
+ @project.present(exportable_params)
end
- def group_members
- return [] unless @current_user.can?(:admin_group, @project.group)
-
- # We need `.where.not(user_id: nil)` here otherwise when a group has an
- # invitee, it would make the following query return 0 rows since a NULL
- # user_id would be present in the subquery
- # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
- non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
-
- GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
+ def exportable_params
+ params = {
+ presenter_class: presenter_class,
+ current_user: @current_user
+ }
+ params[:override_description] = @params[:description] if @params[:description].present?
+ params
end
- def tree_saver
- @tree_saver ||= RelationTreeSaver.new
+ def presenter_class
+ Projects::ImportExport::ProjectExportPresenter
end
end
end
diff --git a/lib/tasks/gitlab/import_export/export.rake b/lib/tasks/gitlab/import_export/export.rake
index ae54636fdd3..c9c212fbe4d 100644
--- a/lib/tasks/gitlab/import_export/export.rake
+++ b/lib/tasks/gitlab/import_export/export.rake
@@ -19,7 +19,6 @@ namespace :gitlab do
if ENV['EXPORT_DEBUG'].present?
ActiveRecord::Base.logger = logger
- Gitlab::Metrics::Exporter::SidekiqExporter.instance.start
logger.level = Logger::DEBUG
else
logger.level = Logger::INFO
diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake
index 6e2d0e75da8..7e2162a7774 100644
--- a/lib/tasks/gitlab/import_export/import.rake
+++ b/lib/tasks/gitlab/import_export/import.rake
@@ -23,7 +23,6 @@ namespace :gitlab do
if ENV['IMPORT_DEBUG'].present?
ActiveRecord::Base.logger = logger
- Gitlab::Metrics::Exporter::SidekiqExporter.instance.start
logger.level = Logger::DEBUG
else
logger.level = Logger::INFO
diff --git a/lib/tasks/sidekiq.rake b/lib/tasks/sidekiq.rake
index e281ebd5d60..d74878835fd 100644
--- a/lib/tasks/sidekiq.rake
+++ b/lib/tasks/sidekiq.rake
@@ -33,6 +33,6 @@ namespace :sidekiq do
task :launchd do
deprecation_warning!
- system(*%w(bin/background_jobs start_no_deamonize))
+ system(*%w(bin/background_jobs start_silent))
end
end