diff options
author | Sean McGivern <sean@gitlab.com> | 2017-06-09 12:48:25 +0100 |
---|---|---|
committer | Sean McGivern <sean@gitlab.com> | 2017-06-16 18:30:01 +0100 |
commit | 9a73b634ab4220f68a8296ccb582a68293874489 (patch) | |
tree | f99d9eb55836dcdb54c18f72d4b1242a5d9acefa /lib | |
parent | 352a9ed56213b6f83a679e72f9554638a0aed1ee (diff) | |
download | gitlab-ce-9a73b634ab4220f68a8296ccb582a68293874489.tar.gz |
Add table for files in merge request diffs
This adds an ID-less table containing one row per file, per merge request
diff. It has a column for each attribute on Gitlab::Git::Diff that is serialised
currently, with the advantage that we can easily query the attributes of this
new table.
It does not migrate existing data, so we have fallback code when the legacy
st_diffs column is present instead. For a merge request diff to be valid, it
should have at most one of:
* Rows in this new table, with the correct merge_request_diff_id.
* A non-NULL st_diffs column.
It may have neither, if the diff is empty.
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/database.rb | 16 | ||||
-rw-r--r-- | lib/gitlab/git/diff.rb | 17 | ||||
-rw-r--r-- | lib/gitlab/import_export.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/import_export.yml | 7 | ||||
-rw-r--r-- | lib/gitlab/import_export/json_hash_builder.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_factory.rb | 5 | ||||
-rw-r--r-- | lib/tasks/migrate/add_limits_mysql.rake | 2 |
7 files changed, 40 insertions, 13 deletions
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index d0bd1299671..0d5a7cf0694 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -83,6 +83,22 @@ module Gitlab end end + def self.bulk_insert(table, rows) + return if rows.empty? + + keys = rows.first.keys + columns = keys.map { |key| connection.quote_column_name(key) } + + tuples = rows.map do |row| + row.values_at(*keys).map { |value| connection.quote(value) } + end + + connection.execute <<-EOF.strip_heredoc + INSERT INTO #{table} (#{columns.join(', ')}) + VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} + EOF + end + # pool_size - The size of the DB pool. # host - An optional host name to use instead of the default one. def self.create_connection_pool(pool_size, host = nil) diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 4b689f0e94f..f825568f194 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -16,11 +16,11 @@ module Gitlab alias_method :renamed_file?, :renamed_file attr_accessor :expanded + attr_writer :too_large alias_method :expanded?, :expanded - # We need this accessor because of `to_hash` and `init_from_hash` - attr_accessor :too_large + SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze class << self # The maximum size of a diff to display. @@ -231,16 +231,10 @@ module Gitlab end end - def serialize_keys - @serialize_keys ||= %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large) - end - def to_hash hash = {} - keys = serialize_keys - - keys.each do |key| + SERIALIZE_KEYS.each do |key| hash[key] = send(key) end @@ -267,6 +261,9 @@ module Gitlab end end + # This is used by `to_hash` and `init_from_hash`. + alias_method :too_large, :too_large? + def too_large! @diff = '' @line_count = 0 @@ -315,7 +312,7 @@ module Gitlab def init_from_hash(hash) raw_diff = hash.symbolize_keys - serialize_keys.each do |key| + SERIALIZE_KEYS.each do |key| send(:"#{key}=", raw_diff[key.to_sym]) end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 27d5a9198b6..3470a09eaf0 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.7'.freeze + VERSION = '0.1.8'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index ff2b1d08c3c..72183e8aad4 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -26,7 +26,8 @@ project_tree: - notes: - :author - :events - - :merge_request_diff + - merge_request_diff: + - :merge_request_diff_files - :events - :timelogs - label_links: @@ -92,6 +93,8 @@ excluded_attributes: - :expired_at merge_request_diff: - :st_diffs + merge_request_diff_files: + - :diff issues: - :milestone_id merge_requests: @@ -113,6 +116,8 @@ methods: - :type merge_request_diff: - :utf8_st_diffs + merge_request_diff_files: + - :utf8_diff merge_requests: - :diff_head_sha project: diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index 48c09dafcb6..b48f63bcd7e 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -83,7 +83,9 @@ module Gitlab # +value+ existing model to be included in the hash # +json_config_hash+ the original hash containing the root model def add_model_value(current_key, value, json_config_hash) - @attributes_finder.parse(value) { |hash| value = { value => hash } } + @attributes_finder.parse(value) do |hash| + value = { value => hash } unless value.is_a?(Hash) + end add_to_array(current_key, json_config_hash, value) end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 695852526cb..20580459046 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -71,6 +71,7 @@ module Gitlab @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] set_st_diff_commits if @relation_name == :merge_request_diff + set_diff if @relation_name == :merge_request_diff_files end def update_user_references @@ -202,6 +203,10 @@ module Gitlab HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits']) end + def set_diff + @relation_hash['diff'] = @relation_hash.delete('utf8_diff') + end + def existing_or_new_object # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake index 761f275d42a..151f42a2222 100644 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ b/lib/tasks/migrate/add_limits_mysql.rake @@ -1,9 +1,11 @@ require Rails.root.join('db/migrate/limits_to_mysql') require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') +require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql') desc "GitLab | Add limits to strings in mysql database" task add_limits_mysql: :environment do puts "Adding limits to schema.rb for mysql" LimitsToMysql.new.up MarkdownCacheLimitsToMysql.new.up + MergeRequestDiffFileLimitsToMysql.new.up end |