diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/api/groups.rb | 4 | ||||
| -rw-r--r-- | lib/api/merge_requests.rb | 4 | ||||
| -rw-r--r-- | lib/api/projects.rb | 17 | ||||
| -rw-r--r-- | lib/banzai/filter/emoji_filter.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/diff/file.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/import_export.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/import_export/group_project_object_builder.rb | 90 | ||||
| -rw-r--r-- | lib/gitlab/import_export/project_tree_restorer.rb | 29 | ||||
| -rw-r--r-- | lib/gitlab/import_export/relation_factory.rb | 74 | ||||
| -rw-r--r-- | lib/gitlab/middleware/read_only/controller.rb | 33 | ||||
| -rw-r--r-- | lib/gitlab/repository_cache_adapter.rb | 10 | ||||
| -rw-r--r-- | lib/tasks/gitlab/import_export.rake | 21 |
12 files changed, 195 insertions, 95 deletions
diff --git a/lib/api/groups.rb b/lib/api/groups.rb index c7f41aba854..f633dd88d06 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -56,6 +56,8 @@ module API def find_group_projects(params) group = find_group!(params[:id]) projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute + projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled] + projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] projects = reorder_projects(projects) paginate(projects) end @@ -191,6 +193,8 @@ module API desc: 'Return only the ID, URL, name, and path of each project' optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' + optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' + optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' use :pagination use :with_custom_attributes diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index af7d2471b34..0f46bc4c98e 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -72,8 +72,8 @@ module API end params :merge_requests_params do - optional :state, type: String, values: %w[opened closed merged all], default: 'all', - desc: 'Return opened, closed, merged, or all merge requests' + optional :state, type: String, values: %w[opened closed locked merged all], default: 'all', + desc: 'Return opened, closed, locked, merged, or all merge requests' optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' optional :sort, type: String, values: %w[asc desc], default: 'desc', diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 3ef3680c5d9..b83da00502d 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -459,6 +459,23 @@ module API conflict!(error.message) end end + + desc 'Transfer a project to a new namespace' + params do + requires :namespace, type: String, desc: 'The ID or path of the new namespace' + end + put ":id/transfer" do + authorize! :change_namespace, user_project + + namespace = find_namespace!(params[:namespace]) + result = ::Projects::TransferService.new(user_project, current_user).execute(namespace) + + if result + present user_project, with: Entities::Project + else + render_api_error!("Failed to transfer project #{user_project.errors.messages}", 400) + end + end end end end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index e1261e7bbbe..4eccd9d5ed5 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -3,10 +3,6 @@ module Banzai # HTML filter that replaces :emoji: and unicode with images. # # Based on HTML::Pipeline::EmojiFilter - # - # Context options: - # :asset_root - # :asset_host class EmojiFilter < HTML::Pipeline::Filter IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 40bcfa20e7d..a9e209d5b71 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -250,7 +250,7 @@ module Gitlab last_line = lines.last - if last_line.new_pos < total_blob_lines(blob) + if last_line.new_pos < total_blob_lines(blob) && !deleted_file? match_line = Gitlab::Diff::Line.new("", 'match', nil, last_line.old_pos, last_line.new_pos) lines.push(match_line) end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index b713fa7e1cd..53fe2f8e436 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.3'.freeze + VERSION = '0.2.4'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb new file mode 100644 index 00000000000..6c2af770119 --- /dev/null +++ b/lib/gitlab/import_export/group_project_object_builder.rb @@ -0,0 +1,90 @@ +module Gitlab + module ImportExport + # Given a class, it finds or creates a new object + # (initializes in the case of Label) at group or project level. + # If it does not exist in the group, it creates it at project level. + # + # Example: + # `GroupProjectObjectBuilder.build(Label, label_attributes)` + # finds or initializes a label with the given attributes. + # + # It also adds some logic around Group Labels/Milestones for edge cases. + class GroupProjectObjectBuilder + def self.build(*args) + Project.transaction do + new(*args).find + end + end + + def initialize(klass, attributes) + @klass = klass < Label ? Label : klass + @attributes = attributes + @group = @attributes['group'] + @project = @attributes['project'] + end + + def find + find_object || @klass.create(project_attributes) + end + + private + + def find_object + @klass.where(where_clause).first + end + + def where_clause + @attributes.slice('title').map do |key, value| + scope_clause = table[:project_id].eq(@project.id) + scope_clause = scope_clause.or(table[:group_id].eq(@group.id)) if @group + + table[key].eq(value).and(scope_clause) + end.reduce(:or) + end + + def table + @table ||= @klass.arel_table + end + + def project_attributes + @attributes.except('group').tap do |atts| + if label? + atts['type'] = 'ProjectLabel' # Always create project labels + elsif milestone? + if atts['group_id'] # Transform new group milestones into project ones + atts['iid'] = nil + atts.delete('group_id') + else + claim_iid + end + end + end + end + + def label? + @klass == Label + end + + def milestone? + @klass == Milestone + end + + # If an existing group milestone used the IID + # claim the IID back and set the group milestone to use one available + # This is necessary to fix situations like the following: + # - Importing into a user namespace project with exported group milestones + # where the IID of the Group milestone could conflict with a project one. + def claim_iid + # The milestone has to be a group milestone, as it's the only case where + # we set the IID as the maximum. The rest of them are fixed. + milestone = @project.milestones.find_by(iid: @attributes['iid']) + + return unless milestone + + milestone.iid = nil + milestone.ensure_project_iid! + milestone.save! + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 4eb67fbe11e..76b99b1de16 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -1,8 +1,8 @@ module Gitlab module ImportExport class ProjectTreeRestorer - # Relations which cannot have both group_id and project_id at the same time - RESTRICT_PROJECT_AND_GROUP = %i(milestone milestones).freeze + # Relations which cannot be saved at project level (and have a group assigned) + GROUP_MODELS = [GroupLabel, Milestone].freeze def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @@ -70,12 +70,23 @@ module Gitlab def save_relation_hash(relation_hash_batch, relation_key) relation_hash = create_relation(relation_key, relation_hash_batch) + remove_group_models(relation_hash) if relation_hash.is_a?(Array) + @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash) # Restore the project again, extra query that skips holding the AR objects in memory @restored_project = Project.find(@project_id) end + # Remove project models that became group models as we found them at group level. + # This no longer required saving them at the root project level. + # For example, in the case of an existing group label that matched the title. + def remove_group_models(relation_hash) + relation_hash.reject! do |value| + GROUP_MODELS.include?(value.class) && value.group_id + end + end + def default_relation_list reader.tree.reject do |model| model.is_a?(Hash) && model[:project_members] @@ -170,7 +181,7 @@ module Gitlab def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: parsed_relation_hash(relation_hash, relation.to_sym), + relation_hash: relation_hash, members_mapper: members_mapper, user: @user, project: @restored_project, @@ -180,18 +191,6 @@ module Gitlab relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end - def parsed_relation_hash(relation_hash, relation_type) - if RESTRICT_PROJECT_AND_GROUP.include?(relation_type) - params = {} - params['group_id'] = restored_project.group.try(:id) if relation_hash['group_id'] - params['project_id'] = restored_project.id if relation_hash['project_id'] - else - params = { 'group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id } - end - - relation_hash.merge(params) - end - def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index c5cf290f191..091e81028bb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -54,6 +54,8 @@ module Gitlab @project = project @imported_object_retries = 0 + @relation_hash['project_id'] = @project.id + # Remove excluded keys from relation_hash # We don't do this in the parsed_relation_hash because of the 'transformed attributes' # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then, @@ -80,15 +82,12 @@ module Gitlab case @relation_name when :merge_request_diff_files then setup_diff when :notes then setup_note - when :project_label, :project_labels then setup_label - when :milestone, :milestones then setup_milestone when 'Ci::Pipeline' then setup_pipeline - else - @relation_hash['project_id'] = @project.id end update_user_references update_project_references + update_group_references remove_duplicate_assignees reset_tokens! @@ -151,39 +150,23 @@ module Gitlab end def update_project_references - project_id = @relation_hash.delete('project_id') - # If source and target are the same, populate them with the new project ID. if @relation_hash['source_project_id'] - @relation_hash['source_project_id'] = same_source_and_target? ? project_id : MergeRequestParser::FORKED_PROJECT_ID + @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID end - # project_id may not be part of the export, but we always need to populate it if required. - @relation_hash['project_id'] = project_id - @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] + @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id'] end def same_source_and_target? @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] end - def setup_label - # If there's no group, move the label to a project label - if @relation_hash['type'] == 'GroupLabel' && @relation_hash['group_id'] - @relation_hash['project_id'] = nil - @relation_name = :group_label - else - @relation_hash['group_id'] = nil - @relation_hash['type'] = 'ProjectLabel' - end - end + def update_group_references + return unless EXISTING_OBJECT_CHECK.include?(@relation_name) + return unless @relation_hash['group_id'] - def setup_milestone - if @relation_hash['group_id'] - @relation_hash['group_id'] = @project.group.id - else - @relation_hash['project_id'] = @project.id - end + @relation_hash['group_id'] = @project.group&.id end def reset_tokens! @@ -271,15 +254,7 @@ module Gitlab end def existing_object - @existing_object ||= - begin - existing_object = find_or_create_object! - - # Done in two steps, as MySQL behaves differently than PostgreSQL using - # the +find_or_create_by+ method and does not return the ID the second time. - existing_object.update!(parsed_relation_hash) - existing_object - end + @existing_object ||= find_or_create_object! end def unknown_service? @@ -288,29 +263,16 @@ module Gitlab end def find_or_create_object! - finder_attributes = if @relation_name == :group_label - %w[title group_id] - elsif parsed_relation_hash['project_id'] - %w[title project_id] - else - %w[title group_id] - end - - finder_hash = parsed_relation_hash.slice(*finder_attributes) - - if label? - label = relation_class.find_or_initialize_by(finder_hash) - parsed_relation_hash.delete('priorities') if label.persisted? - - label.save! - label - else - relation_class.find_or_create_by(finder_hash) + return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature + + # Can't use IDs as validation exists calling `group` or `project` attributes + finder_hash = parsed_relation_hash.tap do |hash| + hash['group'] = @project.group if relation_class.attribute_method?('group_id') + hash['project'] = @project + hash.delete('project_id') end - end - def label? - @relation_name.to_s.include?('label') + GroupProjectObjectBuilder.build(relation_class, finder_hash) end end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 45b644e6510..4a99b7cca5c 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -4,8 +4,18 @@ module Gitlab class Controller DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze APPLICATION_JSON = 'application/json'.freeze + APPLICATION_JSON_TYPES = %W{#{APPLICATION_JSON} application/vnd.git-lfs+json}.freeze ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'.freeze + WHITELISTED_GIT_ROUTES = { + 'projects/git_http' => %w{git_upload_pack git_receive_pack} + }.freeze + + WHITELISTED_GIT_LFS_ROUTES = { + 'projects/lfs_api' => %w{batch}, + 'projects/lfs_locks_api' => %w{verify create unlock} + }.freeze + def initialize(app, env) @app = app @env = env @@ -36,7 +46,7 @@ module Gitlab end def json_request? - request.media_type == APPLICATION_JSON + APPLICATION_JSON_TYPES.include?(request.media_type) end def rack_flash @@ -63,22 +73,27 @@ module Gitlab grack_route || ReadOnly.internal_routes.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route end - def sidekiq_route - request.path.start_with?('/admin/sidekiq') - end - def grack_route # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('.git/git-upload-pack') + return false unless + request.path.end_with?('.git/git-upload-pack', '.git/git-receive-pack') - route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' + WHITELISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end def lfs_route # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('/info/lfs/objects/batch') + unless request.path.end_with?('/info/lfs/objects/batch', + '/info/lfs/locks', '/info/lfs/locks/verify') || + %r{/info/lfs/locks/\d+/unlock\z}.match?(request.path) + return false + end + + WHITELISTED_GIT_LFS_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + end - route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + def sidekiq_route + request.path.start_with?('/admin/sidekiq') end end end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 7f64a8c9e46..2ec871f0754 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -25,6 +25,11 @@ module Gitlab raise NotImplementedError end + # List of cached methods. Should be overridden by the including class + def cached_methods + raise NotImplementedError + end + # Caches the supplied block both in a cache and in an instance variable. # # The cache key and instance variable are named the same way as the value of @@ -67,6 +72,11 @@ module Gitlab # Expires the caches of a specific set of methods def expire_method_caches(methods) methods.each do |key| + unless cached_methods.include?(key.to_sym) + Rails.logger.error "Requested to expire non-existent method '#{key}' for Repository" + next + end + cache.expire(key) ivar = cache_instance_variable_name(key) diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index 44074397c05..900dbf7be24 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -10,15 +10,22 @@ namespace :gitlab do puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) end - desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz' - task bump_test_version: :environment do - Dir.mktmpdir do |tmp_dir| - system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null") - File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') - system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null") + desc 'GitLab | Bumps the Import/Export version in fixtures and project templates' + task bump_version: :environment do + archives = Dir['vendor/project_templates/*.tar.gz'] + archives.push('spec/features/projects/import_export/test_project_export.tar.gz') + + archives.each do |archive| + raise ArgumentError unless File.exist?(archive) + + Dir.mktmpdir do |tmp_dir| + system("tar -zxf #{archive} -C #{tmp_dir} > /dev/null") + File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') + system("tar -zcvf #{archive} -C #{tmp_dir} . > /dev/null") + end end - puts "Updated to #{Gitlab::ImportExport.version}" + puts "Updated #{archives} to #{Gitlab::ImportExport.version}." end end end |