diff options
Diffstat (limited to 'lib')
44 files changed, 1094 insertions, 308 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 928706dfda7..4ad4a1f7867 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -708,8 +708,9 @@ module API class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active - expose :push_events, :issues_events, :merge_requests_events - expose :tag_push_events, :note_events, :pipeline_events + expose :push_events, :issues_events, :confidential_issues_events + expose :merge_requests_events, :tag_push_events, :note_events + expose :pipeline_events, :wiki_page_events expose :job_events # Expose serialized properties expose :properties do |service, options| diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 9ba15893f55..8ad4b2ecbf3 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -69,7 +69,7 @@ module API end def wiki_page - page = user_project.wiki.find_page(params[:slug]) + page = ProjectWiki.new(user_project, current_user).find_page(params[:slug]) page || not_found!('Wiki Page') end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index ccaaeca10d4..79b302aae70 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -190,9 +190,12 @@ module API project = Gitlab::GlRepository.parse(params[:gl_repository]).first user = identify(params[:identifier]) - redirect_message = Gitlab::Checks::ProjectMoved.fetch_redirect_message(user.id, project.id) - if redirect_message - output[:redirected_message] = redirect_message + + # A user is not guaranteed to be returned; an orphaned write deploy + # key could be used + if user + redirect_message = Gitlab::Checks::ProjectMoved.fetch_redirect_message(user.id, project.id) + output[:redirected_message] = redirect_message if redirect_message end output diff --git a/lib/api/members.rb b/lib/api/members.rb index 22e4bdead41..5446f6b54b1 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -59,7 +59,9 @@ module API member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) - if member.persisted? && member.valid? + if !member + not_allowed! # This currently can only be reached in EE + elsif member.persisted? && member.valid? present member.user, with: Entities::Member, member: member else render_validation_error!(member) diff --git a/lib/api/services.rb b/lib/api/services.rb index bbcc851d07a..a7f44e2869c 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -1,5 +1,143 @@ module API class Services < Grape::API + chat_notification_settings = [ + { + required: true, + name: :webhook, + type: String, + desc: 'The chat webhook' + }, + { + required: false, + name: :username, + type: String, + desc: 'The chat username' + }, + { + required: false, + name: :channel, + type: String, + desc: 'The default chat channel' + } + ] + + chat_notification_flags = [ + { + required: false, + name: :notify_only_broken_pipelines, + type: Boolean, + desc: 'Send notifications for broken pipelines' + }, + { + required: false, + name: :notify_only_default_branch, + type: Boolean, + desc: 'Send notifications only for the default branch' + } + ] + + chat_notification_channels = [ + { + required: false, + name: :push_channel, + type: String, + desc: 'The name of the channel to receive push_events notifications' + }, + { + required: false, + name: :issue_channel, + type: String, + desc: 'The name of the channel to receive issues_events notifications' + }, + { + required: false, + name: :confidential_issue_channel, + type: String, + desc: 'The name of the channel to receive confidential_issues_events notifications' + }, + { + required: false, + name: :merge_request_channel, + type: String, + desc: 'The name of the channel to receive merge_requests_events notifications' + }, + { + required: false, + name: :note_channel, + type: String, + desc: 'The name of the channel to receive note_events notifications' + }, + { + required: false, + name: :tag_push_channel, + type: String, + desc: 'The name of the channel to receive tag_push_events notifications' + }, + { + required: false, + name: :pipeline_channel, + type: String, + desc: 'The name of the channel to receive pipeline_events notifications' + }, + { + required: false, + name: :wiki_page_channel, + type: String, + desc: 'The name of the channel to receive wiki_page_events notifications' + } + ] + + chat_notification_events = [ + { + required: false, + name: :push_events, + type: Boolean, + desc: 'Enable notifications for push_events' + }, + { + required: false, + name: :issues_events, + type: Boolean, + desc: 'Enable notifications for issues_events' + }, + { + required: false, + name: :confidential_issues_events, + type: Boolean, + desc: 'Enable notifications for confidential_issues_events' + }, + { + required: false, + name: :merge_requests_events, + type: Boolean, + desc: 'Enable notifications for merge_requests_events' + }, + { + required: false, + name: :note_events, + type: Boolean, + desc: 'Enable notifications for note_events' + }, + { + required: false, + name: :tag_push_events, + type: Boolean, + desc: 'Enable notifications for tag_push_events' + }, + { + required: false, + name: :pipeline_events, + type: Boolean, + desc: 'Enable notifications for pipeline_events' + }, + { + required: false, + name: :wiki_page_events, + type: Boolean, + desc: 'Enable notifications for wiki_page_events' + } + ] + services = { 'asana' => [ { @@ -489,25 +627,11 @@ module API } ], 'slack' => [ - { - required: true, - name: :webhook, - type: String, - desc: 'The Slack webhook. e.g. https://hooks.slack.com/services/...' - }, - { - required: false, - name: :new_issue_url, - type: String, - desc: 'The user name' - }, - { - required: false, - name: :channel, - type: String, - desc: 'The channel name' - } - ], + chat_notification_settings, + chat_notification_flags, + chat_notification_channels, + chat_notification_events + ].flatten, 'microsoft-teams' => [ { required: true, @@ -517,19 +641,11 @@ module API } ], 'mattermost' => [ - { - required: true, - name: :webhook, - type: String, - desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...' - }, - { - required: false, - name: :username, - type: String, - desc: 'The username to use to post the message' - } - ], + chat_notification_settings, + chat_notification_flags, + chat_notification_channels, + chat_notification_events + ].flatten, 'teamcity' => [ { required: true, diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 30a91647b77..287d591e88d 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -18,7 +18,7 @@ module Backup FileUtils.rm_f(backup_tarball) if ENV['STRATEGY'] == 'copy' - cmd = %W(cp -a #{app_files_dir} #{Gitlab.config.backup.path}) + cmd = %W(rsync -a --exclude=lost+found #{app_files_dir} #{Gitlab.config.backup.path}) output, status = Gitlab::Popen.popen(cmd) unless status.zero? @@ -26,10 +26,10 @@ module Backup abort 'Backup failed' end - run_pipeline!([%W(tar -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + run_pipeline!([%W(tar --exclude=lost+found -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) FileUtils.rm_rf(@backup_files_dir) else - run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + run_pipeline!([%W(tar --exclude=lost+found -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) end end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index c1f933ec54b..5c197afd782 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -66,7 +66,7 @@ module Banzai if uri.relative? && uri.path.present? html_attr.value = rebuild_relative_uri(uri).to_s end - rescue URI::Error + rescue URI::Error, Addressable::URI::InvalidURIError # noop end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb new file mode 100644 index 00000000000..de622f657b2 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for cleaning up a concurrent column rename. + class CleanupConcurrentTypeChange + include Database::MigrationHelpers + + RESCHEDULE_DELAY = 10.minutes + + # table - The name of the table the migration is performed for. + # old_column - The name of the old (to drop) column. + # new_column - The name of the new column. + def perform(table, old_column, new_column) + return unless column_exists?(:issues, new_column) + + rows_to_migrate = define_model_for(table) + .where(new_column => nil) + .where + .not(old_column => nil) + + if rows_to_migrate.any? + BackgroundMigrationWorker.perform_in( + RESCHEDULE_DELAY, + 'CleanupConcurrentTypeChange', + [table, old_column, new_column] + ) + else + cleanup_concurrent_column_type_change(table, old_column) + end + end + + # These methods are necessary so we can re-use the migration helpers in + # this class. + def connection + ActiveRecord::Base.connection + end + + def method_missing(name, *args, &block) + connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend + end + + def respond_to_missing?(*args) + connection.respond_to?(*args) || super + end + + def define_model_for(table) + Class.new(ActiveRecord::Base) do + self.table_name = table + end + end + end + end +end diff --git a/lib/gitlab/background_migration/copy_column.rb b/lib/gitlab/background_migration/copy_column.rb new file mode 100644 index 00000000000..a2cb215c230 --- /dev/null +++ b/lib/gitlab/background_migration/copy_column.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # CopyColumn is a simple (reusable) background migration that can be used to + # update the value of a column based on the value of another column in the + # same table. + # + # For this background migration to work the table that is migrated _has_ to + # have an `id` column as the primary key. + class CopyColumn + # table - The name of the table that contains the columns. + # copy_from - The column containing the data to copy. + # copy_to - The column to copy the data to. + # start_id - The start ID of the range of rows to update. + # end_id - The end ID of the range of rows to update. + def perform(table, copy_from, copy_to, start_id, end_id) + return unless connection.column_exists?(table, copy_to) + + quoted_table = connection.quote_table_name(table) + quoted_copy_from = connection.quote_column_name(copy_from) + quoted_copy_to = connection.quote_column_name(copy_to) + + # We're using raw SQL here since this job may be frequently executed. As + # a result dynamically defining models would lead to many unnecessary + # schema information queries. + connection.execute <<-SQL.strip_heredoc + UPDATE #{quoted_table} + SET #{quoted_copy_to} = #{quoted_copy_from} + WHERE id BETWEEN #{start_id} AND #{end_id} + SQL + end + + def connection + ActiveRecord::Base.connection + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb index a1af045a71f..21b626dde56 100644 --- a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb +++ b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb @@ -1,44 +1,12 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab module BackgroundMigration class DeleteConflictingRedirectRoutesRange - class Route < ActiveRecord::Base - self.table_name = 'routes' - end - - class RedirectRoute < ActiveRecord::Base - self.table_name = 'redirect_routes' - end - - # start_id - The start ID of the range of events to process - # end_id - The end ID of the range to process. def perform(start_id, end_id) - return unless migrate? - - conflicts = RedirectRoute.where(routes_match_redirects_clause(start_id, end_id)) - num_rows = conflicts.delete_all - - Rails.logger.info("Gitlab::BackgroundMigration::DeleteConflictingRedirectRoutesRange [#{start_id}, #{end_id}] - Deleted #{num_rows} redirect routes that were conflicting with routes.") - end - - def migrate? - Route.table_exists? && RedirectRoute.table_exists? - end - - def routes_match_redirects_clause(start_id, end_id) - <<~ROUTES_MATCH_REDIRECTS - EXISTS ( - SELECT 1 FROM routes - WHERE ( - LOWER(redirect_routes.path) = LOWER(routes.path) - OR LOWER(redirect_routes.path) LIKE LOWER(CONCAT(routes.path, '/%')) - ) - AND routes.id BETWEEN #{start_id} AND #{end_id} - ) - ROUTES_MATCH_REDIRECTS + # No-op. + # See https://gitlab.com/gitlab-com/infrastructure/issues/3460#note_53223252 end end end diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb index 84ac00f1a5c..7088aa0860a 100644 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -128,8 +128,14 @@ module Gitlab end def process_event(event) - replicate_event(event) - create_push_event_payload(event) if event.push_event? + ActiveRecord::Base.transaction do + replicate_event(event) + create_push_event_payload(event) if event.push_event? + end + rescue ActiveRecord::InvalidForeignKey => e + # A foreign key error means the associated event was removed. In this + # case we'll just skip migrating the event. + Rails.logger.error("Unable to migrate event #{event.id}: #{e}") end def replicate_event(event) @@ -137,9 +143,6 @@ module Gitlab .with_indifferent_access.except(:title, :data) EventForMigration.create!(new_attributes) - rescue ActiveRecord::InvalidForeignKey - # A foreign key error means the associated event was removed. In this - # case we'll just skip migrating the event. end def create_push_event_payload(event) @@ -156,9 +159,6 @@ module Gitlab ref: event.trimmed_ref_name, commit_title: event.commit_title ) - rescue ActiveRecord::InvalidForeignKey - # A foreign key error means the associated event was removed. In this - # case we'll just skip migrating the event. end def find_events(start_id, end_id) diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb new file mode 100644 index 00000000000..8a901a9bf39 --- /dev/null +++ b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Metrics/MethodLength +# rubocop:disable Metrics/ClassLength +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateMergeRequestMetricsWithEventsData + def perform(min_merge_request_id, max_merge_request_id) + insert_metrics_for_range(min_merge_request_id, max_merge_request_id) + update_metrics_with_events_data(min_merge_request_id, max_merge_request_id) + end + + # Inserts merge_request_metrics records for merge_requests without it for + # a given merge request batch. + def insert_metrics_for_range(min, max) + metrics_not_exists_clause = + <<-SQL.strip_heredoc + NOT EXISTS (SELECT 1 FROM merge_request_metrics + WHERE merge_request_metrics.merge_request_id = merge_requests.id) + SQL + + MergeRequest.where(metrics_not_exists_clause).where(id: min..max).each_batch do |batch| + select_sql = batch.select(:id, :created_at, :updated_at).to_sql + + execute("INSERT INTO merge_request_metrics (merge_request_id, created_at, updated_at) #{select_sql}") + end + end + + def update_metrics_with_events_data(min, max) + if Gitlab::Database.postgresql? + # Uses WITH syntax in order to update merged and closed events with a single UPDATE. + # WITH is not supported by MySQL. + update_events_for_range(min, max) + else + update_merged_events_for_range(min, max) + update_closed_events_for_range(min, max) + end + end + + private + + # Updates merge_request_metrics latest_closed_at, latest_closed_by_id and merged_by_id + # based on the latest event records on events table for a given merge request batch. + def update_events_for_range(min, max) + sql = <<-SQL.strip_heredoc + WITH events_for_update AS ( + SELECT DISTINCT ON (target_id, action) target_id, action, author_id, updated_at + FROM events + WHERE target_id BETWEEN #{min} AND #{max} + AND target_type = 'MergeRequest' + AND action IN (#{Event::CLOSED},#{Event::MERGED}) + ORDER BY target_id, action, id DESC + ) + UPDATE merge_request_metrics met + SET latest_closed_at = latest_closed.updated_at, + latest_closed_by_id = latest_closed.author_id, + merged_by_id = latest_merged.author_id + FROM (SELECT * FROM events_for_update WHERE action = #{Event::CLOSED}) AS latest_closed + FULL OUTER JOIN + (SELECT * FROM events_for_update WHERE action = #{Event::MERGED}) AS latest_merged + USING (target_id) + WHERE target_id = merge_request_id; + SQL + + execute(sql) + end + + # Updates merge_request_metrics latest_closed_at, latest_closed_by_id based on the latest closed + # records on events table for a given merge request batch. + def update_closed_events_for_range(min, max) + sql = + <<-SQL.strip_heredoc + UPDATE merge_request_metrics metrics, + (#{select_events(min, max, Event::CLOSED)}) closed_events + SET metrics.latest_closed_by_id = closed_events.author_id, + metrics.latest_closed_at = closed_events.updated_at #{where_matches_closed_events}; + SQL + + execute(sql) + end + + # Updates merge_request_metrics merged_by_id based on the latest merged + # records on events table for a given merge request batch. + def update_merged_events_for_range(min, max) + sql = + <<-SQL.strip_heredoc + UPDATE merge_request_metrics metrics, + (#{select_events(min, max, Event::MERGED)}) merged_events + SET metrics.merged_by_id = merged_events.author_id #{where_matches_merged_events}; + SQL + + execute(sql) + end + + def execute(sql) + @connection ||= ActiveRecord::Base.connection + @connection.execute(sql) + end + + def select_events(min, max, action) + select_max_event_id = <<-SQL.strip_heredoc + SELECT max(id) + FROM events + WHERE action = #{action} + AND target_type = 'MergeRequest' + AND target_id BETWEEN #{min} AND #{max} + GROUP BY target_id + SQL + + <<-SQL.strip_heredoc + SELECT author_id, updated_at, target_id + FROM events + WHERE id IN(#{select_max_event_id}) + SQL + end + + def where_matches_closed_events + <<-SQL.strip_heredoc + WHERE metrics.merge_request_id = closed_events.target_id + AND metrics.latest_closed_at IS NULL + AND metrics.latest_closed_by_id IS NULL + SQL + end + + def where_matches_merged_events + <<-SQL.strip_heredoc + WHERE metrics.merge_request_id = merged_events.target_id + AND metrics.merged_by_id IS NULL + SQL + end + end + end +end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 298409d8b5a..709a901aa77 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -14,7 +14,7 @@ module Gitlab repos_to_import.each do |repo_path| bare_repo = Gitlab::BareRepositoryImport::Repository.new(import_path, repo_path) - if bare_repo.hashed? || bare_repo.wiki? + unless bare_repo.processable? log " * Skipping repo #{bare_repo.repo_path}".color(:yellow) next @@ -55,12 +55,15 @@ module Gitlab name: project_name, path: project_name, skip_disk_validation: true, - import_type: 'gitlab_project', + skip_wiki: bare_repo.wiki_exists?, + import_type: 'bare_repository', namespace_id: group&.id).execute if project.persisted? && mv_repo(project) log " * Created #{project.name} (#{project_full_path})".color(:green) + project.write_repository_config + ProjectCacheWorker.perform_async(project.id) else log " * Failed trying to create #{project.name} (#{project_full_path})".color(:red) diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb index fa7891c8dcc..85b79362196 100644 --- a/lib/gitlab/bare_repository_import/repository.rb +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -6,39 +6,56 @@ module Gitlab def initialize(root_path, repo_path) @root_path = root_path @repo_path = repo_path - @root_path << '/' unless root_path.ends_with?('/') + full_path = + if hashed? && !wiki? + repository.config.get('gitlab.fullpath') + else + repo_relative_path + end + # Split path into 'all/the/namespaces' and 'project_name' - @group_path, _, @project_name = repo_relative_path.rpartition('/') + @group_path, _, @project_name = full_path.to_s.rpartition('/') end def wiki_exists? File.exist?(wiki_path) end - def wiki? - @wiki ||= repo_path.end_with?('.wiki.git') - end - def wiki_path @wiki_path ||= repo_path.sub(/\.git$/, '.wiki.git') end - def hashed? - @hashed ||= group_path.start_with?('@hashed') - end - def project_full_path @project_full_path ||= "#{group_path}/#{project_name}" end + def processable? + return false if wiki? + return false if hashed? && (group_path.blank? || project_name.blank?) + + true + end + private + def wiki? + @wiki ||= repo_path.end_with?('.wiki.git') + end + + def hashed? + @hashed ||= repo_relative_path.include?('@hashed') + end + def repo_relative_path # Remove root path and `.git` at the end repo_path[@root_path.size...-4] end + + def repository + @repository ||= Rugged::Repository.new(repo_path) + end end end end diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index 3a1c0a3455e..dfb2f4d4054 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -21,6 +21,10 @@ module Gitlab end def add_redirect_message + # Don't bother with sending a redirect message for anonymous clones + # because they never see it via the `/internal/post_receive` endpoint + return unless user.present? && project.present? + Gitlab::Redis::SharedState.with do |redis| key = self.class.redirect_message_key(user.id, project.id) redis.setex(key, 5.minutes, redirect_message) diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 72b75791bbb..e25916528f4 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -234,7 +234,7 @@ module Gitlab # Most terminals show bold colored text in the light color variant # Let's mimic that here if @style_mask & STYLE_SWITCHES[:bold] != 0 - fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1') + fg_color.sub!(/fg-([a-z]{2,}+)/, 'fg-l-\1') end css_classes << fg_color end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 76aee5a3deb..0a3ae2c3760 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -13,12 +13,13 @@ module Gitlab end def resolve(user, commit_message, files) + msg = commit_message || default_commit_message + resolution = Gitlab::Git::Conflict::Resolution.new(user, files, msg) args = { source_branch: merge_request.source_branch, - target_branch: merge_request.target_branch, - commit_message: commit_message || default_commit_message + target_branch: merge_request.target_branch } - resolver.resolve_conflicts(@source_repo, user, files, args) + resolver.resolve_conflicts(@source_repo, resolution, args) ensure @merge_request.clear_memoized_shas end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 3f65bc912de..33171f83692 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -385,10 +385,27 @@ module Gitlab # necessary since we copy over old values further down. change_column_default(table, new, old_col.default) if old_col.default - trigger_name = rename_trigger_name(table, old, new) + install_rename_triggers(table, old, new) + + update_column_in_batches(table, new, Arel::Table.new(table)[old]) + + change_column_null(table, new, false) unless old_col.null + + copy_indexes(table, old, new) + copy_foreign_keys(table, old, new) + end + + # Installs triggers in a table that keep a new column in sync with an old + # one. + # + # table - The name of the table to install the trigger in. + # old_column - The name of the old column. + # new_column - The name of the new column. + def install_rename_triggers(table, old_column, new_column) + trigger_name = rename_trigger_name(table, old_column, new_column) quoted_table = quote_table_name(table) - quoted_old = quote_column_name(old) - quoted_new = quote_column_name(new) + quoted_old = quote_column_name(old_column) + quoted_new = quote_column_name(new_column) if Database.postgresql? install_rename_triggers_for_postgresql(trigger_name, quoted_table, @@ -397,13 +414,6 @@ module Gitlab install_rename_triggers_for_mysql(trigger_name, quoted_table, quoted_old, quoted_new) end - - update_column_in_batches(table, new, Arel::Table.new(table)[old]) - - change_column_null(table, new, false) unless old_col.null - - copy_indexes(table, old, new) - copy_foreign_keys(table, old, new) end # Changes the type of a column concurrently. @@ -455,6 +465,97 @@ module Gitlab remove_column(table, old) end + # Changes the column type of a table using a background migration. + # + # Because this method uses a background migration it's more suitable for + # large tables. For small tables it's better to use + # `change_column_type_concurrently` since it can complete its work in a + # much shorter amount of time and doesn't rely on Sidekiq. + # + # Example usage: + # + # class Issue < ActiveRecord::Base + # self.table_name = 'issues' + # + # include EachBatch + # + # def self.to_migrate + # where('closed_at IS NOT NULL') + # end + # end + # + # change_column_type_using_background_migration( + # Issue.to_migrate, + # :closed_at, + # :datetime_with_timezone + # ) + # + # Reverting a migration like this is done exactly the same way, just with + # a different type to migrate to (e.g. `:datetime` in the above example). + # + # relation - An ActiveRecord relation to use for scheduling jobs and + # figuring out what table we're modifying. This relation _must_ + # have the EachBatch module included. + # + # column - The name of the column for which the type will be changed. + # + # new_type - The new type of the column. + # + # batch_size - The number of rows to schedule in a single background + # migration. + # + # interval - The time interval between every background migration. + def change_column_type_using_background_migration( + relation, + column, + new_type, + batch_size: 10_000, + interval: 10.minutes + ) + unless relation.model < EachBatch + raise TypeError, 'The relation must include the EachBatch module' + end + + temp_column = "#{column}_for_type_change" + table = relation.table_name + max_index = 0 + + add_column(table, temp_column, new_type) + install_rename_triggers(table, column, temp_column) + + # Schedule the jobs that will copy the data from the old column to the + # new one. + relation.each_batch(of: batch_size) do |batch, index| + start_id, end_id = batch.pluck('MIN(id), MAX(id)').first + max_index = index + + BackgroundMigrationWorker.perform_in( + index * interval, + 'CopyColumn', + [table, column, temp_column, start_id, end_id] + ) + end + + # Schedule the renaming of the column to happen (initially) 1 hour after + # the last batch finished. + BackgroundMigrationWorker.perform_in( + (max_index * interval) + 1.hour, + 'CleanupConcurrentTypeChange', + [table, column, temp_column] + ) + + if perform_background_migration_inline? + # To ensure the schema is up to date immediately we perform the + # migration inline in dev / test environments. + Gitlab::BackgroundMigration.steal('CopyColumn') + Gitlab::BackgroundMigration.steal('CleanupConcurrentTypeChange') + end + end + + def perform_background_migration_inline? + Rails.env.test? || Rails.env.development? + end + # Performs a concurrent column rename when using PostgreSQL. def install_rename_triggers_for_postgresql(trigger, table, old, new) execute <<-EOF.strip_heredoc diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index cd490aaa291..34b070dd375 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -116,8 +116,10 @@ module Gitlab new_content_sha || old_content_sha end + # Use #itself to check the value wrapped by a BatchLoader instance, rather + # than if the BatchLoader instance itself is falsey. def blob - new_blob || old_blob + new_blob&.itself || old_blob&.itself end attr_writer :highlighted_diff_lines @@ -173,7 +175,7 @@ module Gitlab end def binary? - has_binary_notice? || old_blob&.binary? || new_blob&.binary? + has_binary_notice? || try_blobs(:binary?) end def text? @@ -181,15 +183,15 @@ module Gitlab end def external_storage_error? - old_blob&.external_storage_error? || new_blob&.external_storage_error? + try_blobs(:external_storage_error?) end def stored_externally? - old_blob&.stored_externally? || new_blob&.stored_externally? + try_blobs(:stored_externally?) end def external_storage - old_blob&.external_storage || new_blob&.external_storage + try_blobs(:external_storage) end def content_changed? @@ -204,15 +206,15 @@ module Gitlab end def size - [old_blob&.size, new_blob&.size].compact.sum + valid_blobs.map(&:size).sum end def raw_size - [old_blob&.raw_size, new_blob&.raw_size].compact.sum + valid_blobs.map(&:raw_size).sum end def raw_binary? - old_blob&.raw_binary? || new_blob&.raw_binary? + try_blobs(:raw_binary?) end def raw_text? @@ -235,6 +237,19 @@ module Gitlab private + # The blob instances are instances of BatchLoader, which means calling + # &. directly on them won't work. Object#try also won't work, because Blob + # doesn't inherit from Object, but from BasicObject (via SimpleDelegator). + def try_blobs(meth) + old_blob&.itself&.public_send(meth) || new_blob&.itself&.public_send(meth) + end + + # We can't use #compact for the same reason we can't use &., but calling + # #nil? explicitly does work because it is proxied to the blob itself. + def valid_blobs + [old_blob, new_blob].reject(&:nil?) + end + def text_position_properties(line) { old_line: line.old_line, new_line: line.new_line } end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 582028493e9..6b53eb4533d 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -71,6 +71,16 @@ module Gitlab end end + def encode_binary(s) + return "" if s.nil? + + s.dup.force_encoding(Encoding::ASCII_8BIT) + end + + def binary_stringio(s) + StringIO.new(s || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } + end + private def clean(message) diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index 6bf49a0af18..8463b1eb794 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -34,13 +34,8 @@ module Gitlab def rugged_stats(commit) diff = commit.rugged_diff_from_parent - - diff.each_patch do |p| - # TODO: Use the new Rugged convenience methods when they're released - @additions += p.stat[0] - @deletions += p.stat[1] - @total += p.changes - end + _files_changed, @additions, @deletions = diff.stat + @total = @additions + @deletions end end end diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb index b2a625e08fa..2a9cf10a068 100644 --- a/lib/gitlab/git/conflict/file.rb +++ b/lib/gitlab/git/conflict/file.rb @@ -2,7 +2,9 @@ module Gitlab module Git module Conflict class File - attr_reader :content, :their_path, :our_path, :our_mode, :repository, :commit_oid + attr_reader :their_path, :our_path, :our_mode, :repository, :commit_oid + + attr_accessor :content def initialize(repository, commit_oid, conflict, content) @repository = repository diff --git a/lib/gitlab/git/conflict/resolution.rb b/lib/gitlab/git/conflict/resolution.rb new file mode 100644 index 00000000000..ab9be683e15 --- /dev/null +++ b/lib/gitlab/git/conflict/resolution.rb @@ -0,0 +1,15 @@ +module Gitlab + module Git + module Conflict + class Resolution + attr_reader :user, :files, :commit_message + + def initialize(user, files, commit_message) + @user = user + @files = files + @commit_message = commit_message + end + end + end + end +end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 03e5c0fcd6f..74c9874d590 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -13,37 +13,27 @@ module Gitlab def conflicts @conflicts ||= begin - target_index = @target_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - - # We don't need to do `with_repo_branch_commit` here, because the target - # project always fetches source refs when creating merge request diffs. - conflict_files(@target_repository, target_index) + @target_repository.gitaly_migrate(:conflicts_list_conflict_files) do |is_enabled| + if is_enabled + gitaly_conflicts_client(@target_repository).list_conflict_files + else + rugged_list_conflict_files + end + end end + rescue GRPC::FailedPrecondition => e + raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message) + rescue Rugged::OdbError, GRPC::BadStatus => e + raise Gitlab::Git::CommandError.new(e) end - def resolve_conflicts(source_repository, user, files, source_branch:, target_branch:, commit_message:) - source_repository.with_repo_branch_commit(@target_repository, target_branch) do - index = source_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - conflicts = conflict_files(source_repository, index) - - files.each do |file_params| - conflict_file = conflict_for_path(conflicts, file_params[:old_path], file_params[:new_path]) - - write_resolved_file_to_index(source_repository, index, conflict_file, file_params) - end - - unless index.conflicts.empty? - missing_files = index.conflicts.map { |file| file[:ours][:path] } - - raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" + def resolve_conflicts(source_repository, resolution, source_branch:, target_branch:) + source_repository.gitaly_migrate(:conflicts_resolve_conflicts) do |is_enabled| + if is_enabled + gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch) + else + rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) end - - commit_params = { - message: commit_message, - parents: [@our_commit_oid, @their_commit_oid] - } - - source_repository.commit_index(user, source_branch, index, commit_params) end end @@ -68,6 +58,10 @@ module Gitlab end end + def gitaly_conflicts_client(repository) + repository.gitaly_conflicts_client(@our_commit_oid, @their_commit_oid) + end + def write_resolved_file_to_index(repository, index, file, params) if params[:sections] resolved_lines = file.resolve_lines(params[:sections]) @@ -84,6 +78,40 @@ module Gitlab index.add(path: our_path, oid: oid, mode: file.our_mode) index.conflict_remove(our_path) end + + def rugged_list_conflict_files + target_index = @target_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) + + # We don't need to do `with_repo_branch_commit` here, because the target + # project always fetches source refs when creating merge request diffs. + conflict_files(@target_repository, target_index) + end + + def rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) + source_repository.with_repo_branch_commit(@target_repository, target_branch) do + index = source_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) + conflicts = conflict_files(source_repository, index) + + resolution.files.each do |file_params| + conflict_file = conflict_for_path(conflicts, file_params[:old_path], file_params[:new_path]) + + write_resolved_file_to_index(source_repository, index, conflict_file, file_params) + end + + unless index.conflicts.empty? + missing_files = index.conflicts.map { |file| file[:ours][:path] } + + raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" + end + + commit_params = { + message: resolution.commit_message, + parents: [@our_commit_oid, @their_commit_oid] + } + + source_repository.commit_index(resolution.user, source_branch, index, commit_params) + end + end end end end diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index d948d7895ed..cba638c06db 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -2,6 +2,9 @@ module Gitlab module Git class GitlabProjects include Gitlab::Git::Popen + include Gitlab::Utils::StrongMemoize + + ShardNameNotFoundError = Class.new(StandardError) # Absolute path to directory where repositories are stored. # Example: /home/git/repositories @@ -97,22 +100,13 @@ module Gitlab end def fork_repository(new_shard_path, new_repository_relative_path) - from_path = repository_absolute_path - to_path = File.join(new_shard_path, new_repository_relative_path) - - # The repository cannot already exist - if File.exist?(to_path) - logger.error "fork-repository failed: destination repository <#{to_path}> already exists." - return false + Gitlab::GitalyClient.migrate(:fork_repository) do |is_enabled| + if is_enabled + gitaly_fork_repository(new_shard_path, new_repository_relative_path) + else + git_fork_repository(new_shard_path, new_repository_relative_path) + end end - - # Ensure the namepsace / hashed storage directory exists - FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) - - logger.info "Forking repository from <#{from_path}> to <#{to_path}>." - cmd = %W(git clone --bare --no-local -- #{from_path} #{to_path}) - - run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) end def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil) @@ -253,6 +247,48 @@ module Gitlab known_hosts_file&.close! script&.close! end + + private + + def shard_name + strong_memoize(:shard_name) do + shard_name_from_shard_path(shard_path) + end + end + + def shard_name_from_shard_path(shard_path) + Gitlab.config.repositories.storages.find { |_, info| info['path'] == shard_path }&.first || + raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'") + end + + def git_fork_repository(new_shard_path, new_repository_relative_path) + from_path = repository_absolute_path + to_path = File.join(new_shard_path, new_repository_relative_path) + + # The repository cannot already exist + if File.exist?(to_path) + logger.error "fork-repository failed: destination repository <#{to_path}> already exists." + return false + end + + # Ensure the namepsace / hashed storage directory exists + FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) + + logger.info "Forking repository from <#{from_path}> to <#{to_path}>." + cmd = %W(git clone --bare --no-local -- #{from_path} #{to_path}) + + run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) + end + + def gitaly_fork_repository(new_shard_path, new_repository_relative_path) + target_repository = Gitlab::Git::Repository.new(shard_name_from_shard_path(new_shard_path), new_repository_relative_path, nil) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) + rescue GRPC::BadStatus => e + logger.error "fork-repository failed: #{e.message}" + false + end end end end diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb new file mode 100644 index 00000000000..38e9d2a8554 --- /dev/null +++ b/lib/gitlab/git/remote_mirror.rb @@ -0,0 +1,75 @@ +module Gitlab + module Git + class RemoteMirror + def initialize(repository, ref_name) + @repository = repository + @ref_name = ref_name + end + + def update(only_branches_matching: [], only_tags_matching: []) + local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching) + remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching) + + updated_branches = changed_refs(local_branches, remote_branches) + push_branches(updated_branches.keys) if updated_branches.present? + + delete_refs(local_branches, remote_branches) + + local_tags = refs_obj(@repository.tags, only_refs_matching: only_tags_matching) + remote_tags = refs_obj(@repository.remote_tags(@ref_name), only_refs_matching: only_tags_matching) + + updated_tags = changed_refs(local_tags, remote_tags) + @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present? + + delete_refs(local_tags, remote_tags) + end + + private + + def refs_obj(refs, only_refs_matching: []) + refs.each_with_object({}) do |ref, refs| + next if only_refs_matching.present? && !only_refs_matching.include?(ref.name) + + refs[ref.name] = ref + end + end + + def changed_refs(local_refs, remote_refs) + local_refs.select do |ref_name, ref| + remote_ref = remote_refs[ref_name] + + remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target + end + end + + def push_branches(branches) + default_branch, branches = branches.partition do |branch| + @repository.root_ref == branch + end + + # Push the default branch first so it works fine when remote mirror is empty. + branches.unshift(*default_branch) + + @repository.push_remote_branches(@ref_name, branches) + end + + def delete_refs(local_refs, remote_refs) + refs = refs_to_delete(local_refs, remote_refs) + + @repository.delete_remote_branches(@ref_name, refs.keys) if refs.present? + end + + def refs_to_delete(local_refs, remote_refs) + default_branch_id = @repository.commit.id + + remote_refs.select do |remote_ref_name, remote_ref| + next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo + + remote_ref_id = remote_ref.dereferenced_target.try(:id) + + remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id) + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 603323d0452..aec85f971ca 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -21,6 +21,7 @@ module Gitlab REBASE_WORKTREE_PREFIX = 'rebase'.freeze SQUASH_WORKTREE_PREFIX = 'squash'.freeze GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze + GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) @@ -83,7 +84,7 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage, :gl_repository, :relative_path + attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. @@ -93,6 +94,12 @@ module Gitlab @gl_repository = gl_repository storage_path = Gitlab.config.repositories.storages[@storage]['path'] + @gitlab_projects = Gitlab::Git::GitlabProjects.new( + storage_path, + relative_path, + global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, + logger: Rails.logger + ) @path = File.join(storage_path, @relative_path) @name = @relative_path.split("/").last @attributes = Gitlab::Git::Attributes.new(path) @@ -126,7 +133,7 @@ module Gitlab end def exists? - Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| + Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| if enabled gitaly_repository_client.exists? else @@ -188,7 +195,7 @@ module Gitlab end def local_branches(sort_by: nil) - gitaly_migrate(:local_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + gitaly_migrate(:local_branches) do |is_enabled| if is_enabled gitaly_ref_client.local_branches(sort_by: sort_by) else @@ -919,31 +926,23 @@ module Gitlab # If `mirror_refmap` is present the remote is set as mirror with that mapping def add_remote(remote_name, url, mirror_refmap: nil) - rugged.remotes.create(remote_name, url) - - set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap - rescue Rugged::ConfigError - remote_update(remote_name, url: url) + gitaly_migrate(:remote_add_remote) do |is_enabled| + if is_enabled + gitaly_remote_client.add_remote(remote_name, url, mirror_refmap) + else + rugged_add_remote(remote_name, url, mirror_refmap) + end + end end def remove_remote(remote_name) - # When a remote is deleted all its remote refs are deleted too, but in - # the case of mirrors we map its refs (that would usualy go under - # [remote_name]/) to the top level namespace. We clean the mapping so - # those don't get deleted. - if rugged.config["remote.#{remote_name}.mirror"] - rugged.config.delete("remote.#{remote_name}.fetch") + gitaly_migrate(:remote_remove_remote) do |is_enabled| + if is_enabled + gitaly_remote_client.remove_remote(remote_name) + else + rugged_remove_remote(remote_name) + end end - - rugged.remotes.delete(remote_name) - true - rescue Rugged::ConfigError - false - end - - # Returns true if a remote exists. - def remote_exists?(name) - rugged.remotes[name].present? end # Update the specified remote using the values in the +options+ hash @@ -1274,6 +1273,24 @@ module Gitlab fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) end + def push_remote_branches(remote_name, branch_names, forced: true) + success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names) + + success || gitlab_projects_error + end + + def delete_remote_branches(remote_name, branch_names) + success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) + + success || gitlab_projects_error + end + + def delete_remote_branches(remote_name, branch_names) + success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) + + success || gitlab_projects_error + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end @@ -1298,6 +1315,14 @@ module Gitlab @gitaly_operation_client ||= Gitlab::GitalyClient::OperationService.new(self) end + def gitaly_remote_client + @gitaly_remote_client ||= Gitlab::GitalyClient::RemoteService.new(self) + end + + def gitaly_conflicts_client(our_commit_oid, their_commit_oid) + Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid) + end + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) Gitlab::GitalyClient.migrate(method, status: status, &block) rescue GRPC::NotFound => e @@ -1665,6 +1690,7 @@ module Gitlab cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] cmd << "--after=#{options[:after].iso8601}" if options[:after] cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd << "--max-count=#{options[:max_count]}" if options[:max_count] cmd += %W[--count #{options[:ref]}] cmd += %W[-- #{options[:path]}] if options[:path].present? @@ -1917,9 +1943,36 @@ module Gitlab raise ArgumentError, 'Invalid merge source' end + def rugged_add_remote(remote_name, url, mirror_refmap) + rugged.remotes.create(remote_name, url) + + set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap + rescue Rugged::ConfigError + remote_update(remote_name, url: url) + end + + def rugged_remove_remote(remote_name) + # When a remote is deleted all its remote refs are deleted too, but in + # the case of mirrors we map its refs (that would usualy go under + # [remote_name]/) to the top level namespace. We clean the mapping so + # those don't get deleted. + if rugged.config["remote.#{remote_name}.mirror"] + rugged.config.delete("remote.#{remote_name}.fetch") + end + + rugged.remotes.delete(remote_name) + true + rescue Rugged::ConfigError + false + end + def fetch_remote(remote_name = 'origin', env: nil) run_git(['fetch', remote_name], env: env).last.zero? end + + def gitlab_projects_error + raise CommandError, @gitlab_projects.output + end end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index b753ac46291..4507ea923b4 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -330,22 +330,6 @@ module Gitlab Google::Protobuf::Timestamp.new(seconds: t.to_i) end - def self.encode(s) - return "" if s.nil? - - s.dup.force_encoding(Encoding::ASCII_8BIT) - end - - def self.binary_stringio(s) - io = StringIO.new(s || '') - io.set_encoding(Encoding::ASCII_8BIT) - io - end - - def self.encode_repeated(a) - Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } ) - end - # The default timeout on all Gitaly calls def self.default_timeout return 0 if Sidekiq.server? diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index fb3e27770b4..fed05bb6c64 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class CommitService + include Gitlab::EncodingHelper + # The ID of empty tree. # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze @@ -13,7 +15,7 @@ module Gitlab def ls_files(revision) request = Gitaly::ListFilesRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout) @@ -73,7 +75,7 @@ module Gitlab request = Gitaly::TreeEntryRequest.new( repository: @gitaly_repo, revision: ref, - path: GitalyClient.encode(path), + path: encode_binary(path), limit: limit.to_i ) @@ -98,8 +100,8 @@ module Gitlab def tree_entries(repository, revision, path) request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision), - path: path.present? ? GitalyClient.encode(path) : '.' + revision: encode_binary(revision), + path: path.present? ? encode_binary(path) : '.' ) response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) @@ -112,8 +114,8 @@ module Gitlab type: gitaly_tree_entry.type.downcase, mode: gitaly_tree_entry.mode.to_s(8), name: File.basename(gitaly_tree_entry.path), - path: GitalyClient.encode(gitaly_tree_entry.path), - flat_path: GitalyClient.encode(gitaly_tree_entry.flat_path), + path: encode_binary(gitaly_tree_entry.path), + flat_path: encode_binary(gitaly_tree_entry.flat_path), commit_id: gitaly_tree_entry.commit_oid ) end @@ -128,6 +130,7 @@ module Gitlab request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? request.path = options[:path] if options[:path].present? + request.max_count = options[:max_count] if options[:max_count].present? GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end @@ -135,8 +138,8 @@ module Gitlab def last_commit_for_path(revision, path) request = Gitaly::LastCommitForPathRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision), - path: GitalyClient.encode(path.to_s) + revision: encode_binary(revision), + path: encode_binary(path.to_s) ) gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit @@ -202,8 +205,8 @@ module Gitlab def raw_blame(revision, path) request = Gitaly::RawBlameRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision), - path: GitalyClient.encode(path) + revision: encode_binary(revision), + path: encode_binary(path) ) response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) @@ -213,7 +216,7 @@ module Gitlab def find_commit(revision) request = Gitaly::FindCommitRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) @@ -224,7 +227,7 @@ module Gitlab def patch(revision) request = Gitaly::CommitPatchRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request, timeout: GitalyClient.medium_timeout) @@ -234,7 +237,7 @@ module Gitlab def commit_stats(revision) request = Gitaly::CommitStatsRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout) end @@ -250,9 +253,9 @@ module Gitlab ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] - request.revision = GitalyClient.encode(options[:ref]) if options[:ref] + request.revision = encode_binary(options[:ref]) if options[:ref] - request.paths = GitalyClient.encode_repeated(Array(options[:path])) if options[:path].present? + request.paths = encode_repeated(Array(options[:path])) if options[:path].present? response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) @@ -264,7 +267,7 @@ module Gitlab enum = Enumerator.new do |y| shas.each_slice(20) do |revs| - request.shas = GitalyClient.encode_repeated(revs) + request.shas = encode_repeated(revs) y.yield request @@ -303,7 +306,7 @@ module Gitlab repository: @gitaly_repo, left_commit_id: from_id, right_commit_id: to_id, - paths: options.fetch(:paths, []).compact.map { |path| GitalyClient.encode(path) } + paths: options.fetch(:paths, []).compact.map { |path| encode_binary(path) } } end @@ -314,6 +317,10 @@ module Gitlab end end end + + def encode_repeated(a) + Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| encode_binary(s) } ) + end end end end diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb new file mode 100644 index 00000000000..40f032cf873 --- /dev/null +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -0,0 +1,95 @@ +module Gitlab + module GitalyClient + class ConflictsService + include Gitlab::EncodingHelper + + MAX_MSG_SIZE = 128.kilobytes.freeze + + def initialize(repository, our_commit_oid, their_commit_oid) + @gitaly_repo = repository.gitaly_repository + @repository = repository + @our_commit_oid = our_commit_oid + @their_commit_oid = their_commit_oid + end + + def list_conflict_files + request = Gitaly::ListConflictFilesRequest.new( + repository: @gitaly_repo, + our_commit_oid: @our_commit_oid, + their_commit_oid: @their_commit_oid + ) + response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request) + + files_from_response(response).to_a + end + + def resolve_conflicts(target_repository, resolution, source_branch, target_branch) + reader = binary_stringio(resolution.files.to_json) + + req_enum = Enumerator.new do |y| + header = resolve_conflicts_request_header(target_repository, resolution, source_branch, target_branch) + y.yield Gitaly::ResolveConflictsRequest.new(header: header) + + until reader.eof? + chunk = reader.read(MAX_MSG_SIZE) + + y.yield Gitaly::ResolveConflictsRequest.new(files_json: chunk) + end + end + + response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage) + + if response.resolution_error.present? + raise Gitlab::Git::Conflict::Resolver::ResolutionError, response.resolution_error + end + end + + private + + def resolve_conflicts_request_header(target_repository, resolution, source_branch, target_branch) + Gitaly::ResolveConflictsRequestHeader.new( + repository: @gitaly_repo, + our_commit_oid: @our_commit_oid, + target_repository: target_repository.gitaly_repository, + their_commit_oid: @their_commit_oid, + source_branch: source_branch, + target_branch: target_branch, + commit_message: resolution.commit_message, + user: Gitlab::Git::User.from_gitlab(resolution.user).to_gitaly + ) + end + + def files_from_response(response) + files = [] + + response.each do |msg| + msg.files.each do |gitaly_file| + if gitaly_file.header + files << file_from_gitaly_header(gitaly_file.header) + else + files.last.content << gitaly_file.content + end + end + end + + files + end + + def file_from_gitaly_header(header) + Gitlab::Git::Conflict::File.new( + Gitlab::GitalyClient::Util.git_repository(header.repository), + header.commit_oid, + conflict_from_gitaly_file_header(header), + '' + ) + end + + def conflict_from_gitaly_file_header(header) + { + ours: { path: header.our_path, mode: header.our_mode }, + theirs: { path: header.their_path } + } + end + end + end +end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 400a4af363b..ae1753ff0ae 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class OperationService + include Gitlab::EncodingHelper + def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository @@ -9,7 +11,7 @@ module Gitlab def rm_tag(tag_name, user) request = Gitaly::UserDeleteTagRequest.new( repository: @gitaly_repo, - tag_name: GitalyClient.encode(tag_name), + tag_name: encode_binary(tag_name), user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) @@ -24,9 +26,9 @@ module Gitlab request = Gitaly::UserCreateTagRequest.new( repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - tag_name: GitalyClient.encode(tag_name), - target_revision: GitalyClient.encode(target), - message: GitalyClient.encode(message.to_s) + tag_name: encode_binary(tag_name), + target_revision: encode_binary(target), + message: encode_binary(message.to_s) ) response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request) @@ -44,9 +46,9 @@ module Gitlab def user_create_branch(branch_name, user, start_point) request = Gitaly::UserCreateBranchRequest.new( repository: @gitaly_repo, - branch_name: GitalyClient.encode(branch_name), + branch_name: encode_binary(branch_name), user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - start_point: GitalyClient.encode(start_point) + start_point: encode_binary(start_point) ) response = GitalyClient.call(@repository.storage, :operation_service, :user_create_branch, request) @@ -64,7 +66,7 @@ module Gitlab def user_delete_branch(branch_name, user) request = Gitaly::UserDeleteBranchRequest.new( repository: @gitaly_repo, - branch_name: GitalyClient.encode(branch_name), + branch_name: encode_binary(branch_name), user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) @@ -89,8 +91,8 @@ module Gitlab repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, - branch: GitalyClient.encode(target_branch), - message: GitalyClient.encode(message) + branch: encode_binary(target_branch), + message: encode_binary(message) ) ) @@ -99,6 +101,7 @@ module Gitlab request_enum.push(Gitaly::UserMergeBranchRequest.new(apply: true)) branch_update = response_enum.next.branch_update + return if branch_update.nil? raise Gitlab::Git::CommitError.new('failed to apply merge to branch') unless branch_update.commit_id.present? Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) @@ -111,7 +114,7 @@ module Gitlab repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, - branch: GitalyClient.encode(target_branch) + branch: encode_binary(target_branch) ) branch_update = GitalyClient.call( @@ -152,9 +155,9 @@ module Gitlab repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit: commit.to_gitaly_commit, - branch_name: GitalyClient.encode(branch_name), - message: GitalyClient.encode(message), - start_branch_name: GitalyClient.encode(start_branch_name.to_s), + branch_name: encode_binary(branch_name), + message: encode_binary(message), + start_branch_name: encode_binary(start_branch_name.to_s), start_repository: start_repository.gitaly_repository ) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 066e4e183c0..5bce1009878 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -72,7 +72,7 @@ module Gitlab end def ref_exists?(ref_name) - request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: GitalyClient.encode(ref_name)) + request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: encode_binary(ref_name)) response = GitalyClient.call(@storage, :ref_service, :ref_exists, request) response.value rescue GRPC::InvalidArgument => e @@ -82,7 +82,7 @@ module Gitlab def find_branch(branch_name) request = Gitaly::FindBranchRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(branch_name) + name: encode_binary(branch_name) ) response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request) @@ -96,8 +96,8 @@ module Gitlab def create_branch(ref, start_point) request = Gitaly::CreateBranchRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(ref), - start_point: GitalyClient.encode(start_point) + name: encode_binary(ref), + start_point: encode_binary(start_point) ) response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request) @@ -121,7 +121,7 @@ module Gitlab def delete_branch(branch_name) request = Gitaly::DeleteBranchRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(branch_name) + name: encode_binary(branch_name) ) GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request) diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb new file mode 100644 index 00000000000..9218f6cfd68 --- /dev/null +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -0,0 +1,28 @@ +module Gitlab + module GitalyClient + class RemoteService + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def add_remote(name, url, mirror_refmap) + request = Gitaly::AddRemoteRequest.new( + repository: @gitaly_repo, name: name, url: url, + mirror_refmap: mirror_refmap.to_s + ) + + GitalyClient.call(@storage, :remote_service, :add_remote, request) + end + + def remove_remote(name) + request = Gitaly::RemoveRemoteRequest.new(repository: @gitaly_repo, name: name) + + response = GitalyClient.call(@storage, :remote_service, :remove_remote, request) + + response.result + end + end + end +end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index c1f95396878..d43d80da960 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class RepositoryService + include Gitlab::EncodingHelper + def initialize(repository) @repository = repository @gitaly_repo = repository.gitaly_repository @@ -72,13 +74,29 @@ module Gitlab def find_merge_base(*revisions) request = Gitaly::FindMergeBaseRequest.new( repository: @gitaly_repo, - revisions: revisions.map { |r| GitalyClient.encode(r) } + revisions: revisions.map { |r| encode_binary(r) } ) response = GitalyClient.call(@storage, :repository_service, :find_merge_base, request) response.base.presence end + def fork_repository(source_repository) + request = Gitaly::CreateForkRequest.new( + repository: @gitaly_repo, + source_repository: source_repository.gitaly_repository + ) + + GitalyClient.call( + @storage, + :repository_service, + :create_fork, + request, + remote_storage: source_repository.storage, + timeout: GitalyClient.default_timeout + ) + end + def fetch_source_branch(source_repository, source_branch, local_ref) request = Gitaly::FetchSourceBranchRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index b1a033280b4..a8c6d478de8 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -12,12 +12,18 @@ module Gitlab Gitaly::Repository.new( storage_name: repository_storage, relative_path: relative_path, - gl_repository: gl_repository, + gl_repository: gl_repository.to_s, git_object_directory: git_object_directory.to_s, git_alternate_object_directories: git_alternate_object_directories ) end + def git_repository(gitaly_repository) + Gitlab::Git::Repository.new(gitaly_repository.storage_name, + gitaly_repository.relative_path, + gitaly_repository.gl_repository) + end + def gitlab_tag_from_gitaly_tag(repository, gitaly_tag) if gitaly_tag.target_commit.present? commit = Gitlab::Git::Commit.decorate(repository, gitaly_tag.target_commit) diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 337d225d081..5c5b170a3e0 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -3,6 +3,8 @@ require 'stringio' module Gitlab module GitalyClient class WikiService + include Gitlab::EncodingHelper + MAX_MSG_SIZE = 128.kilobytes.freeze def initialize(repository) @@ -13,12 +15,12 @@ module Gitlab def write_page(name, format, content, commit_details) request = Gitaly::WikiWritePageRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(name), + name: encode_binary(name), format: format.to_s, commit_details: gitaly_commit_details(commit_details) ) - strio = GitalyClient.binary_stringio(content) + strio = binary_stringio(content) enum = Enumerator.new do |y| until strio.eof? @@ -39,13 +41,13 @@ module Gitlab def update_page(page_path, title, format, content, commit_details) request = Gitaly::WikiUpdatePageRequest.new( repository: @gitaly_repo, - page_path: GitalyClient.encode(page_path), - title: GitalyClient.encode(title), + page_path: encode_binary(page_path), + title: encode_binary(title), format: format.to_s, commit_details: gitaly_commit_details(commit_details) ) - strio = GitalyClient.binary_stringio(content) + strio = binary_stringio(content) enum = Enumerator.new do |y| until strio.eof? @@ -63,7 +65,7 @@ module Gitlab def delete_page(page_path, commit_details) request = Gitaly::WikiDeletePageRequest.new( repository: @gitaly_repo, - page_path: GitalyClient.encode(page_path), + page_path: encode_binary(page_path), commit_details: gitaly_commit_details(commit_details) ) @@ -73,9 +75,9 @@ module Gitlab def find_page(title:, version: nil, dir: nil) request = Gitaly::WikiFindPageRequest.new( repository: @gitaly_repo, - title: GitalyClient.encode(title), - revision: GitalyClient.encode(version), - directory: GitalyClient.encode(dir) + title: encode_binary(title), + revision: encode_binary(version), + directory: encode_binary(dir) ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request) @@ -102,8 +104,8 @@ module Gitlab def find_file(name, revision) request = Gitaly::WikiFindFileRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(name), - revision: GitalyClient.encode(revision) + name: encode_binary(name), + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request) @@ -158,9 +160,9 @@ module Gitlab def gitaly_commit_details(commit_details) Gitaly::WikiCommitDetails.new( - name: GitalyClient.encode(commit_details.name), - email: GitalyClient.encode(commit_details.email), - message: GitalyClient.encode(commit_details.message) + name: encode_binary(commit_details.name), + email: encode_binary(commit_details.email), + message: encode_binary(commit_details.message) ) end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index dfcdfc307b6..9148d7571f2 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -21,6 +21,7 @@ module Gitlab gon.revision = Gitlab::REVISION gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') gon.sprite_icons = IconsHelper.sprite_icon_path + gon.sprite_file_icons = IconsHelper.sprite_file_icons_path if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index eeb03625479..60d5fa4d29a 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -7,6 +7,7 @@ module Gitlab module ImportSources ImportSource = Struct.new(:name, :title, :importer) + # We exclude `bare_repository` here as it has no import class associated ImportTable = [ ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index ebd7dc1b100..737081ddc5b 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -34,7 +34,7 @@ module Gitlab private def pod_resource(command) - Pod.new(command, @namespace.name, @kubeclient).generate + Gitlab::Kubernetes::Helm::Pod.new(command, @namespace.name, @kubeclient).generate end end end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index 329b07af5db..c2f9db56824 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -5,7 +5,7 @@ module Gitlab # Class for tracking timing information about method calls class MethodCall @@measurement_enabled_cache = Concurrent::AtomicBoolean.new(false) - @@measurement_enabled_cache_expires_at = Concurrent::AtomicFixnum.new(Time.now.to_i) + @@measurement_enabled_cache_expires_at = Concurrent::AtomicReference.new(Time.now.to_i) MUTEX = Mutex.new BASE_LABELS = { module: nil, method: nil }.freeze attr_reader :real_time, :cpu_time, :call_count, :labels diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 7037e2e61cc..ca48c6df602 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -82,7 +82,10 @@ module Gitlab end def issues - issues = IssuesFinder.new(current_user).execute.where(project_id: project_ids_relation) + issues = IssuesFinder.new(current_user).execute + unless default_project_filter + issues = issues.where(project_id: project_ids_relation) + end issues = if query =~ /#(\d+)\z/ diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 30df7e4a831..94a481a0f2e 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -8,7 +8,7 @@ end module Gitlab class Seeder def self.quiet - mute_mailer unless Rails.env.test? + mute_mailer SeedFu.quiet = true diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 9cdd3d22f18..40650fc5ee7 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -306,47 +306,6 @@ module Gitlab end end - # Push branch to remote repository - # - # storage - project's storage path - # project_name - project's disk path - # remote_name - remote name - # branch_names - remote branch names to push - # forced - should we use --force flag - # - # Ex. - # push_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test' 'upstream', ['feature']) - # - def push_remote_branches(storage, project_name, remote_name, branch_names, forced: true) - cmd = gitlab_projects(storage, "#{project_name}.git") - - success = cmd.push_branches(remote_name, git_timeout, forced, branch_names) - - raise Error, cmd.output unless success - - success - end - - # Delete branch from remote repository - # - # storage - project's storage path - # project_name - project's disk path - # remote_name - remote name - # branch_names - remote branch names - # - # Ex. - # delete_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test', 'upstream', ['feature']) - # - def delete_remote_branches(storage, project_name, remote_name, branch_names) - cmd = gitlab_projects(storage, "#{project_name}.git") - - success = cmd.delete_remote_branches(remote_name, branch_names) - - raise Error, cmd.output unless success - - success - end - protected def gitlab_shell_path diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 11472ce6cce..6ced06a863d 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -57,11 +57,17 @@ module Gitlab } end - def highest_allowed_level + def allowed_levels restricted_levels = current_application_settings.restricted_visibility_levels - allowed_levels = self.values - restricted_levels - allowed_levels.max || PRIVATE + self.values - restricted_levels + end + + def closest_allowed_level(target_level) + highest_allowed_level = allowed_levels.select { |level| level <= target_level }.max + + # If all levels are restricted, fall back to PRIVATE + highest_allowed_level || PRIVATE end def allowed_for?(user, level) |