diff options
20 files changed, 564 insertions, 78 deletions
@@ -391,7 +391,7 @@ gem 'vmstat', '~> 2.3.0' gem 'sys-filesystem', '~> 1.1.6' # Gitaly GRPC client -gem 'gitaly', '~> 0.21.0' +gem 'gitaly', '~> 0.23.0' gem 'toml-rb', '~> 0.3.15', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 5a327a14c4a..9f90965a567 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -269,7 +269,7 @@ GEM po_to_json (>= 1.0.0) rails (>= 3.2.0) gherkin-ruby (0.3.2) - gitaly (0.21.0) + gitaly (0.23.0) google-protobuf (~> 3.1) grpc (~> 1.0) github-linguist (4.7.6) @@ -978,7 +978,7 @@ DEPENDENCIES gettext (~> 3.2.2) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.2.0) - gitaly (~> 0.21.0) + gitaly (~> 0.23.0) github-linguist (~> 4.7.0) gitlab-flowdock-git-hook (~> 1.0.1) gitlab-markup (~> 1.5.1) diff --git a/app/assets/javascripts/commons/bootstrap.js b/app/assets/javascripts/commons/bootstrap.js index 510bedbf641..389587a2596 100644 --- a/app/assets/javascripts/commons/bootstrap.js +++ b/app/assets/javascripts/commons/bootstrap.js @@ -9,6 +9,7 @@ import 'bootstrap-sass/assets/javascripts/bootstrap/tab'; import 'bootstrap-sass/assets/javascripts/bootstrap/transition'; import 'bootstrap-sass/assets/javascripts/bootstrap/tooltip'; import 'bootstrap-sass/assets/javascripts/bootstrap/popover'; +import 'bootstrap-sass/assets/javascripts/bootstrap/button'; // custom jQuery functions $.fn.extend({ diff --git a/app/models/key.rb b/app/models/key.rb index cb8f10f6d55..49bc26122fa 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -16,8 +16,6 @@ class Key < ActiveRecord::Base presence: true, length: { maximum: 5000 }, format: { with: /\A(ssh|ecdsa)-.*\Z/ } - validates :key, - format: { without: /\n|\r/, message: 'should be a single line' } validates :fingerprint, uniqueness: true, presence: { message: 'cannot be generated' } @@ -31,6 +29,7 @@ class Key < ActiveRecord::Base after_destroy :post_destroy_hook def key=(value) + value&.delete!("\n\r") value.strip! unless value.blank? write_attribute(:key, value) end diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index ec87aee9310..d9d746ccf41 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -85,11 +85,7 @@ class MergeRequestDiff < ActiveRecord::Base def raw_diffs(options = {}) if options[:ignore_whitespace_change] - @diffs_no_whitespace ||= - Gitlab::Git::Compare.new( - repository.raw_repository, - safe_start_commit_sha, - head_commit_sha).diffs(options) + @diffs_no_whitespace ||= compare.diffs(options) else @raw_diffs ||= {} @raw_diffs[options] ||= load_diffs(options) diff --git a/changelogs/unreleased/fix-oauth-checkboxes.yml b/changelogs/unreleased/fix-oauth-checkboxes.yml new file mode 100644 index 00000000000..2839ccc42cb --- /dev/null +++ b/changelogs/unreleased/fix-oauth-checkboxes.yml @@ -0,0 +1,4 @@ +--- +title: Fixed sign-in restrictions buttons not toggling active state +merge_request: +author: diff --git a/changelogs/unreleased/reorganise-issues-indexes-for-sorting.yml b/changelogs/unreleased/reorganise-issues-indexes-for-sorting.yml new file mode 100644 index 00000000000..5bfe55e562f --- /dev/null +++ b/changelogs/unreleased/reorganise-issues-indexes-for-sorting.yml @@ -0,0 +1,4 @@ +--- +title: Re-organise "issues" indexes for faster ordering +merge_request: +author: diff --git a/db/migrate/20170803130232_reorganise_issues_indexes_for_faster_sorting.rb b/db/migrate/20170803130232_reorganise_issues_indexes_for_faster_sorting.rb new file mode 100644 index 00000000000..eb7d1be1732 --- /dev/null +++ b/db/migrate/20170803130232_reorganise_issues_indexes_for_faster_sorting.rb @@ -0,0 +1,43 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class ReorganiseIssuesIndexesForFasterSorting < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + REMOVE_INDEX_COLUMNS = %i[project_id created_at due_date updated_at].freeze + + ADD_INDEX_COLUMNS = [ + %i[project_id created_at id state], + %i[project_id due_date id state], + %i[project_id updated_at id state] + ].freeze + + TABLE = :issues + + def up + add_indexes(ADD_INDEX_COLUMNS) + remove_indexes(REMOVE_INDEX_COLUMNS) + end + + def down + add_indexes(REMOVE_INDEX_COLUMNS) + remove_indexes(ADD_INDEX_COLUMNS) + end + + def add_indexes(columns) + columns.each do |column| + add_concurrent_index(TABLE, column) unless index_exists?(TABLE, column) + end + end + + def remove_indexes(columns) + columns.each do |column| + remove_concurrent_index(TABLE, column) if index_exists?(TABLE, column) + end + end +end diff --git a/db/post_migrate/20170703130158_schedule_merge_request_diff_migrations.rb b/db/post_migrate/20170703130158_schedule_merge_request_diff_migrations.rb new file mode 100644 index 00000000000..17a9dc293f1 --- /dev/null +++ b/db/post_migrate/20170703130158_schedule_merge_request_diff_migrations.rb @@ -0,0 +1,33 @@ +class ScheduleMergeRequestDiffMigrations < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 2500 + MIGRATION = 'DeserializeMergeRequestDiffsAndCommits' + + disable_ddl_transaction! + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + include ::EachBatch + end + + # Assuming that there are 5 million rows affected (which is more than on + # GitLab.com), and that each batch of 2,500 rows takes up to 5 minutes, then + # we can migrate all the rows in 7 days. + # + # On staging, plucking the IDs themselves takes 5 seconds. + def up + non_empty = 'st_commits IS NOT NULL OR st_diffs IS NOT NULL' + + MergeRequestDiff.where(non_empty).each_batch(of: BATCH_SIZE) do |relation, index| + range = relation.pluck('MIN(id)', 'MAX(id)').first + + BackgroundMigrationWorker.perform_in(index * 5.minutes, MIGRATION, range) + end + end + + def down + end +end diff --git a/db/schema.rb b/db/schema.rb index 4ba218e1e0d..f2f35acef95 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170728101014) do +ActiveRecord::Schema.define(version: 20170803130232) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -641,12 +641,13 @@ ActiveRecord::Schema.define(version: 20170728101014) do add_index "issues", ["assignee_id"], name: "index_issues_on_assignee_id", using: :btree add_index "issues", ["author_id"], name: "index_issues_on_author_id", using: :btree add_index "issues", ["confidential"], name: "index_issues_on_confidential", using: :btree - add_index "issues", ["created_at"], name: "index_issues_on_created_at", using: :btree add_index "issues", ["deleted_at"], name: "index_issues_on_deleted_at", using: :btree add_index "issues", ["description"], name: "index_issues_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"} - add_index "issues", ["due_date"], name: "index_issues_on_due_date", using: :btree add_index "issues", ["milestone_id"], name: "index_issues_on_milestone_id", using: :btree + add_index "issues", ["project_id", "created_at", "id", "state"], name: "index_issues_on_project_id_and_created_at_and_id_and_state", using: :btree + add_index "issues", ["project_id", "due_date", "id", "state"], name: "index_issues_on_project_id_and_due_date_and_id_and_state", using: :btree add_index "issues", ["project_id", "iid"], name: "index_issues_on_project_id_and_iid", unique: true, using: :btree + add_index "issues", ["project_id", "updated_at", "id", "state"], name: "index_issues_on_project_id_and_updated_at_and_id_and_state", using: :btree add_index "issues", ["relative_position"], name: "index_issues_on_relative_position", using: :btree add_index "issues", ["state"], name: "index_issues_on_state", using: :btree add_index "issues", ["title"], name: "index_issues_on_title_trigram", using: :gin, opclasses: {"title"=>"gin_trgm_ops"} diff --git a/doc/administration/reply_by_email_postfix_setup.md b/doc/administration/reply_by_email_postfix_setup.md index 3b8c716eff5..a1bb3851951 100644 --- a/doc/administration/reply_by_email_postfix_setup.md +++ b/doc/administration/reply_by_email_postfix_setup.md @@ -177,6 +177,20 @@ Courier, which we will install later to add IMAP authentication, requires mailbo ```sh sudo apt-get install courier-imap ``` + + And start `imapd`: + ```sh + imapd start + ``` + +1. The courier-authdaemon isn't started after installation. Without it, imap authentication will fail: + ```sh + sudo service courier-authdaemon start + ``` + You can also configure courier-authdaemon to start on boot: + ```sh + sudo systemctl enable courier-authdaemon + ``` ## Configure Postfix to receive email from the internet diff --git a/doc/api/README.md b/doc/api/README.md index f63d395b10e..8acb2145f1a 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -77,6 +77,38 @@ controller-specific endpoints. GraphQL has a number of benefits: It will co-exist with the current v4 REST API. If we have a v5 API, this should be a compatibility layer on top of GraphQL. +## Basic usage + +API requests should be prefixed with `api` and the API version. The API version +is defined in [`lib/api.rb`][lib-api-url]. For example, the root of the v4 API +is at `/api/v4`. + +For endpoints that require [authentication](#authentication), you need to pass +a `private_token` parameter via query string or header. If passed as a header, +the header name must be `PRIVATE-TOKEN` (uppercase and with a dash instead of +an underscore). + +Example of a valid API request: + +``` +GET /projects?private_token=9koXpg98eAheJpvBs5tK +``` + +Example of a valid API request using cURL and authentication via header: + +```shell +curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects" +``` + +Example of a valid API request using cURL and authentication via a query string: + +```shell +curl "https://gitlab.example.com/api/v4/projects?private_token=9koXpg98eAheJpvBs5tK" +``` + +The API uses JSON to serialize data. You don't need to specify `.json` at the +end of an API URL. + ## Authentication Most API requests require authentication via a session cookie or token. For @@ -207,37 +239,6 @@ GET /projects?private_token=9koXpg98eAheJpvBs5tK&sudo=23 curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --header "SUDO: 23" "https://gitlab.example.com/api/v4/projects" ``` -## Basic usage - -API requests should be prefixed with `api` and the API version. The API version -is defined in [`lib/api.rb`][lib-api-url]. - -For endpoints that require [authentication](#authentication), you need to pass -a `private_token` parameter via query string or header. If passed as a header, -the header name must be `PRIVATE-TOKEN` (uppercase and with a dash instead of -an underscore). - -Example of a valid API request: - -``` -GET /projects?private_token=9koXpg98eAheJpvBs5tK -``` - -Example of a valid API request using cURL and authentication via header: - -```shell -curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects" -``` - -Example of a valid API request using cURL and authentication via a query string: - -```shell -curl "https://gitlab.example.com/api/v4/projects?private_token=9koXpg98eAheJpvBs5tK" -``` - -The API uses JSON to serialize data. You don't need to specify `.json` at the -end of an API URL. - ## Status codes The API is designed to return different status codes according to context and diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb new file mode 100644 index 00000000000..0fbc6b70989 --- /dev/null +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -0,0 +1,107 @@ +module Gitlab + module BackgroundMigration + class DeserializeMergeRequestDiffsAndCommits + attr_reader :diff_ids, :commit_rows, :file_rows + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + end + + BUFFER_ROWS = 1000 + + def perform(start_id, stop_id) + merge_request_diffs = MergeRequestDiff + .select(:id, :st_commits, :st_diffs) + .where('st_commits IS NOT NULL OR st_diffs IS NOT NULL') + .where(id: start_id..stop_id) + + reset_buffers! + + merge_request_diffs.each do |merge_request_diff| + commits, files = single_diff_rows(merge_request_diff) + + diff_ids << merge_request_diff.id + commit_rows.concat(commits) + file_rows.concat(files) + + if diff_ids.length > BUFFER_ROWS || + commit_rows.length > BUFFER_ROWS || + file_rows.length > BUFFER_ROWS + + flush_buffers! + end + end + + flush_buffers! + end + + private + + def reset_buffers! + @diff_ids = [] + @commit_rows = [] + @file_rows = [] + end + + def flush_buffers! + if diff_ids.any? + MergeRequestDiff.transaction do + Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows) + Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows) + + MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil) + end + end + + reset_buffers! + end + + def single_diff_rows(merge_request_diff) + sha_attribute = Gitlab::Database::ShaAttribute.new + commits = YAML.load(merge_request_diff.st_commits) rescue [] + + commit_rows = commits.map.with_index do |commit, index| + commit_hash = commit.to_hash.with_indifferent_access.except(:parent_ids) + sha = commit_hash.delete(:id) + + commit_hash.merge( + merge_request_diff_id: merge_request_diff.id, + relative_order: index, + sha: sha_attribute.type_cast_for_database(sha) + ) + end + + diffs = YAML.load(merge_request_diff.st_diffs) rescue [] + diffs = [] unless valid_raw_diffs?(diffs) + + file_rows = diffs.map.with_index do |diff, index| + diff_hash = diff.to_hash.with_indifferent_access.merge( + binary: false, + merge_request_diff_id: merge_request_diff.id, + relative_order: index + ) + + # Compatibility with old diffs created with Psych. + diff_hash.tap do |hash| + diff_text = hash[:diff] + + if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only? + hash[:binary] = true + hash[:diff] = [diff_text].pack('m0') + end + end + end + + [commit_rows, file_rows] + end + + # Unlike MergeRequestDiff#valid_raw_diff?, don't count Rugged objects as + # valid, because we don't render them usefully anyway. + def valid_raw_diffs?(diffs) + return false unless diffs.respond_to?(:each) + + diffs.all? { |diff| diff.is_a?(Hash) } + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 70d793f8e4a..ffe2c8b91bb 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -300,17 +300,14 @@ module Gitlab raw_log(options).map { |c| Commit.decorate(c) } end - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382 def count_commits(options) - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd += %W[--count #{options[:ref]}] - cmd += %W[-- #{options[:path]}] if options[:path].present? - - raw_output = IO.popen(cmd) { |io| io.read } - - raw_output.to_i + gitaly_migrate(:count_commits) do |is_enabled| + if is_enabled + count_commits_by_gitaly(options) + else + count_commits_by_shelling_out(options) + end + end end def sha_from_ref(ref) @@ -1005,6 +1002,22 @@ module Gitlab gitaly_ref_client.tags end + def count_commits_by_gitaly(options) + gitaly_commit_client.commit_count(options[:ref], options) + end + + def count_commits_by_shelling_out(options) + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd += %W[--count #{options[:ref]}] + cmd += %W[-- #{options[:path]}] if options[:path].present? + + raw_output = IO.popen(cmd) { |io| io.read } + + raw_output.to_i + end + def gitaly_migrate(method, &block) Gitlab::GitalyClient.migrate(method, &block) rescue GRPC::NotFound => e diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index a834781b1f1..b1424a458e9 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -85,11 +85,14 @@ module Gitlab end end - def commit_count(ref) + def commit_count(ref, options = {}) request = Gitaly::CountCommitsRequest.new( repository: @gitaly_repo, revision: ref ) + request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? + request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? + request.path = options[:path] if options[:path].present? GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count end diff --git a/spec/features/admin/admin_settings_spec.rb b/spec/features/admin/admin_settings_spec.rb index c1eced417cf..c9591a7d854 100644 --- a/spec/features/admin/admin_settings_spec.rb +++ b/spec/features/admin/admin_settings_spec.rb @@ -69,6 +69,14 @@ feature 'Admin updates settings' do expect(find('#service_push_channel').value).to eq '#test_channel' end + context 'sign-in restrictions', :js do + it 'de-activates oauth sign-in source' do + find('.btn', text: 'GitLab.com').click + + expect(find('.btn', text: 'GitLab.com')).not_to have_css('.active') + end + end + def check_all_events page.check('Active') page.check('Push') diff --git a/spec/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits_spec.rb b/spec/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits_spec.rb new file mode 100644 index 00000000000..18843cbe992 --- /dev/null +++ b/spec/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits_spec.rb @@ -0,0 +1,188 @@ +require 'spec_helper' + +describe Gitlab::BackgroundMigration::DeserializeMergeRequestDiffsAndCommits do + describe '#perform' do + set(:merge_request) { create(:merge_request) } + set(:merge_request_diff) { merge_request.merge_request_diff } + let(:updated_merge_request_diff) { MergeRequestDiff.find(merge_request_diff.id) } + + def diffs_to_hashes(diffs) + diffs.as_json(only: Gitlab::Git::Diff::SERIALIZE_KEYS).map(&:with_indifferent_access) + end + + def quote_yaml(value) + MergeRequestDiff.connection.quote(YAML.dump(value)) + end + + def convert_to_yaml(merge_request_diff_id, commits, diffs) + MergeRequestDiff.where(id: merge_request_diff_id).update_all( + "st_commits = #{quote_yaml(commits)}, st_diffs = #{quote_yaml(diffs)}" + ) + end + + shared_examples 'updated MR diff' do + before do + convert_to_yaml(merge_request_diff.id, commits, diffs) + + MergeRequestDiffCommit.delete_all + MergeRequestDiffFile.delete_all + + subject.perform(merge_request_diff.id, merge_request_diff.id) + end + + it 'creates correct entries in the merge_request_diff_commits table' do + expect(updated_merge_request_diff.merge_request_diff_commits.count).to eq(commits.count) + expect(updated_merge_request_diff.commits.map(&:to_hash)).to eq(commits) + end + + it 'creates correct entries in the merge_request_diff_files table' do + expect(updated_merge_request_diff.merge_request_diff_files.count).to eq(expected_diffs.count) + expect(diffs_to_hashes(updated_merge_request_diff.raw_diffs)).to eq(expected_diffs) + end + + it 'sets the st_commits and st_diffs columns to nil' do + expect(updated_merge_request_diff.st_commits_before_type_cast).to be_nil + expect(updated_merge_request_diff.st_diffs_before_type_cast).to be_nil + end + end + + context 'when the diff IDs passed do not exist' do + it 'does not raise' do + expect { subject.perform(0, 0) }.not_to raise_exception + end + end + + context 'when the merge request diff has no serialised commits or diffs' do + before do + merge_request_diff.update(st_commits: nil, st_diffs: nil) + end + + it 'does not raise' do + expect { subject.perform(merge_request_diff.id, merge_request_diff.id) } + .not_to raise_exception + end + end + + context 'processing multiple merge request diffs' do + let(:start_id) { described_class::MergeRequestDiff.minimum(:id) } + let(:stop_id) { described_class::MergeRequestDiff.maximum(:id) } + + before do + merge_request.reload_diff(true) + + convert_to_yaml(start_id, merge_request_diff.commits, merge_request_diff.diffs) + convert_to_yaml(stop_id, updated_merge_request_diff.commits, updated_merge_request_diff.diffs) + + MergeRequestDiffCommit.delete_all + MergeRequestDiffFile.delete_all + end + + context 'when BUFFER_ROWS is exceeded' do + before do + stub_const("#{described_class}::BUFFER_ROWS", 1) + end + + it 'updates and continues' do + expect(described_class::MergeRequestDiff).to receive(:transaction).twice + + subject.perform(start_id, stop_id) + end + end + + context 'when BUFFER_ROWS is not exceeded' do + it 'only updates once' do + expect(described_class::MergeRequestDiff).to receive(:transaction).once + + subject.perform(start_id, stop_id) + end + end + end + + context 'when the merge request diff update fails' do + before do + allow(described_class::MergeRequestDiff) + .to receive(:update_all).and_raise(ActiveRecord::Rollback) + end + + it 'does not add any diff commits' do + expect { subject.perform(merge_request_diff.id, merge_request_diff.id) } + .not_to change { MergeRequestDiffCommit.count } + end + + it 'does not add any diff files' do + expect { subject.perform(merge_request_diff.id, merge_request_diff.id) } + .not_to change { MergeRequestDiffFile.count } + end + end + + context 'when the merge request diff has valid commits and diffs' do + let(:commits) { merge_request_diff.commits.map(&:to_hash) } + let(:diffs) { diffs_to_hashes(merge_request_diff.merge_request_diff_files) } + let(:expected_diffs) { diffs } + + include_examples 'updated MR diff' + end + + context 'when the merge request diffs have binary content' do + let(:commits) { merge_request_diff.commits.map(&:to_hash) } + let(:expected_diffs) { diffs } + + # The start of a PDF created by Illustrator + let(:binary_string) do + "\x25\x50\x44\x46\x2d\x31\x2e\x35\x0d\x25\xe2\xe3\xcf\xd3\x0d\x0a".force_encoding(Encoding::BINARY) + end + + let(:diffs) do + [ + { + 'diff' => binary_string, + 'new_path' => 'path', + 'old_path' => 'path', + 'a_mode' => '100644', + 'b_mode' => '100644', + 'new_file' => false, + 'renamed_file' => false, + 'deleted_file' => false, + 'too_large' => false + } + ] + end + + include_examples 'updated MR diff' + end + + context 'when the merge request diff has commits, but no diffs' do + let(:commits) { merge_request_diff.commits.map(&:to_hash) } + let(:diffs) { [] } + let(:expected_diffs) { diffs } + + include_examples 'updated MR diff' + end + + context 'when the merge request diffs have invalid content' do + let(:commits) { merge_request_diff.commits.map(&:to_hash) } + let(:diffs) { ['--broken-diff'] } + let(:expected_diffs) { [] } + + include_examples 'updated MR diff' + end + + context 'when the merge request diffs are Rugged::Patch instances' do + let(:commits) { merge_request_diff.commits.map(&:to_hash) } + let(:first_commit) { merge_request.project.repository.commit(merge_request_diff.head_commit_sha) } + let(:diffs) { first_commit.diff_from_parent.patches } + let(:expected_diffs) { [] } + + include_examples 'updated MR diff' + end + + context 'when the merge request diffs are Rugged::Diff::Delta instances' do + let(:commits) { merge_request_diff.commits.map(&:to_hash) } + let(:first_commit) { merge_request.project.repository.commit(merge_request_diff.head_commit_sha) } + let(:diffs) { first_commit.diff_from_parent.deltas } + let(:expected_diffs) { [] } + + include_examples 'updated MR diff' + end + end +end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 8e4a1f31ced..9bfad0c9bdf 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -361,20 +361,20 @@ describe Gitlab::Git::Repository, seed_helper: true do end describe '#commit_count' do - shared_examples 'counting commits' do + shared_examples 'simple commit counting' do it { expect(repository.commit_count("master")).to eq(25) } it { expect(repository.commit_count("feature")).to eq(9) } end context 'when Gitaly commit_count feature is enabled' do - it_behaves_like 'counting commits' + it_behaves_like 'simple commit counting' it_behaves_like 'wrapping gRPC errors', Gitlab::GitalyClient::CommitService, :commit_count do subject { repository.commit_count('master') } end end context 'when Gitaly commit_count feature is disabled', skip_gitaly_mock: true do - it_behaves_like 'counting commits' + it_behaves_like 'simple commit counting' end end @@ -797,29 +797,39 @@ describe Gitlab::Git::Repository, seed_helper: true do end describe '#count_commits' do - context 'with after timestamp' do - it 'returns the number of commits after timestamp' do - options = { ref: 'master', limit: nil, after: Time.iso8601('2013-03-03T20:15:01+00:00') } + shared_examples 'extended commit counting' do + context 'with after timestamp' do + it 'returns the number of commits after timestamp' do + options = { ref: 'master', limit: nil, after: Time.iso8601('2013-03-03T20:15:01+00:00') } - expect(repository.count_commits(options)).to eq(25) + expect(repository.count_commits(options)).to eq(25) + end end - end - context 'with before timestamp' do - it 'returns the number of commits after timestamp' do - options = { ref: 'feature', limit: nil, before: Time.iso8601('2015-03-03T20:15:01+00:00') } + context 'with before timestamp' do + it 'returns the number of commits before timestamp' do + options = { ref: 'feature', limit: nil, before: Time.iso8601('2015-03-03T20:15:01+00:00') } - expect(repository.count_commits(options)).to eq(9) + expect(repository.count_commits(options)).to eq(9) + end end - end - context 'with path' do - it 'returns the number of commits with path ' do - options = { ref: 'master', limit: nil, path: "encoding" } + context 'with path' do + it 'returns the number of commits with path ' do + options = { ref: 'master', limit: nil, path: "encoding" } - expect(repository.count_commits(options)).to eq(2) + expect(repository.count_commits(options)).to eq(2) + end end end + + context 'when Gitaly count_commits feature is enabled' do + it_behaves_like 'extended commit counting' + end + + context 'when Gitaly count_commits feature is disabled', skip_gitaly_mock: true do + it_behaves_like 'extended commit counting' + end end describe "branch_names_contains" do diff --git a/spec/migrations/schedule_merge_request_diff_migrations_spec.rb b/spec/migrations/schedule_merge_request_diff_migrations_spec.rb new file mode 100644 index 00000000000..f95bd6e3511 --- /dev/null +++ b/spec/migrations/schedule_merge_request_diff_migrations_spec.rb @@ -0,0 +1,59 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20170703130158_schedule_merge_request_diff_migrations') + +describe ScheduleMergeRequestDiffMigrations, :migration, :sidekiq do + matcher :be_scheduled_migration do |time, *expected| + match do |migration| + BackgroundMigrationWorker.jobs.any? do |job| + job['args'] == [migration, expected] && + job['at'].to_i == time.to_i + end + end + + failure_message do |migration| + "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!" + end + end + + let(:merge_request_diffs) { table(:merge_request_diffs) } + let(:merge_requests) { table(:merge_requests) } + let(:projects) { table(:projects) } + + before do + stub_const("#{described_class.name}::BATCH_SIZE", 1) + + projects.create!(id: 1, name: 'gitlab', path: 'gitlab') + + merge_requests.create!(id: 1, target_project_id: 1, source_project_id: 1, target_branch: 'feature', source_branch: 'master') + + merge_request_diffs.create!(id: 1, merge_request_id: 1, st_commits: YAML.dump([]), st_diffs: nil) + merge_request_diffs.create!(id: 2, merge_request_id: 1, st_commits: nil, st_diffs: YAML.dump([])) + merge_request_diffs.create!(id: 3, merge_request_id: 1, st_commits: nil, st_diffs: nil) + merge_request_diffs.create!(id: 4, merge_request_id: 1, st_commits: YAML.dump([]), st_diffs: YAML.dump([])) + end + + it 'correctly schedules background migrations' do + Sidekiq::Testing.fake! do + Timecop.freeze do + migrate! + + expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes.from_now, 1, 1) + expect(described_class::MIGRATION).to be_scheduled_migration(10.minutes.from_now, 2, 2) + expect(described_class::MIGRATION).to be_scheduled_migration(15.minutes.from_now, 4, 4) + expect(BackgroundMigrationWorker.jobs.size).to eq 3 + end + end + end + + it 'schedules background migrations' do + Sidekiq::Testing.inline! do + non_empty = 'st_commits IS NOT NULL OR st_diffs IS NOT NULL' + + expect(merge_request_diffs.where(non_empty).count).to eq 3 + + migrate! + + expect(merge_request_diffs.where(non_empty).count).to eq 0 + end + end +end diff --git a/spec/models/key_spec.rb b/spec/models/key_spec.rb index 882522c0a97..0daeb337168 100644 --- a/spec/models/key_spec.rb +++ b/spec/models/key_spec.rb @@ -92,15 +92,17 @@ describe Key, :mailer do expect(key).not_to be_valid end - it 'rejects the unfingerprintable key (not a key)' do - expect(build(:key, key: 'ssh-rsa an-invalid-key==')).not_to be_valid + it 'accepts a key with newline charecters after stripping them' do + key = build(:key) + key.key = key.key.insert(100, "\n") + key.key = key.key.insert(40, "\r\n") + expect(key).to be_valid end - it 'rejects the multiple line key' do - key = build(:key) - key.key.tr!(' ', "\n") - expect(key).not_to be_valid + it 'rejects the unfingerprintable key (not a key)' do + expect(build(:key, key: 'ssh-rsa an-invalid-key==')).not_to be_valid end + end context 'callbacks' do |