diff options
Diffstat (limited to 'db')
32 files changed, 743 insertions, 20 deletions
diff --git a/db/fixtures/development/17_cycle_analytics.rb b/db/fixtures/development/17_cycle_analytics.rb index d7be6f5950f..7b9a4bad449 100644 --- a/db/fixtures/development/17_cycle_analytics.rb +++ b/db/fixtures/development/17_cycle_analytics.rb @@ -1,5 +1,5 @@ require './spec/support/sidekiq' -require './spec/support/test_env' +require './spec/support/helpers/test_env' class Gitlab::Seeder::CycleAnalytics def initialize(project, perf: false) diff --git a/db/migrate/20161220141214_remove_dot_git_from_group_names.rb b/db/migrate/20161220141214_remove_dot_git_from_group_names.rb index bddc234db25..17357b67ab7 100644 --- a/db/migrate/20161220141214_remove_dot_git_from_group_names.rb +++ b/db/migrate/20161220141214_remove_dot_git_from_group_names.rb @@ -59,17 +59,17 @@ class RemoveDotGitFromGroupNames < ActiveRecord::Migration end def move_namespace(group_id, path_was, path) - repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row| - Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path + repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row| + row['repository_storage'] end.compact # Move the namespace directory in all storages paths used by member projects - repository_storage_paths.each do |repository_storage_path| + repository_storages.each do |repository_storage| # Ensure old directory exists before moving it - gitlab_shell.add_namespace(repository_storage_path, path_was) + gitlab_shell.add_namespace(repository_storage, path_was) - unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path) - Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}" + unless gitlab_shell.mv_namespace(repository_storage, path_was, path) + Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}" # if we cannot move namespace directory we should rollback # db changes in order to prevent out of sync between db and fs diff --git a/db/migrate/20161226122833_remove_dot_git_from_usernames.rb b/db/migrate/20161226122833_remove_dot_git_from_usernames.rb index 7c28d934c29..8986cd8cb4b 100644 --- a/db/migrate/20161226122833_remove_dot_git_from_usernames.rb +++ b/db/migrate/20161226122833_remove_dot_git_from_usernames.rb @@ -53,8 +53,8 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration select_all("SELECT id, path FROM routes WHERE path = '#{quote_string(path)}'").present? end - def path_exists?(path, repository_storage_path) - repository_storage_path && gitlab_shell.exists?(repository_storage_path, path) + def path_exists?(shard, repository_storage_path) + repository_storage_path && gitlab_shell.exists?(shard, repository_storage_path) end # Accepts invalid path like test.git and returns test_git or @@ -70,8 +70,8 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration def check_routes(base, counter, path) route_exists = route_exists?(path) - Gitlab.config.repositories.storages.each_value do |storage| - if route_exists || path_exists?(path, storage.legacy_disk_path) + Gitlab.config.repositories.storages.each do |shard, storage| + if route_exists || path_exists?(shard, storage.legacy_disk_path) counter += 1 path = "#{base}#{counter}" @@ -83,17 +83,17 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration end def move_namespace(namespace_id, path_was, path) - repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row| - Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path + repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row| + row['repository_storage'] end.compact - # Move the namespace directory in all storages paths used by member projects - repository_storage_paths.each do |repository_storage_path| + # Move the namespace directory in all storages used by member projects + repository_storages.each do |repository_storage| # Ensure old directory exists before moving it - gitlab_shell.add_namespace(repository_storage_path, path_was) + gitlab_shell.add_namespace(repository_storage, path_was) - unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path) - Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}" + unless gitlab_shell.mv_namespace(repository_storage, path_was, path) + Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}" # if we cannot move namespace directory we should rollback # db changes in order to prevent out of sync between db and fs diff --git a/db/migrate/20170301101006_add_ci_runner_namespaces.rb b/db/migrate/20170301101006_add_ci_runner_namespaces.rb new file mode 100644 index 00000000000..deaf03e928b --- /dev/null +++ b/db/migrate/20170301101006_add_ci_runner_namespaces.rb @@ -0,0 +1,17 @@ +class AddCiRunnerNamespaces < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + create_table :ci_runner_namespaces do |t| + t.integer :runner_id + t.integer :namespace_id + + t.index [:runner_id, :namespace_id], unique: true + t.index :namespace_id + t.foreign_key :ci_runners, column: :runner_id, on_delete: :cascade + t.foreign_key :namespaces, column: :namespace_id, on_delete: :cascade + end + end +end diff --git a/db/migrate/20170906133745_add_runners_token_to_groups.rb b/db/migrate/20170906133745_add_runners_token_to_groups.rb new file mode 100644 index 00000000000..852f4cba670 --- /dev/null +++ b/db/migrate/20170906133745_add_runners_token_to_groups.rb @@ -0,0 +1,9 @@ +class AddRunnersTokenToGroups < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :namespaces, :runners_token, :string + end +end diff --git a/db/migrate/20180326202229_create_ci_build_trace_chunks.rb b/db/migrate/20180326202229_create_ci_build_trace_chunks.rb new file mode 100644 index 00000000000..fb3f5786e85 --- /dev/null +++ b/db/migrate/20180326202229_create_ci_build_trace_chunks.rb @@ -0,0 +1,17 @@ +class CreateCiBuildTraceChunks < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + create_table :ci_build_trace_chunks, id: :bigserial do |t| + t.integer :build_id, null: false + t.integer :chunk_index, null: false + t.integer :data_store, null: false + t.binary :raw_data + + t.foreign_key :ci_builds, column: :build_id, on_delete: :cascade + t.index [:build_id, :chunk_index], unique: true + end + end +end diff --git a/db/migrate/20180403035759_create_project_ci_cd_settings.rb b/db/migrate/20180403035759_create_project_ci_cd_settings.rb new file mode 100644 index 00000000000..06856af6204 --- /dev/null +++ b/db/migrate/20180403035759_create_project_ci_cd_settings.rb @@ -0,0 +1,68 @@ +class CreateProjectCiCdSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + unless table_exists?(:project_ci_cd_settings) + create_table(:project_ci_cd_settings) do |t| + t.integer(:project_id, null: false) + t.boolean(:group_runners_enabled, default: true, null: false) + end + end + + disable_statement_timeout + + # This particular INSERT will take between 10 and 20 seconds. + execute 'INSERT INTO project_ci_cd_settings (project_id) SELECT id FROM projects' + + # We add the index and foreign key separately so the above INSERT statement + # takes as little time as possible. + add_concurrent_index(:project_ci_cd_settings, :project_id, unique: true) + + add_foreign_key_with_retry + end + + def down + drop_table :project_ci_cd_settings + end + + def add_foreign_key_with_retry + if Gitlab::Database.mysql? + # When using MySQL we don't support online upgrades, thus projects can't + # be deleted while we are running this migration. + return add_project_id_foreign_key + end + + # Between the initial INSERT and the addition of the foreign key some + # projects may have been removed, leaving orphaned rows in our new settings + # table. + loop do + remove_orphaned_settings + + begin + add_project_id_foreign_key + break + rescue ActiveRecord::InvalidForeignKey + say 'project_ci_cd_settings contains some orphaned rows, retrying...' + end + end + end + + def add_project_id_foreign_key + add_concurrent_foreign_key(:project_ci_cd_settings, :projects, column: :project_id) + end + + def remove_orphaned_settings + execute <<~SQL + DELETE FROM project_ci_cd_settings + WHERE NOT EXISTS ( + SELECT 1 + FROM projects + WHERE projects.id = project_ci_cd_settings.project_id + ) + SQL + end +end diff --git a/db/migrate/20180406204716_add_limits_ci_build_trace_chunks_raw_data_for_mysql.rb b/db/migrate/20180406204716_add_limits_ci_build_trace_chunks_raw_data_for_mysql.rb new file mode 100644 index 00000000000..0f2734853e6 --- /dev/null +++ b/db/migrate/20180406204716_add_limits_ci_build_trace_chunks_raw_data_for_mysql.rb @@ -0,0 +1,13 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. +require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql') + +class AddLimitsCiBuildTraceChunksRawDataForMysql < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + LimitsCiBuildTraceChunksRawDataForMysql.new.up + end +end diff --git a/db/migrate/20180413022611_create_missing_namespace_for_internal_users.rb b/db/migrate/20180413022611_create_missing_namespace_for_internal_users.rb new file mode 100644 index 00000000000..8fc558be733 --- /dev/null +++ b/db/migrate/20180413022611_create_missing_namespace_for_internal_users.rb @@ -0,0 +1,66 @@ +class CreateMissingNamespaceForInternalUsers < ActiveRecord::Migration + DOWNTIME = false + + def up + connection.exec_query(users_query.to_sql).rows.each do |id, username| + create_namespace(id, username) + # When testing locally I've noticed that these internal users are missing + # the notification email, for more details visit the below link: + # https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18357#note_68327560 + set_notification_email(id) + end + end + + def down + # no-op + end + + private + + def users + @users ||= Arel::Table.new(:users) + end + + def namespaces + @namespaces ||= Arel::Table.new(:namespaces) + end + + def users_query + condition = users[:ghost].eq(true) + + if column_exists?(:users, :support_bot) + condition = condition.or(users[:support_bot].eq(true)) + end + + users.join(namespaces, Arel::Nodes::OuterJoin) + .on(namespaces[:type].eq(nil).and(namespaces[:owner_id].eq(users[:id]))) + .where(namespaces[:owner_id].eq(nil)) + .where(condition) + .project(users[:id], users[:username]) + end + + def create_namespace(user_id, username) + path = Uniquify.new.string(username) do |str| + query = "SELECT id FROM namespaces WHERE parent_id IS NULL AND path='#{str}' LIMIT 1" + connection.exec_query(query).present? + end + + insert_query = "INSERT INTO namespaces(owner_id, path, name) VALUES(#{user_id}, '#{path}', '#{path}')" + namespace_id = connection.insert_sql(insert_query) + + create_route(namespace_id) + end + + def create_route(namespace_id) + return unless namespace_id + + row = connection.exec_query("SELECT id, path FROM namespaces WHERE id=#{namespace_id}").first + id, path = row.values_at('id', 'path') + + execute("INSERT INTO routes(source_id, source_type, path, name) VALUES(#{id}, 'Namespace', '#{path}', '#{path}')") + end + + def set_notification_email(user_id) + execute "UPDATE users SET notification_email = email WHERE notification_email IS NULL AND id = #{user_id}" + end +end diff --git a/db/migrate/20180417101040_add_tmp_stage_priority_index_to_ci_builds.rb b/db/migrate/20180417101040_add_tmp_stage_priority_index_to_ci_builds.rb new file mode 100644 index 00000000000..ee82c70ecf8 --- /dev/null +++ b/db/migrate/20180417101040_add_tmp_stage_priority_index_to_ci_builds.rb @@ -0,0 +1,16 @@ +class AddTmpStagePriorityIndexToCiBuilds < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index(:ci_builds, [:stage_id, :stage_idx], + where: 'stage_idx IS NOT NULL', name: 'tmp_build_stage_position_index') + end + + def down + remove_concurrent_index_by_name(:ci_builds, 'tmp_build_stage_position_index') + end +end diff --git a/db/migrate/20180417101940_add_index_to_ci_stage.rb b/db/migrate/20180417101940_add_index_to_ci_stage.rb new file mode 100644 index 00000000000..9dac78db774 --- /dev/null +++ b/db/migrate/20180417101940_add_index_to_ci_stage.rb @@ -0,0 +1,9 @@ +class AddIndexToCiStage < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :ci_stages, :position, :integer + end +end diff --git a/db/migrate/20180420010616_cleanup_build_stage_migration.rb b/db/migrate/20180420010616_cleanup_build_stage_migration.rb new file mode 100644 index 00000000000..0342695ec3d --- /dev/null +++ b/db/migrate/20180420010616_cleanup_build_stage_migration.rb @@ -0,0 +1,28 @@ +class CleanupBuildStageMigration < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class Build < ActiveRecord::Base + include EachBatch + + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled + end + + def up + Gitlab::BackgroundMigration.steal('MigrateBuildStage') + + Build.where('stage_id IS NULL').each_batch(of: 50) do |batch| + range = batch.pluck('MIN(id)', 'MAX(id)').first + + Gitlab::BackgroundMigration::MigrateBuildStage.new.perform(*range) + end + end + + def down + # noop + end +end diff --git a/db/migrate/20180424090541_add_enforce_terms_to_application_settings.rb b/db/migrate/20180424090541_add_enforce_terms_to_application_settings.rb new file mode 100644 index 00000000000..306cd737771 --- /dev/null +++ b/db/migrate/20180424090541_add_enforce_terms_to_application_settings.rb @@ -0,0 +1,9 @@ +class AddEnforceTermsToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :application_settings, :enforce_terms, :boolean, default: false + end +end diff --git a/db/migrate/20180424134533_create_application_setting_terms.rb b/db/migrate/20180424134533_create_application_setting_terms.rb new file mode 100644 index 00000000000..f29335cfc51 --- /dev/null +++ b/db/migrate/20180424134533_create_application_setting_terms.rb @@ -0,0 +1,13 @@ +class CreateApplicationSettingTerms < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + create_table :application_setting_terms do |t| + t.integer :cached_markdown_version + t.text :terms, null: false + t.text :terms_html + end + end +end diff --git a/db/migrate/20180425075446_create_term_agreements.rb b/db/migrate/20180425075446_create_term_agreements.rb new file mode 100644 index 00000000000..22a9d7b574d --- /dev/null +++ b/db/migrate/20180425075446_create_term_agreements.rb @@ -0,0 +1,28 @@ +class CreateTermAgreements < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + create_table :term_agreements do |t| + t.references :term, index: true, null: false + t.foreign_key :application_setting_terms, column: :term_id + t.references :user, index: true, null: false, foreign_key: { on_delete: :cascade } + t.boolean :accepted, default: false, null: false + + t.timestamps_with_timezone null: false + end + + add_index :term_agreements, [:user_id, :term_id], + unique: true, + name: 'term_agreements_unique_index' + end + + def down + remove_index :term_agreements, name: 'term_agreements_unique_index' + + drop_table :term_agreements + end +end diff --git a/db/migrate/20180425131009_assure_commits_count_for_merge_request_diff.rb b/db/migrate/20180425131009_assure_commits_count_for_merge_request_diff.rb new file mode 100644 index 00000000000..0e991c23bfa --- /dev/null +++ b/db/migrate/20180425131009_assure_commits_count_for_merge_request_diff.rb @@ -0,0 +1,27 @@ +class AssureCommitsCountForMergeRequestDiff < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + include ::EachBatch + end + + def up + Gitlab::BackgroundMigration.steal('AddMergeRequestDiffCommitsCount') + + MergeRequestDiff.where(commits_count: nil).each_batch(of: 50) do |batch| + range = batch.pluck('MIN(id)', 'MAX(id)').first + + Gitlab::BackgroundMigration::AddMergeRequestDiffCommitsCount.new.perform(*range) + end + end + + def down + # noop + end +end diff --git a/db/migrate/20180426102016_add_accepted_term_to_users.rb b/db/migrate/20180426102016_add_accepted_term_to_users.rb new file mode 100644 index 00000000000..3d446f66214 --- /dev/null +++ b/db/migrate/20180426102016_add_accepted_term_to_users.rb @@ -0,0 +1,23 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddAcceptedTermToUsers < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + change_table :users do |t| + t.references :accepted_term, + null: true + end + add_concurrent_foreign_key :users, :application_setting_terms, column: :accepted_term_id + end + + def down + remove_foreign_key :users, column: :accepted_term_id + remove_column :users, :accepted_term_id + end +end diff --git a/db/migrate/20180430101916_add_runner_type_to_ci_runners.rb b/db/migrate/20180430101916_add_runner_type_to_ci_runners.rb new file mode 100644 index 00000000000..42409349b75 --- /dev/null +++ b/db/migrate/20180430101916_add_runner_type_to_ci_runners.rb @@ -0,0 +1,9 @@ +class AddRunnerTypeToCiRunners < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :ci_runners, :runner_type, :smallint + end +end diff --git a/db/migrate/20180502122856_create_project_mirror_data.rb b/db/migrate/20180502122856_create_project_mirror_data.rb new file mode 100644 index 00000000000..d449f944844 --- /dev/null +++ b/db/migrate/20180502122856_create_project_mirror_data.rb @@ -0,0 +1,20 @@ +class CreateProjectMirrorData < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + return if table_exists?(:project_mirror_data) + + create_table :project_mirror_data do |t| + t.references :project, index: true, foreign_key: { on_delete: :cascade } + t.string :status + t.string :jid + t.text :last_error + end + end + + def down + drop_table(:project_mirror_data) if table_exists?(:project_mirror_data) + end +end diff --git a/db/migrate/20180503131624_create_remote_mirrors.rb b/db/migrate/20180503131624_create_remote_mirrors.rb new file mode 100644 index 00000000000..7800186455f --- /dev/null +++ b/db/migrate/20180503131624_create_remote_mirrors.rb @@ -0,0 +1,33 @@ +class CreateRemoteMirrors < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + return if table_exists?(:remote_mirrors) + + create_table :remote_mirrors do |t| + t.references :project, index: true, foreign_key: { on_delete: :cascade } + t.string :url + t.boolean :enabled, default: true + t.string :update_status + t.datetime :last_update_at + t.datetime :last_successful_update_at + t.datetime :last_update_started_at + t.string :last_error + t.boolean :only_protected_branches, default: false, null: false + t.string :remote_name + t.text :encrypted_credentials + t.string :encrypted_credentials_iv + t.string :encrypted_credentials_salt + + t.timestamps null: false + end + end + + def down + drop_table(:remote_mirrors) if table_exists?(:remote_mirrors) + end +end diff --git a/db/migrate/20180503141722_add_remote_mirror_available_overridden_to_projects.rb b/db/migrate/20180503141722_add_remote_mirror_available_overridden_to_projects.rb new file mode 100644 index 00000000000..841393971f4 --- /dev/null +++ b/db/migrate/20180503141722_add_remote_mirror_available_overridden_to_projects.rb @@ -0,0 +1,15 @@ +class AddRemoteMirrorAvailableOverriddenToProjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column(:projects, :remote_mirror_available_overridden, :boolean) unless column_exists?(:projects, :remote_mirror_available_overridden) + end + + def down + remove_column(:projects, :remote_mirror_available_overridden) if column_exists?(:projects, :remote_mirror_available_overridden) + end +end diff --git a/db/migrate/20180503150427_add_index_to_namespaces_runners_token.rb b/db/migrate/20180503150427_add_index_to_namespaces_runners_token.rb new file mode 100644 index 00000000000..4c4e576d49f --- /dev/null +++ b/db/migrate/20180503150427_add_index_to_namespaces_runners_token.rb @@ -0,0 +1,20 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddIndexToNamespacesRunnersToken < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :namespaces, :runners_token, unique: true + end + + def down + if index_exists?(:namespaces, :runners_token, unique: true) + remove_index :namespaces, :runners_token + end + end +end diff --git a/db/migrate/20180503175054_add_indexes_to_project_mirror_data.rb b/db/migrate/20180503175054_add_indexes_to_project_mirror_data.rb new file mode 100644 index 00000000000..17570269b2e --- /dev/null +++ b/db/migrate/20180503175054_add_indexes_to_project_mirror_data.rb @@ -0,0 +1,17 @@ +class AddIndexesToProjectMirrorData < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :project_mirror_data, :jid + add_concurrent_index :project_mirror_data, :status + end + + def down + remove_index :project_mirror_data, :jid if index_exists? :project_mirror_data, :jid + remove_index :project_mirror_data, :status if index_exists? :project_mirror_data, :status + end +end diff --git a/db/migrate/20180503193542_add_indexes_to_remote_mirror.rb b/db/migrate/20180503193542_add_indexes_to_remote_mirror.rb new file mode 100644 index 00000000000..9a9decffdab --- /dev/null +++ b/db/migrate/20180503193542_add_indexes_to_remote_mirror.rb @@ -0,0 +1,15 @@ +class AddIndexesToRemoteMirror < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :remote_mirrors, :last_successful_update_at unless index_exists?(:remote_mirrors, :last_successful_update_at) + end + + def down + remove_index :remote_mirrors, :last_successful_update_at if index_exists? :remote_mirrors, :last_successful_update_at + end +end diff --git a/db/migrate/20180503193953_add_mirror_available_to_application_settings.rb b/db/migrate/20180503193953_add_mirror_available_to_application_settings.rb new file mode 100644 index 00000000000..25b9905b1a9 --- /dev/null +++ b/db/migrate/20180503193953_add_mirror_available_to_application_settings.rb @@ -0,0 +1,15 @@ +class AddMirrorAvailableToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default(:application_settings, :mirror_available, :boolean, default: true, allow_null: false) unless column_exists?(:application_settings, :mirror_available) + end + + def down + remove_column(:application_settings, :mirror_available) if column_exists?(:application_settings, :mirror_available) + end +end diff --git a/db/migrate/20180503200320_enable_prometheus_metrics_by_default.rb b/db/migrate/20180503200320_enable_prometheus_metrics_by_default.rb new file mode 100644 index 00000000000..2c8f86ff0f4 --- /dev/null +++ b/db/migrate/20180503200320_enable_prometheus_metrics_by_default.rb @@ -0,0 +1,11 @@ +class EnablePrometheusMetricsByDefault < ActiveRecord::Migration + DOWNTIME = false + + def up + change_column_default :application_settings, :prometheus_metrics_enabled, true + end + + def down + change_column_default :application_settings, :prometheus_metrics_enabled, false + end +end diff --git a/db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql.rb b/db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql.rb new file mode 100644 index 00000000000..e1771912c3c --- /dev/null +++ b/db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql.rb @@ -0,0 +1,9 @@ +class LimitsCiBuildTraceChunksRawDataForMysql < ActiveRecord::Migration + def up + return unless Gitlab::Database.mysql? + + # Mysql needs MEDIUMTEXT type (up to 16MB) rather than TEXT (up to 64KB) + # Because 'raw_data' is always capped by Ci::BuildTraceChunk::CHUNK_SIZE, which is 128KB + change_column :ci_build_trace_chunks, :raw_data, :binary, limit: 16.megabytes - 1 #MEDIUMTEXT + end +end diff --git a/db/post_migrate/20180409170809_populate_missing_project_ci_cd_settings.rb b/db/post_migrate/20180409170809_populate_missing_project_ci_cd_settings.rb new file mode 100644 index 00000000000..3b0fdb3aeea --- /dev/null +++ b/db/post_migrate/20180409170809_populate_missing_project_ci_cd_settings.rb @@ -0,0 +1,34 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class PopulateMissingProjectCiCdSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + # MySQL does not support online upgrades, thus there can't be any missing + # rows. + return if Gitlab::Database.mysql? + + # Projects created after the initial migration but before the code started + # using ProjectCiCdSetting won't have a corresponding row in + # project_ci_cd_settings, so let's fix that. + execute <<~SQL + INSERT INTO project_ci_cd_settings (project_id) + SELECT id + FROM projects + WHERE NOT EXISTS ( + SELECT 1 + FROM project_ci_cd_settings + WHERE project_ci_cd_settings.project_id = projects.id + ) + SQL + end + + def down + # There's nothing to revert for this migration. + end +end diff --git a/db/post_migrate/20180420080616_schedule_stages_index_migration.rb b/db/post_migrate/20180420080616_schedule_stages_index_migration.rb new file mode 100644 index 00000000000..1d0daad002f --- /dev/null +++ b/db/post_migrate/20180420080616_schedule_stages_index_migration.rb @@ -0,0 +1,29 @@ +class ScheduleStagesIndexMigration < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + MIGRATION = 'MigrateStageIndex'.freeze + BATCH_SIZE = 10000 + + disable_ddl_transaction! + + class Stage < ActiveRecord::Base + include EachBatch + self.table_name = 'ci_stages' + end + + def up + disable_statement_timeout + + Stage.all.tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + MIGRATION, + 5.minutes, + batch_size: BATCH_SIZE) + end + end + + def down + # noop + end +end diff --git a/db/post_migrate/20180430143705_backfill_runner_type_for_ci_runners_post_migrate.rb b/db/post_migrate/20180430143705_backfill_runner_type_for_ci_runners_post_migrate.rb new file mode 100644 index 00000000000..38af5aae924 --- /dev/null +++ b/db/post_migrate/20180430143705_backfill_runner_type_for_ci_runners_post_migrate.rb @@ -0,0 +1,23 @@ +class BackfillRunnerTypeForCiRunnersPostMigrate < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + INSTANCE_RUNNER_TYPE = 1 + PROJECT_RUNNER_TYPE = 3 + + disable_ddl_transaction! + + def up + update_column_in_batches(:ci_runners, :runner_type, INSTANCE_RUNNER_TYPE) do |table, query| + query.where(table[:is_shared].eq(true)).where(table[:runner_type].eq(nil)) + end + + update_column_in_batches(:ci_runners, :runner_type, PROJECT_RUNNER_TYPE) do |table, query| + query.where(table[:is_shared].eq(false)).where(table[:runner_type].eq(nil)) + end + end + + def down + end +end diff --git a/db/post_migrate/20180502134117_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb b/db/post_migrate/20180502134117_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb new file mode 100644 index 00000000000..e39cd33c414 --- /dev/null +++ b/db/post_migrate/20180502134117_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb @@ -0,0 +1,38 @@ +class MigrateImportAttributesDataFromProjectsToProjectMirrorData < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + UP_MIGRATION = 'PopulateImportState'.freeze + DOWN_MIGRATION = 'RollbackImportStateData'.freeze + + BATCH_SIZE = 1000 + DELAY_INTERVAL = 5.minutes + + disable_ddl_transaction! + + class Project < ActiveRecord::Base + include EachBatch + + self.table_name = 'projects' + end + + class ProjectImportState < ActiveRecord::Base + include EachBatch + + self.table_name = 'project_mirror_data' + end + + def up + projects = Project.where.not(import_status: :none) + + queue_background_migration_jobs_by_range_at_intervals(projects, UP_MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + end + + def down + import_state = ProjectImportState.where.not(status: :none) + + queue_background_migration_jobs_by_range_at_intervals(import_state, DOWN_MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + end + +end diff --git a/db/schema.rb b/db/schema.rb index df621956c80..65e9cc4ea08 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180418053107) do +ActiveRecord::Schema.define(version: 20180503200320) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -40,6 +40,12 @@ ActiveRecord::Schema.define(version: 20180418053107) do t.text "new_project_guidelines_html" end + create_table "application_setting_terms", force: :cascade do |t| + t.integer "cached_markdown_version" + t.text "terms", null: false + t.text "terms_html" + end + create_table "application_settings", force: :cascade do |t| t.integer "default_projects_limit" t.boolean "signup_enabled" @@ -128,7 +134,7 @@ ActiveRecord::Schema.define(version: 20180418053107) do t.integer "cached_markdown_version" t.boolean "clientside_sentry_enabled", default: false, null: false t.string "clientside_sentry_dsn" - t.boolean "prometheus_metrics_enabled", default: false, null: false + t.boolean "prometheus_metrics_enabled", default: true, null: false t.boolean "help_page_hide_commercial_content", default: false t.string "help_page_support_url" t.integer "performance_bar_allowed_group_id" @@ -158,6 +164,8 @@ ActiveRecord::Schema.define(version: 20180418053107) do t.string "auto_devops_domain" t.boolean "pages_domain_verification_enabled", default: true, null: false t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false + t.boolean "enforce_terms", default: false + t.boolean "mirror_available", default: true, null: false end create_table "audit_events", force: :cascade do |t| @@ -246,6 +254,15 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_index "chat_teams", ["namespace_id"], name: "index_chat_teams_on_namespace_id", unique: true, using: :btree + create_table "ci_build_trace_chunks", id: :bigserial, force: :cascade do |t| + t.integer "build_id", null: false + t.integer "chunk_index", null: false + t.integer "data_store", null: false + t.binary "raw_data" + end + + add_index "ci_build_trace_chunks", ["build_id", "chunk_index"], name: "index_ci_build_trace_chunks_on_build_id_and_chunk_index", unique: true, using: :btree + create_table "ci_build_trace_section_names", force: :cascade do |t| t.integer "project_id", null: false t.string "name", null: false @@ -322,6 +339,7 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_index "ci_builds", ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree add_index "ci_builds", ["protected"], name: "index_ci_builds_on_protected", using: :btree add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree + add_index "ci_builds", ["stage_id", "stage_idx"], name: "tmp_build_stage_position_index", where: "(stage_idx IS NOT NULL)", using: :btree add_index "ci_builds", ["stage_id"], name: "index_ci_builds_on_stage_id", using: :btree add_index "ci_builds", ["status", "type", "runner_id"], name: "index_ci_builds_on_status_and_type_and_runner_id", using: :btree add_index "ci_builds", ["status"], name: "index_ci_builds_on_status", using: :btree @@ -443,6 +461,14 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_index "ci_pipelines", ["status"], name: "index_ci_pipelines_on_status", using: :btree add_index "ci_pipelines", ["user_id"], name: "index_ci_pipelines_on_user_id", using: :btree + create_table "ci_runner_namespaces", force: :cascade do |t| + t.integer "runner_id" + t.integer "namespace_id" + end + + add_index "ci_runner_namespaces", ["namespace_id"], name: "index_ci_runner_namespaces_on_namespace_id", using: :btree + add_index "ci_runner_namespaces", ["runner_id", "namespace_id"], name: "index_ci_runner_namespaces_on_runner_id_and_namespace_id", unique: true, using: :btree + create_table "ci_runner_projects", force: :cascade do |t| t.integer "runner_id", null: false t.datetime "created_at" @@ -471,6 +497,7 @@ ActiveRecord::Schema.define(version: 20180418053107) do t.integer "access_level", default: 0, null: false t.string "ip_address" t.integer "maximum_timeout" + t.integer "runner_type", limit: 2 end add_index "ci_runners", ["contacted_at"], name: "index_ci_runners_on_contacted_at", using: :btree @@ -486,6 +513,7 @@ ActiveRecord::Schema.define(version: 20180418053107) do t.string "name" t.integer "status" t.integer "lock_version" + t.integer "position" end add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", unique: true, using: :btree @@ -1259,6 +1287,7 @@ ActiveRecord::Schema.define(version: 20180418053107) do t.boolean "require_two_factor_authentication", default: false, null: false t.integer "two_factor_grace_period", default: 48, null: false t.integer "cached_markdown_version" + t.string "runners_token" end add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree @@ -1269,6 +1298,7 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_index "namespaces", ["path"], name: "index_namespaces_on_path", using: :btree add_index "namespaces", ["path"], name: "index_namespaces_on_path_trigram", using: :gin, opclasses: {"path"=>"gin_trgm_ops"} add_index "namespaces", ["require_two_factor_authentication"], name: "index_namespaces_on_require_two_factor_authentication", using: :btree + add_index "namespaces", ["runners_token"], name: "index_namespaces_on_runners_token", unique: true, using: :btree add_index "namespaces", ["type"], name: "index_namespaces_on_type", using: :btree create_table "notes", force: :cascade do |t| @@ -1436,6 +1466,13 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_index "project_auto_devops", ["project_id"], name: "index_project_auto_devops_on_project_id", unique: true, using: :btree + create_table "project_ci_cd_settings", force: :cascade do |t| + t.integer "project_id", null: false + t.boolean "group_runners_enabled", default: true, null: false + end + + add_index "project_ci_cd_settings", ["project_id"], name: "index_project_ci_cd_settings_on_project_id", unique: true, using: :btree + create_table "project_custom_attributes", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -1491,6 +1528,17 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree + create_table "project_mirror_data", force: :cascade do |t| + t.integer "project_id" + t.string "status" + t.string "jid" + t.text "last_error" + end + + add_index "project_mirror_data", ["jid"], name: "index_project_mirror_data_on_jid", using: :btree + add_index "project_mirror_data", ["project_id"], name: "index_project_mirror_data_on_project_id", using: :btree + add_index "project_mirror_data", ["status"], name: "index_project_mirror_data_on_status", using: :btree + create_table "project_statistics", force: :cascade do |t| t.integer "project_id", null: false t.integer "namespace_id", null: false @@ -1555,6 +1603,7 @@ ActiveRecord::Schema.define(version: 20180418053107) do t.boolean "merge_requests_rebase_enabled", default: false, null: false t.integer "jobs_cache_index" t.boolean "pages_https_only", default: true + t.boolean "remote_mirror_available_overridden" end add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree @@ -1660,6 +1709,27 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_index "releases", ["project_id", "tag"], name: "index_releases_on_project_id_and_tag", using: :btree add_index "releases", ["project_id"], name: "index_releases_on_project_id", using: :btree + create_table "remote_mirrors", force: :cascade do |t| + t.integer "project_id" + t.string "url" + t.boolean "enabled", default: true + t.string "update_status" + t.datetime "last_update_at" + t.datetime "last_successful_update_at" + t.datetime "last_update_started_at" + t.string "last_error" + t.boolean "only_protected_branches", default: false, null: false + t.string "remote_name" + t.text "encrypted_credentials" + t.string "encrypted_credentials_iv" + t.string "encrypted_credentials_salt" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + + add_index "remote_mirrors", ["last_successful_update_at"], name: "index_remote_mirrors_on_last_successful_update_at", using: :btree + add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree + create_table "routes", force: :cascade do |t| t.integer "source_id", null: false t.string "source_type", null: false @@ -1797,6 +1867,18 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_index "tags", ["name"], name: "index_tags_on_name", unique: true, using: :btree + create_table "term_agreements", force: :cascade do |t| + t.integer "term_id", null: false + t.integer "user_id", null: false + t.boolean "accepted", default: false, null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + end + + add_index "term_agreements", ["term_id"], name: "index_term_agreements_on_term_id", using: :btree + add_index "term_agreements", ["user_id", "term_id"], name: "term_agreements_unique_index", unique: true, using: :btree + add_index "term_agreements", ["user_id"], name: "index_term_agreements_on_user_id", using: :btree + create_table "timelogs", force: :cascade do |t| t.integer "time_spent", null: false t.integer "user_id" @@ -1985,6 +2067,7 @@ ActiveRecord::Schema.define(version: 20180418053107) do t.string "preferred_language" t.string "rss_token" t.integer "theme_id", limit: 2 + t.integer "accepted_term_id" end add_index "users", ["admin"], name: "index_users_on_admin", using: :btree @@ -2059,6 +2142,7 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_foreign_key "boards", "namespaces", column: "group_id", on_delete: :cascade add_foreign_key "boards", "projects", name: "fk_f15266b5f9", on_delete: :cascade add_foreign_key "chat_teams", "namespaces", on_delete: :cascade + add_foreign_key "ci_build_trace_chunks", "ci_builds", column: "build_id", on_delete: :cascade add_foreign_key "ci_build_trace_section_names", "projects", on_delete: :cascade add_foreign_key "ci_build_trace_sections", "ci_build_trace_section_names", column: "section_name_id", name: "fk_264e112c66", on_delete: :cascade add_foreign_key "ci_build_trace_sections", "ci_builds", column: "build_id", name: "fk_4ebe41f502", on_delete: :cascade @@ -2078,6 +2162,8 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify add_foreign_key "ci_pipelines", "projects", name: "fk_86635dbd80", on_delete: :cascade + add_foreign_key "ci_runner_namespaces", "ci_runners", column: "runner_id", on_delete: :cascade + add_foreign_key "ci_runner_namespaces", "namespaces", on_delete: :cascade add_foreign_key "ci_runner_projects", "projects", name: "fk_4478a6f1e4", on_delete: :cascade add_foreign_key "ci_stages", "ci_pipelines", column: "pipeline_id", name: "fk_fb57e6cc56", on_delete: :cascade add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade @@ -2162,12 +2248,14 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_foreign_key "project_authorizations", "projects", on_delete: :cascade add_foreign_key "project_authorizations", "users", on_delete: :cascade add_foreign_key "project_auto_devops", "projects", on_delete: :cascade + add_foreign_key "project_ci_cd_settings", "projects", name: "fk_24c15d2f2e", on_delete: :cascade add_foreign_key "project_custom_attributes", "projects", on_delete: :cascade add_foreign_key "project_deploy_tokens", "deploy_tokens", on_delete: :cascade add_foreign_key "project_deploy_tokens", "projects", on_delete: :cascade add_foreign_key "project_features", "projects", name: "fk_18513d9b92", on_delete: :cascade add_foreign_key "project_group_links", "projects", name: "fk_daa8cee94c", on_delete: :cascade add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade + add_foreign_key "project_mirror_data", "projects", on_delete: :cascade add_foreign_key "project_statistics", "projects", on_delete: :cascade add_foreign_key "protected_branch_merge_access_levels", "protected_branches", name: "fk_8a3072ccb3", on_delete: :cascade add_foreign_key "protected_branch_push_access_levels", "protected_branches", name: "fk_9ffc86a3d9", on_delete: :cascade @@ -2178,10 +2266,13 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade add_foreign_key "push_event_payloads", "events", name: "fk_36c74129da", on_delete: :cascade add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade + add_foreign_key "remote_mirrors", "projects", on_delete: :cascade add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade add_foreign_key "subscriptions", "projects", on_delete: :cascade add_foreign_key "system_note_metadata", "notes", name: "fk_d83a918cb1", on_delete: :cascade + add_foreign_key "term_agreements", "application_setting_terms", column: "term_id" + add_foreign_key "term_agreements", "users", on_delete: :cascade add_foreign_key "timelogs", "issues", name: "fk_timelogs_issues_issue_id", on_delete: :cascade add_foreign_key "timelogs", "merge_requests", name: "fk_timelogs_merge_requests_merge_request_id", on_delete: :cascade add_foreign_key "todos", "notes", name: "fk_91d1f47b13", on_delete: :cascade @@ -2195,6 +2286,7 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_foreign_key "user_interacted_projects", "projects", name: "fk_722ceba4f7", on_delete: :cascade add_foreign_key "user_interacted_projects", "users", name: "fk_0894651f08", on_delete: :cascade add_foreign_key "user_synced_attributes_metadata", "users", on_delete: :cascade + add_foreign_key "users", "application_setting_terms", column: "accepted_term_id", name: "fk_789cd90b35", on_delete: :cascade add_foreign_key "users_star_projects", "projects", name: "fk_22cd27ddfc", on_delete: :cascade add_foreign_key "web_hook_logs", "web_hooks", on_delete: :cascade add_foreign_key "web_hooks", "projects", name: "fk_0c8ca6d9d1", on_delete: :cascade |