Merge branch 'master' into expiration-date-on-memberships

20 files changed, 356 insertions, 8 deletions

diff --git a/db/migrate/20140407135544_fix_namespaces.rb b/db/migrate/20140407135544_fix_namespaces.rb
index 91374966698..0026ce645a6 100644
--- a/db/migrate/20140407135544_fix_namespaces.rb
+++ b/db/migrate/20140407135544_fix_namespaces.rb
@@ -1,8 +1,14 @@
 # rubocop:disable all
 class FixNamespaces < ActiveRecord::Migration
+  DOWNTIME = false
+
   def up
-    Namespace.where('name <> path and type is null').each do |namespace|
-      namespace.update_attribute(:name, namespace.path)
+    namespaces = exec_query('SELECT id, path FROM namespaces WHERE name <> path and type is null')
+
+    namespaces.each do |row|
+      id = row['id']
+      path = row['path']
+      exec_query("UPDATE namespaces SET name = '#{path}' WHERE id = #{id}")
     end
   end
 
diff --git a/db/migrate/20160705055254_move_from_developers_can_merge_to_protected_branches_merge_access.rb b/db/migrate/20160705055254_move_from_developers_can_merge_to_protected_branches_merge_access.rb
index fa93936ced7..1db0df92bec 100644
--- a/db/migrate/20160705055254_move_from_developers_can_merge_to_protected_branches_merge_access.rb
+++ b/db/migrate/20160705055254_move_from_developers_can_merge_to_protected_branches_merge_access.rb
@@ -14,7 +14,7 @@ class MoveFromDevelopersCanMergeToProtectedBranchesMergeAccess < ActiveRecord::M
   def up
     execute <<-HEREDOC
       INSERT into protected_branch_merge_access_levels (protected_branch_id, access_level, created_at, updated_at)
-        SELECT id, (CASE WHEN developers_can_merge THEN 1 ELSE 0 END), now(), now()
+        SELECT id, (CASE WHEN developers_can_merge THEN 30 ELSE 40 END), now(), now()
           FROM protected_branches
     HEREDOC
   end
@@ -23,7 +23,7 @@ class MoveFromDevelopersCanMergeToProtectedBranchesMergeAccess < ActiveRecord::M
     execute <<-HEREDOC
       UPDATE protected_branches SET developers_can_merge = TRUE
         WHERE id IN (SELECT protected_branch_id FROM protected_branch_merge_access_levels
-                       WHERE access_level = 1);
+                       WHERE access_level = 30);
     HEREDOC
   end
 end
diff --git a/db/migrate/20160705055308_move_from_developers_can_push_to_protected_branches_push_access.rb b/db/migrate/20160705055308_move_from_developers_can_push_to_protected_branches_push_access.rb
index 56f6159d1d8..5c3e189bb5b 100644
--- a/db/migrate/20160705055308_move_from_developers_can_push_to_protected_branches_push_access.rb
+++ b/db/migrate/20160705055308_move_from_developers_can_push_to_protected_branches_push_access.rb
@@ -14,7 +14,7 @@ class MoveFromDevelopersCanPushToProtectedBranchesPushAccess < ActiveRecord::Mig
   def up
     execute <<-HEREDOC
       INSERT into protected_branch_push_access_levels (protected_branch_id, access_level, created_at, updated_at)
-        SELECT id, (CASE WHEN developers_can_push THEN 1 ELSE 0 END), now(), now()
+        SELECT id, (CASE WHEN developers_can_push THEN 30 ELSE 40 END), now(), now()
           FROM protected_branches
     HEREDOC
   end
@@ -23,7 +23,7 @@ class MoveFromDevelopersCanPushToProtectedBranchesPushAccess < ActiveRecord::Mig
     execute <<-HEREDOC
       UPDATE protected_branches SET developers_can_push = TRUE
         WHERE id IN (SELECT protected_branch_id FROM protected_branch_push_access_levels
-                       WHERE access_level = 1);
+                       WHERE access_level = 30);
     HEREDOC
   end
 end
diff --git a/db/migrate/20160705055809_remove_developers_can_push_from_protected_branches.rb b/db/migrate/20160705055809_remove_developers_can_push_from_protected_branches.rb
index f563f660ddf..52a9819c628 100644
--- a/db/migrate/20160705055809_remove_developers_can_push_from_protected_branches.rb
+++ b/db/migrate/20160705055809_remove_developers_can_push_from_protected_branches.rb
@@ -14,6 +14,6 @@ class RemoveDevelopersCanPushFromProtectedBranches < ActiveRecord::Migration
   end
 
   def down
-    add_column_with_default(:protected_branches, :developers_can_push, :boolean, default: false, null: false)
+    add_column_with_default(:protected_branches, :developers_can_push, :boolean, default: false, allow_null: false)
   end
 end
diff --git a/db/migrate/20160705055813_remove_developers_can_merge_from_protected_branches.rb b/db/migrate/20160705055813_remove_developers_can_merge_from_protected_branches.rb
index aa71e06d36e..4a7bde7f9f3 100644
--- a/db/migrate/20160705055813_remove_developers_can_merge_from_protected_branches.rb
+++ b/db/migrate/20160705055813_remove_developers_can_merge_from_protected_branches.rb
@@ -14,6 +14,6 @@ class RemoveDevelopersCanMergeFromProtectedBranches < ActiveRecord::Migration
   end
 
   def down
-    add_column_with_default(:protected_branches, :developers_can_merge, :boolean, default: false, null: false)
+    add_column_with_default(:protected_branches, :developers_can_merge, :boolean, default: false, allow_null: false)
   end
 end
diff --git a/db/migrate/20160716115711_add_queued_at_to_ci_builds.rb b/db/migrate/20160716115711_add_queued_at_to_ci_builds.rb
new file mode 100644
index 00000000000..756910a1fa0
--- /dev/null
+++ b/db/migrate/20160716115711_add_queued_at_to_ci_builds.rb
@@ -0,0 +1,9 @@
+class AddQueuedAtToCiBuilds < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_builds, :queued_at, :timestamp
+  end
+end
diff --git a/db/migrate/20160725083350_add_external_url_to_enviroments.rb b/db/migrate/20160725083350_add_external_url_to_enviroments.rb
new file mode 100644
index 00000000000..21a8abd310b
--- /dev/null
+++ b/db/migrate/20160725083350_add_external_url_to_enviroments.rb
@@ -0,0 +1,9 @@
+class AddExternalUrlToEnviroments < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column(:environments, :external_url, :string)
+  end
+end
diff --git a/db/migrate/20160727163552_create_user_agent_details.rb b/db/migrate/20160727163552_create_user_agent_details.rb
new file mode 100644
index 00000000000..ed4ccfedc0a
--- /dev/null
+++ b/db/migrate/20160727163552_create_user_agent_details.rb
@@ -0,0 +1,18 @@
+class CreateUserAgentDetails < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    create_table :user_agent_details do |t|
+      t.string :user_agent, null: false
+      t.string :ip_address, null: false
+      t.integer :subject_id, null: false
+      t.string :subject_type, null: false
+      t.boolean :submitted, default: false, null: false
+
+      t.timestamps null: false
+    end
+  end
+end
diff --git a/db/migrate/20160727191041_create_boards.rb b/db/migrate/20160727191041_create_boards.rb
new file mode 100644
index 00000000000..56afbd4e030
--- /dev/null
+++ b/db/migrate/20160727191041_create_boards.rb
@@ -0,0 +1,13 @@
+class CreateBoards < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :boards do |t|
+      t.references :project, index: true, foreign_key: true, null: false
+
+      t.timestamps null: false
+    end
+  end
+end
diff --git a/db/migrate/20160727193336_create_lists.rb b/db/migrate/20160727193336_create_lists.rb
new file mode 100644
index 00000000000..61d501215f2
--- /dev/null
+++ b/db/migrate/20160727193336_create_lists.rb
@@ -0,0 +1,16 @@
+class CreateLists < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :lists do |t|
+      t.references :board, index: true, foreign_key: true, null: false
+      t.references :label, index: true, foreign_key: true
+      t.integer :list_type, null: false, default: 1
+      t.integer :position
+
+      t.timestamps null: false
+    end
+  end
+end
diff --git a/db/migrate/20160728081025_add_pipeline_events_to_web_hooks.rb b/db/migrate/20160728081025_add_pipeline_events_to_web_hooks.rb
new file mode 100644
index 00000000000..b800e6d7283
--- /dev/null
+++ b/db/migrate/20160728081025_add_pipeline_events_to_web_hooks.rb
@@ -0,0 +1,16 @@
+class AddPipelineEventsToWebHooks < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:web_hooks, :pipeline_events, :boolean,
+                            default: false, allow_null: false)
+  end
+
+  def down
+    remove_column(:web_hooks, :pipeline_events)
+  end
+end
diff --git a/db/migrate/20160728103734_add_pipeline_events_to_services.rb b/db/migrate/20160728103734_add_pipeline_events_to_services.rb
new file mode 100644
index 00000000000..bcd24fe1566
--- /dev/null
+++ b/db/migrate/20160728103734_add_pipeline_events_to_services.rb
@@ -0,0 +1,16 @@
+class AddPipelineEventsToServices < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:services, :pipeline_events, :boolean,
+                            default: false, allow_null: false)
+  end
+
+  def down
+    remove_column(:services, :pipeline_events)
+  end
+end
diff --git a/db/migrate/20160729173930_remove_project_id_from_spam_logs.rb b/db/migrate/20160729173930_remove_project_id_from_spam_logs.rb
new file mode 100644
index 00000000000..e28ab31d629
--- /dev/null
+++ b/db/migrate/20160729173930_remove_project_id_from_spam_logs.rb
@@ -0,0 +1,29 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveProjectIdFromSpamLogs < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = true
+
+  # When a migration requires downtime you **must** uncomment the following
+  # constant and define a short and easy to understand explanation as to why the
+  # migration requires downtime.
+  DOWNTIME_REASON = 'Removing a column that contains data that is not used anywhere.'
+
+  # When using the methods "add_concurrent_index" or "add_column_with_default"
+  # you must disable the use of transactions as these methods can not run in an
+  # existing transaction. When using "add_concurrent_index" make sure that this
+  # method is the _only_ method called in the migration, any other changes
+  # should go in a separate migration. This ensures that upon failure _only_ the
+  # index creation fails and can be retried or reverted easily.
+  #
+  # To disable transactions uncomment the following line and remove these
+  # comments:
+  # disable_ddl_transaction!
+
+  def change
+    remove_column :spam_logs, :project_id, :integer
+  end
+end
diff --git a/db/migrate/20160801163709_add_submitted_as_ham_to_spam_logs.rb b/db/migrate/20160801163709_add_submitted_as_ham_to_spam_logs.rb
new file mode 100644
index 00000000000..296f1dfac7b
--- /dev/null
+++ b/db/migrate/20160801163709_add_submitted_as_ham_to_spam_logs.rb
@@ -0,0 +1,20 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddSubmittedAsHamToSpamLogs < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  # When a migration requires downtime you **must** uncomment the following
+  # constant and define a short and easy to understand explanation as to why the
+  # migration requires downtime.
+  # DOWNTIME_REASON = ''
+
+  disable_ddl_transaction!
+
+  def change
+    add_column_with_default :spam_logs, :submitted_as_ham, :boolean, default: false
+  end
+end
diff --git a/db/migrate/20160802010328_remove_builds_enable_index_on_projects.rb b/db/migrate/20160802010328_remove_builds_enable_index_on_projects.rb
new file mode 100644
index 00000000000..5fd51cb65f1
--- /dev/null
+++ b/db/migrate/20160802010328_remove_builds_enable_index_on_projects.rb
@@ -0,0 +1,9 @@
+class RemoveBuildsEnableIndexOnProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    remove_index :projects, column: :builds_enabled if index_exists?(:projects, :builds_enabled)
+  end
+end
diff --git a/db/migrate/20160803161903_add_unique_index_to_lists_label_id.rb b/db/migrate/20160803161903_add_unique_index_to_lists_label_id.rb
new file mode 100644
index 00000000000..baf2e70b127
--- /dev/null
+++ b/db/migrate/20160803161903_add_unique_index_to_lists_label_id.rb
@@ -0,0 +1,15 @@
+class AddUniqueIndexToListsLabelId < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :lists, [:board_id, :label_id], unique: true
+  end
+
+  def down
+    remove_index :lists, column: [:board_id, :label_id] if index_exists?(:lists, [:board_id, :label_id], unique: true)
+  end
+end
diff --git a/db/migrate/20160804150737_add_timestamps_to_members_again.rb b/db/migrate/20160804150737_add_timestamps_to_members_again.rb
new file mode 100644
index 00000000000..6691ba57fbb
--- /dev/null
+++ b/db/migrate/20160804150737_add_timestamps_to_members_again.rb
@@ -0,0 +1,21 @@
+# rubocop:disable all
+# 20141121133009_add_timestamps_to_members.rb was meant to ensure that all
+# rows in the members table had created_at and updated_at set, following an
+# error in a previous migration. This failed to set all rows in at least one
+# case: https://gitlab.com/gitlab-org/gitlab-ce/issues/20568
+#
+# Why this happened is lost in the mists of time, so repeat the SQL query
+# without speculation, just in case more than one person was affected.
+class AddTimestampsToMembersAgain < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    execute "UPDATE members SET created_at = NOW() WHERE created_at IS NULL"
+    execute "UPDATE members SET updated_at = NOW() WHERE updated_at IS NULL"
+  end
+
+  def down
+    # no change
+  end
+
+end
diff --git a/db/migrate/20160805041956_add_deleted_at_to_namespaces.rb b/db/migrate/20160805041956_add_deleted_at_to_namespaces.rb
new file mode 100644
index 00000000000..a853de3abfb
--- /dev/null
+++ b/db/migrate/20160805041956_add_deleted_at_to_namespaces.rb
@@ -0,0 +1,12 @@
+class AddDeletedAtToNamespaces < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def change
+    add_column :namespaces, :deleted_at, :datetime
+    add_concurrent_index :namespaces, :deleted_at
+  end
+end
diff --git a/db/migrate/20160810102349_remove_ci_runner_trigram_indexes.rb b/db/migrate/20160810102349_remove_ci_runner_trigram_indexes.rb
new file mode 100644
index 00000000000..0cfb637804b
--- /dev/null
+++ b/db/migrate/20160810102349_remove_ci_runner_trigram_indexes.rb
@@ -0,0 +1,27 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveCiRunnerTrigramIndexes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  # Disabled for the "down" method so the indexes can be re-created concurrently.
+  disable_ddl_transaction!
+
+  def up
+    return unless Gitlab::Database.postgresql?
+
+    transaction do
+      execute 'DROP INDEX IF EXISTS index_ci_runners_on_token_trigram;'
+      execute 'DROP INDEX IF EXISTS index_ci_runners_on_description_trigram;'
+    end
+  end
+
+  def down
+    return unless Gitlab::Database.postgresql?
+
+    execute 'CREATE INDEX CONCURRENTLY index_ci_runners_on_token_trigram ON ci_runners USING gin(token gin_trgm_ops);'
+    execute 'CREATE INDEX CONCURRENTLY index_ci_runners_on_description_trigram ON ci_runners USING gin(description gin_trgm_ops);'
+  end
+end
diff --git a/db/migrate/20160810142633_remove_redundant_indexes.rb b/db/migrate/20160810142633_remove_redundant_indexes.rb
new file mode 100644
index 00000000000..8641c6ffa8f
--- /dev/null
+++ b/db/migrate/20160810142633_remove_redundant_indexes.rb
@@ -0,0 +1,112 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveRedundantIndexes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    indexes = [
+      [:ci_taggings, 'ci_taggings_idx'],
+      [:audit_events, 'index_audit_events_on_author_id'],
+      [:audit_events, 'index_audit_events_on_type'],
+      [:ci_builds, 'index_ci_builds_on_erased_by_id'],
+      [:ci_builds, 'index_ci_builds_on_project_id_and_commit_id'],
+      [:ci_builds, 'index_ci_builds_on_type'],
+      [:ci_commits, 'index_ci_commits_on_project_id'],
+      [:ci_commits, 'index_ci_commits_on_project_id_and_committed_at'],
+      [:ci_commits, 'index_ci_commits_on_project_id_and_committed_at_and_id'],
+      [:ci_commits, 'index_ci_commits_on_project_id_and_sha'],
+      [:ci_commits, 'index_ci_commits_on_sha'],
+      [:ci_events, 'index_ci_events_on_created_at'],
+      [:ci_events, 'index_ci_events_on_is_admin'],
+      [:ci_events, 'index_ci_events_on_project_id'],
+      [:ci_jobs, 'index_ci_jobs_on_deleted_at'],
+      [:ci_jobs, 'index_ci_jobs_on_project_id'],
+      [:ci_projects, 'index_ci_projects_on_gitlab_id'],
+      [:ci_projects, 'index_ci_projects_on_shared_runners_enabled'],
+      [:ci_services, 'index_ci_services_on_project_id'],
+      [:ci_sessions, 'index_ci_sessions_on_session_id'],
+      [:ci_sessions, 'index_ci_sessions_on_updated_at'],
+      [:ci_tags, 'index_ci_tags_on_name'],
+      [:ci_triggers, 'index_ci_triggers_on_deleted_at'],
+      [:identities, 'index_identities_on_created_at_and_id'],
+      [:issues, 'index_issues_on_title'],
+      [:keys, 'index_keys_on_created_at_and_id'],
+      [:members, 'index_members_on_created_at_and_id'],
+      [:members, 'index_members_on_type'],
+      [:milestones, 'index_milestones_on_created_at_and_id'],
+      [:namespaces, 'index_namespaces_on_visibility_level'],
+      [:projects, 'index_projects_on_builds_enabled_and_shared_runners_enabled'],
+      [:services, 'index_services_on_category'],
+      [:services, 'index_services_on_created_at_and_id'],
+      [:services, 'index_services_on_default'],
+      [:snippets, 'index_snippets_on_created_at'],
+      [:snippets, 'index_snippets_on_created_at_and_id'],
+      [:todos, 'index_todos_on_state'],
+      [:web_hooks, 'index_web_hooks_on_created_at_and_id'],
+
+      # These indexes _may_ be used but they can be replaced by other existing
+      # indexes.
+
+      # There's already a composite index on (project_id, iid) which means that
+      # a separate index for _just_ project_id is not needed.
+      [:issues, 'index_issues_on_project_id'],
+
+      # These are all composite indexes for the columns (created_at, id). In all
+      # these cases there's already a standalone index for "created_at" which
+      # can be used instead.
+      #
+      # Because the "id" column of these composite indexes is never needed (due
+      # to "id" already being indexed as its a primary key) these composite
+      # indexes are useless.
+      [:issues, 'index_issues_on_created_at_and_id'],
+      [:merge_requests, 'index_merge_requests_on_created_at_and_id'],
+      [:namespaces, 'index_namespaces_on_created_at_and_id'],
+      [:notes, 'index_notes_on_created_at_and_id'],
+      [:projects, 'index_projects_on_created_at_and_id'],
+      [:users, 'index_users_on_created_at_and_id'],
+    ]
+
+    transaction do
+      indexes.each do |(table, index)|
+        remove_index(table, name: index) if index_exists_by_name?(table, index)
+      end
+    end
+
+    add_concurrent_index(:users, :created_at)
+    add_concurrent_index(:projects, :created_at)
+    add_concurrent_index(:namespaces, :created_at)
+  end
+
+  def down
+    # We're only restoring the composite indexes that could be replaced with
+    # individual ones, just in case somebody would ever want to revert.
+    transaction do
+      remove_index(:users, :created_at)
+      remove_index(:projects, :created_at)
+      remove_index(:namespaces, :created_at)
+    end
+
+    [:issues, :merge_requests, :namespaces, :notes, :projects, :users].each do |table|
+      add_concurrent_index(table, [:created_at, :id],
+                           name: "index_#{table}_on_created_at_and_id")
+    end
+  end
+
+  # Rails' index_exists? doesn't work when you only give it a table and index
+  # name. As such we have to use some extra code to check if an index exists for
+  # a given name.
+  def index_exists_by_name?(table, index)
+    indexes_for_table[table].include?(index)
+  end
+
+  def indexes_for_table
+    @indexes_for_table ||= Hash.new do |hash, table_name|
+      hash[table_name] = indexes(table_name).map(&:name)
+    end
+  end
+end