Merge branch 'master' into feature/gb/migrate-pipeline-stages

* master: (524 commits)
  Improve user experience around slash commands in instant comments
  Fix LFS timeouts when trying to save large files
  retryable? is now available for CommitStatus
  Resolve "Documentation of `.gitlab-ci.yml` states incorrect example for variables"
  Fix test failures
  Add slugify project path to CI enviroment variables
  Fixed typo: craeted -> created
  32118 Make New environment empty state btn lowercase
  Expose import_status in Projects API
  32832 Remove overflow from comment form for confidential issues and vertically aligns confidential issue icon
  Fix test failures
  Allow manual bypass of auto_sign_in_with_provider
  Fix keys seed
  Allow users to be hard-deleted from the API
  fixup some classnames and media queries
  Enable the Style/PreferredHashMethods cop
  Lint our factories creation in addition to their build
  Don’t schedule workers from inside transactions
  Allow scheduling from after_commit hooks
  Forbid Sidekiq scheduling in transactions
  ...

Conflicts:
	db/schema.rb
	spec/spec_helper.rb

29 files changed, 535 insertions, 34 deletions

diff --git a/db/fixtures/development/11_keys.rb b/db/fixtures/development/11_keys.rb
index 51e22137d6f..c405ecfdaf3 100644
--- a/db/fixtures/development/11_keys.rb
+++ b/db/fixtures/development/11_keys.rb
@@ -1,17 +1,26 @@
 require './spec/support/sidekiq'
 
+
 # Creating keys runs a gitlab-shell worker. Since we may not have the right
 # gitlab-shell path set (yet) we need to disable this for these fixtures.
 Sidekiq::Testing.disable! do
   Gitlab::Seeder.quiet do
+    # We want to run `add_to_shell` immediately instead of after the commit, so
+    # that it falls under `Sidekiq::Testing.disable!`.
+    Key.skip_callback(:commit, :after, :add_to_shell)
+
     User.first(10).each do |user|
       key = "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt#{user.id + 100}6k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0="
 
-      user.keys.create(
+      key = user.keys.create(
         title: "Sample key #{user.id}",
         key: key
       )
 
+      Sidekiq::Worker.skipping_transaction_check do
+        key.add_to_shell
+      end
+
       print '.'
     end
   end
diff --git a/db/fixtures/development/14_pipelines.rb b/db/fixtures/development/14_pipelines.rb
index 3c42f7db6d5..5de5339b70e 100644
--- a/db/fixtures/development/14_pipelines.rb
+++ b/db/fixtures/development/14_pipelines.rb
@@ -98,7 +98,7 @@ class Gitlab::Seeder::Pipelines
 
 
   def create_pipeline!(project, ref, commit)
-    project.pipelines.create(sha: commit.id, ref: ref)
+    project.pipelines.create(sha: commit.id, ref: ref, source: :push)
   end
 
   def build_create!(pipeline, opts = {})
@@ -112,6 +112,10 @@ class Gitlab::Seeder::Pipelines
 
       setup_artifacts(build)
       setup_build_log(build)
+
+      build.project.environments.
+        find_or_create_by(name: build.expanded_environment_name)
+
       build.save
     end
   end
diff --git a/db/fixtures/development/17_cycle_analytics.rb b/db/fixtures/development/17_cycle_analytics.rb
index 0d7eb1a7c93..7c1d758dada 100644
--- a/db/fixtures/development/17_cycle_analytics.rb
+++ b/db/fixtures/development/17_cycle_analytics.rb
@@ -190,7 +190,7 @@ class Gitlab::Seeder::CycleAnalytics
       service = Ci::CreatePipelineService.new(merge_request.project,
                                               @user,
                                               ref: "refs/heads/#{merge_request.source_branch}")
-      pipeline = service.execute(ignore_skip_ci: true, save_on_errors: false)
+      pipeline = service.execute(:push, ignore_skip_ci: true, save_on_errors: false)
 
       pipeline.run!
       Timecop.travel rand(1..6).hours.from_now
@@ -212,12 +212,9 @@ class Gitlab::Seeder::CycleAnalytics
     merge_requests.each do |merge_request|
       Timecop.travel 12.hours.from_now
 
-      CreateDeploymentService.new(merge_request.project, @user, {
-                                    environment: 'production',
-                                    ref: 'master',
-                                    tag: false,
-                                    sha: @project.repository.commit('master').sha
-                                  }).execute
+      job = merge_request.head_pipeline.builds.where.not(environment: nil).last
+
+      CreateDeploymentService.new(job).execute
     end
   end
 end
diff --git a/db/fixtures/development/21_conversational_development_index_metrics.rb b/db/fixtures/development/21_conversational_development_index_metrics.rb
new file mode 100644
index 00000000000..4cd0a82ed1a
--- /dev/null
+++ b/db/fixtures/development/21_conversational_development_index_metrics.rb
@@ -0,0 +1,40 @@
+Gitlab::Seeder.quiet do
+  conversational_development_index_metric = ConversationalDevelopmentIndex::Metric.new(
+    leader_issues: 10.2,
+    instance_issues: 3.2,
+
+    leader_notes: 25.3,
+    instance_notes: 23.2,
+
+    leader_milestones: 16.2,
+    instance_milestones: 5.5,
+
+    leader_boards: 5.2,
+    instance_boards: 3.2,
+
+    leader_merge_requests: 5.2,
+    instance_merge_requests: 3.2,
+
+    leader_ci_pipelines: 25.1,
+    instance_ci_pipelines: 21.3,
+
+    leader_environments: 3.3,
+    instance_environments: 2.2,
+
+    leader_deployments: 41.3,
+    instance_deployments: 15.2,
+
+    leader_projects_prometheus_active: 0.31,
+    instance_projects_prometheus_active: 0.30,
+
+    leader_service_desk_issues: 15.8,
+    instance_service_desk_issues: 15.1
+  )
+
+  if conversational_development_index_metric.save
+    print '.'
+  else
+    puts conversational_development_index_metric.errors.full_messages
+    print 'F'
+  end
+end
diff --git a/db/migrate/20160615191922_set_missing_stage_on_ci_builds.rb b/db/migrate/20160615191922_set_missing_stage_on_ci_builds.rb
index bd0463886bc..4d6a61bd614 100644
--- a/db/migrate/20160615191922_set_missing_stage_on_ci_builds.rb
+++ b/db/migrate/20160615191922_set_missing_stage_on_ci_builds.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class SetMissingStageOnCiBuilds < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20160721081015_drop_and_readd_has_external_wiki_in_projects.rb b/db/migrate/20160721081015_drop_and_readd_has_external_wiki_in_projects.rb
index 1eb99feb40c..b2a2ce41391 100644
--- a/db/migrate/20160721081015_drop_and_readd_has_external_wiki_in_projects.rb
+++ b/db/migrate/20160721081015_drop_and_readd_has_external_wiki_in_projects.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class DropAndReaddHasExternalWikiInProjects < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20160901141443_set_confidential_issues_events_on_webhooks.rb b/db/migrate/20160901141443_set_confidential_issues_events_on_webhooks.rb
index f1a1f001cb3..febd2c0e65e 100644
--- a/db/migrate/20160901141443_set_confidential_issues_events_on_webhooks.rb
+++ b/db/migrate/20160901141443_set_confidential_issues_events_on_webhooks.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class SetConfidentialIssuesEventsOnWebhooks < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20160919144305_add_type_to_labels.rb b/db/migrate/20160919144305_add_type_to_labels.rb
index 66172bda6ff..2d2725ccf59 100644
--- a/db/migrate/20160919144305_add_type_to_labels.rb
+++ b/db/migrate/20160919144305_add_type_to_labels.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class AddTypeToLabels < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20161018124658_make_project_owners_masters.rb b/db/migrate/20161018124658_make_project_owners_masters.rb
index a576bb7b622..fe11699c196 100644
--- a/db/migrate/20161018124658_make_project_owners_masters.rb
+++ b/db/migrate/20161018124658_make_project_owners_masters.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class MakeProjectOwnersMasters < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20161227192806_rename_slack_and_mattermost_notification_services.rb b/db/migrate/20161227192806_rename_slack_and_mattermost_notification_services.rb
index 50ad7437227..c7cada6dfc5 100644
--- a/db/migrate/20161227192806_rename_slack_and_mattermost_notification_services.rb
+++ b/db/migrate/20161227192806_rename_slack_and_mattermost_notification_services.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class RenameSlackAndMattermostNotificationServices < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20170317203554_index_routes_path_for_like.rb b/db/migrate/20170317203554_index_routes_path_for_like.rb
index 7ac09b2abe5..8d3609135d0 100644
--- a/db/migrate/20170317203554_index_routes_path_for_like.rb
+++ b/db/migrate/20170317203554_index_routes_path_for_like.rb
@@ -21,9 +21,8 @@ class IndexRoutesPathForLike < ActiveRecord::Migration
 
   def down
     return unless Gitlab::Database.postgresql?
+    return unless index_exists?(:routes, :path, name: INDEX_NAME)
 
-    if index_exists?(:routes, :path, name: INDEX_NAME)
-      execute("DROP INDEX CONCURRENTLY #{INDEX_NAME};")
-    end
+    remove_concurrent_index_by_name(:routes, INDEX_NAME)
   end
 end
diff --git a/db/migrate/20170320173259_migrate_assignees.rb b/db/migrate/20170320173259_migrate_assignees.rb
index 23e7500a32d..7b61e811317 100644
--- a/db/migrate/20170320173259_migrate_assignees.rb
+++ b/db/migrate/20170320173259_migrate_assignees.rb
@@ -1,6 +1,4 @@
-# See http://doc.gitlab.com/ce/development/migration_style_guide.html
-# for more information on how to write migrations for GitLab.
-
+# rubocop:disable Migration/UpdateColumnInBatches
 class MigrateAssignees < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20170402231018_remove_index_for_users_current_sign_in_at.rb b/db/migrate/20170402231018_remove_index_for_users_current_sign_in_at.rb
index 9d4380ef960..84635fa39b9 100644
--- a/db/migrate/20170402231018_remove_index_for_users_current_sign_in_at.rb
+++ b/db/migrate/20170402231018_remove_index_for_users_current_sign_in_at.rb
@@ -11,13 +11,7 @@ class RemoveIndexForUsersCurrentSignInAt < ActiveRecord::Migration
   disable_ddl_transaction!
 
   def up
-    if index_exists? :users, :current_sign_in_at
-      if Gitlab::Database.postgresql?
-        execute 'DROP INDEX CONCURRENTLY index_users_on_current_sign_in_at;'
-      else
-        remove_concurrent_index :users, :current_sign_in_at
-      end
-    end
+    remove_concurrent_index :users, :current_sign_in_at
   end
 
   def down
diff --git a/db/migrate/20170425112628_remove_foreigh_key_ci_trigger_schedules.rb b/db/migrate/20170425112628_remove_foreigh_key_ci_trigger_schedules.rb
index 6116ca59ee4..1587eee06ae 100644
--- a/db/migrate/20170425112628_remove_foreigh_key_ci_trigger_schedules.rb
+++ b/db/migrate/20170425112628_remove_foreigh_key_ci_trigger_schedules.rb
@@ -4,10 +4,20 @@ class RemoveForeighKeyCiTriggerSchedules < ActiveRecord::Migration
   DOWNTIME = false
 
   def up
-    remove_foreign_key :ci_trigger_schedules, column: :trigger_id
+    if fk_on_trigger_schedules?
+      remove_foreign_key :ci_trigger_schedules, column: :trigger_id
+    end
   end
 
   def down
     # no op, the foreign key should not have been here
   end
+
+  private
+
+  # Not made more generic and lifted to the helpers as Rails 5 will provide
+  # such an API
+  def fk_on_trigger_schedules?
+    connection.foreign_keys(:ci_trigger_schedules).include?("ci_triggers")
+  end
 end
diff --git a/db/migrate/20170427103502_create_web_hook_logs.rb b/db/migrate/20170427103502_create_web_hook_logs.rb
new file mode 100644
index 00000000000..3643c52180c
--- /dev/null
+++ b/db/migrate/20170427103502_create_web_hook_logs.rb
@@ -0,0 +1,22 @@
+# rubocop:disable all
+class CreateWebHookLogs < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :web_hook_logs do |t|
+      t.references :web_hook, null: false, index: true, foreign_key: { on_delete: :cascade }
+
+      t.string :trigger
+      t.string :url
+      t.text :request_headers
+      t.text :request_data
+      t.text :response_headers
+      t.text :response_body
+      t.string :response_status
+      t.float :execution_duration
+      t.string :internal_error_message
+
+      t.timestamps null: false
+    end
+  end
+end
diff --git a/db/migrate/20170503140201_reschedule_project_authorizations.rb b/db/migrate/20170503140201_reschedule_project_authorizations.rb
new file mode 100644
index 00000000000..fa45adadbae
--- /dev/null
+++ b/db/migrate/20170503140201_reschedule_project_authorizations.rb
@@ -0,0 +1,44 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RescheduleProjectAuthorizations < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class User < ActiveRecord::Base
+    self.table_name = 'users'
+  end
+
+  def up
+    offset = 0
+    batch = 5000
+    start = Time.now
+
+    loop do
+      relation = User.where('id > ?', offset)
+      user_ids = relation.limit(batch).reorder(id: :asc).pluck(:id)
+
+      break if user_ids.empty?
+
+      offset = user_ids.last
+
+      # This will schedule each batch 5 minutes after the previous batch was
+      # scheduled. This smears out the load over time, instead of immediately
+      # scheduling a million jobs.
+      Sidekiq::Client.push_bulk(
+        'queue' => 'authorized_projects',
+        'args' => user_ids.zip,
+        'class' => 'AuthorizedProjectsWorker',
+        'at' => start.to_i
+      )
+
+      start += 5.minutes
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/migrate/20170503140202_turn_nested_groups_into_regular_groups_for_mysql.rb b/db/migrate/20170503140202_turn_nested_groups_into_regular_groups_for_mysql.rb
new file mode 100644
index 00000000000..c67690642c9
--- /dev/null
+++ b/db/migrate/20170503140202_turn_nested_groups_into_regular_groups_for_mysql.rb
@@ -0,0 +1,123 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+# This migration depends on code external to it. For example, it relies on
+# updating a namespace to also rename directories (uploads, GitLab pages, etc).
+# The alternative is to copy hundreds of lines of code into this migration,
+# adjust them where needed, etc; something which doesn't work well at all.
+class TurnNestedGroupsIntoRegularGroupsForMysql < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def run_migration?
+    Gitlab::Database.mysql?
+  end
+
+  def up
+    return unless run_migration?
+
+    # For all sub-groups we need to give the right people access. We do this as
+    # follows:
+    #
+    # 1. Get all the ancestors for the current namespace
+    # 2. Get all the members of these namespaces, along with their higher access
+    #    level
+    # 3. Give these members access to the current namespace
+    Namespace.unscoped.where('parent_id IS NOT NULL').find_each do |namespace|
+      rows = []
+      existing = namespace.members.pluck(:user_id)
+
+      all_members_for(namespace).each do |member|
+        next if existing.include?(member[:user_id])
+
+        rows << {
+          access_level: member[:access_level],
+          source_id: namespace.id,
+          source_type: 'Namespace',
+          user_id: member[:user_id],
+          notification_level: 3, # global
+          type: 'GroupMember',
+          created_at: Time.current,
+          updated_at: Time.current
+        }
+      end
+
+      bulk_insert_members(rows)
+
+      # This method relies on the parent to determine the proper path.
+      # Because we reset "parent_id" this method will not return the right path
+      # when moving namespaces.
+      full_path_was = namespace.send(:full_path_was)
+
+      namespace.define_singleton_method(:full_path_was) { full_path_was }
+
+      namespace.update!(parent_id: nil, path: new_path_for(namespace))
+    end
+  end
+
+  def down
+    # There is no way to go back from regular groups to nested groups.
+  end
+
+  # Generates a new (unique) path for a namespace.
+  def new_path_for(namespace)
+    counter = 1
+    base = namespace.full_path.tr('/', '-')
+    new_path = base
+
+    while Namespace.unscoped.where(path: new_path).exists?
+      new_path = base + "-#{counter}"
+      counter += 1
+    end
+
+    new_path
+  end
+
+  # Returns an Array containing all the ancestors of the current namespace.
+  #
+  # This method is not particularly efficient, but it's probably still faster
+  # than using the "routes" table. Most importantly of all, it _only_ depends
+  # on the namespaces table and the "parent_id" column.
+  def ancestors_for(namespace)
+    ancestors = []
+    current = namespace
+
+    while current&.parent_id
+      # We're using find_by(id: ...) here to deal with cases where the
+      # parent_id may point to a missing row.
+      current = Namespace.unscoped.select([:id, :parent_id]).
+        find_by(id: current.parent_id)
+
+      ancestors << current.id if current
+    end
+
+    ancestors
+  end
+
+  # Returns a relation containing all the members that have access to any of
+  # the current namespace's parent namespaces.
+  def all_members_for(namespace)
+    Member.
+      unscoped.
+      select(['user_id', 'MAX(access_level) AS access_level']).
+      where(source_type: 'Namespace', source_id: ancestors_for(namespace)).
+      group(:user_id)
+  end
+
+  def bulk_insert_members(rows)
+    return if rows.empty?
+
+    keys = rows.first.keys
+
+    tuples = rows.map do |row|
+      row.map { |(_, value)| connection.quote(value) }
+    end
+
+    execute <<-EOF.strip_heredoc
+    INSERT INTO members (#{keys.join(', ')})
+    VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')}
+    EOF
+  end
+end
diff --git a/db/migrate/20170503185032_index_redirect_routes_path_for_like.rb b/db/migrate/20170503185032_index_redirect_routes_path_for_like.rb
index 5b8b6c828be..8eb20faa03a 100644
--- a/db/migrate/20170503185032_index_redirect_routes_path_for_like.rb
+++ b/db/migrate/20170503185032_index_redirect_routes_path_for_like.rb
@@ -21,9 +21,8 @@ class IndexRedirectRoutesPathForLike < ActiveRecord::Migration
 
   def down
     return unless Gitlab::Database.postgresql?
+    return unless index_exists?(:redirect_routes, :path, name: INDEX_NAME)
 
-    if index_exists?(:redirect_routes, :path, name: INDEX_NAME)
-      execute("DROP INDEX CONCURRENTLY #{INDEX_NAME};")
-    end
+    remove_concurrent_index_by_name(:redirect_routes, INDEX_NAME)
   end
 end
diff --git a/db/migrate/20170504182103_add_index_project_group_links_group_id.rb b/db/migrate/20170504182103_add_index_project_group_links_group_id.rb
new file mode 100644
index 00000000000..62bf641daa6
--- /dev/null
+++ b/db/migrate/20170504182103_add_index_project_group_links_group_id.rb
@@ -0,0 +1,19 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddIndexProjectGroupLinksGroupId < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :project_group_links, :group_id
+  end
+
+  def down
+    remove_concurrent_index :project_group_links, :group_id
+  end
+end
diff --git a/db/migrate/20170523121229_create_conversational_development_index_metrics.rb b/db/migrate/20170523121229_create_conversational_development_index_metrics.rb
new file mode 100644
index 00000000000..9f9ec526055
--- /dev/null
+++ b/db/migrate/20170523121229_create_conversational_development_index_metrics.rb
@@ -0,0 +1,39 @@
+class CreateConversationalDevelopmentIndexMetrics < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :conversational_development_index_metrics do |t|
+      t.float :leader_issues, null: false
+      t.float :instance_issues, null: false
+
+      t.float :leader_notes, null: false
+      t.float :instance_notes, null: false
+
+      t.float :leader_milestones, null: false
+      t.float :instance_milestones, null: false
+
+      t.float :leader_boards, null: false
+      t.float :instance_boards, null: false
+
+      t.float :leader_merge_requests, null: false
+      t.float :instance_merge_requests, null: false
+
+      t.float :leader_ci_pipelines, null: false
+      t.float :instance_ci_pipelines, null: false
+
+      t.float :leader_environments, null: false
+      t.float :instance_environments, null: false
+
+      t.float :leader_deployments, null: false
+      t.float :instance_deployments, null: false
+
+      t.float :leader_projects_prometheus_active, null: false
+      t.float :instance_projects_prometheus_active, null: false
+
+      t.float :leader_service_desk_issues, null: false
+      t.float :instance_service_desk_issues, null: false
+
+      t.timestamps null: false
+    end
+  end
+end
diff --git a/db/migrate/20170524125940_add_source_to_ci_pipeline.rb b/db/migrate/20170524125940_add_source_to_ci_pipeline.rb
new file mode 100644
index 00000000000..1fa3d48037b
--- /dev/null
+++ b/db/migrate/20170524125940_add_source_to_ci_pipeline.rb
@@ -0,0 +1,9 @@
+class AddSourceToCiPipeline < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_pipelines, :source, :integer
+  end
+end
diff --git a/db/migrate/20170524161101_add_protected_to_ci_variables.rb b/db/migrate/20170524161101_add_protected_to_ci_variables.rb
new file mode 100644
index 00000000000..99d4861e889
--- /dev/null
+++ b/db/migrate/20170524161101_add_protected_to_ci_variables.rb
@@ -0,0 +1,15 @@
+class AddProtectedToCiVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:ci_variables, :protected, :boolean, default: false)
+  end
+
+  def down
+    remove_column(:ci_variables, :protected)
+  end
+end
diff --git a/db/migrate/20170525174156_create_feature_tables.rb b/db/migrate/20170525174156_create_feature_tables.rb
new file mode 100644
index 00000000000..a083c89c85f
--- /dev/null
+++ b/db/migrate/20170525174156_create_feature_tables.rb
@@ -0,0 +1,26 @@
+class CreateFeatureTables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def self.up
+    create_table :features do |t|
+      t.string :key, null: false
+      t.timestamps null: false
+    end
+    add_index :features, :key, unique: true
+
+    create_table :feature_gates do |t|
+      t.string :feature_key, null: false
+      t.string :key, null: false
+      t.string :value
+      t.timestamps null: false
+    end
+    add_index :feature_gates, [:feature_key, :key, :value], unique: true
+  end
+
+  def self.down
+    drop_table :feature_gates
+    drop_table :features
+  end
+end
diff --git a/db/post_migrate/20170131214021_reset_users_authorized_projects_populated.rb b/db/post_migrate/20170131214021_reset_users_authorized_projects_populated.rb
index b518038e93a..82f8147547e 100644
--- a/db/post_migrate/20170131214021_reset_users_authorized_projects_populated.rb
+++ b/db/post_migrate/20170131214021_reset_users_authorized_projects_populated.rb
@@ -1,6 +1,4 @@
-# See http://doc.gitlab.com/ce/development/migration_style_guide.html
-# for more information on how to write migrations for GitLab.
-
+# rubocop:disable Migration/UpdateColumnInBatches
 class ResetUsersAuthorizedProjectsPopulated < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/post_migrate/20170309171644_reset_relative_position_for_issue.rb b/db/post_migrate/20170309171644_reset_relative_position_for_issue.rb
index b61dd7cfc61..b1c9eed1148 100644
--- a/db/post_migrate/20170309171644_reset_relative_position_for_issue.rb
+++ b/db/post_migrate/20170309171644_reset_relative_position_for_issue.rb
@@ -1,6 +1,4 @@
-# See http://doc.gitlab.com/ce/development/migration_style_guide.html
-# for more information on how to write migrations for GitLab.
-
+# rubocop:disable Migration/UpdateColumnInBatches
 class ResetRelativePositionForIssue < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/post_migrate/20170502070007_enable_auto_cancel_pending_pipelines_for_all.rb b/db/post_migrate/20170502070007_enable_auto_cancel_pending_pipelines_for_all.rb
index a19b73fc114..3c13a3d2518 100644
--- a/db/post_migrate/20170502070007_enable_auto_cancel_pending_pipelines_for_all.rb
+++ b/db/post_migrate/20170502070007_enable_auto_cancel_pending_pipelines_for_all.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class EnableAutoCancelPendingPipelinesForAll < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/post_migrate/20170503120310_remove_users_authorized_projects_populated.rb b/db/post_migrate/20170503120310_remove_users_authorized_projects_populated.rb
new file mode 100644
index 00000000000..1b44334395f
--- /dev/null
+++ b/db/post_migrate/20170503120310_remove_users_authorized_projects_populated.rb
@@ -0,0 +1,15 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveUsersAuthorizedProjectsPopulated < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def change
+    remove_column :users, :authorized_projects_populated, :boolean
+  end
+end
diff --git a/db/post_migrate/20170523083112_migrate_old_artifacts.rb b/db/post_migrate/20170523083112_migrate_old_artifacts.rb
new file mode 100644
index 00000000000..f2690bd0017
--- /dev/null
+++ b/db/post_migrate/20170523083112_migrate_old_artifacts.rb
@@ -0,0 +1,72 @@
+class MigrateOldArtifacts < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  # This uses special heuristic to find potential candidates for data migration
+  # Read more about this here: https://gitlab.com/gitlab-org/gitlab-ce/issues/32036#note_30422345
+  
+  def up
+    builds_with_artifacts.find_each do |build|
+      build.migrate_artifacts!
+    end
+  end
+
+  def down
+  end
+
+  private
+
+  def builds_with_artifacts
+    Build.with_artifacts
+      .joins('JOIN projects ON projects.id = ci_builds.project_id')
+      .where('ci_builds.id < ?', min_id)
+      .where('projects.ci_id IS NOT NULL')
+      .select('id', 'created_at', 'project_id', 'projects.ci_id AS ci_id')
+  end
+
+  def min_id
+    Build.joins('JOIN projects ON projects.id = ci_builds.project_id')
+      .where('projects.ci_id IS NULL')
+      .pluck('coalesce(min(ci_builds.id), 0)')
+      .first
+  end
+
+  class Build < ActiveRecord::Base
+    self.table_name = 'ci_builds'
+
+    scope :with_artifacts, -> { where.not(artifacts_file: [nil, '']) }
+
+    def migrate_artifacts!
+      return unless File.exist?(source_artifacts_path)
+      return if File.exist?(target_artifacts_path)
+
+      ensure_target_path
+
+      FileUtils.move(source_artifacts_path, target_artifacts_path)
+    end
+
+    private
+
+    def source_artifacts_path
+      @source_artifacts_path ||= 
+        File.join(Gitlab.config.artifacts.path,
+          created_at.utc.strftime('%Y_%m'),
+          ci_id.to_s, id.to_s)
+    end
+
+    def target_artifacts_path
+      @target_artifacts_path ||= 
+        File.join(Gitlab.config.artifacts.path,
+          created_at.utc.strftime('%Y_%m'),
+          project_id.to_s, id.to_s)
+    end
+
+    def ensure_target_path
+      directory = File.dirname(target_artifacts_path)
+      FileUtils.mkdir_p(directory) unless Dir.exist?(directory)
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 8d57159d2e0..47484e6b8fe 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -284,6 +284,7 @@ ActiveRecord::Schema.define(version: 20170526190948) do
     t.integer "lock_version"
     t.integer "auto_canceled_by_id"
     t.integer "pipeline_schedule_id"
+    t.integer "source"
   end
 
   add_index "ci_pipelines", ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id", using: :btree
@@ -366,6 +367,7 @@ ActiveRecord::Schema.define(version: 20170526190948) do
     t.string "encrypted_value_salt"
     t.string "encrypted_value_iv"
     t.integer "project_id", null: false
+    t.boolean "protected", default: false, null: false
   end
 
   add_index "ci_variables", ["project_id"], name: "index_ci_variables_on_project_id", using: :btree
@@ -380,6 +382,31 @@ ActiveRecord::Schema.define(version: 20170526190948) do
   add_index "container_repositories", ["project_id", "name"], name: "index_container_repositories_on_project_id_and_name", unique: true, using: :btree
   add_index "container_repositories", ["project_id"], name: "index_container_repositories_on_project_id", using: :btree
 
+  create_table "conversational_development_index_metrics", force: :cascade do |t|
+    t.float "leader_issues", null: false
+    t.float "instance_issues", null: false
+    t.float "leader_notes", null: false
+    t.float "instance_notes", null: false
+    t.float "leader_milestones", null: false
+    t.float "instance_milestones", null: false
+    t.float "leader_boards", null: false
+    t.float "instance_boards", null: false
+    t.float "leader_merge_requests", null: false
+    t.float "instance_merge_requests", null: false
+    t.float "leader_ci_pipelines", null: false
+    t.float "instance_ci_pipelines", null: false
+    t.float "leader_environments", null: false
+    t.float "instance_environments", null: false
+    t.float "leader_deployments", null: false
+    t.float "instance_deployments", null: false
+    t.float "leader_projects_prometheus_active", null: false
+    t.float "instance_projects_prometheus_active", null: false
+    t.float "leader_service_desk_issues", null: false
+    t.float "instance_service_desk_issues", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
   create_table "deploy_keys_projects", force: :cascade do |t|
     t.integer "deploy_key_id", null: false
     t.integer "project_id", null: false
@@ -451,6 +478,24 @@ ActiveRecord::Schema.define(version: 20170526190948) do
   add_index "events", ["target_id"], name: "index_events_on_target_id", using: :btree
   add_index "events", ["target_type"], name: "index_events_on_target_type", using: :btree
 
+  create_table "feature_gates", force: :cascade do |t|
+    t.string "feature_key", null: false
+    t.string "key", null: false
+    t.string "value"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "feature_gates", ["feature_key", "key", "value"], name: "index_feature_gates_on_feature_key_and_key_and_value", unique: true, using: :btree
+
+  create_table "features", force: :cascade do |t|
+    t.string "key", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "features", ["key"], name: "index_features_on_key", unique: true, using: :btree
+
   create_table "forked_project_links", force: :cascade do |t|
     t.integer "forked_to_project_id", null: false
     t.integer "forked_from_project_id", null: false
@@ -939,6 +984,8 @@ ActiveRecord::Schema.define(version: 20170526190948) do
     t.date "expires_at"
   end
 
+  add_index "project_group_links", ["group_id"], name: "index_project_group_links_on_group_id", using: :btree
+
   create_table "project_import_data", force: :cascade do |t|
     t.integer "project_id"
     t.text "data"
@@ -1366,7 +1413,6 @@ ActiveRecord::Schema.define(version: 20170526190948) do
     t.boolean "external", default: false
     t.string "incoming_email_token"
     t.string "organization"
-    t.boolean "authorized_projects_populated"
     t.boolean "require_two_factor_authentication_from_group", default: false, null: false
     t.integer "two_factor_grace_period", default: 48, null: false
     t.boolean "ghost"
@@ -1402,6 +1448,23 @@ ActiveRecord::Schema.define(version: 20170526190948) do
   add_index "users_star_projects", ["project_id"], name: "index_users_star_projects_on_project_id", using: :btree
   add_index "users_star_projects", ["user_id", "project_id"], name: "index_users_star_projects_on_user_id_and_project_id", unique: true, using: :btree
 
+  create_table "web_hook_logs", force: :cascade do |t|
+    t.integer "web_hook_id", null: false
+    t.string "trigger"
+    t.string "url"
+    t.text "request_headers"
+    t.text "request_data"
+    t.text "response_headers"
+    t.text "response_body"
+    t.string "response_status"
+    t.float "execution_duration"
+    t.string "internal_error_message"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "web_hook_logs", ["web_hook_id"], name: "index_web_hook_logs_on_web_hook_id", using: :btree
+
   create_table "web_hooks", force: :cascade do |t|
     t.string "url", limit: 2000
     t.integer "project_id"
@@ -1419,8 +1482,8 @@ ActiveRecord::Schema.define(version: 20170526190948) do
     t.string "token"
     t.boolean "pipeline_events", default: false, null: false
     t.boolean "confidential_issues_events", default: false, null: false
-    t.boolean "job_events", default: false, null: false
     t.boolean "repository_update_events", default: false, null: false
+    t.boolean "job_events", default: false, null: false
   end
 
   add_index "web_hooks", ["project_id"], name: "index_web_hooks_on_project_id", using: :btree
@@ -1467,4 +1530,5 @@ ActiveRecord::Schema.define(version: 20170526190948) do
   add_foreign_key "timelogs", "merge_requests", name: "fk_timelogs_merge_requests_merge_request_id", on_delete: :cascade
   add_foreign_key "trending_projects", "projects", on_delete: :cascade
   add_foreign_key "u2f_registrations", "users"
+  add_foreign_key "web_hook_logs", "web_hooks", on_delete: :cascade
 end