Merge remote-tracking branch 'upstream/master' into feature/1376-allow-write-access-deploy-keys

* upstream/master: (538 commits)
  Reject blank environment vcariables in Gitlab::Git::RevList
  Add online terminal documentation
  Add changelog entry
  Add terminal UI and controller actions
  Fix specs
  Even out padding on plus button in breadcrumb menu
  Update font size of detail page header to 14px
  Update CHANGELOG.md for 8.13.10
  Update CHANGELOG.md for 8.14.5
  Fix Route#rename_children behavior
  Remove inline-block styling from status
  Add terminals to the Kubernetes deployment service
  Add a ReactiveCaching concern for use in the KubernetesService
  Add xterm.js 2.1.0 and a wrapper class to the asset pipeline
  Remove unnecessary hidden svg elements for icons.
  Fix consistent typo in environment.js
  Use a block to insert extra check for authenticate_build!
  Align milestone column header with count number
  Add Wiki import to BB importer
  Make CI badge hitboxes better match container
  ...

16 files changed, 364 insertions, 28 deletions

diff --git a/db/migrate/20141006143943_move_slack_service_to_webhook.rb b/db/migrate/20141006143943_move_slack_service_to_webhook.rb
index 8cb120f7007..42e88d6d6e3 100644
--- a/db/migrate/20141006143943_move_slack_service_to_webhook.rb
+++ b/db/migrate/20141006143943_move_slack_service_to_webhook.rb
@@ -1,7 +1,11 @@
 # rubocop:disable all
 class MoveSlackServiceToWebhook < ActiveRecord::Migration
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Move old fields "token" and "subdomain" to one single field "webhook"'
+
   def change
-    SlackService.all.each do |slack_service|
+    SlackNotificationService.all.each do |slack_service|
       if ["token", "subdomain"].all? { |property| slack_service.properties.key? property }
         token = slack_service.properties['token']
         subdomain = slack_service.properties['subdomain']
diff --git a/db/migrate/20160823083941_add_column_scopes_to_personal_access_tokens.rb b/db/migrate/20160823083941_add_column_scopes_to_personal_access_tokens.rb
new file mode 100644
index 00000000000..91479de840b
--- /dev/null
+++ b/db/migrate/20160823083941_add_column_scopes_to_personal_access_tokens.rb
@@ -0,0 +1,19 @@
+# The default needs to be `[]`, but all existing access tokens need to have `scopes` set to `['api']`.
+# It's easier to achieve this by adding the column with the `['api']` default, and then changing the default to
+# `[]`.
+
+class AddColumnScopesToPersonalAccessTokens < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :personal_access_tokens, :scopes, :string, default: ['api'].to_yaml
+  end
+
+  def down
+    remove_column :personal_access_tokens, :scopes
+  end
+end
diff --git a/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb b/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb
index 453a44e271a..77e0c40d850 100644
--- a/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb
+++ b/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb
@@ -47,14 +47,14 @@ class MigrateProcessCommitWorkerJobs < ActiveRecord::Migration
 
         hash = {
           id: commit.oid,
-          message: commit.message,
+          message: encode(commit.message),
           parent_ids: commit.parent_ids,
           authored_date: commit.author[:time],
-          author_name: commit.author[:name],
-          author_email: commit.author[:email],
+          author_name: encode(commit.author[:name]),
+          author_email: encode(commit.author[:email]),
           committed_date: commit.committer[:time],
-          committer_email: commit.committer[:email],
-          committer_name: commit.committer[:name]
+          committer_email: encode(commit.committer[:email]),
+          committer_name: encode(commit.committer[:name])
         }
 
         payload['args'][2] = hash
@@ -89,4 +89,14 @@ class MigrateProcessCommitWorkerJobs < ActiveRecord::Migration
       end
     end
   end
+
+  def encode(data)
+    encoding = Encoding::UTF_8
+
+    if data.encoding == encoding
+      data
+    else
+      data.encode(encoding, invalid: :replace, undef: :replace)
+    end
+  end
 end
diff --git a/db/migrate/20161130095245_fill_routes_table.rb b/db/migrate/20161130095245_fill_routes_table.rb
index 6754e583000..c3536d6d911 100644
--- a/db/migrate/20161130095245_fill_routes_table.rb
+++ b/db/migrate/20161130095245_fill_routes_table.rb
@@ -16,6 +16,6 @@ class FillRoutesTable < ActiveRecord::Migration
   end
 
   def down
-    Route.delete_all(source_type: 'Namespace')
+    execute("DELETE FROM routes WHERE source_type = 'Namespace'")
   end
 end
diff --git a/db/migrate/20161130101252_fill_projects_routes_table.rb b/db/migrate/20161130101252_fill_projects_routes_table.rb
index 14700583be5..56ba6fcdbe3 100644
--- a/db/migrate/20161130101252_fill_projects_routes_table.rb
+++ b/db/migrate/20161130101252_fill_projects_routes_table.rb
@@ -8,15 +8,23 @@ class FillProjectsRoutesTable < ActiveRecord::Migration
   DOWNTIME_REASON = 'No new projects should be created during data copy'
 
   def up
-    execute <<-EOF
-      INSERT INTO routes
-      (source_id, source_type, path)
-      (SELECT projects.id, 'Project', concat(namespaces.path, '/', projects.path) FROM projects
-      INNER JOIN namespaces ON projects.namespace_id = namespaces.id)
-    EOF
+    if Gitlab::Database.postgresql?
+      execute <<-EOF
+        INSERT INTO routes (source_id, source_type, path)
+        (SELECT DISTINCT ON (namespaces.path, projects.path) projects.id, 'Project', concat(namespaces.path, '/', projects.path)
+         FROM projects INNER JOIN namespaces ON projects.namespace_id = namespaces.id
+         ORDER BY namespaces.path, projects.path, projects.id DESC)
+      EOF
+    else
+      execute <<-EOF
+        INSERT INTO routes (source_id, source_type, path)
+        (SELECT projects.id, 'Project', concat(namespaces.path, '/', projects.path)
+         FROM projects INNER JOIN namespaces ON projects.namespace_id = namespaces.id)
+      EOF
+    end
   end
 
   def down
-    Route.delete_all(source_type: 'Project')
+    execute("DELETE FROM routes WHERE source_type = 'Project'")
   end
 end
diff --git a/db/migrate/20161202152031_remove_duplicates_from_routes.rb b/db/migrate/20161202152031_remove_duplicates_from_routes.rb
index 510796e05f2..d73b0847506 100644
--- a/db/migrate/20161202152031_remove_duplicates_from_routes.rb
+++ b/db/migrate/20161202152031_remove_duplicates_from_routes.rb
@@ -7,20 +7,21 @@ class RemoveDuplicatesFromRoutes < ActiveRecord::Migration
   DOWNTIME = false
 
   def up
-    select_all("SELECT path FROM #{quote_table_name(:routes)} GROUP BY path HAVING COUNT(*) > 1").each do |row|
-      path = connection.quote(row['path'])
-      execute(%Q{
-        DELETE FROM #{quote_table_name(:routes)}
-        WHERE path = #{path}
-        AND id != (
-          SELECT id FROM (
-            SELECT max(id) AS id
-            FROM #{quote_table_name(:routes)}
-            WHERE path = #{path}
-          ) max_ids
-        )
-      })
-    end
+    # We can skip this migration when running a PostgreSQL database because
+    # we use an optimized query in the "FillProjectsRoutesTable" migration
+    # to fill these values that avoid duplicate entries in the routes table.
+    return unless Gitlab::Database.mysql?
+
+    execute <<-EOF
+      DELETE duplicated_rows.*
+      FROM routes AS duplicated_rows
+        INNER JOIN (
+          SELECT path, MAX(id) as max_id
+          FROM routes
+          GROUP BY path
+          HAVING COUNT(*) > 1
+        ) AS good_rows ON good_rows.path = duplicated_rows.path AND good_rows.max_id <> duplicated_rows.id;
+    EOF
   end
 
   def down
diff --git a/db/migrate/20161206153749_remove_uniq_path_index_from_namespace.rb b/db/migrate/20161206153749_remove_uniq_path_index_from_namespace.rb
new file mode 100644
index 00000000000..2977917f2d1
--- /dev/null
+++ b/db/migrate/20161206153749_remove_uniq_path_index_from_namespace.rb
@@ -0,0 +1,36 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveUniqPathIndexFromNamespace < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    constraint_name = 'namespaces_path_key'
+
+    transaction do
+      if index_exists?(:namespaces, :path)
+        remove_index(:namespaces, :path)
+      end
+
+      # In some bizarre cases PostgreSQL might have a separate unique constraint
+      # that we'll need to drop.
+      if constraint_exists?(constraint_name) && Gitlab::Database.postgresql?
+        execute("ALTER TABLE namespaces DROP CONSTRAINT IF EXISTS #{constraint_name};")
+      end
+    end
+  end
+
+  def down
+    unless index_exists?(:namespaces, :path)
+      add_concurrent_index(:namespaces, :path, unique: true)
+    end
+  end
+
+  def constraint_exists?(name)
+    indexes(:namespaces).map(&:name).include?(name)
+  end
+end
diff --git a/db/migrate/20161206153751_add_path_index_to_namespace.rb b/db/migrate/20161206153751_add_path_index_to_namespace.rb
new file mode 100644
index 00000000000..b0bac7d121e
--- /dev/null
+++ b/db/migrate/20161206153751_add_path_index_to_namespace.rb
@@ -0,0 +1,20 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddPathIndexToNamespace < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    add_concurrent_index :namespaces, :path
+  end
+
+  def down
+    if index_exists?(:namespaces, :path)
+      remove_index :namespaces, :path
+    end
+  end
+end
diff --git a/db/migrate/20161206153753_remove_uniq_name_index_from_namespace.rb b/db/migrate/20161206153753_remove_uniq_name_index_from_namespace.rb
new file mode 100644
index 00000000000..cc9d4974baa
--- /dev/null
+++ b/db/migrate/20161206153753_remove_uniq_name_index_from_namespace.rb
@@ -0,0 +1,36 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveUniqNameIndexFromNamespace < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    constraint_name = 'namespaces_name_key'
+
+    transaction do
+      if index_exists?(:namespaces, :name)
+        remove_index(:namespaces, :name)
+      end
+
+      # In some bizarre cases PostgreSQL might have a separate unique constraint
+      # that we'll need to drop.
+      if constraint_exists?(constraint_name) && Gitlab::Database.postgresql?
+        execute("ALTER TABLE namespaces DROP CONSTRAINT IF EXISTS #{constraint_name};")
+      end
+    end
+  end
+
+  def down
+    unless index_exists?(:namespaces, :name)
+      add_concurrent_index(:namespaces, :name, unique: true)
+    end
+  end
+
+  def constraint_exists?(name)
+    indexes(:namespaces).map(&:name).include?(name)
+  end
+end
diff --git a/db/migrate/20161206153754_add_name_index_to_namespace.rb b/db/migrate/20161206153754_add_name_index_to_namespace.rb
new file mode 100644
index 00000000000..b3f3cb68a99
--- /dev/null
+++ b/db/migrate/20161206153754_add_name_index_to_namespace.rb
@@ -0,0 +1,20 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddNameIndexToNamespace < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    add_concurrent_index(:namespaces, [:name, :parent_id], unique: true)
+  end
+
+  def down
+    if index_exists?(:namespaces, [:name, :parent_id])
+      remove_index :namespaces, [:name, :parent_id]
+    end
+  end
+end
diff --git a/db/migrate/20161207231620_fixup_environment_name_uniqueness.rb b/db/migrate/20161207231620_fixup_environment_name_uniqueness.rb
new file mode 100644
index 00000000000..b74552e762d
--- /dev/null
+++ b/db/migrate/20161207231620_fixup_environment_name_uniqueness.rb
@@ -0,0 +1,53 @@
+class FixupEnvironmentNameUniqueness < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Renaming non-unique environments'
+
+  def up
+    environments = Arel::Table.new(:environments)
+
+    # Get all [project_id, name] pairs that occur more than once
+    finder_sql = environments.
+      group(environments[:project_id], environments[:name]).
+      having(Arel.sql("COUNT(1)").gt(1)).
+      project(environments[:project_id], environments[:name]).
+      to_sql
+
+    conflicting = connection.exec_query(finder_sql)
+
+    conflicting.rows.each do |project_id, name|
+      fix_duplicates(project_id, name)
+    end
+  end
+
+  def down
+    # Nothing to do
+  end
+
+  # Rename conflicting environments by appending "-#{id}" to all but the first
+  def fix_duplicates(project_id, name)
+    environments = Arel::Table.new(:environments)
+    finder_sql = environments.
+      where(environments[:project_id].eq(project_id)).
+      where(environments[:name].eq(name)).
+      order(environments[:id].asc).
+      project(environments[:id], environments[:name]).
+      to_sql
+
+    # Now we have the data for all the conflicting rows
+    conflicts = connection.exec_query(finder_sql).rows
+    conflicts.shift # Leave the first row alone
+
+    conflicts.each do |id, name|
+      update_sql =
+        Arel::UpdateManager.new(ActiveRecord::Base).
+        table(environments).
+        set(environments[:name] => name + "-" + id.to_s).
+        where(environments[:id].eq(id)).
+        to_sql
+
+      connection.exec_update(update_sql, self.class.name, [])
+    end
+  end
+end
diff --git a/db/migrate/20161207231621_create_environment_name_unique_index.rb b/db/migrate/20161207231621_create_environment_name_unique_index.rb
new file mode 100644
index 00000000000..ac680c8d10f
--- /dev/null
+++ b/db/migrate/20161207231621_create_environment_name_unique_index.rb
@@ -0,0 +1,18 @@
+class CreateEnvironmentNameUniqueIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Making a non-unique index into a unique index'
+
+  def up
+    remove_index :environments, [:project_id, :name]
+    add_concurrent_index :environments, [:project_id, :name], unique: true
+  end
+
+  def down
+    remove_index :environments, [:project_id, :name], unique: true
+    add_concurrent_index :environments, [:project_id, :name]
+  end
+end
diff --git a/db/migrate/20161207231626_add_environment_slug.rb b/db/migrate/20161207231626_add_environment_slug.rb
new file mode 100644
index 00000000000..7153e6a32b1
--- /dev/null
+++ b/db/migrate/20161207231626_add_environment_slug.rb
@@ -0,0 +1,60 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddEnvironmentSlug < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Adding NOT NULL column environments.slug with dependent data'
+
+  # Used to generate random suffixes for the slug
+  NUMBERS = '0'..'9'
+  SUFFIX_CHARS = ('a'..'z').to_a + NUMBERS.to_a
+
+  def up
+    environments = Arel::Table.new(:environments)
+
+    add_column :environments, :slug, :string
+    finder = environments.project(:id, :name)
+
+    connection.exec_query(finder.to_sql).rows.each do |id, name|
+      updater = Arel::UpdateManager.new(ActiveRecord::Base).
+        table(environments).
+        set(environments[:slug] => generate_slug(name)).
+        where(environments[:id].eq(id))
+
+      connection.exec_update(updater.to_sql, self.class.name, [])
+    end
+
+    change_column_null :environments, :slug, false
+  end
+
+  def down
+    remove_column :environments, :slug
+  end
+
+  # Copy of the Environment#generate_slug implementation
+  def generate_slug(name)
+    # Lowercase letters and numbers only
+    slugified = name.to_s.downcase.gsub(/[^a-z0-9]/, '-')
+
+    # Must start with a letter
+    slugified = "env-" + slugified if NUMBERS.cover?(slugified[0])
+
+    # Maximum length: 24 characters (OpenShift limitation)
+    slugified = slugified[0..23]
+
+    # Cannot end with a "-" character (Kubernetes label limitation)
+    slugified = slugified[0..-2] if slugified[-1] == "-"
+
+    # Add a random suffix, shortening the current string if necessary, if it
+    # has been slugified. This ensures uniqueness.
+    slugified = slugified[0..16] + "-" + random_suffix if slugified != name
+
+    slugified
+  end
+
+  def random_suffix
+    (0..5).map { SUFFIX_CHARS.sample }.join
+  end
+end
diff --git a/db/migrate/20161209153400_add_unique_index_for_environment_slug.rb b/db/migrate/20161209153400_add_unique_index_for_environment_slug.rb
new file mode 100644
index 00000000000..e9fcef1cd45
--- /dev/null
+++ b/db/migrate/20161209153400_add_unique_index_for_environment_slug.rb
@@ -0,0 +1,15 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddUniqueIndexForEnvironmentSlug < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Adding a *unique* index to environments.slug'
+
+  disable_ddl_transaction!
+
+  def change
+    add_concurrent_index :environments, [:project_id, :slug], unique: true
+  end
+end
diff --git a/db/migrate/20161212142807_add_lower_path_index_to_routes.rb b/db/migrate/20161212142807_add_lower_path_index_to_routes.rb
new file mode 100644
index 00000000000..6958500306f
--- /dev/null
+++ b/db/migrate/20161212142807_add_lower_path_index_to_routes.rb
@@ -0,0 +1,22 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddLowerPathIndexToRoutes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    return unless Gitlab::Database.postgresql?
+
+    execute 'CREATE INDEX CONCURRENTLY index_on_routes_lower_path ON routes (LOWER(path));'
+  end
+
+  def down
+    return unless Gitlab::Database.postgresql?
+
+    remove_index :routes, name: :index_on_routes_lower_path
+  end
+end
diff --git a/db/migrate/20161213172958_change_slack_service_to_slack_notification_service.rb b/db/migrate/20161213172958_change_slack_service_to_slack_notification_service.rb
new file mode 100644
index 00000000000..a7278d7b5a6
--- /dev/null
+++ b/db/migrate/20161213172958_change_slack_service_to_slack_notification_service.rb
@@ -0,0 +1,14 @@
+class ChangeSlackServiceToSlackNotificationService < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Rename SlackService to SlackNotificationService'
+
+  def up
+    execute("UPDATE services SET type = 'SlackNotificationService' WHERE type = 'SlackService'")
+  end
+
+  def down
+    execute("UPDATE services SET type = 'SlackService' WHERE type = 'SlackNotificationService'")
+  end
+end