Merge remote-tracking branch 'origin/master' into ci-commit-as-pipeline

# Conflicts:
#	db/schema.rb

6 files changed, 159 insertions, 1 deletions

diff --git a/db/migrate/20160302151724_add_import_credentials_to_project_import_data.rb b/db/migrate/20160302151724_add_import_credentials_to_project_import_data.rb
new file mode 100644
index 00000000000..ffcd64266e3
--- /dev/null
+++ b/db/migrate/20160302151724_add_import_credentials_to_project_import_data.rb
@@ -0,0 +1,7 @@
+class AddImportCredentialsToProjectImportData < ActiveRecord::Migration
+  def change
+    add_column :project_import_data, :encrypted_credentials, :text
+    add_column :project_import_data, :encrypted_credentials_iv, :string
+    add_column :project_import_data, :encrypted_credentials_salt, :string
+  end
+end
diff --git a/db/migrate/20160302152808_remove_wrong_import_url_from_projects.rb b/db/migrate/20160302152808_remove_wrong_import_url_from_projects.rb
new file mode 100644
index 00000000000..8a351cf27a3
--- /dev/null
+++ b/db/migrate/20160302152808_remove_wrong_import_url_from_projects.rb
@@ -0,0 +1,131 @@
+# Loops through old importer projects that kept a token/password in the import URL
+# and encrypts the credentials into a separate field in project#import_data
+# #down method not supported
+class RemoveWrongImportUrlFromProjects < ActiveRecord::Migration
+
+  class ProjectImportDataFake
+    extend AttrEncrypted
+    attr_accessor :credentials
+    attr_encrypted :credentials, key: Gitlab::Application.secrets.db_key_base, marshal: true, encode: true, :mode => :per_attribute_iv_and_salt
+  end
+
+  def up
+    say("Encrypting and migrating project import credentials...")
+
+    # This should cover GitHub, GitLab, Bitbucket user:password, token@domain, and other similar URLs.
+    in_transaction(message: "Projects including GitHub and GitLab projects with an unsecured URL.") { process_projects_with_wrong_url }
+
+    in_transaction(message: "Migrating Bitbucket credentials...") { process_project(import_type: 'bitbucket', credentials_keys: ['bb_session']) }
+
+    in_transaction(message: "Migrating FogBugz credentials...") { process_project(import_type: 'fogbugz', credentials_keys: ['fb_session']) }
+
+  end
+
+  def process_projects_with_wrong_url
+    projects_with_wrong_import_url.each do |project|
+      begin
+        import_url = Gitlab::ImportUrl.new(project["import_url"])
+
+        update_import_url(import_url, project)
+        update_import_data(import_url, project)
+      rescue URI::InvalidURIError
+        nullify_import_url(project)
+      end
+    end
+  end
+
+  def process_project(import_type:, credentials_keys: [])
+    unencrypted_import_data(import_type: import_type).each do |data|
+      replace_data_credentials(data, credentials_keys)
+    end
+  end
+
+  def replace_data_credentials(data, credentials_keys)
+    data_hash = JSON.load(data['data']) if data['data']
+    unless data_hash.blank?
+      encrypted_data_hash = encrypt_data(data_hash, credentials_keys)
+      unencrypted_data = data_hash.empty? ? ' NULL ' :  quote(data_hash.to_json)
+      update_with_encrypted_data(encrypted_data_hash, data['id'], unencrypted_data)
+    end
+  end
+
+  def encrypt_data(data_hash, credentials_keys)
+    new_data_hash = {}
+    credentials_keys.each do |key|
+      new_data_hash[key.to_sym] = data_hash.delete(key) if data_hash[key]
+    end
+    new_data_hash.deep_symbolize_keys
+  end
+
+  def in_transaction(message:)
+    say_with_time(message) do
+      ActiveRecord::Base.transaction do
+        yield
+      end
+    end
+  end
+
+  def update_import_data(import_url, project)
+    fake_import_data = ProjectImportDataFake.new
+    fake_import_data.credentials = import_url.credentials
+    import_data_id = project['import_data_id']
+    if import_data_id
+      execute(update_import_data_sql(import_data_id, fake_import_data))
+    else
+      execute(insert_import_data_sql(project['id'], fake_import_data))
+    end
+  end
+
+  def update_with_encrypted_data(data_hash, import_data_id, unencrypted_data = ' NULL ')
+    fake_import_data = ProjectImportDataFake.new
+    fake_import_data.credentials = data_hash
+    execute(update_import_data_sql(import_data_id, fake_import_data, unencrypted_data))
+  end
+
+  def update_import_url(import_url, project)
+    execute("UPDATE projects SET import_url = #{quote(import_url.sanitized_url)} WHERE id = #{project['id']}")
+  end
+
+  def nullify_import_url(project)
+    execute("UPDATE projects SET import_url = NULL WHERE id = #{project['id']}")
+  end
+
+  def insert_import_data_sql(project_id, fake_import_data)
+    %(
+      INSERT INTO project_import_data
+                  (encrypted_credentials,
+                   project_id,
+                   encrypted_credentials_iv,
+                   encrypted_credentials_salt)
+      VALUES      ( #{quote(fake_import_data.encrypted_credentials)},
+                    '#{project_id}',
+                    #{quote(fake_import_data.encrypted_credentials_iv)},
+                    #{quote(fake_import_data.encrypted_credentials_salt)})
+    ).squish
+  end
+
+  def update_import_data_sql(id, fake_import_data, data = 'NULL')
+    %(
+      UPDATE project_import_data
+      SET    encrypted_credentials = #{quote(fake_import_data.encrypted_credentials)},
+             encrypted_credentials_iv = #{quote(fake_import_data.encrypted_credentials_iv)},
+             encrypted_credentials_salt = #{quote(fake_import_data.encrypted_credentials_salt)},
+             data = #{data}
+      WHERE  id = '#{id}'
+    ).squish
+  end
+
+  #GitHub projects with token, and any user:password@ based URL
+  def projects_with_wrong_import_url
+    select_all("SELECT p.id, p.import_url, i.id as import_data_id FROM projects p LEFT JOIN project_import_data i on p.id = i.project_id WHERE p.import_url <> '' AND p.import_url LIKE '%//%@%'")
+  end
+
+  # All imports with data for import_type
+  def unencrypted_import_data(import_type: )
+    select_all("SELECT i.id, p.import_url, i.data FROM projects p INNER JOIN project_import_data i ON p.id = i.project_id WHERE p.import_url <> '' AND p.import_type = '#{import_type}' ")
+  end
+
+  def quote(value)
+    ActiveRecord::Base.connection.quote(value)
+  end
+end
diff --git a/db/migrate/20160315135439_project_add_repository_check.rb b/db/migrate/20160315135439_project_add_repository_check.rb
new file mode 100644
index 00000000000..8687d5d6296
--- /dev/null
+++ b/db/migrate/20160315135439_project_add_repository_check.rb
@@ -0,0 +1,8 @@
+class ProjectAddRepositoryCheck < ActiveRecord::Migration
+  def change
+    add_column :projects, :last_repository_check_failed, :boolean
+    add_index :projects, :last_repository_check_failed
+
+    add_column :projects, :last_repository_check_at, :datetime
+  end
+end
diff --git a/db/migrate/20160328115649_migrate_new_notification_setting.rb b/db/migrate/20160328115649_migrate_new_notification_setting.rb
index 0a110869027..3c81b2c37bf 100644
--- a/db/migrate/20160328115649_migrate_new_notification_setting.rb
+++ b/db/migrate/20160328115649_migrate_new_notification_setting.rb
@@ -7,7 +7,7 @@
 #
 class MigrateNewNotificationSetting < ActiveRecord::Migration
   def up
-    timestamp = Time.now
+    timestamp = Time.now.strftime('%F %T')
     execute "INSERT INTO notification_settings ( user_id, source_id, source_type, level, created_at, updated_at ) SELECT user_id, source_id, source_type, notification_level, '#{timestamp}', '#{timestamp}' FROM members WHERE user_id IS NOT NULL"
   end
 
diff --git a/db/migrate/20160412140240_add_repository_checks_enabled_setting.rb b/db/migrate/20160412140240_add_repository_checks_enabled_setting.rb
new file mode 100644
index 00000000000..ebfa4bcbc7b
--- /dev/null
+++ b/db/migrate/20160412140240_add_repository_checks_enabled_setting.rb
@@ -0,0 +1,5 @@
+class AddRepositoryChecksEnabledSetting < ActiveRecord::Migration
+  def change
+    add_column :application_settings, :repository_checks_enabled, :boolean, default: true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f53478a8594..36a05e0d676 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -77,6 +77,7 @@ ActiveRecord::Schema.define(version: 20160412175417) do
     t.string   "akismet_api_key"
     t.boolean  "email_author_in_body",              default: false
     t.integer  "default_group_visibility"
+    t.boolean  "repository_checks_enabled",         default: true
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -710,6 +711,9 @@ ActiveRecord::Schema.define(version: 20160412175417) do
   create_table "project_import_data", force: :cascade do |t|
     t.integer "project_id"
     t.text    "data"
+    t.text    "encrypted_credentials"
+    t.text    "encrypted_credentials_iv"
+    t.text    "encrypted_credentials_salt"
   end
 
   create_table "projects", force: :cascade do |t|
@@ -750,6 +754,8 @@ ActiveRecord::Schema.define(version: 20160412175417) do
     t.boolean  "public_builds",          default: true,     null: false
     t.string   "main_language"
     t.integer  "pushes_since_gc",        default: 0
+    t.boolean  "last_repository_check_failed"
+    t.datetime "last_repository_check_at"
   end
 
   add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree
@@ -759,6 +765,7 @@ ActiveRecord::Schema.define(version: 20160412175417) do
   add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree
   add_index "projects", ["description"], name: "index_projects_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
   add_index "projects", ["last_activity_at"], name: "index_projects_on_last_activity_at", using: :btree
+  add_index "projects", ["last_repository_check_failed"], name: "index_projects_on_last_repository_check_failed", using: :btree
   add_index "projects", ["name"], name: "index_projects_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"}
   add_index "projects", ["namespace_id"], name: "index_projects_on_namespace_id", using: :btree
   add_index "projects", ["path"], name: "index_projects_on_path", using: :btree