Merge commit '46e4ed6bd0c8c256bce6d35b4bb992d77fd09971' into feature/multi-level-container-registry-imagesfeature/multi-level-container-registry-images

* commit '46e4ed6bd0c8c256bce6d35b4bb992d77fd09971': (28 commits)
  Award emoji button smiley animation
  Introduced empty/error UX states to environments monitoring.
  Github import rake task
  Remove individual modal width styles
  Fix RuboCop for removing index
  Link to docs site for file in doc/
  Disable invalid service templates (again)
  Show CI status as Favicon on Pipelines, Job and MR pages
  STL file viewer
  Wait for the PDF to be loaded before doing anything
  remove unnecessary lease as cron job
  Search for opened MRs - include reopened  MRs
  ProjectsFinder should handle more options
  Clearly show who triggered the pipeline in email
  Make it possible to preview pipeline success/failed emails
  Add remove_concurrent_index to database helper
  Add more tests for subgroups feature
  Large features by the 1st, small ones by the 3rd
  Ask people to create EE MRs on the 7th
  fix project authorizations migration issue
  ...

11 files changed, 251 insertions, 7 deletions

diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb
index b888ede6fe8..8a54f7f3f05 100644
--- a/lib/api/deploy_keys.rb
+++ b/lib/api/deploy_keys.rb
@@ -47,6 +47,7 @@ module API
       params do
         requires :key, type: String, desc: 'The new deploy key'
         requires :title, type: String, desc: 'The name of the deploy key'
+        optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository"
       end
       post ":id/deploy_keys" do
         params[:key].strip!
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 8f3799417e3..605769eddde 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -142,7 +142,7 @@ module API
       end
       get ":id/projects" do
         group = find_group!(params[:id])
-        projects = GroupProjectsFinder.new(group).execute(current_user)
+        projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute
         projects = filter_projects(projects)
         entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project
         present paginate(projects), with: entity, current_user: current_user
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 0fbe1669d45..766fbea53e6 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -84,7 +84,7 @@ module API
       end
       get do
         entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
-        present_projects ProjectsFinder.new.execute(current_user), with: entity, statistics: params[:statistics]
+        present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics]
       end
 
       desc 'Create new project' do
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 530ca0b5235..992a751b37d 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -341,7 +341,7 @@ module API
         not_found!('User') unless user
 
         events = user.events.
-          merge(ProjectsFinder.new.execute(current_user)).
+          merge(ProjectsFinder.new(current_user: current_user).execute).
           references(:project).
           with_associations.
           recent
diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb
index c5b37622d79..9b27411ae21 100644
--- a/lib/api/v3/groups.rb
+++ b/lib/api/v3/groups.rb
@@ -151,7 +151,7 @@ module API
         end
         get ":id/projects" do
           group = find_group!(params[:id])
-          projects = GroupProjectsFinder.new(group).execute(current_user)
+          projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute
           projects = filter_projects(projects)
           entity = params[:simple] ? ::API::Entities::BasicProjectDetails : Entities::Project
           present paginate(projects), with: entity, current_user: current_user
diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb
index b753dbab381..ba9748ada59 100644
--- a/lib/api/v3/projects.rb
+++ b/lib/api/v3/projects.rb
@@ -107,7 +107,7 @@ module API
         end
         get '/visible' do
           entity = current_user ? ::API::V3::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails
-          present_projects ProjectsFinder.new.execute(current_user), with: entity
+          present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity
         end
 
         desc 'Get a projects list for authenticated user' do
diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb
index 5e18cecc431..f4cda3b2eba 100644
--- a/lib/api/v3/users.rb
+++ b/lib/api/v3/users.rb
@@ -138,7 +138,7 @@ module API
           not_found!('User') unless user
 
           events = user.events.
-            merge(ProjectsFinder.new.execute(current_user)).
+            merge(ProjectsFinder.new(current_user: current_user).execute).
             references(:project).
             with_associations.
             recent
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index fc445ab9483..525aa920328 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -26,6 +26,30 @@ module Gitlab
         add_index(table_name, column_name, options)
       end
 
+      # Removes an existed index, concurrently when supported
+      #
+      # On PostgreSQL this method removes an index concurrently.
+      #
+      # Example:
+      #
+      #     remove_concurrent_index :users, :some_column
+      #
+      # See Rails' `remove_index` for more info on the available arguments.
+      def remove_concurrent_index(table_name, column_name, options = {})
+        if transaction_open?
+          raise 'remove_concurrent_index can not be run inside a transaction, ' \
+            'you can disable transactions by calling disable_ddl_transaction! ' \
+            'in the body of your migration class'
+        end
+
+        if Database.postgresql?
+          options = options.merge({ algorithm: :concurrently })
+          disable_statement_timeout
+        end
+
+        remove_index(table_name, options.merge({ column: column_name }))
+      end
+
       # Adds a foreign key with only minimal locking on the tables involved.
       #
       # This method only requires minimal locking when using PostgreSQL. When
diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb
index 11e5f1b645c..ca8d3271541 100644
--- a/lib/gitlab/sidekiq_status.rb
+++ b/lib/gitlab/sidekiq_status.rb
@@ -72,6 +72,8 @@ module Gitlab
     # job_ids - The Sidekiq job IDs to check.
     #
     # Returns an array of true or false indicating job completion.
+    # true = job is still running
+    # false = job completed
     def self.job_status(job_ids)
       keys = job_ids.map { |jid| key_for(jid) }
 
@@ -82,6 +84,17 @@ module Gitlab
       end
     end
 
+    # Returns the JIDs that are completed
+    #
+    # job_ids - The Sidekiq job IDs to check.
+    #
+    # Returns an array of completed JIDs
+    def self.completed_jids(job_ids)
+      Sidekiq.redis do |redis|
+        job_ids.reject { |jid| redis.exists(key_for(jid)) }
+      end
+    end
+
     def self.key_for(jid)
       STATUS_KEY % jid
     end
diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb
index d47609f490d..00983b3284a 100644
--- a/lib/gitlab/sidekiq_status/client_middleware.rb
+++ b/lib/gitlab/sidekiq_status/client_middleware.rb
@@ -2,7 +2,9 @@ module Gitlab
   module SidekiqStatus
     class ClientMiddleware
       def call(_, job, _, _)
-        Gitlab::SidekiqStatus.set(job['jid'])
+        status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION
+
+        Gitlab::SidekiqStatus.set(job['jid'], status_expiration)
         yield
       end
     end
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
new file mode 100644
index 00000000000..350afeb5c0b
--- /dev/null
+++ b/lib/tasks/import.rake
@@ -0,0 +1,204 @@
+require 'benchmark'
+require 'rainbow/ext/string'
+require_relative '../gitlab/shell_adapter'
+require_relative '../gitlab/github_import/importer'
+
+class NewImporter < ::Gitlab::GithubImport::Importer
+  def execute
+    # Same as ::Gitlab::GithubImport::Importer#execute, but showing some progress.
+    puts 'Importing repository...'.color(:aqua)
+    import_repository unless project.repository_exists?
+
+    puts 'Importing labels...'.color(:aqua)
+    import_labels
+
+    puts 'Importing milestones...'.color(:aqua)
+    import_milestones
+
+    puts 'Importing pull requests...'.color(:aqua)
+    import_pull_requests
+
+    puts 'Importing issues...'.color(:aqua)
+    import_issues
+
+    puts 'Importing issue comments...'.color(:aqua)
+    import_comments(:issues)
+
+    puts 'Importing pull request comments...'.color(:aqua)
+    import_comments(:pull_requests)
+
+    puts 'Importing wiki...'.color(:aqua)
+    import_wiki
+
+    # Gitea doesn't have a Release API yet
+    # See https://github.com/go-gitea/gitea/issues/330
+    unless project.gitea_import?
+      import_releases
+    end
+
+    handle_errors
+
+    project.repository.after_import
+    project.import_finish
+
+    true
+  end
+
+  def import_repository
+    begin
+      raise 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url)
+
+      gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url)
+    rescue => e
+      project.repository.before_import if project.repository_exists?
+
+      raise "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
+    end
+  end
+end
+
+class GithubImport
+  def self.run!(*args)
+    new(*args).run!
+  end
+
+  def initialize(token, gitlab_username, project_path, extras)
+    @token = token
+    @project_path = project_path
+    @current_user = User.find_by_username(gitlab_username)
+    @github_repo = extras.empty? ? nil : extras.first
+  end
+
+  def run!
+    @repo = GithubRepos.new(@token, @current_user, @github_repo).choose_one!
+
+    raise 'No repo found!' unless @repo
+
+    show_warning!
+
+    @project = Project.find_by_full_path(@project_path) || new_project
+
+    import!
+  end
+
+  private
+
+  def show_warning!
+    puts "This will import GH #{@repo.full_name.bright} into GL #{@project_path.bright} as #{@current_user.name}"
+    puts "Permission checks are ignored. Press any key to continue.".color(:red)
+
+    STDIN.getch
+
+    puts 'Starting the import...'.color(:green)
+  end
+
+  def import!
+    import_url = @project.import_url.gsub(/\:\/\/(.*@)?/, "://#{@token}@")
+    @project.update(import_url: import_url)
+
+    @project.import_start
+
+    timings = Benchmark.measure do
+      NewImporter.new(@project).execute
+    end
+
+    puts "Import finished. Timings: #{timings}".color(:green)
+  end
+
+  def new_project
+    Project.transaction do
+      namespace_path, _sep, name = @project_path.rpartition('/')
+      namespace = find_or_create_namespace(namespace_path)
+
+      Project.create!(
+        import_url: "https://#{@token}@github.com/#{@repo.full_name}.git",
+        name: name,
+        path: name,
+        description: @repo.description,
+        namespace: namespace,
+        visibility_level: visibility_level,
+        import_type: 'github',
+        import_source: @repo.full_name,
+        creator: @current_user
+      )
+    end
+  end
+
+  def find_or_create_namespace(names)
+    return @current_user.namespace if names == @current_user.namespace_path
+    return @current_user.namespace unless @current_user.can_create_group?
+
+    names = params[:target_namespace].presence || names
+    full_path_namespace = Namespace.find_by_full_path(names)
+
+    return full_path_namespace if full_path_namespace
+
+    names.split('/').inject(nil) do |parent, name|
+      begin
+        namespace = Group.create!(name: name,
+                                  path: name,
+                                  owner: @current_user,
+                                  parent: parent)
+        namespace.add_owner(@current_user)
+
+        namespace
+      rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid
+        Namespace.where(parent: parent).find_by_path_or_name(name)
+      end
+    end
+  end
+
+  def full_path_namespace(names)
+    @full_path_namespace ||= Namespace.find_by_full_path(names)
+  end
+
+  def visibility_level
+    @repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility
+  end
+end
+
+class GithubRepos
+  def initialize(token, current_user, github_repo)
+    @token = token
+    @current_user = current_user
+    @github_repo = github_repo
+  end
+
+  def choose_one!
+    return found_github_repo if @github_repo
+
+    repos.each do |repo|
+      print "ID: #{repo[:id].to_s.bright} ".color(:green)
+      puts "- Name: #{repo[:full_name]}".color(:green)
+    end
+
+    print 'ID? '.bright
+
+    repos.find { |repo| repo[:id] == repo_id }
+  end
+
+  def found_github_repo
+    repos.find { |repo| repo[:full_name] == @github_repo }
+  end
+
+  def repo_id
+    @repo_id ||= STDIN.gets.chomp.to_i
+  end
+
+  def repos
+    @repos ||= client.repos
+  end
+
+  def client
+    @client ||= Gitlab::GithubImport::Client.new(@token, {})
+  end
+end
+
+namespace :import do
+  desc 'Import a GitHub project - Example: import:github[ToKeN,root,root/blah,my/github_repo] (optional my/github_repo)'
+  task :github, [:token, :gitlab_username, :project_path] => :environment do |_t, args|
+    abort 'Project path must be: namespace(s)/project_name'.color(:red) unless args.project_path.include?('/')
+
+    GithubImport.run!(args.token, args.gitlab_username, args.project_path, args.extras)
+  end
+end