Add latest changes from gitlab-org/gitlab@13-1-stable-ee

8 files changed, 317 insertions, 45 deletions

diff --git a/app/controllers/import/base_controller.rb b/app/controllers/import/base_controller.rb
index 04919a4b9d0..afdea4f7c9d 100644
--- a/app/controllers/import/base_controller.rb
+++ b/app/controllers/import/base_controller.rb
@@ -1,10 +1,86 @@
 # frozen_string_literal: true
 
 class Import::BaseController < ApplicationController
+  include ActionView::Helpers::SanitizeHelper
+
   before_action :import_rate_limit, only: [:create]
 
+  def status
+    respond_to do |format|
+      format.json do
+        render json: { imported_projects: serialized_imported_projects,
+                       provider_repos: serialized_provider_repos,
+                       incompatible_repos: serialized_incompatible_repos,
+                       namespaces: serialized_namespaces }
+      end
+      format.html
+    end
+  end
+
+  def realtime_changes
+    Gitlab::PollingInterval.set_header(response, interval: 3_000)
+
+    render json: already_added_projects.to_json(only: [:id], methods: [:import_status])
+  end
+
+  protected
+
+  def importable_repos
+    raise NotImplementedError
+  end
+
+  def incompatible_repos
+    []
+  end
+
+  def provider_name
+    raise NotImplementedError
+  end
+
+  def provider_url
+    raise NotImplementedError
+  end
+
   private
 
+  def filter_attribute
+    :name
+  end
+
+  def sanitized_filter_param
+    @filter ||= sanitize(params[:filter])
+  end
+
+  def filtered(collection)
+    return collection unless sanitized_filter_param
+
+    collection.select { |item| item[filter_attribute].include?(sanitized_filter_param) }
+  end
+
+  def serialized_provider_repos
+    Import::ProviderRepoSerializer.new(current_user: current_user).represent(importable_repos, provider: provider_name, provider_url: provider_url)
+  end
+
+  def serialized_incompatible_repos
+    Import::ProviderRepoSerializer.new(current_user: current_user).represent(incompatible_repos, provider: provider_name, provider_url: provider_url)
+  end
+
+  def serialized_imported_projects
+    ProjectSerializer.new.represent(already_added_projects, serializer: :import, provider_url: provider_url)
+  end
+
+  def already_added_projects
+    @already_added_projects ||= filtered(find_already_added_projects(provider_name))
+  end
+
+  def serialized_namespaces
+    NamespaceSerializer.new.represent(namespaces)
+  end
+
+  def namespaces
+    current_user.manageable_groups_with_routes
+  end
+
   # rubocop: disable CodeReuse/ActiveRecord
   def find_already_added_projects(import_type)
     current_user.created_projects.where(import_type: import_type).with_import_state
diff --git a/app/controllers/import/bitbucket_controller.rb b/app/controllers/import/bitbucket_controller.rb
index c37e799de62..4886aeb5e3f 100644
--- a/app/controllers/import/bitbucket_controller.rb
+++ b/app/controllers/import/bitbucket_controller.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Import::BitbucketController < Import::BaseController
+  extend ::Gitlab::Utils::Override
+
   include ActionView::Helpers::SanitizeHelper
 
   before_action :verify_bitbucket_import_enabled
@@ -10,7 +12,7 @@ class Import::BitbucketController < Import::BaseController
   rescue_from Bitbucket::Error::Unauthorized, with: :bitbucket_unauthorized
 
   def callback
-    response = client.auth_code.get_token(params[:code], redirect_uri: users_import_bitbucket_callback_url)
+    response = oauth_client.auth_code.get_token(params[:code], redirect_uri: users_import_bitbucket_callback_url)
 
     session[:bitbucket_token]         = response.token
     session[:bitbucket_expires_at]    = response.expires_at
@@ -22,9 +24,10 @@ class Import::BitbucketController < Import::BaseController
 
   # rubocop: disable CodeReuse/ActiveRecord
   def status
+    return super if Feature.enabled?(:new_import_ui)
+
     bitbucket_client = Bitbucket::Client.new(credentials)
     repos = bitbucket_client.repos(filter: sanitized_filter_param)
-
     @repos, @incompatible_repos = repos.partition { |repo| repo.valid? }
 
     @already_added_projects = find_already_added_projects('bitbucket')
@@ -38,6 +41,10 @@ class Import::BitbucketController < Import::BaseController
     render json: find_jobs('bitbucket')
   end
 
+  def realtime_changes
+    super
+  end
+
   def create
     bitbucket_client = Bitbucket::Client.new(credentials)
 
@@ -59,7 +66,7 @@ class Import::BitbucketController < Import::BaseController
       project = Gitlab::BitbucketImport::ProjectCreator.new(repo, project_name, target_namespace, current_user, credentials).execute
 
       if project.persisted?
-        render json: ProjectSerializer.new.represent(project)
+        render json: ProjectSerializer.new.represent(project, serializer: :import)
       else
         render json: { errors: project_save_error(project) }, status: :unprocessable_entity
       end
@@ -68,16 +75,50 @@ class Import::BitbucketController < Import::BaseController
     end
   end
 
+  protected
+
+  # rubocop: disable CodeReuse/ActiveRecord
+  override :importable_repos
+  def importable_repos
+    already_added_projects_names = already_added_projects.map(&:import_source)
+
+    bitbucket_repos.reject { |repo| already_added_projects_names.include?(repo.full_name) || !repo.valid? }
+  end
+  # rubocop: enable CodeReuse/ActiveRecord
+
+  override :incompatible_repos
+  def incompatible_repos
+    bitbucket_repos.reject { |repo| repo.valid? }
+  end
+
+  override :provider_name
+  def provider_name
+    :bitbucket
+  end
+
+  override :provider_url
+  def provider_url
+    provider.url
+  end
+
   private
 
-  def client
-    @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options)
+  def oauth_client
+    @oauth_client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options)
   end
 
   def provider
     Gitlab::Auth::OAuth::Provider.config_for('bitbucket')
   end
 
+  def client
+    @client ||= Bitbucket::Client.new(credentials)
+  end
+
+  def bitbucket_repos
+    @bitbucket_repos ||= client.repos(filter: sanitized_filter_param).to_a
+  end
+
   def options
     OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys
   end
@@ -91,7 +132,7 @@ class Import::BitbucketController < Import::BaseController
   end
 
   def go_to_bitbucket_for_permissions
-    redirect_to client.auth_code.authorize_url(redirect_uri: users_import_bitbucket_callback_url)
+    redirect_to oauth_client.auth_code.authorize_url(redirect_uri: users_import_bitbucket_callback_url)
   end
 
   def bitbucket_unauthorized
diff --git a/app/controllers/import/bitbucket_server_controller.rb b/app/controllers/import/bitbucket_server_controller.rb
index 5fb7b5dccc5..9aa8110257d 100644
--- a/app/controllers/import/bitbucket_server_controller.rb
+++ b/app/controllers/import/bitbucket_server_controller.rb
@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
 class Import::BitbucketServerController < Import::BaseController
+  extend ::Gitlab::Utils::Override
+
   include ActionView::Helpers::SanitizeHelper
 
   before_action :verify_bitbucket_server_import_enabled
   before_action :bitbucket_auth, except: [:new, :configure]
   before_action :validate_import_params, only: [:create]
 
+  rescue_from BitbucketServer::Connection::ConnectionError, with: :bitbucket_connection_error
+
   # As a basic sanity check to prevent URL injection, restrict project
   # repository input and repository slugs to allowed characters. For Bitbucket:
   #
@@ -24,7 +28,7 @@ class Import::BitbucketServerController < Import::BaseController
   end
 
   def create
-    repo = bitbucket_client.repo(@project_key, @repo_slug)
+    repo = client.repo(@project_key, @repo_slug)
 
     unless repo
       return render json: { errors: _("Project %{project_repo} could not be found") % { project_repo: "#{@project_key}/#{@repo_slug}" } }, status: :unprocessable_entity
@@ -38,15 +42,13 @@ class Import::BitbucketServerController < Import::BaseController
       project = Gitlab::BitbucketServerImport::ProjectCreator.new(@project_key, @repo_slug, repo, project_name, target_namespace, current_user, credentials).execute
 
       if project.persisted?
-        render json: ProjectSerializer.new.represent(project)
+        render json: ProjectSerializer.new.represent(project, serializer: :import)
       else
         render json: { errors: project_save_error(project) }, status: :unprocessable_entity
       end
     else
       render json: { errors: _('This namespace has already been taken! Please choose another one.') }, status: :unprocessable_entity
     end
-  rescue BitbucketServer::Connection::ConnectionError => error
-    render json: { errors: _("Unable to connect to server: %{error}") % { error: error } }, status: :unprocessable_entity
   end
 
   def configure
@@ -59,7 +61,9 @@ class Import::BitbucketServerController < Import::BaseController
 
   # rubocop: disable CodeReuse/ActiveRecord
   def status
-    @collection = bitbucket_client.repos(page_offset: page_offset, limit: limit_per_page, filter: sanitized_filter_param)
+    return super if Feature.enabled?(:new_import_ui)
+
+    @collection = client.repos(page_offset: page_offset, limit: limit_per_page, filter: sanitized_filter_param)
     @repos, @incompatible_repos = @collection.partition { |repo| repo.valid? }
 
     # Use the import URL to filter beyond what BaseService#find_already_added_projects
@@ -67,10 +71,6 @@ class Import::BitbucketServerController < Import::BaseController
     already_added_projects_names = @already_added_projects.pluck(:import_source)
 
     @repos.reject! { |repo| already_added_projects_names.include?(repo.browse_url) }
-  rescue BitbucketServer::Connection::ConnectionError => error
-    flash[:alert] = _("Unable to connect to server: %{error}") % { error: error }
-    clear_session_data
-    redirect_to new_import_bitbucket_server_path
   end
   # rubocop: enable CodeReuse/ActiveRecord
 
@@ -78,6 +78,38 @@ class Import::BitbucketServerController < Import::BaseController
     render json: find_jobs('bitbucket_server')
   end
 
+  def realtime_changes
+    super
+  end
+
+  protected
+
+  # rubocop: disable CodeReuse/ActiveRecord
+  override :importable_repos
+  def importable_repos
+    # Use the import URL to filter beyond what BaseService#find_already_added_projects
+    already_added_projects = filter_added_projects('bitbucket_server', bitbucket_repos.map(&:browse_url))
+    already_added_projects_names = already_added_projects.map(&:import_source)
+
+    bitbucket_repos.reject { |repo| already_added_projects_names.include?(repo.browse_url) || !repo.valid? }
+  end
+  # rubocop: enable CodeReuse/ActiveRecord
+
+  override :incompatible_repos
+  def incompatible_repos
+    bitbucket_repos.reject { |repo| repo.valid? }
+  end
+
+  override :provider_name
+  def provider_name
+    :bitbucket_server
+  end
+
+  override :provider_url
+  def provider_url
+    session[bitbucket_server_url_key]
+  end
+
   private
 
   # rubocop: disable CodeReuse/ActiveRecord
@@ -86,8 +118,12 @@ class Import::BitbucketServerController < Import::BaseController
   end
   # rubocop: enable CodeReuse/ActiveRecord
 
-  def bitbucket_client
-    @bitbucket_client ||= BitbucketServer::Client.new(credentials)
+  def client
+    @client ||= BitbucketServer::Client.new(credentials)
+  end
+
+  def bitbucket_repos
+    @bitbucket_repos ||= client.repos(page_offset: page_offset, limit: limit_per_page, filter: sanitized_filter_param).to_a
   end
 
   def validate_import_params
@@ -153,4 +189,23 @@ class Import::BitbucketServerController < Import::BaseController
   def sanitized_filter_param
     sanitize(params[:filter])
   end
+
+  def bitbucket_connection_error(error)
+    flash[:alert] = _("Unable to connect to server: %{error}") % { error: error }
+    clear_session_data
+
+    respond_to do |format|
+      format.json do
+        render json: {
+          error: {
+            message: _("Unable to connect to server: %{error}") % { error: error },
+            redirect: new_import_bitbucket_server_path
+          }
+        }, status: :unprocessable_entity
+      end
+      format.html do
+        redirect_to new_import_bitbucket_server_path
+      end
+    end
+  end
 end
diff --git a/app/controllers/import/fogbugz_controller.rb b/app/controllers/import/fogbugz_controller.rb
index 4fb6efde7ff..91779a5d6cc 100644
--- a/app/controllers/import/fogbugz_controller.rb
+++ b/app/controllers/import/fogbugz_controller.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Import::FogbugzController < Import::BaseController
+  extend ::Gitlab::Utils::Override
+
   before_action :verify_fogbugz_import_enabled
   before_action :user_map, only: [:new_user_map, :create_user_map]
   before_action :verify_blocked_uri, only: :callback
@@ -48,6 +50,8 @@ class Import::FogbugzController < Import::BaseController
       return redirect_to new_import_fogbugz_path
     end
 
+    return super if Feature.enabled?(:new_import_ui)
+
     @repos = client.repos
 
     @already_added_projects = find_already_added_projects('fogbugz')
@@ -57,6 +61,10 @@ class Import::FogbugzController < Import::BaseController
   end
   # rubocop: enable CodeReuse/ActiveRecord
 
+  def realtime_changes
+    super
+  end
+
   def jobs
     render json: find_jobs('fogbugz')
   end
@@ -69,12 +77,35 @@ class Import::FogbugzController < Import::BaseController
     project = Gitlab::FogbugzImport::ProjectCreator.new(repo, fb_session, current_user.namespace, current_user, umap).execute
 
     if project.persisted?
-      render json: ProjectSerializer.new.represent(project)
+      render json: ProjectSerializer.new.represent(project, serializer: :import)
     else
       render json: { errors: project_save_error(project) }, status: :unprocessable_entity
     end
   end
 
+  protected
+
+  # rubocop: disable CodeReuse/ActiveRecord
+  override :importable_repos
+  def importable_repos
+    repos = client.repos
+
+    already_added_projects_names = already_added_projects.map(&:import_source)
+
+    repos.reject { |repo| already_added_projects_names.include? repo.name }
+  end
+  # rubocop: enable CodeReuse/ActiveRecord
+
+  override :provider_name
+  def provider_name
+    :fogbugz
+  end
+
+  override :provider_url
+  def provider_url
+    session[:fogbugz_uri]
+  end
+
   private
 
   def client
diff --git a/app/controllers/import/github_controller.rb b/app/controllers/import/github_controller.rb
index 4e8ceae75bd..097edcd6075 100644
--- a/app/controllers/import/github_controller.rb
+++ b/app/controllers/import/github_controller.rb
@@ -76,7 +76,7 @@ class Import::GithubController < Import::BaseController
 
   def serialized_provider_repos
     repos = client_repos.reject { |repo| already_added_project_names.include? repo.full_name }
-    ProviderRepoSerializer.new(current_user: current_user).represent(repos, provider: provider, provider_url: provider_url)
+    Import::ProviderRepoSerializer.new(current_user: current_user).represent(repos, provider: provider, provider_url: provider_url)
   end
 
   def serialized_namespaces
diff --git a/app/controllers/import/gitlab_controller.rb b/app/controllers/import/gitlab_controller.rb
index 5ec8e9e6fc5..a95a67e208c 100644
--- a/app/controllers/import/gitlab_controller.rb
+++ b/app/controllers/import/gitlab_controller.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Import::GitlabController < Import::BaseController
+  extend ::Gitlab::Utils::Override
+
   MAX_PROJECT_PAGES = 15
   PER_PAGE_PROJECTS = 100
 
@@ -16,6 +18,8 @@ class Import::GitlabController < Import::BaseController
 
   # rubocop: disable CodeReuse/ActiveRecord
   def status
+    return super if Feature.enabled?(:new_import_ui)
+
     @repos = client.projects(starting_page: 1, page_limit: MAX_PROJECT_PAGES, per_page: PER_PAGE_PROJECTS)
 
     @already_added_projects = find_already_added_projects('gitlab')
@@ -37,7 +41,7 @@ class Import::GitlabController < Import::BaseController
       project = Gitlab::GitlabImport::ProjectCreator.new(repo, target_namespace, current_user, access_params).execute
 
       if project.persisted?
-        render json: ProjectSerializer.new.represent(project)
+        render json: ProjectSerializer.new.represent(project, serializer: :import)
       else
         render json: { errors: project_save_error(project) }, status: :unprocessable_entity
       end
@@ -46,6 +50,29 @@ class Import::GitlabController < Import::BaseController
     end
   end
 
+  protected
+
+  # rubocop: disable CodeReuse/ActiveRecord
+  override :importable_repos
+  def importable_repos
+    repos = client.projects(starting_page: 1, page_limit: MAX_PROJECT_PAGES, per_page: PER_PAGE_PROJECTS)
+
+    already_added_projects_names = already_added_projects.map(&:import_source)
+
+    repos.reject { |repo| already_added_projects_names.include? repo["path_with_namespace"] }
+  end
+  # rubocop: enable CodeReuse/ActiveRecord
+
+  override :provider_name
+  def provider_name
+    :gitlab
+  end
+
+  override :provider_url
+  def provider_url
+    'https://gitlab.com'
+  end
+
   private
 
   def client
diff --git a/app/controllers/import/gitlab_groups_controller.rb b/app/controllers/import/gitlab_groups_controller.rb
new file mode 100644
index 00000000000..330af68385e
--- /dev/null
+++ b/app/controllers/import/gitlab_groups_controller.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+class Import::GitlabGroupsController < ApplicationController
+  include WorkhorseImportExportUpload
+
+  before_action :ensure_group_import_enabled
+  before_action :import_rate_limit, only: %i[create]
+
+  def create
+    unless file_is_valid?(group_params[:file])
+      return redirect_back_or_default(options: { alert: s_('GroupImport|Unable to process group import file') })
+    end
+
+    group_data = group_params.except(:file).merge(
+      visibility_level: closest_allowed_visibility_level,
+      import_export_upload: ImportExportUpload.new(import_file: group_params[:file])
+    )
+
+    group = ::Groups::CreateService.new(current_user, group_data).execute
+
+    if group.persisted?
+      if Groups::ImportExport::ImportService.new(group: group, user: current_user).async_execute
+        redirect_to(
+          group_path(group),
+          notice: s_("GroupImport|Group '%{group_name}' is being imported.") % { group_name: group.name }
+        )
+      else
+        redirect_to group_path(group), alert: _("Group import could not be scheduled")
+      end
+    else
+      redirect_back_or_default(
+        options: { alert: s_("GroupImport|Group could not be imported: %{errors}") % { errors: group.errors.full_messages.to_sentence } }
+      )
+    end
+  end
+
+  private
+
+  def group_params
+    params.permit(:path, :name, :parent_id, :file)
+  end
+
+  def closest_allowed_visibility_level
+    if group_params[:parent_id].present?
+      parent_group = Group.find(group_params[:parent_id])
+
+      Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level)
+    else
+      Gitlab::VisibilityLevel::PRIVATE
+    end
+  end
+
+  def ensure_group_import_enabled
+    render_404 unless Feature.enabled?(:group_import_export, @group, default_enabled: true)
+  end
+
+  def import_rate_limit
+    if Gitlab::ApplicationRateLimiter.throttled?(:group_import, scope: current_user)
+      Gitlab::ApplicationRateLimiter.log_request(request, :group_import_request_limit, current_user)
+
+      flash[:alert] = _('This endpoint has been requested too many times. Try again later.')
+      redirect_to new_group_path
+    end
+  end
+end
diff --git a/app/controllers/import/gitlab_projects_controller.rb b/app/controllers/import/gitlab_projects_controller.rb
index 6a3715a4675..39d053347f0 100644
--- a/app/controllers/import/gitlab_projects_controller.rb
+++ b/app/controllers/import/gitlab_projects_controller.rb
@@ -1,14 +1,11 @@
 # frozen_string_literal: true
 
 class Import::GitlabProjectsController < Import::BaseController
-  include WorkhorseRequest
+  include WorkhorseImportExportUpload
 
   before_action :whitelist_query_limiting, only: [:create]
   before_action :verify_gitlab_project_import_enabled
 
-  skip_before_action :verify_authenticity_token, only: [:authorize]
-  before_action :verify_workhorse_api!, only: [:authorize]
-
   def new
     @namespace = Namespace.find(project_params[:namespace_id])
     return render_404 unless current_user.can?(:create_projects, @namespace)
@@ -17,7 +14,7 @@ class Import::GitlabProjectsController < Import::BaseController
   end
 
   def create
-    unless file_is_valid?
+    unless file_is_valid?(project_params[:file])
       return redirect_back_or_default(options: { alert: _("You need to upload a GitLab project export archive (ending in .gz).") })
     end
 
@@ -33,28 +30,8 @@ class Import::GitlabProjectsController < Import::BaseController
     end
   end
 
-  def authorize
-    set_workhorse_internal_api_content_type
-
-    authorized = ImportExportUploader.workhorse_authorize(
-      has_length: false,
-      maximum_size: Gitlab::CurrentSettings.max_attachment_size.megabytes.to_i)
-
-    render json: authorized
-  rescue SocketError
-    render json: _("Error uploading file"), status: :internal_server_error
-  end
-
   private
 
-  def file_is_valid?
-    return false unless project_params[:file].is_a?(::UploadedFile)
-
-    filename = project_params[:file].original_filename
-
-    ImportExportUploader::EXTENSION_WHITELIST.include?(File.extname(filename).delete('.'))
-  end
-
   def verify_gitlab_project_import_enabled
     render_404 unless gitlab_project_import_enabled?
   end