summaryrefslogtreecommitdiff
path: root/app/controllers/import
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-06-18 11:18:50 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2020-06-18 11:18:50 +0000
commit8c7f4e9d5f36cff46365a7f8c4b9c21578c1e781 (patch)
treea77e7fe7a93de11213032ed4ab1f33a3db51b738 /app/controllers/import
parent00b35af3db1abfe813a778f643dad221aad51fca (diff)
downloadgitlab-ce-8c7f4e9d5f36cff46365a7f8c4b9c21578c1e781.tar.gz
Add latest changes from gitlab-org/gitlab@13-1-stable-ee
Diffstat (limited to 'app/controllers/import')
-rw-r--r--app/controllers/import/base_controller.rb76
-rw-r--r--app/controllers/import/bitbucket_controller.rb53
-rw-r--r--app/controllers/import/bitbucket_server_controller.rb77
-rw-r--r--app/controllers/import/fogbugz_controller.rb33
-rw-r--r--app/controllers/import/github_controller.rb2
-rw-r--r--app/controllers/import/gitlab_controller.rb29
-rw-r--r--app/controllers/import/gitlab_groups_controller.rb65
-rw-r--r--app/controllers/import/gitlab_projects_controller.rb27
8 files changed, 317 insertions, 45 deletions
diff --git a/app/controllers/import/base_controller.rb b/app/controllers/import/base_controller.rb
index 04919a4b9d0..afdea4f7c9d 100644
--- a/app/controllers/import/base_controller.rb
+++ b/app/controllers/import/base_controller.rb
@@ -1,10 +1,86 @@
# frozen_string_literal: true
class Import::BaseController < ApplicationController
+ include ActionView::Helpers::SanitizeHelper
+
before_action :import_rate_limit, only: [:create]
+ def status
+ respond_to do |format|
+ format.json do
+ render json: { imported_projects: serialized_imported_projects,
+ provider_repos: serialized_provider_repos,
+ incompatible_repos: serialized_incompatible_repos,
+ namespaces: serialized_namespaces }
+ end
+ format.html
+ end
+ end
+
+ def realtime_changes
+ Gitlab::PollingInterval.set_header(response, interval: 3_000)
+
+ render json: already_added_projects.to_json(only: [:id], methods: [:import_status])
+ end
+
+ protected
+
+ def importable_repos
+ raise NotImplementedError
+ end
+
+ def incompatible_repos
+ []
+ end
+
+ def provider_name
+ raise NotImplementedError
+ end
+
+ def provider_url
+ raise NotImplementedError
+ end
+
private
+ def filter_attribute
+ :name
+ end
+
+ def sanitized_filter_param
+ @filter ||= sanitize(params[:filter])
+ end
+
+ def filtered(collection)
+ return collection unless sanitized_filter_param
+
+ collection.select { |item| item[filter_attribute].include?(sanitized_filter_param) }
+ end
+
+ def serialized_provider_repos
+ Import::ProviderRepoSerializer.new(current_user: current_user).represent(importable_repos, provider: provider_name, provider_url: provider_url)
+ end
+
+ def serialized_incompatible_repos
+ Import::ProviderRepoSerializer.new(current_user: current_user).represent(incompatible_repos, provider: provider_name, provider_url: provider_url)
+ end
+
+ def serialized_imported_projects
+ ProjectSerializer.new.represent(already_added_projects, serializer: :import, provider_url: provider_url)
+ end
+
+ def already_added_projects
+ @already_added_projects ||= filtered(find_already_added_projects(provider_name))
+ end
+
+ def serialized_namespaces
+ NamespaceSerializer.new.represent(namespaces)
+ end
+
+ def namespaces
+ current_user.manageable_groups_with_routes
+ end
+
# rubocop: disable CodeReuse/ActiveRecord
def find_already_added_projects(import_type)
current_user.created_projects.where(import_type: import_type).with_import_state
diff --git a/app/controllers/import/bitbucket_controller.rb b/app/controllers/import/bitbucket_controller.rb
index c37e799de62..4886aeb5e3f 100644
--- a/app/controllers/import/bitbucket_controller.rb
+++ b/app/controllers/import/bitbucket_controller.rb
@@ -1,6 +1,8 @@
# frozen_string_literal: true
class Import::BitbucketController < Import::BaseController
+ extend ::Gitlab::Utils::Override
+
include ActionView::Helpers::SanitizeHelper
before_action :verify_bitbucket_import_enabled
@@ -10,7 +12,7 @@ class Import::BitbucketController < Import::BaseController
rescue_from Bitbucket::Error::Unauthorized, with: :bitbucket_unauthorized
def callback
- response = client.auth_code.get_token(params[:code], redirect_uri: users_import_bitbucket_callback_url)
+ response = oauth_client.auth_code.get_token(params[:code], redirect_uri: users_import_bitbucket_callback_url)
session[:bitbucket_token] = response.token
session[:bitbucket_expires_at] = response.expires_at
@@ -22,9 +24,10 @@ class Import::BitbucketController < Import::BaseController
# rubocop: disable CodeReuse/ActiveRecord
def status
+ return super if Feature.enabled?(:new_import_ui)
+
bitbucket_client = Bitbucket::Client.new(credentials)
repos = bitbucket_client.repos(filter: sanitized_filter_param)
-
@repos, @incompatible_repos = repos.partition { |repo| repo.valid? }
@already_added_projects = find_already_added_projects('bitbucket')
@@ -38,6 +41,10 @@ class Import::BitbucketController < Import::BaseController
render json: find_jobs('bitbucket')
end
+ def realtime_changes
+ super
+ end
+
def create
bitbucket_client = Bitbucket::Client.new(credentials)
@@ -59,7 +66,7 @@ class Import::BitbucketController < Import::BaseController
project = Gitlab::BitbucketImport::ProjectCreator.new(repo, project_name, target_namespace, current_user, credentials).execute
if project.persisted?
- render json: ProjectSerializer.new.represent(project)
+ render json: ProjectSerializer.new.represent(project, serializer: :import)
else
render json: { errors: project_save_error(project) }, status: :unprocessable_entity
end
@@ -68,16 +75,50 @@ class Import::BitbucketController < Import::BaseController
end
end
+ protected
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ override :importable_repos
+ def importable_repos
+ already_added_projects_names = already_added_projects.map(&:import_source)
+
+ bitbucket_repos.reject { |repo| already_added_projects_names.include?(repo.full_name) || !repo.valid? }
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ override :incompatible_repos
+ def incompatible_repos
+ bitbucket_repos.reject { |repo| repo.valid? }
+ end
+
+ override :provider_name
+ def provider_name
+ :bitbucket
+ end
+
+ override :provider_url
+ def provider_url
+ provider.url
+ end
+
private
- def client
- @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options)
+ def oauth_client
+ @oauth_client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options)
end
def provider
Gitlab::Auth::OAuth::Provider.config_for('bitbucket')
end
+ def client
+ @client ||= Bitbucket::Client.new(credentials)
+ end
+
+ def bitbucket_repos
+ @bitbucket_repos ||= client.repos(filter: sanitized_filter_param).to_a
+ end
+
def options
OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys
end
@@ -91,7 +132,7 @@ class Import::BitbucketController < Import::BaseController
end
def go_to_bitbucket_for_permissions
- redirect_to client.auth_code.authorize_url(redirect_uri: users_import_bitbucket_callback_url)
+ redirect_to oauth_client.auth_code.authorize_url(redirect_uri: users_import_bitbucket_callback_url)
end
def bitbucket_unauthorized
diff --git a/app/controllers/import/bitbucket_server_controller.rb b/app/controllers/import/bitbucket_server_controller.rb
index 5fb7b5dccc5..9aa8110257d 100644
--- a/app/controllers/import/bitbucket_server_controller.rb
+++ b/app/controllers/import/bitbucket_server_controller.rb
@@ -1,12 +1,16 @@
# frozen_string_literal: true
class Import::BitbucketServerController < Import::BaseController
+ extend ::Gitlab::Utils::Override
+
include ActionView::Helpers::SanitizeHelper
before_action :verify_bitbucket_server_import_enabled
before_action :bitbucket_auth, except: [:new, :configure]
before_action :validate_import_params, only: [:create]
+ rescue_from BitbucketServer::Connection::ConnectionError, with: :bitbucket_connection_error
+
# As a basic sanity check to prevent URL injection, restrict project
# repository input and repository slugs to allowed characters. For Bitbucket:
#
@@ -24,7 +28,7 @@ class Import::BitbucketServerController < Import::BaseController
end
def create
- repo = bitbucket_client.repo(@project_key, @repo_slug)
+ repo = client.repo(@project_key, @repo_slug)
unless repo
return render json: { errors: _("Project %{project_repo} could not be found") % { project_repo: "#{@project_key}/#{@repo_slug}" } }, status: :unprocessable_entity
@@ -38,15 +42,13 @@ class Import::BitbucketServerController < Import::BaseController
project = Gitlab::BitbucketServerImport::ProjectCreator.new(@project_key, @repo_slug, repo, project_name, target_namespace, current_user, credentials).execute
if project.persisted?
- render json: ProjectSerializer.new.represent(project)
+ render json: ProjectSerializer.new.represent(project, serializer: :import)
else
render json: { errors: project_save_error(project) }, status: :unprocessable_entity
end
else
render json: { errors: _('This namespace has already been taken! Please choose another one.') }, status: :unprocessable_entity
end
- rescue BitbucketServer::Connection::ConnectionError => error
- render json: { errors: _("Unable to connect to server: %{error}") % { error: error } }, status: :unprocessable_entity
end
def configure
@@ -59,7 +61,9 @@ class Import::BitbucketServerController < Import::BaseController
# rubocop: disable CodeReuse/ActiveRecord
def status
- @collection = bitbucket_client.repos(page_offset: page_offset, limit: limit_per_page, filter: sanitized_filter_param)
+ return super if Feature.enabled?(:new_import_ui)
+
+ @collection = client.repos(page_offset: page_offset, limit: limit_per_page, filter: sanitized_filter_param)
@repos, @incompatible_repos = @collection.partition { |repo| repo.valid? }
# Use the import URL to filter beyond what BaseService#find_already_added_projects
@@ -67,10 +71,6 @@ class Import::BitbucketServerController < Import::BaseController
already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.reject! { |repo| already_added_projects_names.include?(repo.browse_url) }
- rescue BitbucketServer::Connection::ConnectionError => error
- flash[:alert] = _("Unable to connect to server: %{error}") % { error: error }
- clear_session_data
- redirect_to new_import_bitbucket_server_path
end
# rubocop: enable CodeReuse/ActiveRecord
@@ -78,6 +78,38 @@ class Import::BitbucketServerController < Import::BaseController
render json: find_jobs('bitbucket_server')
end
+ def realtime_changes
+ super
+ end
+
+ protected
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ override :importable_repos
+ def importable_repos
+ # Use the import URL to filter beyond what BaseService#find_already_added_projects
+ already_added_projects = filter_added_projects('bitbucket_server', bitbucket_repos.map(&:browse_url))
+ already_added_projects_names = already_added_projects.map(&:import_source)
+
+ bitbucket_repos.reject { |repo| already_added_projects_names.include?(repo.browse_url) || !repo.valid? }
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ override :incompatible_repos
+ def incompatible_repos
+ bitbucket_repos.reject { |repo| repo.valid? }
+ end
+
+ override :provider_name
+ def provider_name
+ :bitbucket_server
+ end
+
+ override :provider_url
+ def provider_url
+ session[bitbucket_server_url_key]
+ end
+
private
# rubocop: disable CodeReuse/ActiveRecord
@@ -86,8 +118,12 @@ class Import::BitbucketServerController < Import::BaseController
end
# rubocop: enable CodeReuse/ActiveRecord
- def bitbucket_client
- @bitbucket_client ||= BitbucketServer::Client.new(credentials)
+ def client
+ @client ||= BitbucketServer::Client.new(credentials)
+ end
+
+ def bitbucket_repos
+ @bitbucket_repos ||= client.repos(page_offset: page_offset, limit: limit_per_page, filter: sanitized_filter_param).to_a
end
def validate_import_params
@@ -153,4 +189,23 @@ class Import::BitbucketServerController < Import::BaseController
def sanitized_filter_param
sanitize(params[:filter])
end
+
+ def bitbucket_connection_error(error)
+ flash[:alert] = _("Unable to connect to server: %{error}") % { error: error }
+ clear_session_data
+
+ respond_to do |format|
+ format.json do
+ render json: {
+ error: {
+ message: _("Unable to connect to server: %{error}") % { error: error },
+ redirect: new_import_bitbucket_server_path
+ }
+ }, status: :unprocessable_entity
+ end
+ format.html do
+ redirect_to new_import_bitbucket_server_path
+ end
+ end
+ end
end
diff --git a/app/controllers/import/fogbugz_controller.rb b/app/controllers/import/fogbugz_controller.rb
index 4fb6efde7ff..91779a5d6cc 100644
--- a/app/controllers/import/fogbugz_controller.rb
+++ b/app/controllers/import/fogbugz_controller.rb
@@ -1,6 +1,8 @@
# frozen_string_literal: true
class Import::FogbugzController < Import::BaseController
+ extend ::Gitlab::Utils::Override
+
before_action :verify_fogbugz_import_enabled
before_action :user_map, only: [:new_user_map, :create_user_map]
before_action :verify_blocked_uri, only: :callback
@@ -48,6 +50,8 @@ class Import::FogbugzController < Import::BaseController
return redirect_to new_import_fogbugz_path
end
+ return super if Feature.enabled?(:new_import_ui)
+
@repos = client.repos
@already_added_projects = find_already_added_projects('fogbugz')
@@ -57,6 +61,10 @@ class Import::FogbugzController < Import::BaseController
end
# rubocop: enable CodeReuse/ActiveRecord
+ def realtime_changes
+ super
+ end
+
def jobs
render json: find_jobs('fogbugz')
end
@@ -69,12 +77,35 @@ class Import::FogbugzController < Import::BaseController
project = Gitlab::FogbugzImport::ProjectCreator.new(repo, fb_session, current_user.namespace, current_user, umap).execute
if project.persisted?
- render json: ProjectSerializer.new.represent(project)
+ render json: ProjectSerializer.new.represent(project, serializer: :import)
else
render json: { errors: project_save_error(project) }, status: :unprocessable_entity
end
end
+ protected
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ override :importable_repos
+ def importable_repos
+ repos = client.repos
+
+ already_added_projects_names = already_added_projects.map(&:import_source)
+
+ repos.reject { |repo| already_added_projects_names.include? repo.name }
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ override :provider_name
+ def provider_name
+ :fogbugz
+ end
+
+ override :provider_url
+ def provider_url
+ session[:fogbugz_uri]
+ end
+
private
def client
diff --git a/app/controllers/import/github_controller.rb b/app/controllers/import/github_controller.rb
index 4e8ceae75bd..097edcd6075 100644
--- a/app/controllers/import/github_controller.rb
+++ b/app/controllers/import/github_controller.rb
@@ -76,7 +76,7 @@ class Import::GithubController < Import::BaseController
def serialized_provider_repos
repos = client_repos.reject { |repo| already_added_project_names.include? repo.full_name }
- ProviderRepoSerializer.new(current_user: current_user).represent(repos, provider: provider, provider_url: provider_url)
+ Import::ProviderRepoSerializer.new(current_user: current_user).represent(repos, provider: provider, provider_url: provider_url)
end
def serialized_namespaces
diff --git a/app/controllers/import/gitlab_controller.rb b/app/controllers/import/gitlab_controller.rb
index 5ec8e9e6fc5..a95a67e208c 100644
--- a/app/controllers/import/gitlab_controller.rb
+++ b/app/controllers/import/gitlab_controller.rb
@@ -1,6 +1,8 @@
# frozen_string_literal: true
class Import::GitlabController < Import::BaseController
+ extend ::Gitlab::Utils::Override
+
MAX_PROJECT_PAGES = 15
PER_PAGE_PROJECTS = 100
@@ -16,6 +18,8 @@ class Import::GitlabController < Import::BaseController
# rubocop: disable CodeReuse/ActiveRecord
def status
+ return super if Feature.enabled?(:new_import_ui)
+
@repos = client.projects(starting_page: 1, page_limit: MAX_PROJECT_PAGES, per_page: PER_PAGE_PROJECTS)
@already_added_projects = find_already_added_projects('gitlab')
@@ -37,7 +41,7 @@ class Import::GitlabController < Import::BaseController
project = Gitlab::GitlabImport::ProjectCreator.new(repo, target_namespace, current_user, access_params).execute
if project.persisted?
- render json: ProjectSerializer.new.represent(project)
+ render json: ProjectSerializer.new.represent(project, serializer: :import)
else
render json: { errors: project_save_error(project) }, status: :unprocessable_entity
end
@@ -46,6 +50,29 @@ class Import::GitlabController < Import::BaseController
end
end
+ protected
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ override :importable_repos
+ def importable_repos
+ repos = client.projects(starting_page: 1, page_limit: MAX_PROJECT_PAGES, per_page: PER_PAGE_PROJECTS)
+
+ already_added_projects_names = already_added_projects.map(&:import_source)
+
+ repos.reject { |repo| already_added_projects_names.include? repo["path_with_namespace"] }
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ override :provider_name
+ def provider_name
+ :gitlab
+ end
+
+ override :provider_url
+ def provider_url
+ 'https://gitlab.com'
+ end
+
private
def client
diff --git a/app/controllers/import/gitlab_groups_controller.rb b/app/controllers/import/gitlab_groups_controller.rb
new file mode 100644
index 00000000000..330af68385e
--- /dev/null
+++ b/app/controllers/import/gitlab_groups_controller.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+class Import::GitlabGroupsController < ApplicationController
+ include WorkhorseImportExportUpload
+
+ before_action :ensure_group_import_enabled
+ before_action :import_rate_limit, only: %i[create]
+
+ def create
+ unless file_is_valid?(group_params[:file])
+ return redirect_back_or_default(options: { alert: s_('GroupImport|Unable to process group import file') })
+ end
+
+ group_data = group_params.except(:file).merge(
+ visibility_level: closest_allowed_visibility_level,
+ import_export_upload: ImportExportUpload.new(import_file: group_params[:file])
+ )
+
+ group = ::Groups::CreateService.new(current_user, group_data).execute
+
+ if group.persisted?
+ if Groups::ImportExport::ImportService.new(group: group, user: current_user).async_execute
+ redirect_to(
+ group_path(group),
+ notice: s_("GroupImport|Group '%{group_name}' is being imported.") % { group_name: group.name }
+ )
+ else
+ redirect_to group_path(group), alert: _("Group import could not be scheduled")
+ end
+ else
+ redirect_back_or_default(
+ options: { alert: s_("GroupImport|Group could not be imported: %{errors}") % { errors: group.errors.full_messages.to_sentence } }
+ )
+ end
+ end
+
+ private
+
+ def group_params
+ params.permit(:path, :name, :parent_id, :file)
+ end
+
+ def closest_allowed_visibility_level
+ if group_params[:parent_id].present?
+ parent_group = Group.find(group_params[:parent_id])
+
+ Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level)
+ else
+ Gitlab::VisibilityLevel::PRIVATE
+ end
+ end
+
+ def ensure_group_import_enabled
+ render_404 unless Feature.enabled?(:group_import_export, @group, default_enabled: true)
+ end
+
+ def import_rate_limit
+ if Gitlab::ApplicationRateLimiter.throttled?(:group_import, scope: current_user)
+ Gitlab::ApplicationRateLimiter.log_request(request, :group_import_request_limit, current_user)
+
+ flash[:alert] = _('This endpoint has been requested too many times. Try again later.')
+ redirect_to new_group_path
+ end
+ end
+end
diff --git a/app/controllers/import/gitlab_projects_controller.rb b/app/controllers/import/gitlab_projects_controller.rb
index 6a3715a4675..39d053347f0 100644
--- a/app/controllers/import/gitlab_projects_controller.rb
+++ b/app/controllers/import/gitlab_projects_controller.rb
@@ -1,14 +1,11 @@
# frozen_string_literal: true
class Import::GitlabProjectsController < Import::BaseController
- include WorkhorseRequest
+ include WorkhorseImportExportUpload
before_action :whitelist_query_limiting, only: [:create]
before_action :verify_gitlab_project_import_enabled
- skip_before_action :verify_authenticity_token, only: [:authorize]
- before_action :verify_workhorse_api!, only: [:authorize]
-
def new
@namespace = Namespace.find(project_params[:namespace_id])
return render_404 unless current_user.can?(:create_projects, @namespace)
@@ -17,7 +14,7 @@ class Import::GitlabProjectsController < Import::BaseController
end
def create
- unless file_is_valid?
+ unless file_is_valid?(project_params[:file])
return redirect_back_or_default(options: { alert: _("You need to upload a GitLab project export archive (ending in .gz).") })
end
@@ -33,28 +30,8 @@ class Import::GitlabProjectsController < Import::BaseController
end
end
- def authorize
- set_workhorse_internal_api_content_type
-
- authorized = ImportExportUploader.workhorse_authorize(
- has_length: false,
- maximum_size: Gitlab::CurrentSettings.max_attachment_size.megabytes.to_i)
-
- render json: authorized
- rescue SocketError
- render json: _("Error uploading file"), status: :internal_server_error
- end
-
private
- def file_is_valid?
- return false unless project_params[:file].is_a?(::UploadedFile)
-
- filename = project_params[:file].original_filename
-
- ImportExportUploader::EXTENSION_WHITELIST.include?(File.extname(filename).delete('.'))
- end
-
def verify_gitlab_project_import_enabled
render_404 unless gitlab_project_import_enabled?
end