diff options
| author | Douwe Maan <douwe@gitlab.com> | 2016-06-17 14:28:58 +0000 |
|---|---|---|
| committer | Douwe Maan <douwe@gitlab.com> | 2016-06-17 14:28:58 +0000 |
| commit | 402b651a552030c90af421ddd2a74f6cbf298413 (patch) | |
| tree | c7aa3e6ff87eff5dccdb0b91dbd64675d5b6d211 | |
| parent | 2a747d386dbdc05453fce6b8be3f483e8cd9e796 (diff) | |
| parent | 2d4556c5d208e9ae805b0467c1c7281ae6a36ebe (diff) | |
| download | gitlab-ce-feature/project-import.tar.gz | |
Merge branch 'feature/project-export-ui-experimental' into 'feature/project-import'
feature/project-import
Experimental UI for exporting and importing a project
Part of https://gitlab.com/gitlab-org/gitlab-ce/issues/3050
Screenshots of both the export and import processes:
## Export
1 - Project settings
2 - Flash after clicking on export
3 - Email received with download link
4 - The project settings export screen changes so we can either delete the file or download it again (it won't generate a new export, unless we delete it first)
5 - After delete flash
## Import
1 - New project page with new gitlab export option
2 - Next step importing - choosing a file
3 - Import in progress
4 - Import successful
See merge request !4012
45 files changed, 503 insertions, 218 deletions
diff --git a/CHANGELOG b/CHANGELOG index 39532e88138..75b48ad3207 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -90,6 +90,7 @@ v 8.9.0 (unreleased) - Improved UX of date pickers on issue & milestone forms - Cache on the database if a project has an active external issue tracker. - Put project Labels and Milestones pages links under Issues and Merge Requests tabs as subnav + - GitLab project import and export functionality - All classes in the Banzai::ReferenceParser namespace are now instrumented - Remove deprecated issues_tracker and issues_tracker_id from project model - Allow users to create confidential issues in private projects @@ -221,7 +221,7 @@ gem 'jquery-turbolinks', '~> 2.1.0' gem 'addressable', '~> 2.3.8' gem 'bootstrap-sass', '~> 3.3.0' -gem 'font-awesome-rails', '~> 4.2' +gem 'font-awesome-rails', '~> 4.6.1' gem 'gitlab_emoji', '~> 0.3.0' gem 'gon', '~> 6.0.1' gem 'jquery-atwho-rails', '~> 1.3.2' diff --git a/Gemfile.lock b/Gemfile.lock index a22bd1b1f52..3c599d04eda 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -246,7 +246,7 @@ GEM fog-xml (0.1.2) fog-core nokogiri (~> 1.5, >= 1.5.11) - font-awesome-rails (4.5.0.1) + font-awesome-rails (4.6.1.0) railties (>= 3.2, < 5.1) foreman (0.78.0) thor (~> 0.19.1) @@ -866,7 +866,7 @@ DEPENDENCIES fog-google (~> 0.3) fog-local (~> 0.3) fog-openstack (~> 0.1) - font-awesome-rails (~> 4.2) + font-awesome-rails (~> 4.6.1) foreman fuubar (~> 2.0.0) gemnasium-gitlab-service (~> 0.2) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index cd6ae507cf1..726acbdb3ed 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -24,7 +24,7 @@ class ApplicationController < ActionController::Base protect_from_forgery with: :exception helper_method :abilities, :can?, :current_application_settings - helper_method :import_sources_enabled?, :github_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :gitorious_import_enabled?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled? + helper_method :import_sources_enabled?, :github_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :gitorious_import_enabled?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled? rescue_from Encoding::CompatibilityError do |exception| log_exception(exception) @@ -326,6 +326,10 @@ class ApplicationController < ActionController::Base current_application_settings.import_sources.include?('git') end + def gitlab_project_import_enabled? + current_application_settings.import_sources.include?('gitlab_project') + end + def two_factor_authentication_required? current_application_settings.require_two_factor_authentication end diff --git a/app/controllers/import/gitlab_projects_controller.rb b/app/controllers/import/gitlab_projects_controller.rb new file mode 100644 index 00000000000..f99aa490d3e --- /dev/null +++ b/app/controllers/import/gitlab_projects_controller.rb @@ -0,0 +1,48 @@ +class Import::GitlabProjectsController < Import::BaseController + before_action :verify_gitlab_project_import_enabled + + def new + @namespace_id = project_params[:namespace_id] + @namespace_name = Namespace.find(project_params[:namespace_id]).name + @path = project_params[:path] + end + + def create + unless file_is_valid? + return redirect_back_or_default(options: { alert: "You need to upload a GitLab project export archive." }) + end + + @project = Gitlab::ImportExport::ProjectCreator.new(project_params[:namespace_id], + current_user, + File.expand_path(project_params[:file].path), + project_params[:path]).execute + + if @project.saved? + redirect_to( + project_path(@project), + notice: "Project '#{@project.name}' is being imported." + ) + else + redirect_to( + new_import_gitlab_project_path, + alert: "Project could not be imported: #{@project.errors.full_messages.join(', ')}" + ) + end + end + + private + + def file_is_valid? + project_params[:file] && project_params[:file].respond_to?(:read) + end + + def verify_gitlab_project_import_enabled + render_404 unless gitlab_project_import_enabled? + end + + def project_params + params.permit( + :path, :namespace_id, :file + ) + end +end diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index a6479c42d94..673adca6ade 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -7,7 +7,7 @@ class ProjectsController < Projects::ApplicationController before_action :assign_ref_vars, :tree, only: [:show], if: :repo_exists? # Authorize - before_action :authorize_admin_project!, only: [:edit, :update, :housekeeping] + before_action :authorize_admin_project!, only: [:edit, :update, :housekeeping, :download_export, :export, :remove_export, :generate_new_export] before_action :event_filter, only: [:show, :activity] layout :determine_layout @@ -185,6 +185,48 @@ class ProjectsController < Projects::ApplicationController ) end + def export + @project.add_export_job(current_user: current_user) + + redirect_to( + edit_project_path(@project), + notice: "Project export started. A download link will be sent by email." + ) + end + + def download_export + export_project_path = @project.export_project_path + + if export_project_path + send_file export_project_path, disposition: 'attachment' + else + redirect_to( + edit_project_path(@project), + alert: "Project export link has expired. Please generate a new export from your project settings." + ) + end + end + + def remove_export + if @project.remove_exports + flash[:notice] = "Project export has been deleted." + else + flash[:alert] = "Project export could not be deleted." + end + redirect_to(edit_project_path(@project)) + end + + def generate_new_export + if @project.remove_exports + export + else + redirect_to( + edit_project_path(@project), + alert: "Project export could not be deleted." + ) + end + end + def toggle_star current_user.toggle_star(@project) @project.reload diff --git a/app/mailers/emails/projects.rb b/app/mailers/emails/projects.rb index 689fb3e0ffb..e0af7081411 100644 --- a/app/mailers/emails/projects.rb +++ b/app/mailers/emails/projects.rb @@ -9,6 +9,19 @@ module Emails subject: subject("Project was moved")) end + def project_was_exported_email(current_user, project) + @project = project + mail(to: current_user.notification_email, + subject: subject("Project was exported")) + end + + def project_was_not_exported_email(current_user, project, errors) + @project = project + @errors = errors + mail(to: current_user.notification_email, + subject: subject("Project export error")) + end + def repository_push_email(project_id, opts = {}) @message = Gitlab::Email::Message::RepositoryPush.new(self, project_id, opts) diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index a744f937918..d914b0b26eb 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -123,7 +123,7 @@ class ApplicationSetting < ActiveRecord::Base default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], - import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], + import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, diff --git a/app/models/project.rb b/app/models/project.rb index 6a92a2c0448..064b3c1fc23 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -352,14 +352,9 @@ class Project < ActiveRecord::Base joins(join_body).reorder('join_note_counts.amount DESC') end - def create_from_import_job(current_user_id:, tmp_file:, namespace_id:, project_path:) - job_id = ProjectImportWorker.perform_async(current_user_id, tmp_file, namespace_id, project_path) - - if job_id - Rails.logger.info "Import job started for export #{tmp_file} with job ID #{job_id}" - else - Rails.logger.error "Import job failed to start for #{tmp_file}" - end + # Deletes gitlab project export files older than 24 hours + def remove_gitlab_exports! + Gitlab::Popen.popen(%W(find #{Gitlab::ImportExport.storage_path} -not -path #{Gitlab::ImportExport.storage_path} -mmin +1440 -delete)) end end @@ -464,7 +459,7 @@ class Project < ActiveRecord::Base end def import? - external_import? || forked? + external_import? || forked? || gitlab_project_import? end def no_import? @@ -495,6 +490,10 @@ class Project < ActiveRecord::Base Gitlab::UrlSanitizer.new(import_url).masked_url end + def gitlab_project_import? + import_type == 'gitlab_project' + end + def check_limit unless creator.can_create_project? or namespace.kind == 'group' projects_limit = creator.projects_limit @@ -1091,8 +1090,8 @@ class Project < ActiveRecord::Base @errors = original_errors end - def add_export_job(current_user_id:) - job_id = ProjectExportWorker.perform_async(current_user_id, self.id) + def add_export_job(current_user:) + job_id = ProjectExportWorker.perform_async(current_user.id, self.id) if job_id Rails.logger.info "Export job started for project ID #{self.id} with job ID #{job_id}" @@ -1100,4 +1099,17 @@ class Project < ActiveRecord::Base Rails.logger.error "Export job failed to start for project ID #{self.id}" end end + + def export_path + File.join(Gitlab::ImportExport.storage_path, path_with_namespace) + end + + def export_project_path + Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) } + end + + def remove_exports + _, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete)) + status.zero? + end end diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb index f804ac171c4..e70b400829a 100644 --- a/app/services/notification_service.rb +++ b/app/services/notification_service.rb @@ -266,6 +266,14 @@ class NotificationService end end + def project_exported(project, current_user) + mailer.project_was_exported_email(current_user, project).deliver_later + end + + def project_not_exported(project, current_user, errors) + mailer.project_was_not_exported_email(current_user, project, errors).deliver_later + end + protected # Get project users with WATCH notification level diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 61cac5419ad..55956be2844 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -80,16 +80,18 @@ module Projects def after_create_actions log_info("#{@project.owner.name} created a new project \"#{@project.name_with_namespace}\"") - @project.create_wiki if @project.wiki_enabled? + unless @project.gitlab_project_import? + @project.create_wiki if @project.wiki_enabled? - @project.build_missing_services + @project.build_missing_services - @project.create_labels + @project.create_labels + end event_service.create_project(@project, current_user) system_hook_service.execute_hooks_for(@project, :create) - unless @project.group + unless @project.group || @project.gitlab_project_import? @project.team << [current_user, :master, current_user] end end diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index 25524c1c060..d6752377ce5 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -12,8 +12,9 @@ module Projects def save_all if [version_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save) Gitlab::ImportExport::Saver.save(shared: @shared) + notify_success else - cleanup_and_notify_worker + cleanup_and_notify end end @@ -37,10 +38,20 @@ module Projects Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared) end - def cleanup_and_notify_worker + def cleanup_and_notify FileUtils.rm_rf(@shared.export_path) + + notify_error raise Gitlab::ImportExport::Error.new(@shared.errors.join(', ')) end + + def notify_success + notification_service.project_exported(@project, @current_user) + end + + def notify_error + notification_service.project_not_exported(@project, @current_user, @shared.errors.join(', ')) + end end end end diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb index c4838d31f2f..9159ec08959 100644 --- a/app/services/projects/import_service.rb +++ b/app/services/projects/import_service.rb @@ -9,26 +9,31 @@ module Projects 'fogbugz', 'gitlab', 'github', - 'google_code' + 'google_code', + 'gitlab_project' ] def execute - if unknown_url? - # In this case, we only want to import issues, not a repository. - create_repository - else - import_repository - end + add_repository_to_project unless project.gitlab_project_import? import_data success - rescue Error => e + rescue => e error(e.message) end private + def add_repository_to_project + if unknown_url? + # In this case, we only want to import issues, not a repository. + create_repository + else + import_repository + end + end + def create_repository unless project.create_repository raise Error, 'The repository could not be created.' @@ -46,7 +51,7 @@ module Projects def import_data return unless has_importer? - project.repository.before_import + project.repository.before_import unless project.gitlab_project_import? unless importer.execute raise Error, 'The remote data could not be imported.' @@ -58,6 +63,8 @@ module Projects end def importer + return Gitlab::ImportExport::Importer.new(project) if @project.gitlab_project_import? + class_name = "Gitlab::#{project.import_type.camelize}Import::Importer" class_name.constantize.new(project) end diff --git a/app/views/import/gitlab_projects/new.html.haml b/app/views/import/gitlab_projects/new.html.haml new file mode 100644 index 00000000000..44e2653ca4a --- /dev/null +++ b/app/views/import/gitlab_projects/new.html.haml @@ -0,0 +1,25 @@ +- page_title "GitLab Import" +- header_title "Projects", root_path +%h3.page-title + = icon('gitlab') + Import an exported GitLab project +%hr + += form_tag import_gitlab_project_path, class: 'form-horizontal', multipart: true do + %p + Project will be imported as + %strong + #{@namespace_name}/#{@path} + + %p + To move or copy an entire GitLab project from another GitLab installation to this one, navigate to the original project's settings page, generate an export file, and upload it here. + .form-group + = hidden_field_tag :namespace_id, @namespace_id + = hidden_field_tag :path, @path + = label_tag :file, class: 'control-label' do + %span GitLab project export + .col-sm-10 + = file_field_tag :file, class: '' + + .form-actions + = submit_tag 'Import project', class: 'btn btn-create' diff --git a/app/views/notify/project_was_exported_email.html.haml b/app/views/notify/project_was_exported_email.html.haml new file mode 100644 index 00000000000..b28fea35ad5 --- /dev/null +++ b/app/views/notify/project_was_exported_email.html.haml @@ -0,0 +1,8 @@ +%p + Project #{@project.name} was exported successfully. +%p + The project export can be downloaded from: + = link_to download_export_namespace_project_url(@project.namespace, @project) do + = @project.name_with_namespace + " export" +%p + The download link will expire in 24 hours. diff --git a/app/views/notify/project_was_exported_email.text.erb b/app/views/notify/project_was_exported_email.text.erb new file mode 100644 index 00000000000..42c4d176876 --- /dev/null +++ b/app/views/notify/project_was_exported_email.text.erb @@ -0,0 +1,6 @@ +Project <%= @project.name %> was exported successfully. + +The project export can be downloaded from: +<%= download_export_namespace_project_url(@project.namespace, @project) %> + +The download link will expire in 24 hours. diff --git a/app/views/notify/project_was_not_exported_email.html.haml b/app/views/notify/project_was_not_exported_email.html.haml new file mode 100644 index 00000000000..c9e9ade2cf1 --- /dev/null +++ b/app/views/notify/project_was_not_exported_email.html.haml @@ -0,0 +1,9 @@ +%p + Project #{@project.name} couldn't be exported. +%p + The errors we encountered were: + + %ul + - @errors.each do |error| + %li + error diff --git a/app/views/notify/project_was_not_exported_email.text.erb b/app/views/notify/project_was_not_exported_email.text.erb new file mode 100644 index 00000000000..a07f6edacf7 --- /dev/null +++ b/app/views/notify/project_was_not_exported_email.text.erb @@ -0,0 +1,6 @@ +Project <%= @project.name %> couldn't be exported. + +The errors we encountered were: + +- @errors.each do |error| +<%= error %>
\ No newline at end of file diff --git a/app/views/projects/edit.html.haml b/app/views/projects/edit.html.haml index 8449fe1e4e0..27a94fe02dc 100644 --- a/app/views/projects/edit.html.haml +++ b/app/views/projects/edit.html.haml @@ -120,6 +120,42 @@ = link_to 'Housekeeping', housekeeping_namespace_project_path(@project.namespace, @project), method: :post, class: "btn btn-save" %hr + .row.prepend-top-default + .col-lg-3 + %h4.prepend-top-0 + Export project + %p.append-bottom-0 + %p + Export this project with all its related data in order to move your project to a new GitLab instance. Once the export is finished, you can import the file from the "New Project" page. + %p + Once the exported file is ready, you will receive a notification email with a download link. + + .col-lg-9 + + - if @project.export_project_path + = link_to 'Download export', download_export_namespace_project_path(@project.namespace, @project), + method: :get, class: "btn btn-default" + = link_to 'Generate new export', generate_new_export_namespace_project_path(@project.namespace, @project), + method: :post, class: "btn btn-default" + - else + = link_to 'Export project', export_namespace_project_path(@project.namespace, @project), + method: :post, class: "btn btn-default" + + .bs-callout.bs-callout-info + %p.append-bottom-0 + %p + The following items will be exported: + %ul + %li Project and wiki repositories + %li Project uploads + %li Project configuration including web hooks and services + %li Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities + %p + The following items will NOT be exported: + %ul + %li Build traces and artifacts + %li LFS objects + %hr - if can? current_user, :archive_project, @project .row.prepend-top-default .col-lg-3 diff --git a/app/views/projects/imports/show.html.haml b/app/views/projects/imports/show.html.haml index c0d1ce0d120..4d8ee562e6a 100644 --- a/app/views/projects/imports/show.html.haml +++ b/app/views/projects/imports/show.html.haml @@ -7,7 +7,7 @@ Forking in progress. - else Import in progress. - - unless @project.forked? + - if @project.external_import? %p.monospace git clone --bare #{@project.safe_import_url} %p Please wait while we import the repository for you. Refresh at will. :javascript diff --git a/app/views/projects/new.html.haml b/app/views/projects/new.html.haml index 7e8b8f83467..3c1c6060504 100644 --- a/app/views/projects/new.html.haml +++ b/app/views/projects/new.html.haml @@ -84,7 +84,12 @@ - if git_import_enabled? = link_to "#", class: 'btn js-toggle-button import_git' do %i.fa.fa-git - %span Any repo by URL + %span Repo by URL + + - if gitlab_project_import_enabled? + = link_to new_import_gitlab_project_path, class: 'btn import_gitlab_project project-submit' do + %i.fa.fa-gitlab + %span GitLab export .js-toggle-content.hide = render "shared/import_form", f: f @@ -115,6 +120,33 @@ e.preventDefault(); var import_modal = $(this).next(".modal").show(); }); + $('.modal-header .close').bind('click', function() { $(".modal").hide(); }); + + $('.import_gitlab_project').bind('click', function() { + var _href = $("a.import_gitlab_project").attr("href"); + $(".import_gitlab_project").attr("href", _href + '?namespace_id=' + $("#project_namespace_id").val() + '&path=' + $("#project_path").val()); + }); + + $('.import_gitlab_project').attr('disabled',true) + $('.import_gitlab_project').attr('title', 'Project path required.'); + + $('.import_gitlab_project').click(function( event ) { + if($('.import_gitlab_project').attr('disabled')) { + event.preventDefault(); + new Flash("Please enter a path for the project to be imported to."); + } + }); + + $('#project_path').keyup(function(){ + if($(this).val().length !=0) { + $('.import_gitlab_project').attr('disabled', false); + $('.import_gitlab_project').attr('title',''); + $(".flash-container").html("") + } else { + $('.import_gitlab_project').attr('disabled',true); + $('.import_gitlab_project').attr('title', 'Project path required.'); + } + }) diff --git a/app/workers/gitlab_remove_project_export_worker.rb b/app/workers/gitlab_remove_project_export_worker.rb new file mode 100644 index 00000000000..1d91897d520 --- /dev/null +++ b/app/workers/gitlab_remove_project_export_worker.rb @@ -0,0 +1,9 @@ +class GitlabRemoveProjectExportWorker + include Sidekiq::Worker + + sidekiq_options queue: :default + + def perform + Project.remove_gitlab_exports! + end +end diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index 3616b37d2ad..39f6037e077 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -1,12 +1,12 @@ class ProjectExportWorker include Sidekiq::Worker - # TODO: enable retry - disabled for QA purposes - sidekiq_options queue: :gitlab_shell, retry: false + sidekiq_options queue: :gitlab_shell, retry: true def perform(current_user_id, project_id) current_user = User.find(current_user_id) project = Project.find(project_id) + ::Projects::ImportExport::ExportService.new(project, current_user).execute end end diff --git a/app/workers/project_import_worker.rb b/app/workers/project_import_worker.rb deleted file mode 100644 index b2902c3278e..00000000000 --- a/app/workers/project_import_worker.rb +++ /dev/null @@ -1,24 +0,0 @@ -class ProjectImportWorker - include Sidekiq::Worker - - # TODO: enabled retry - disabled for QA purposes - sidekiq_options queue: :gitlab_shell, retry: false - - def perform(current_user_id, tmp_file, namespace_id, path) - current_user = User.find(current_user_id) - - project = Gitlab::ImportExport::ImportService.execute(archive_file: tmp_file, - owner: current_user, - namespace_id: namespace_id, - project_path: path) - if project - project.repository.after_import - else - logger.error("There was an error during the import: #{tmp_file}") - end - end - - def logger - Sidekiq.logger - end -end diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 916fd33e767..09ffc319065 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -291,6 +291,9 @@ Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker' Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker' +Settings.cron_jobs['gitlab_remove_project_export_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['gitlab_remove_project_export_worker']['cron'] ||= '0 * * * *' +Settings.cron_jobs['gitlab_remove_project_export_worker']['job_class'] = 'GitlabRemoveProjectExportWorker' # # GitLab Shell diff --git a/config/routes.rb b/config/routes.rb index 92ed4a62ede..09bd9ac55a9 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -459,6 +459,10 @@ Rails.application.routes.draw do post :housekeeping post :toggle_star post :markdown_preview + post :export + post :remove_export + post :generate_new_export + get :download_export get :autocomplete_sources get :activity end diff --git a/features/dashboard/new_project.feature b/features/dashboard/new_project.feature index 76392068357..56b4a639c01 100644 --- a/features/dashboard/new_project.feature +++ b/features/dashboard/new_project.feature @@ -14,7 +14,7 @@ Background: @javascript Scenario: I should see instructions on how to import from Git URL Given I see "New Project" page - When I click on "Any repo by URL" + When I click on "Repo by URL" Then I see instructions on how to import from Git URL @javascript diff --git a/features/steps/dashboard/new_project.rb b/features/steps/dashboard/new_project.rb index 5308e77fb19..29e6b9f1a01 100644 --- a/features/steps/dashboard/new_project.rb +++ b/features/steps/dashboard/new_project.rb @@ -20,7 +20,8 @@ class Spinach::Features::NewProject < Spinach::FeatureSteps expect(page).to have_link('GitLab.com') expect(page).to have_link('Gitorious.org') expect(page).to have_link('Google Code') - expect(page).to have_link('Any repo by URL') + expect(page).to have_link('Repo by URL') + expect(page).to have_link('GitLab export') end step 'I click on "Import project from GitHub"' do @@ -37,7 +38,7 @@ class Spinach::Features::NewProject < Spinach::FeatureSteps end end - step 'I click on "Any repo by URL"' do + step 'I click on "Repo by URL"' do first('.import_git').click end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 5e7532f57ae..28c34429c1f 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -36,7 +36,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], - import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], + import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index 77c33db4b59..3d0418261bb 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo["name"], path: repo["path"], @@ -22,8 +22,6 @@ module Gitlab import_source: repo["path_with_namespace"], import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") ).execute - - project end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb new file mode 100644 index 00000000000..0e70d9282d5 --- /dev/null +++ b/lib/gitlab/import_export/file_importer.rb @@ -0,0 +1,30 @@ +module Gitlab + module ImportExport + class FileImporter + include Gitlab::ImportExport::CommandLineUtil + + def self.import(*args) + new(*args).import + end + + def initialize(archive_file:, shared:) + @archive_file = archive_file + @shared = shared + end + + def import + FileUtils.mkdir_p(@shared.export_path) + decompress_archive + rescue => e + @shared.error(e) + false + end + + private + + def decompress_archive + untar_zxf(archive: @archive_file, dir: @shared.export_path) + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 3796fc8cd02..164ab6238c4 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -30,8 +30,6 @@ project_tree: # Only include the following attributes for the models specified. included_attributes: project: - - :name - - :path - :description - :issues_enabled - :merge_requests_enabled diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb deleted file mode 100644 index db71f72efec..00000000000 --- a/lib/gitlab/import_export/import_service.rb +++ /dev/null @@ -1,69 +0,0 @@ -module Gitlab - module ImportExport - class ImportService - - def self.execute(*args) - new(*args).execute - end - - def initialize(archive_file:, owner:, namespace_id:, project_path:) - @archive_file = archive_file - @current_user = owner - @namespace = Namespace.find(namespace_id) - @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace(project_path), project_path: project_path) - end - - def execute - Gitlab::ImportExport::Importer.import(archive_file: @archive_file, - shared: @shared) - if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) - project_tree.project - else - project_tree.project.destroy if project_tree.project - nil - end - end - - private - - def check_version! - Gitlab::ImportExport::VersionChecker.check!(shared: @shared) - end - - def project_tree - @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user, - shared: @shared, - namespace_id: @namespace.id) - end - - def repo_restorer - Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, - shared: @shared, - project: project_tree.project) - end - - def wiki_restorer - Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, - shared: @shared, - project: ProjectWiki.new(project_tree.project), - wiki: true) - end - - def uploads_restorer - Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.project, shared: @shared) - end - - def path_with_namespace(project_path) - File.join(@namespace.path, project_path) - end - - def repo_path - File.join(@shared.export_path, 'project.bundle') - end - - def wiki_repo_path - File.join(@shared.export_path, 'project.wiki.bundle') - end - end - end -end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 8020aab3da9..d209e04f7be 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -1,29 +1,63 @@ module Gitlab module ImportExport class Importer - include Gitlab::ImportExport::CommandLineUtil - def self.import(*args) - new(*args).import + def initialize(project) + @archive_file = project.import_source + @current_user = project.creator + @project = project + @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace) end - def initialize(archive_file:, shared:) - @archive_file = archive_file - @shared = shared + def execute + Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + shared: @shared) + if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) + project_tree.restored_project + else + raise Projects::ImportService::Error.new(@shared.errors.join(', ')) + end end - def import - FileUtils.mkdir_p(@shared.export_path) - decompress_archive - rescue => e - @shared.error(e) - false + private + + def check_version! + Gitlab::ImportExport::VersionChecker.check!(shared: @shared) end - private + def project_tree + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user, + shared: @shared, + project: @project) + end + + def repo_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, + shared: @shared, + project: project_tree.restored_project) + end + + def wiki_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, + shared: @shared, + project: ProjectWiki.new(project_tree.restored_project), + wiki: true) + end + + def uploads_restorer + Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + + def path_with_namespace + File.join(@project.namespace.path, @project.path) + end + + def repo_path + File.join(@shared.export_path, 'project.bundle') + end - def decompress_archive - untar_zxf(archive: @archive_file, dir: @shared.export_path) + def wiki_repo_path + File.join(@shared.export_path, 'project.wiki.bundle') end end end diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb new file mode 100644 index 00000000000..89388d1984b --- /dev/null +++ b/lib/gitlab/import_export/project_creator.rb @@ -0,0 +1,24 @@ +module Gitlab + module ImportExport + class ProjectCreator + + def initialize(namespace_id, current_user, file, project_path) + @namespace_id = namespace_id + @current_user = current_user + @file = file + @project_path = project_path + end + + def execute + ::Projects::CreateService.new( + @current_user, + name: @project_path, + path: @project_path, + namespace_id: @namespace_id, + import_type: "gitlab_project", + import_source: @file + ).execute + end + end + end +end diff --git a/lib/gitlab/import_export/project_factory.rb b/lib/gitlab/import_export/project_factory.rb deleted file mode 100644 index 6cd4736649b..00000000000 --- a/lib/gitlab/import_export/project_factory.rb +++ /dev/null @@ -1,41 +0,0 @@ -module Gitlab - module ImportExport - module ProjectFactory - extend self - - def create(project_params:, user:, namespace_id:) - project = Project.new(project_params.except('id')) - project.creator = user - check_namespace(namespace_id, project, user) - end - - def check_namespace(namespace_id, project, user) - if namespace_id - # Find matching namespace and check if it allowed - # for current user if namespace_id passed. - if allowed_namespace?(user, namespace_id) - project.namespace_id = namespace_id - else - project.namespace_id = nil - deny_namespace(project) - end - else - # Set current user namespace if namespace_id is nil - project.namespace_id = user.namespace_id - end - project - end - - private - - def allowed_namespace?(user, namespace_id) - namespace = Namespace.find_by(id: namespace_id) - user.can?(:create_projects, namespace) - end - - def deny_namespace(project) - project.errors.add(:namespace, "is not valid") - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 75a261bb121..dd71b92c522 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -2,12 +2,11 @@ module Gitlab module ImportExport class ProjectTreeRestorer - def initialize(user:, shared:, namespace_id:) + def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @user = user - @project_path = shared.opts[:project_path] - @namespace_id = namespace_id @shared = shared + @project = project end def restore @@ -20,8 +19,8 @@ module Gitlab false end - def project - @project ||= create_project + def restored_project + @restored_project ||= restore_project end private @@ -29,7 +28,7 @@ module Gitlab def members_mapper @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, user: @user, - project: project) + project: restored_project) end # Loops through the tree of models defined in import_export.yml and @@ -46,7 +45,7 @@ module Gitlab relation_key = relation.is_a?(Hash) ? relation.keys.first : relation relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s]) - saved << project.update_attribute(relation_key, relation_hash) + saved << restored_project.update_attribute(relation_key, relation_hash) end saved.all? end @@ -57,14 +56,12 @@ module Gitlab end end - def create_project + def restore_project + return @project unless @tree_hash + project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } - project = Gitlab::ImportExport::ProjectFactory.create( - project_params: project_params, user: @user, namespace_id: @namespace_id) - project.path = @project_path - project.name = @project_path - project.save! - project + @project.update(project_params) + @project end # Given a relation hash containing one or more models and its relationships, @@ -96,7 +93,7 @@ module Gitlab def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: relation_hash.merge('project_id' => project.id), + relation_hash: relation_hash.merge('project_id' => restored_project.id), members_mapper: members_mapper, user: @user) end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 4e4ce4f14a9..b872780f20a 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -57,10 +57,10 @@ module Gitlab author = @relation_hash.delete('author') - update_note_for_missing_author(author['name']) if missing_author? + update_note_for_missing_author(author['name']) if missing_author?(old_author_id) end - def missing_author? + def missing_author?(old_author_id) !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id) end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index ef4d9c24067..546dae4d122 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -11,7 +11,7 @@ module Gitlab end def restore - return false unless File.exist?(@path_to_bundle) || wiki? + return wiki? unless File.exist?(@path_to_bundle) FileUtils.mkdir_p(path_to_repo) diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 4f467760862..cf5c62c5e3c 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -2,8 +2,8 @@ module Gitlab module ImportExport class VersionChecker - def self.restore(*args) - new(*args).check + def self.check!(*args) + new(*args).check! end def initialize(shared:) diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index ccfdfbe73e8..948d43582cf 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -20,7 +20,8 @@ module Gitlab 'Gitorious.org' => 'gitorious', 'Google Code' => 'google_code', 'FogBugz' => 'fogbugz', - 'Any repo by URL' => 'git', + 'Repo by URL' => 'git', + 'GitLab export' => 'gitlab_project' } end diff --git a/spec/features/projects/import_export/import_file_spec.rb b/spec/features/projects/import_export/import_file_spec.rb new file mode 100644 index 00000000000..c5fb0fc783b --- /dev/null +++ b/spec/features/projects/import_export/import_file_spec.rb @@ -0,0 +1,49 @@ +require 'spec_helper' + +feature 'project import', feature: true, js: true do + include Select2Helper + + let(:user) { create(:admin) } + let!(:namespace) { create(:namespace, name: "asd", owner: user) } + let(:file) { File.join(Rails.root, 'spec', 'features', 'projects', 'import_export', 'test_project_export.tar.gz') } + let(:export_path) { "#{Dir::tmpdir}/import_file_spec" } + let(:project) { Project.last } + + background do + allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) + login_as(user) + end + + after(:each) do + FileUtils.rm_rf(export_path, secure: true) + end + + scenario 'user imports an exported project successfully' do + expect(Project.all.count).to be_zero + + visit new_project_path + + select2('2', from: '#project_namespace_id') + fill_in :project_path, with:'test-project-path', visible: true + click_link 'GitLab export' + + expect(page).to have_content('GitLab project export') + expect(URI.parse(current_url).query).to eq('namespace_id=2&path=test-project-path') + + attach_file('file', file) + + click_on 'Import project' # import starts + + expect(project).not_to be_nil + expect(project.issues).not_to be_empty + expect(project.merge_requests).not_to be_empty + expect(project.repo_exists?).to be true + expect(wiki_exists?).to be true + expect(project.import_status).to eq('finished') + end + + def wiki_exists? + wiki = ProjectWiki.new(project) + File.exist?(wiki.repository.path_to_repo) && !wiki.repository.empty? + end +end diff --git a/spec/features/projects/import_export/test_project_export.tar.gz b/spec/features/projects/import_export/test_project_export.tar.gz Binary files differnew file mode 100644 index 00000000000..1fd04416d95 --- /dev/null +++ b/spec/features/projects/import_export/test_project_export.tar.gz diff --git a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb index bc7993b9d75..7a40a43f8ae 100644 --- a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb @@ -6,7 +6,8 @@ describe Gitlab::ImportExport::ProjectTreeRestorer, services: true do let(:user) { create(:user) } let(:namespace) { create(:namespace, owner: user) } let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: "", project_path: 'path') } - let(:project_tree_restorer) { described_class.new(user: user, shared: shared, namespace_id: namespace.id) } + let(:project) { create(:empty_project, name: 'project', path: 'project') } + let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) } let(:restored_project_json) { project_tree_restorer.restore } before do diff --git a/spec/lib/gitlab/import_export/project_tree_saver_spec.rb b/spec/lib/gitlab/import_export/project_tree_saver_spec.rb index 6e6adfd60eb..8d29b2f8fd1 100644 --- a/spec/lib/gitlab/import_export/project_tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project_tree_saver_spec.rb @@ -30,7 +30,7 @@ describe Gitlab::ImportExport::ProjectTreeSaver, services: true do end it 'saves the correct json' do - expect(saved_project_json).to include({ "name" => project.name }) + expect(saved_project_json).to include({ "visibility_level" => 20 }) end it 'has events' do @@ -94,7 +94,7 @@ describe Gitlab::ImportExport::ProjectTreeSaver, services: true do end it 'has pipeline builds' do - expect(saved_project_json['pipelines'].first['statuses'].first['type']).to eq('Ci::Build') + expect(saved_project_json['pipelines'].first['statuses'].count { |hash| hash['type'] == 'Ci::Build'}).to eq(1) end it 'has pipeline commits' do |