diff options
Diffstat (limited to 'app')
21 files changed, 311 insertions, 57 deletions
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index cd6ae507cf1..726acbdb3ed 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -24,7 +24,7 @@ class ApplicationController < ActionController::Base protect_from_forgery with: :exception helper_method :abilities, :can?, :current_application_settings - helper_method :import_sources_enabled?, :github_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :gitorious_import_enabled?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled? + helper_method :import_sources_enabled?, :github_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :gitorious_import_enabled?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled? rescue_from Encoding::CompatibilityError do |exception| log_exception(exception) @@ -326,6 +326,10 @@ class ApplicationController < ActionController::Base current_application_settings.import_sources.include?('git') end + def gitlab_project_import_enabled? + current_application_settings.import_sources.include?('gitlab_project') + end + def two_factor_authentication_required? current_application_settings.require_two_factor_authentication end diff --git a/app/controllers/import/gitlab_projects_controller.rb b/app/controllers/import/gitlab_projects_controller.rb new file mode 100644 index 00000000000..f99aa490d3e --- /dev/null +++ b/app/controllers/import/gitlab_projects_controller.rb @@ -0,0 +1,48 @@ +class Import::GitlabProjectsController < Import::BaseController + before_action :verify_gitlab_project_import_enabled + + def new + @namespace_id = project_params[:namespace_id] + @namespace_name = Namespace.find(project_params[:namespace_id]).name + @path = project_params[:path] + end + + def create + unless file_is_valid? + return redirect_back_or_default(options: { alert: "You need to upload a GitLab project export archive." }) + end + + @project = Gitlab::ImportExport::ProjectCreator.new(project_params[:namespace_id], + current_user, + File.expand_path(project_params[:file].path), + project_params[:path]).execute + + if @project.saved? + redirect_to( + project_path(@project), + notice: "Project '#{@project.name}' is being imported." + ) + else + redirect_to( + new_import_gitlab_project_path, + alert: "Project could not be imported: #{@project.errors.full_messages.join(', ')}" + ) + end + end + + private + + def file_is_valid? + project_params[:file] && project_params[:file].respond_to?(:read) + end + + def verify_gitlab_project_import_enabled + render_404 unless gitlab_project_import_enabled? + end + + def project_params + params.permit( + :path, :namespace_id, :file + ) + end +end diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index a6479c42d94..673adca6ade 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -7,7 +7,7 @@ class ProjectsController < Projects::ApplicationController before_action :assign_ref_vars, :tree, only: [:show], if: :repo_exists? # Authorize - before_action :authorize_admin_project!, only: [:edit, :update, :housekeeping] + before_action :authorize_admin_project!, only: [:edit, :update, :housekeeping, :download_export, :export, :remove_export, :generate_new_export] before_action :event_filter, only: [:show, :activity] layout :determine_layout @@ -185,6 +185,48 @@ class ProjectsController < Projects::ApplicationController ) end + def export + @project.add_export_job(current_user: current_user) + + redirect_to( + edit_project_path(@project), + notice: "Project export started. A download link will be sent by email." + ) + end + + def download_export + export_project_path = @project.export_project_path + + if export_project_path + send_file export_project_path, disposition: 'attachment' + else + redirect_to( + edit_project_path(@project), + alert: "Project export link has expired. Please generate a new export from your project settings." + ) + end + end + + def remove_export + if @project.remove_exports + flash[:notice] = "Project export has been deleted." + else + flash[:alert] = "Project export could not be deleted." + end + redirect_to(edit_project_path(@project)) + end + + def generate_new_export + if @project.remove_exports + export + else + redirect_to( + edit_project_path(@project), + alert: "Project export could not be deleted." + ) + end + end + def toggle_star current_user.toggle_star(@project) @project.reload diff --git a/app/mailers/emails/projects.rb b/app/mailers/emails/projects.rb index 689fb3e0ffb..e0af7081411 100644 --- a/app/mailers/emails/projects.rb +++ b/app/mailers/emails/projects.rb @@ -9,6 +9,19 @@ module Emails subject: subject("Project was moved")) end + def project_was_exported_email(current_user, project) + @project = project + mail(to: current_user.notification_email, + subject: subject("Project was exported")) + end + + def project_was_not_exported_email(current_user, project, errors) + @project = project + @errors = errors + mail(to: current_user.notification_email, + subject: subject("Project export error")) + end + def repository_push_email(project_id, opts = {}) @message = Gitlab::Email::Message::RepositoryPush.new(self, project_id, opts) diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index a744f937918..d914b0b26eb 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -123,7 +123,7 @@ class ApplicationSetting < ActiveRecord::Base default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], - import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], + import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, diff --git a/app/models/project.rb b/app/models/project.rb index 6a92a2c0448..064b3c1fc23 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -352,14 +352,9 @@ class Project < ActiveRecord::Base joins(join_body).reorder('join_note_counts.amount DESC') end - def create_from_import_job(current_user_id:, tmp_file:, namespace_id:, project_path:) - job_id = ProjectImportWorker.perform_async(current_user_id, tmp_file, namespace_id, project_path) - - if job_id - Rails.logger.info "Import job started for export #{tmp_file} with job ID #{job_id}" - else - Rails.logger.error "Import job failed to start for #{tmp_file}" - end + # Deletes gitlab project export files older than 24 hours + def remove_gitlab_exports! + Gitlab::Popen.popen(%W(find #{Gitlab::ImportExport.storage_path} -not -path #{Gitlab::ImportExport.storage_path} -mmin +1440 -delete)) end end @@ -464,7 +459,7 @@ class Project < ActiveRecord::Base end def import? - external_import? || forked? + external_import? || forked? || gitlab_project_import? end def no_import? @@ -495,6 +490,10 @@ class Project < ActiveRecord::Base Gitlab::UrlSanitizer.new(import_url).masked_url end + def gitlab_project_import? + import_type == 'gitlab_project' + end + def check_limit unless creator.can_create_project? or namespace.kind == 'group' projects_limit = creator.projects_limit @@ -1091,8 +1090,8 @@ class Project < ActiveRecord::Base @errors = original_errors end - def add_export_job(current_user_id:) - job_id = ProjectExportWorker.perform_async(current_user_id, self.id) + def add_export_job(current_user:) + job_id = ProjectExportWorker.perform_async(current_user.id, self.id) if job_id Rails.logger.info "Export job started for project ID #{self.id} with job ID #{job_id}" @@ -1100,4 +1099,17 @@ class Project < ActiveRecord::Base Rails.logger.error "Export job failed to start for project ID #{self.id}" end end + + def export_path + File.join(Gitlab::ImportExport.storage_path, path_with_namespace) + end + + def export_project_path + Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) } + end + + def remove_exports + _, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete)) + status.zero? + end end diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb index f804ac171c4..e70b400829a 100644 --- a/app/services/notification_service.rb +++ b/app/services/notification_service.rb @@ -266,6 +266,14 @@ class NotificationService end end + def project_exported(project, current_user) + mailer.project_was_exported_email(current_user, project).deliver_later + end + + def project_not_exported(project, current_user, errors) + mailer.project_was_not_exported_email(current_user, project, errors).deliver_later + end + protected # Get project users with WATCH notification level diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 61cac5419ad..55956be2844 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -80,16 +80,18 @@ module Projects def after_create_actions log_info("#{@project.owner.name} created a new project \"#{@project.name_with_namespace}\"") - @project.create_wiki if @project.wiki_enabled? + unless @project.gitlab_project_import? + @project.create_wiki if @project.wiki_enabled? - @project.build_missing_services + @project.build_missing_services - @project.create_labels + @project.create_labels + end event_service.create_project(@project, current_user) system_hook_service.execute_hooks_for(@project, :create) - unless @project.group + unless @project.group || @project.gitlab_project_import? @project.team << [current_user, :master, current_user] end end diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index 25524c1c060..d6752377ce5 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -12,8 +12,9 @@ module Projects def save_all if [version_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save) Gitlab::ImportExport::Saver.save(shared: @shared) + notify_success else - cleanup_and_notify_worker + cleanup_and_notify end end @@ -37,10 +38,20 @@ module Projects Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared) end - def cleanup_and_notify_worker + def cleanup_and_notify FileUtils.rm_rf(@shared.export_path) + + notify_error raise Gitlab::ImportExport::Error.new(@shared.errors.join(', ')) end + + def notify_success + notification_service.project_exported(@project, @current_user) + end + + def notify_error + notification_service.project_not_exported(@project, @current_user, @shared.errors.join(', ')) + end end end end diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb index c4838d31f2f..9159ec08959 100644 --- a/app/services/projects/import_service.rb +++ b/app/services/projects/import_service.rb @@ -9,26 +9,31 @@ module Projects 'fogbugz', 'gitlab', 'github', - 'google_code' + 'google_code', + 'gitlab_project' ] def execute - if unknown_url? - # In this case, we only want to import issues, not a repository. - create_repository - else - import_repository - end + add_repository_to_project unless project.gitlab_project_import? import_data success - rescue Error => e + rescue => e error(e.message) end private + def add_repository_to_project + if unknown_url? + # In this case, we only want to import issues, not a repository. + create_repository + else + import_repository + end + end + def create_repository unless project.create_repository raise Error, 'The repository could not be created.' @@ -46,7 +51,7 @@ module Projects def import_data return unless has_importer? - project.repository.before_import + project.repository.before_import unless project.gitlab_project_import? unless importer.execute raise Error, 'The remote data could not be imported.' @@ -58,6 +63,8 @@ module Projects end def importer + return Gitlab::ImportExport::Importer.new(project) if @project.gitlab_project_import? + class_name = "Gitlab::#{project.import_type.camelize}Import::Importer" class_name.constantize.new(project) end diff --git a/app/views/import/gitlab_projects/new.html.haml b/app/views/import/gitlab_projects/new.html.haml new file mode 100644 index 00000000000..44e2653ca4a --- /dev/null +++ b/app/views/import/gitlab_projects/new.html.haml @@ -0,0 +1,25 @@ +- page_title "GitLab Import" +- header_title "Projects", root_path +%h3.page-title + = icon('gitlab') + Import an exported GitLab project +%hr + += form_tag import_gitlab_project_path, class: 'form-horizontal', multipart: true do + %p + Project will be imported as + %strong + #{@namespace_name}/#{@path} + + %p + To move or copy an entire GitLab project from another GitLab installation to this one, navigate to the original project's settings page, generate an export file, and upload it here. + .form-group + = hidden_field_tag :namespace_id, @namespace_id + = hidden_field_tag :path, @path + = label_tag :file, class: 'control-label' do + %span GitLab project export + .col-sm-10 + = file_field_tag :file, class: '' + + .form-actions + = submit_tag 'Import project', class: 'btn btn-create' diff --git a/app/views/notify/project_was_exported_email.html.haml b/app/views/notify/project_was_exported_email.html.haml new file mode 100644 index 00000000000..b28fea35ad5 --- /dev/null +++ b/app/views/notify/project_was_exported_email.html.haml @@ -0,0 +1,8 @@ +%p + Project #{@project.name} was exported successfully. +%p + The project export can be downloaded from: + = link_to download_export_namespace_project_url(@project.namespace, @project) do + = @project.name_with_namespace + " export" +%p + The download link will expire in 24 hours. diff --git a/app/views/notify/project_was_exported_email.text.erb b/app/views/notify/project_was_exported_email.text.erb new file mode 100644 index 00000000000..42c4d176876 --- /dev/null +++ b/app/views/notify/project_was_exported_email.text.erb @@ -0,0 +1,6 @@ +Project <%= @project.name %> was exported successfully. + +The project export can be downloaded from: +<%= download_export_namespace_project_url(@project.namespace, @project) %> + +The download link will expire in 24 hours. diff --git a/app/views/notify/project_was_not_exported_email.html.haml b/app/views/notify/project_was_not_exported_email.html.haml new file mode 100644 index 00000000000..c9e9ade2cf1 --- /dev/null +++ b/app/views/notify/project_was_not_exported_email.html.haml @@ -0,0 +1,9 @@ +%p + Project #{@project.name} couldn't be exported. +%p + The errors we encountered were: + + %ul + - @errors.each do |error| + %li + error diff --git a/app/views/notify/project_was_not_exported_email.text.erb b/app/views/notify/project_was_not_exported_email.text.erb new file mode 100644 index 00000000000..a07f6edacf7 --- /dev/null +++ b/app/views/notify/project_was_not_exported_email.text.erb @@ -0,0 +1,6 @@ +Project <%= @project.name %> couldn't be exported. + +The errors we encountered were: + +- @errors.each do |error| +<%= error %>
\ No newline at end of file diff --git a/app/views/projects/edit.html.haml b/app/views/projects/edit.html.haml index 8449fe1e4e0..27a94fe02dc 100644 --- a/app/views/projects/edit.html.haml +++ b/app/views/projects/edit.html.haml @@ -120,6 +120,42 @@ = link_to 'Housekeeping', housekeeping_namespace_project_path(@project.namespace, @project), method: :post, class: "btn btn-save" %hr + .row.prepend-top-default + .col-lg-3 + %h4.prepend-top-0 + Export project + %p.append-bottom-0 + %p + Export this project with all its related data in order to move your project to a new GitLab instance. Once the export is finished, you can import the file from the "New Project" page. + %p + Once the exported file is ready, you will receive a notification email with a download link. + + .col-lg-9 + + - if @project.export_project_path + = link_to 'Download export', download_export_namespace_project_path(@project.namespace, @project), + method: :get, class: "btn btn-default" + = link_to 'Generate new export', generate_new_export_namespace_project_path(@project.namespace, @project), + method: :post, class: "btn btn-default" + - else + = link_to 'Export project', export_namespace_project_path(@project.namespace, @project), + method: :post, class: "btn btn-default" + + .bs-callout.bs-callout-info + %p.append-bottom-0 + %p + The following items will be exported: + %ul + %li Project and wiki repositories + %li Project uploads + %li Project configuration including web hooks and services + %li Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities + %p + The following items will NOT be exported: + %ul + %li Build traces and artifacts + %li LFS objects + %hr - if can? current_user, :archive_project, @project .row.prepend-top-default .col-lg-3 diff --git a/app/views/projects/imports/show.html.haml b/app/views/projects/imports/show.html.haml index c0d1ce0d120..4d8ee562e6a 100644 --- a/app/views/projects/imports/show.html.haml +++ b/app/views/projects/imports/show.html.haml @@ -7,7 +7,7 @@ Forking in progress. - else Import in progress. - - unless @project.forked? + - if @project.external_import? %p.monospace git clone --bare #{@project.safe_import_url} %p Please wait while we import the repository for you. Refresh at will. :javascript diff --git a/app/views/projects/new.html.haml b/app/views/projects/new.html.haml index 7e8b8f83467..3c1c6060504 100644 --- a/app/views/projects/new.html.haml +++ b/app/views/projects/new.html.haml @@ -84,7 +84,12 @@ - if git_import_enabled? = link_to "#", class: 'btn js-toggle-button import_git' do %i.fa.fa-git - %span Any repo by URL + %span Repo by URL + + - if gitlab_project_import_enabled? + = link_to new_import_gitlab_project_path, class: 'btn import_gitlab_project project-submit' do + %i.fa.fa-gitlab + %span GitLab export .js-toggle-content.hide = render "shared/import_form", f: f @@ -115,6 +120,33 @@ e.preventDefault(); var import_modal = $(this).next(".modal").show(); }); + $('.modal-header .close').bind('click', function() { $(".modal").hide(); }); + + $('.import_gitlab_project').bind('click', function() { + var _href = $("a.import_gitlab_project").attr("href"); + $(".import_gitlab_project").attr("href", _href + '?namespace_id=' + $("#project_namespace_id").val() + '&path=' + $("#project_path").val()); + }); + + $('.import_gitlab_project').attr('disabled',true) + $('.import_gitlab_project').attr('title', 'Project path required.'); + + $('.import_gitlab_project').click(function( event ) { + if($('.import_gitlab_project').attr('disabled')) { + event.preventDefault(); + new Flash("Please enter a path for the project to be imported to."); + } + }); + + $('#project_path').keyup(function(){ + if($(this).val().length !=0) { + $('.import_gitlab_project').attr('disabled', false); + $('.import_gitlab_project').attr('title',''); + $(".flash-container").html("") + } else { + $('.import_gitlab_project').attr('disabled',true); + $('.import_gitlab_project').attr('title', 'Project path required.'); + } + }) diff --git a/app/workers/gitlab_remove_project_export_worker.rb b/app/workers/gitlab_remove_project_export_worker.rb new file mode 100644 index 00000000000..1d91897d520 --- /dev/null +++ b/app/workers/gitlab_remove_project_export_worker.rb @@ -0,0 +1,9 @@ +class GitlabRemoveProjectExportWorker + include Sidekiq::Worker + + sidekiq_options queue: :default + + def perform + Project.remove_gitlab_exports! + end +end diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index 3616b37d2ad..39f6037e077 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -1,12 +1,12 @@ class ProjectExportWorker include Sidekiq::Worker - # TODO: enable retry - disabled for QA purposes - sidekiq_options queue: :gitlab_shell, retry: false + sidekiq_options queue: :gitlab_shell, retry: true def perform(current_user_id, project_id) current_user = User.find(current_user_id) project = Project.find(project_id) + ::Projects::ImportExport::ExportService.new(project, current_user).execute end end diff --git a/app/workers/project_import_worker.rb b/app/workers/project_import_worker.rb deleted file mode 100644 index b2902c3278e..00000000000 --- a/app/workers/project_import_worker.rb +++ /dev/null @@ -1,24 +0,0 @@ -class ProjectImportWorker - include Sidekiq::Worker - - # TODO: enabled retry - disabled for QA purposes - sidekiq_options queue: :gitlab_shell, retry: false - - def perform(current_user_id, tmp_file, namespace_id, path) - current_user = User.find(current_user_id) - - project = Gitlab::ImportExport::ImportService.execute(archive_file: tmp_file, - owner: current_user, - namespace_id: namespace_id, - project_path: path) - if project - project.repository.after_import - else - logger.error("There was an error during the import: #{tmp_file}") - end - end - - def logger - Sidekiq.logger - end -end |