Merge branch 'feature/project-export' into 'master'

Export project functionality

This is a MR for the export functionality of https://gitlab.com/gitlab-org/gitlab-ce/issues/3050, which adds the ability to export single projects.

- [x] members
- DB data
  - [x] issues
  - [x] issue comments
  - [x] merge requests
  - [x] merge request diff
  - [x] merge request comments
  - [x] labels
  - [x] milestones
  - [x] snippets
  - [x] releases
  - [x] events
  - [x] commit statuses
  - [x] CI builds
- File system data
  - [x] Git repository
  - [x] wiki
  - [x] uploads
  - [ ] ~~CI build traces~~
  - [ ] ~~CI build artifacts~~
  - [ ] ~~LFS objects~~
- DB configuration
  - [x] services
  - [x] web hooks
  - [x] protected branches
  - [x] deploy keys
  - [x] CI variables
  - [x] CI triggers

See merge request !3114

9 files changed, 61 insertions, 14 deletions

diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index a744f937918..d914b0b26eb 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -123,7 +123,7 @@ class ApplicationSetting < ActiveRecord::Base
       default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
       default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
       restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
-      import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
+      import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project],
       shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
       max_artifacts_size: Settings.artifacts['max_size'],
       require_two_factor_authentication: false,
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index a5f2ac59001..5b264ecffc5 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -170,6 +170,10 @@ module Ci
       builds.where.not(environment: nil).success.pluck(:environment).uniq
     end
 
+    def notes
+      Note.for_commit_id(sha)
+    end
+
     private
 
     def build_builds_for_stages(stages, user, status, trigger_request)
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index e53c483b904..ab13db4b297 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -1,5 +1,6 @@
 class CommitStatus < ActiveRecord::Base
   include Statuseable
+  include Importable
 
   self.table_name = 'ci_builds'
 
@@ -7,7 +8,7 @@ class CommitStatus < ActiveRecord::Base
   belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id, touch: true
   belongs_to :user
 
-  validates :pipeline, presence: true
+  validates :pipeline, presence: true, unless: :importing?
 
   validates_presence_of :name
 
diff --git a/app/models/concerns/importable.rb b/app/models/concerns/importable.rb
new file mode 100644
index 00000000000..019ef755849
--- /dev/null
+++ b/app/models/concerns/importable.rb
@@ -0,0 +1,6 @@
+module Importable
+  extend ActiveSupport::Concern
+
+  attr_accessor :importing
+  alias_method :importing?, :importing
+end
diff --git a/app/models/member.rb b/app/models/member.rb
index cea6d259760..4ee3f1bb5c2 100644
--- a/app/models/member.rb
+++ b/app/models/member.rb
@@ -1,5 +1,6 @@
 class Member < ActiveRecord::Base
   include Sortable
+  include Importable
   include Gitlab::Access
 
   attr_accessor :raw_invite_token
@@ -41,11 +42,11 @@ class Member < ActiveRecord::Base
 
   before_validation :generate_invite_token, on: :create, if: -> (member) { member.invite_email.present? }
 
-  after_create :send_invite, if: :invite?
-  after_create :send_request, if: :request?
-  after_create :create_notification_setting, unless: :pending?
-  after_create :post_create_hook, unless: :pending?
-  after_update :post_update_hook, unless: :pending?
+  after_create :send_invite, if: :invite?, unless: :importing?
+  after_create :send_request, if: :request?, unless: :importing?
+  after_create :create_notification_setting, unless: [:pending?, :importing?]
+  after_create :post_create_hook, unless: [:pending?, :importing?]
+  after_update :post_update_hook, unless: [:pending?, :importing?]
   after_destroy :post_destroy_hook, unless: :pending?
   after_destroy :post_decline_request, if: :request?
 
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 7b8858b24d6..73bf182ec9f 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -4,6 +4,7 @@ class MergeRequest < ActiveRecord::Base
   include Referable
   include Sortable
   include Taskable
+  include Importable
 
   belongs_to :target_project, foreign_key: :target_project_id, class_name: "Project"
   belongs_to :source_project, foreign_key: :source_project_id, class_name: "Project"
@@ -13,7 +14,7 @@ class MergeRequest < ActiveRecord::Base
 
   serialize :merge_params, Hash
 
-  after_create :create_merge_request_diff
+  after_create :create_merge_request_diff, unless: :importing
   after_update :update_merge_request_diff
 
   delegate :commits, :diffs, :real_size, to: :merge_request_diff, prefix: nil
@@ -95,12 +96,12 @@ class MergeRequest < ActiveRecord::Base
     end
   end
 
-  validates :source_project, presence: true, unless: :allow_broken
+  validates :source_project, presence: true, unless: [:allow_broken, :importing?]
   validates :source_branch, presence: true
   validates :target_project, presence: true
   validates :target_branch, presence: true
   validates :merge_user, presence: true, if: :merge_when_build_succeeds?
-  validate :validate_branches, unless: :allow_broken
+  validate :validate_branches, unless: [:allow_broken, :importing?]
   validate :validate_fork
 
   scope :by_branch, ->(branch_name) { where("(source_branch LIKE :branch) OR (target_branch LIKE :branch)", branch: branch_name) }
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index 7d5103748f5..aca377cc600 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -1,5 +1,6 @@
 class MergeRequestDiff < ActiveRecord::Base
   include Sortable
+  include Importable
 
   # Prevent store of diff if commits amount more then 500
   COMMITS_SAFE_SIZE = 100
@@ -22,7 +23,7 @@ class MergeRequestDiff < ActiveRecord::Base
   serialize :st_commits
   serialize :st_diffs
 
-  after_create :reload_content
+  after_create :reload_content, unless: :importing?
 
   def reload_content
     reload_commits
diff --git a/app/models/note.rb b/app/models/note.rb
index 4b6748053ff..8d164647550 100644
--- a/app/models/note.rb
+++ b/app/models/note.rb
@@ -4,6 +4,7 @@ class Note < ActiveRecord::Base
   include Participable
   include Mentionable
   include Awardable
+  include Importable
 
   default_value_for :system, false
 
@@ -28,11 +29,11 @@ class Note < ActiveRecord::Base
   validates :attachment, file_size: { maximum: :max_attachment_size }
 
   validates :noteable_type, presence: true
-  validates :noteable_id, presence: true, unless: :for_commit?
+  validates :noteable_id, presence: true, unless: [:for_commit?, :importing?]
   validates :commit_id, presence: true, if: :for_commit?
   validates :author, presence: true
 
-  validate unless: :for_commit? do |note|
+  validate unless: [:for_commit?, :importing?] do |note|
     unless note.noteable.try(:project) == note.project
       errors.add(:invalid_project, 'Note and noteable project mismatch')
     end
diff --git a/app/models/project.rb b/app/models/project.rb
index 8eef22356e2..ca3bc04e2dd 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -367,6 +367,11 @@ class Project < ActiveRecord::Base
 
       joins(join_body).reorder('join_note_counts.amount DESC')
     end
+
+    # Deletes gitlab project export files older than 24 hours
+    def remove_gitlab_exports!
+      Gitlab::Popen.popen(%W(find #{Gitlab::ImportExport.storage_path} -not -path #{Gitlab::ImportExport.storage_path} -mmin +1440 -delete))
+    end
   end
 
   def team
@@ -470,7 +475,7 @@ class Project < ActiveRecord::Base
   end
 
   def import?
-    external_import? || forked?
+    external_import? || forked? || gitlab_project_import?
   end
 
   def no_import?
@@ -501,6 +506,10 @@ class Project < ActiveRecord::Base
     Gitlab::UrlSanitizer.new(import_url).masked_url
   end
 
+  def gitlab_project_import?
+    import_type == 'gitlab_project'
+  end
+
   def check_limit
     unless creator.can_create_project? or namespace.kind == 'group'
       projects_limit = creator.projects_limit
@@ -1096,4 +1105,27 @@ class Project < ActiveRecord::Base
   ensure
     @errors = original_errors
   end
+
+  def add_export_job(current_user:)
+    job_id = ProjectExportWorker.perform_async(current_user.id, self.id)
+
+    if job_id
+      Rails.logger.info "Export job started for project ID #{self.id} with job ID #{job_id}"
+    else
+      Rails.logger.error "Export job failed to start for project ID #{self.id}"
+    end
+  end
+
+  def export_path
+    File.join(Gitlab::ImportExport.storage_path, path_with_namespace)
+  end
+
+  def export_project_path
+    Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) }
+  end
+
+  def remove_exports
+    _, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
+    status.zero?
+  end
 end