Merge remote-tracking branch 'upstream/master' into feature/runner-lock-on-project

* upstream/master: (353 commits)
  Put some admin settings in dropdown
  Add styleguide on configuration settings documentation
  Remove Duplicated keys add UNIQUE index to fingerprint
  Avoid autoload issue such as 'Mail::Parsers::AddressStruct'
  Move appearance settings as sub tab to application settings
  use rails root join
  fixed a couple of errors spotted in production
  Fix RangeError exceptions when referring to issues or merge requests outside of max database values
  Fix bug in `WikiLinkFilter`.
  Grammar and typographic changes to artifacts documentation
  Tweak grammar
  Small frontend code fixes and restore 8a2d88f commit
  Warn about admin privilege to disable GitHub Webhooks
  Listing GH Webhooks doesn't stop import process for non GH admin users
  fixup! updated docs for api endpoint award emoji
  Update CHANGELOG
  Ensure Todos counters doesn't count Todos for projects pending delete
  Add endpoints for award emoji on notes
  Sort API endpoints and implement feedback
  Add endpoints for Award Emoji
  ...

5 files changed, 186 insertions, 51 deletions

diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb
index f804ac171c4..19832a19b2b 100644
--- a/app/services/notification_service.rb
+++ b/app/services/notification_service.rb
@@ -29,9 +29,10 @@ class NotificationService
   #  * issue assignee if their notification level is not Disabled
   #  * project team members with notification level higher then Participating
   #  * watchers of the issue's labels
+  #  * users with custom level checked with "new issue"
   #
   def new_issue(issue, current_user)
-    new_resource_email(issue, issue.project, 'new_issue_email')
+    new_resource_email(issue, issue.project, :new_issue_email)
   end
 
   # When we close an issue we should send an email to:
@@ -39,18 +40,20 @@ class NotificationService
   #  * issue author if their notification level is not Disabled
   #  * issue assignee if their notification level is not Disabled
   #  * project team members with notification level higher then Participating
+  #  * users with custom level checked with "close issue"
   #
   def close_issue(issue, current_user)
-    close_resource_email(issue, issue.project, current_user, 'closed_issue_email')
+    close_resource_email(issue, issue.project, current_user, :closed_issue_email)
   end
 
   # When we reassign an issue we should send an email to:
   #
   #  * issue old assignee if their notification level is not Disabled
   #  * issue new assignee if their notification level is not Disabled
+  #  * users with custom level checked with "reassign issue"
   #
   def reassigned_issue(issue, current_user)
-    reassign_resource_email(issue, issue.project, current_user, 'reassigned_issue_email')
+    reassign_resource_email(issue, issue.project, current_user, :reassigned_issue_email)
   end
 
   # When we add labels to an issue we should send an email to:
@@ -58,7 +61,7 @@ class NotificationService
   #  * watchers of the issue's labels
   #
   def relabeled_issue(issue, added_labels, current_user)
-    relabeled_resource_email(issue, added_labels, current_user, 'relabeled_issue_email')
+    relabeled_resource_email(issue, added_labels, current_user, :relabeled_issue_email)
   end
 
   # When create a merge request we should send an email to:
@@ -66,18 +69,20 @@ class NotificationService
   #  * mr assignee if their notification level is not Disabled
   #  * project team members with notification level higher then Participating
   #  * watchers of the mr's labels
+  #  * users with custom level checked with "new merge request"
   #
   def new_merge_request(merge_request, current_user)
-    new_resource_email(merge_request, merge_request.target_project, 'new_merge_request_email')
+    new_resource_email(merge_request, merge_request.target_project, :new_merge_request_email)
   end
 
   # When we reassign a merge_request we should send an email to:
   #
   #  * merge_request old assignee if their notification level is not Disabled
   #  * merge_request assignee if their notification level is not Disabled
+  #  * users with custom level checked with "reassign merge request"
   #
   def reassigned_merge_request(merge_request, current_user)
-    reassign_resource_email(merge_request, merge_request.target_project, current_user, 'reassigned_merge_request_email')
+    reassign_resource_email(merge_request, merge_request.target_project, current_user, :reassigned_merge_request_email)
   end
 
   # When we add labels to a merge request we should send an email to:
@@ -85,15 +90,15 @@ class NotificationService
   #  * watchers of the mr's labels
   #
   def relabeled_merge_request(merge_request, added_labels, current_user)
-    relabeled_resource_email(merge_request, added_labels, current_user, 'relabeled_merge_request_email')
+    relabeled_resource_email(merge_request, added_labels, current_user, :relabeled_merge_request_email)
   end
 
   def close_mr(merge_request, current_user)
-    close_resource_email(merge_request, merge_request.target_project, current_user, 'closed_merge_request_email')
+    close_resource_email(merge_request, merge_request.target_project, current_user, :closed_merge_request_email)
   end
 
   def reopen_issue(issue, current_user)
-    reopen_resource_email(issue, issue.project, current_user, 'issue_status_changed_email', 'reopened')
+    reopen_resource_email(issue, issue.project, current_user, :issue_status_changed_email, 'reopened')
   end
 
   def merge_mr(merge_request, current_user)
@@ -101,7 +106,7 @@ class NotificationService
       merge_request,
       merge_request.target_project,
       current_user,
-      'merged_merge_request_email'
+      :merged_merge_request_email
     )
   end
 
@@ -110,7 +115,7 @@ class NotificationService
       merge_request,
       merge_request.target_project,
       current_user,
-      'merge_request_status_email',
+      :merge_request_status_email,
       'reopened'
     )
   end
@@ -153,6 +158,9 @@ class NotificationService
     # Merge project watchers
     recipients = add_project_watchers(recipients, note.project)
 
+    # Merge project with custom notification
+    recipients = add_custom_notifications(recipients, note.project, :new_note)
+
     # Reject users with Mention notification level, except those mentioned in _this_ note.
     recipients = reject_mention_users(recipients - mentioned_users, note.project)
     recipients = recipients + mentioned_users
@@ -222,7 +230,7 @@ class NotificationService
   end
 
   def accept_group_invite(group_member)
-    mailer.member_invite_accepted_email(group_member.id).deliver_later
+    mailer.member_invite_accepted_email(group_member.real_source_type, group_member.id).deliver_later
   end
 
   def decline_group_invite(group_member)
@@ -266,14 +274,41 @@ class NotificationService
     end
   end
 
+  def project_exported(project, current_user)
+    mailer.project_was_exported_email(current_user, project).deliver_later
+  end
+
+  def project_not_exported(project, current_user, errors)
+    mailer.project_was_not_exported_email(current_user, project, errors).deliver_later
+  end
+
   protected
 
+  # Get project/group users with CUSTOM notification level
+  def add_custom_notifications(recipients, project, action)
+    user_ids = []
+
+    # Users with a notification setting on group or project
+    user_ids += notification_settings_for(project, :custom, action)
+    user_ids += notification_settings_for(project.group, :custom, action)
+
+    # Users with global level custom
+    users_with_project_level_global = notification_settings_for(project, :global)
+    users_with_group_level_global   = notification_settings_for(project.group, :global)
+
+    global_users_ids = users_with_project_level_global.concat(users_with_group_level_global)
+    user_ids += users_with_global_level_custom(global_users_ids, action)
+
+    recipients.concat(User.find(user_ids))
+  end
+
   # Get project users with WATCH notification level
   def project_watchers(project)
-    project_members = project_member_notification(project)
+    project_members = notification_settings_for(project)
+
+    users_with_project_level_global = notification_settings_for(project, :global)
+    users_with_group_level_global   = notification_settings_for(project.group, :global)
 
-    users_with_project_level_global = project_member_notification(project, :global)
-    users_with_group_level_global = group_member_notification(project, :global)
     users = users_with_global_level_watch([users_with_project_level_global, users_with_group_level_global].flatten.uniq)
 
     users_with_project_setting = select_project_member_setting(project, users_with_project_level_global, users)
@@ -282,33 +317,39 @@ class NotificationService
     User.where(id: users_with_project_setting.concat(users_with_group_setting).uniq).to_a
   end
 
-  def project_member_notification(project, notification_level=nil)
+  def notification_settings_for(resource, notification_level = nil, action = nil)
+    return [] unless resource
+
     if notification_level
-      project.notification_settings.where(level: NotificationSetting.levels[notification_level]).pluck(:user_id)
+      settings = resource.notification_settings.where(level: NotificationSetting.levels[notification_level])
+      settings = settings.select { |setting| setting.events[action] } if action.present?
+      settings.map(&:user_id)
     else
-      project.notification_settings.pluck(:user_id)
+      resource.notification_settings.pluck(:user_id)
     end
   end
 
-  def group_member_notification(project, notification_level)
-    if project.group
-      project.group.notification_settings.where(level: NotificationSetting.levels[notification_level]).pluck(:user_id)
-    else
-      []
-    end
+  def users_with_global_level_watch(ids)
+    settings_with_global_level_of(:watch, ids).pluck(:user_id)
   end
 
-  def users_with_global_level_watch(ids)
+  def users_with_global_level_custom(ids, action)
+    settings = settings_with_global_level_of(:custom, ids)
+    settings = settings.select { |setting| setting.events[action] }
+    settings.map(&:user_id)
+  end
+
+  def settings_with_global_level_of(level, ids)
     NotificationSetting.where(
       user_id: ids,
       source_type: nil,
-      level: NotificationSetting.levels[:watch]
-    ).pluck(:user_id)
+      level: NotificationSetting.levels[level]
+    )
   end
 
   # Build a list of users based on project notifcation settings
   def select_project_member_setting(project, global_setting, users_global_level_watch)
-    users = project_member_notification(project, :watch)
+    users = notification_settings_for(project, :watch)
 
     # If project setting is global, add to watch list if global setting is watch
     global_setting.each do |user_id|
@@ -322,7 +363,7 @@ class NotificationService
 
   # Build a list of users based on group notification settings
   def select_group_member_setting(project, project_members, global_setting, users_global_level_watch)
-    uids = group_member_notification(project, :watch)
+    uids = notification_settings_for(project, :watch)
 
     # Group setting is watch, add to users list if user is not project member
     users = []
@@ -343,7 +384,7 @@ class NotificationService
   end
 
   def add_project_watchers(recipients, project)
-    recipients.concat(project_watchers(project)).compact.uniq
+    recipients.concat(project_watchers(project)).compact
   end
 
   # Remove users with disabled notifications from array
@@ -428,7 +469,7 @@ class NotificationService
   end
 
   def new_resource_email(target, project, method)
-    recipients = build_recipients(target, project, target.author, action: :new)
+    recipients = build_recipients(target, project, target.author, action: "new")
 
     recipients.each do |recipient|
       mailer.send(method, recipient.id, target.id).deliver_later
@@ -436,7 +477,8 @@ class NotificationService
   end
 
   def close_resource_email(target, project, current_user, method)
-    recipients = build_recipients(target, project, current_user)
+    action = method == :merged_merge_request_email ? "merge" : "close"
+    recipients = build_recipients(target, project, current_user, action: action)
 
     recipients.each do |recipient|
       mailer.send(method, recipient.id, target.id, current_user.id).deliver_later
@@ -447,7 +489,7 @@ class NotificationService
     previous_assignee_id = previous_record(target, 'assignee_id')
     previous_assignee = User.find_by(id: previous_assignee_id) if previous_assignee_id
 
-    recipients = build_recipients(target, project, current_user, action: :reassign, previous_assignee: previous_assignee)
+    recipients = build_recipients(target, project, current_user, action: "reassign", previous_assignee: previous_assignee)
 
     recipients.each do |recipient|
       mailer.send(
@@ -470,7 +512,7 @@ class NotificationService
   end
 
   def reopen_resource_email(target, project, current_user, method, status)
-    recipients = build_recipients(target, project, current_user)
+    recipients = build_recipients(target, project, current_user, action: "reopen")
 
     recipients.each do |recipient|
       mailer.send(method, recipient.id, target.id, status, current_user.id).deliver_later
@@ -478,14 +520,20 @@ class NotificationService
   end
 
   def build_recipients(target, project, current_user, action: nil, previous_assignee: nil)
+    custom_action = build_custom_key(action, target)
+
     recipients = target.participants(current_user)
     recipients = add_project_watchers(recipients, project)
+
+    recipients = add_custom_notifications(recipients, project, custom_action)
     recipients = reject_mention_users(recipients, project)
 
+    recipients = recipients.uniq
+
     # Re-assign is considered as a mention of the new assignee so we add the
     # new assignee to the list of recipients after we rejected users with
     # the "on mention" notification level
-    if action == :reassign
+    if [:reassign_merge_request, :reassign_issue].include?(custom_action)
       recipients << previous_assignee if previous_assignee
       recipients << target.assignee
     end
@@ -493,7 +541,7 @@ class NotificationService
     recipients = reject_muted_users(recipients, project)
     recipients = add_subscribed_users(recipients, target)
 
-    if action == :new
+    if [:new_issue, :new_merge_request].include?(custom_action)
       recipients = add_labels_subscribers(recipients, target)
     end
 
@@ -523,4 +571,10 @@ class NotificationService
       end
     end
   end
+
+  # Build event key to search on custom notification level
+  # Check NotificationSetting::EMAIL_EVENTS
+  def build_custom_key(action, object)
+    "#{action}_#{object.class.name.underscore}".to_sym
+  end
 end
diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb
index 61cac5419ad..55956be2844 100644
--- a/app/services/projects/create_service.rb
+++ b/app/services/projects/create_service.rb
@@ -80,16 +80,18 @@ module Projects
     def after_create_actions
       log_info("#{@project.owner.name} created a new project \"#{@project.name_with_namespace}\"")
 
-      @project.create_wiki if @project.wiki_enabled?
+      unless @project.gitlab_project_import?
+        @project.create_wiki if @project.wiki_enabled?
 
-      @project.build_missing_services
+        @project.build_missing_services
 
-      @project.create_labels
+        @project.create_labels
+      end
 
       event_service.create_project(@project, current_user)
       system_hook_service.execute_hooks_for(@project, :create)
 
-      unless @project.group
+      unless @project.group || @project.gitlab_project_import?
         @project.team << [current_user, :master, current_user]
       end
     end
diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb
new file mode 100644
index 00000000000..80c7193efcb
--- /dev/null
+++ b/app/services/projects/import_export/export_service.rb
@@ -0,0 +1,57 @@
+module Projects
+  module ImportExport
+    class ExportService < BaseService
+
+      def execute(_options = {})
+        @shared = Gitlab::ImportExport::Shared.new(relative_path: File.join(project.path_with_namespace, 'work'))
+        save_all
+      end
+
+      private
+
+      def save_all
+        if [version_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save)
+          Gitlab::ImportExport::Saver.save(shared: @shared)
+          notify_success
+        else
+          cleanup_and_notify
+        end
+      end
+
+      def version_saver
+        Gitlab::ImportExport::VersionSaver.new(shared: @shared)
+      end
+
+      def project_tree_saver
+        Gitlab::ImportExport::ProjectTreeSaver.new(project: project, shared: @shared)
+      end
+
+      def uploads_saver
+        Gitlab::ImportExport::UploadsSaver.new(project: project, shared: @shared)
+      end
+
+      def repo_saver
+        Gitlab::ImportExport::RepoSaver.new(project: project, shared: @shared)
+      end
+
+      def wiki_repo_saver
+        Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared)
+      end
+
+      def cleanup_and_notify
+        FileUtils.rm_rf(@shared.export_path)
+
+        notify_error
+        raise Gitlab::ImportExport::Error.new(@shared.errors.join(', '))
+      end
+
+      def notify_success
+        notification_service.project_exported(@project, @current_user)
+      end
+
+      def notify_error
+        notification_service.project_not_exported(@project, @current_user, @shared.errors)
+      end
+    end
+  end
+end
diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb
index c4838d31f2f..9159ec08959 100644
--- a/app/services/projects/import_service.rb
+++ b/app/services/projects/import_service.rb
@@ -9,26 +9,31 @@ module Projects
       'fogbugz',
       'gitlab',
       'github',
-      'google_code'
+      'google_code',
+      'gitlab_project'
     ]
 
     def execute
-      if unknown_url?
-        # In this case, we only want to import issues, not a repository.
-        create_repository
-      else
-        import_repository
-      end
+      add_repository_to_project unless project.gitlab_project_import?
 
       import_data
 
       success
-    rescue Error => e
+    rescue => e
       error(e.message)
     end
 
     private
 
+    def add_repository_to_project
+      if unknown_url?
+        # In this case, we only want to import issues, not a repository.
+        create_repository
+      else
+        import_repository
+      end
+    end
+
     def create_repository
       unless project.create_repository
         raise Error, 'The repository could not be created.'
@@ -46,7 +51,7 @@ module Projects
     def import_data
       return unless has_importer?
 
-      project.repository.before_import
+      project.repository.before_import unless project.gitlab_project_import?
 
       unless importer.execute
         raise Error, 'The remote data could not be imported.'
@@ -58,6 +63,8 @@ module Projects
     end
 
     def importer
+      return Gitlab::ImportExport::Importer.new(project) if @project.gitlab_project_import?
+
       class_name = "Gitlab::#{project.import_type.camelize}Import::Importer"
       class_name.constantize.new(project)
     end
diff --git a/app/services/todo_service.rb b/app/services/todo_service.rb
index e1f9ea64dc4..540bf54b920 100644
--- a/app/services/todo_service.rb
+++ b/app/services/todo_service.rb
@@ -1,6 +1,6 @@
 # TodoService class
 #
-# Used for creating todos after certain user actions
+# Used for creating/updating todos after certain user actions
 #
 # Ex.
 #   TodoService.new.new_issue(issue, current_user)
@@ -137,6 +137,15 @@ class TodoService
   def mark_pending_todos_as_done(target, user)
     attributes = attributes_for_target(target)
     pending_todos(user, attributes).update_all(state: :done)
+    user.update_todos_count_cache
+  end
+
+  # When user marks some todos as done
+  def mark_todos_as_done(todos, current_user)
+    todos = current_user.todos.where(id: todos.map(&:id)) unless todos.respond_to?(:update_all)
+
+    todos.update_all(state: :done)
+    current_user.update_todos_count_cache
   end
 
   # When user marks an issue as todo
@@ -151,6 +160,7 @@ class TodoService
     Array(users).map do |user|
       next if pending_todos(user, attributes).exists?
       Todo.create(attributes.merge(user_id: user.id))
+      user.update_todos_count_cache
     end
   end
 
@@ -161,11 +171,16 @@ class TodoService
 
   def update_issuable(issuable, author)
     # Skip toggling a task list item in a description
-    return if issuable.tasks? && issuable.updated_tasks.any?
+    return if toggling_tasks?(issuable)
 
     create_mention_todos(issuable.project, issuable, author)
   end
 
+  def toggling_tasks?(issuable)
+    issuable.previous_changes.include?('description') &&
+      issuable.tasks? && issuable.updated_tasks.any?
+  end
+
   def handle_note(note, author)
     # Skip system notes, and notes on project snippet
     return if note.system? || note.for_snippet?