Merge remote-tracking branch 'upstream/master' into feature/runner-lock-on-project

* upstream/master: (353 commits)
  Put some admin settings in dropdown
  Add styleguide on configuration settings documentation
  Remove Duplicated keys add UNIQUE index to fingerprint
  Avoid autoload issue such as 'Mail::Parsers::AddressStruct'
  Move appearance settings as sub tab to application settings
  use rails root join
  fixed a couple of errors spotted in production
  Fix RangeError exceptions when referring to issues or merge requests outside of max database values
  Fix bug in `WikiLinkFilter`.
  Grammar and typographic changes to artifacts documentation
  Tweak grammar
  Small frontend code fixes and restore 8a2d88f commit
  Warn about admin privilege to disable GitHub Webhooks
  Listing GH Webhooks doesn't stop import process for non GH admin users
  fixup! updated docs for api endpoint award emoji
  Update CHANGELOG
  Ensure Todos counters doesn't count Todos for projects pending delete
  Add endpoints for award emoji on notes
  Sort API endpoints and implement feedback
  Add endpoints for Award Emoji
  ...

44 files changed, 1504 insertions, 123 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 6cd909f6115..0e7a1cc2623 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -26,38 +26,40 @@ module API
     # Ensure the namespace is right, otherwise we might load Grape::API::Helpers
     helpers ::API::Helpers
 
-    mount ::API::Groups
+    mount ::API::AwardEmoji
+    mount ::API::Branches
+    mount ::API::Builds
+    mount ::API::CommitStatuses
+    mount ::API::Commits
+    mount ::API::DeployKeys
+    mount ::API::Files
+    mount ::API::Gitignores
     mount ::API::GroupMembers
-    mount ::API::Users
-    mount ::API::Projects
-    mount ::API::Repositories
+    mount ::API::Groups
+    mount ::API::Internal
     mount ::API::Issues
-    mount ::API::Milestones
-    mount ::API::Session
+    mount ::API::Keys
+    mount ::API::Labels
+    mount ::API::Licenses
     mount ::API::MergeRequests
+    mount ::API::Milestones
+    mount ::API::Namespaces
     mount ::API::Notes
-    mount ::API::Internal
-    mount ::API::SystemHooks
-    mount ::API::ProjectSnippets
-    mount ::API::ProjectMembers
-    mount ::API::DeployKeys
     mount ::API::ProjectHooks
+    mount ::API::ProjectMembers
+    mount ::API::ProjectSnippets
+    mount ::API::Projects
+    mount ::API::Repositories
+    mount ::API::Runners
     mount ::API::Services
-    mount ::API::Files
-    mount ::API::Commits
-    mount ::API::CommitStatuses
-    mount ::API::Namespaces
-    mount ::API::Branches
-    mount ::API::Labels
+    mount ::API::Session
     mount ::API::Settings
-    mount ::API::Keys
+    mount ::API::SidekiqMetrics
+    mount ::API::Subscriptions
+    mount ::API::SystemHooks
     mount ::API::Tags
     mount ::API::Triggers
-    mount ::API::Builds
+    mount ::API::Users
     mount ::API::Variables
-    mount ::API::Runners
-    mount ::API::Licenses
-    mount ::API::Subscriptions
-    mount ::API::Gitignores
   end
 end
diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb
new file mode 100644
index 00000000000..985590312e3
--- /dev/null
+++ b/lib/api/award_emoji.rb
@@ -0,0 +1,116 @@
+module API
+  class AwardEmoji < Grape::API
+    before { authenticate! }
+    AWARDABLES = [Issue, MergeRequest]
+
+    resource :projects do
+      AWARDABLES.each do |awardable_type|
+        awardable_string = awardable_type.to_s.underscore.pluralize
+        awardable_id_string = "#{awardable_type.to_s.underscore}_id"
+
+        [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji",
+          ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji"
+        ].each do |endpoint|
+
+          # Get a list of project +awardable+ award emoji
+          #
+          # Parameters:
+          #   id (required)           - The ID of a project
+          #   awardable_id (required) - The ID of an issue or MR
+          # Example Request:
+          #   GET /projects/:id/issues/:awardable_id/award_emoji
+          get endpoint do
+            if can_read_awardable?
+              awards = paginate(awardable.award_emoji)
+              present awards, with: Entities::AwardEmoji
+            else
+              not_found!("Award Emoji")
+            end
+          end
+
+          # Get a specific award emoji
+          #
+          # Parameters:
+          #   id (required)           - The ID of a project
+          #   awardable_id (required) - The ID of an issue or MR
+          #   award_id (required)     - The ID of the award
+          # Example Request:
+          #   GET /projects/:id/issues/:awardable_id/award_emoji/:award_id
+          get "#{endpoint}/:award_id" do
+            if can_read_awardable?
+              present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji
+            else
+              not_found!("Award Emoji")
+            end
+          end
+
+          # Award a new Emoji
+          #
+          # Parameters:
+          #   id (required) - The ID of a project
+          #   awardable_id (required) - The ID of an issue or mr
+          #   name (required) - The name of a award_emoji (without colons)
+          # Example Request:
+          #   POST /projects/:id/issues/:awardable_id/award_emoji
+          post endpoint do
+            required_attributes! [:name]
+
+            not_found!('Award Emoji') unless can_read_awardable?
+
+            award = awardable.award_emoji.new(name: params[:name], user: current_user)
+
+            if award.save
+              present award, with: Entities::AwardEmoji
+            else
+              not_found!("Award Emoji #{award.errors.messages}")
+            end
+          end
+
+          # Delete a +awardables+ award emoji
+          #
+          # Parameters:
+          #   id (required) - The ID of a project
+          #   awardable_id (required) - The ID of an issue or MR
+          #   award_emoji_id (required) - The ID of an award emoji
+          # Example Request:
+          #   DELETE /projects/:id/issues/:issue_id/notes/:note_id/award_emoji/:award_id
+          delete "#{endpoint}/:award_id" do
+            award = awardable.award_emoji.find(params[:award_id])
+
+            unauthorized! unless award.user == current_user || current_user.admin?
+
+            award.destroy
+            present award, with: Entities::AwardEmoji
+          end
+        end
+      end
+    end
+
+    helpers do
+      def can_read_awardable?
+        ability = "read_#{awardable.class.to_s.underscore}".to_sym
+
+        can?(current_user, ability, awardable)
+      end
+
+      def awardable
+        @awardable ||=
+          begin
+            if params.include?(:note_id)
+              noteable.notes.find(params[:note_id])
+            else
+              noteable
+            end
+          end
+      end
+
+      def noteable
+        if params.include?(:issue_id)
+          user_project.issues.find(params[:issue_id])
+        else
+          user_project.merge_requests.find(params[:merge_request_id])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index ef5356d00d6..3fde7aa989f 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -225,6 +225,14 @@ module API
       expose(:downvote?)  { |note| false }
     end
 
+    class AwardEmoji < Grape::Entity
+      expose :id
+      expose :name
+      expose :user, using: Entities::UserBasic
+      expose :created_at, :updated_at
+      expose :awardable_id, :awardable_type
+    end
+
     class MRNote < Grape::Entity
       expose :note
       expose :author, using: Entities::UserBasic
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index de5959e3aae..77e407b54c5 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -9,9 +9,13 @@ module API
       [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value)
     end
 
+    def find_user_by_private_token
+      token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s
+      User.find_by_authentication_token(token_string) || User.find_by_personal_access_token(token_string)
+    end
+
     def current_user
-      private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s
-      @current_user ||= (User.find_by(authentication_token: private_token) || doorkeeper_guard)
+      @current_user ||= (find_user_by_private_token || doorkeeper_guard)
 
       unless @current_user && Gitlab::UserAccess.allowed?(@current_user)
         return nil
@@ -33,7 +37,7 @@ module API
       identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER]
 
       # Regex for integers
-      if !!(identifier =~ /^[0-9]+$/)
+      if !!(identifier =~ /\A[0-9]+\z/)
         identifier.to_i
       else
         identifier
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index d4fcfd3d4d3..8bfa998dc53 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -144,7 +144,7 @@ module API
 
     helpers do
       def noteable_read_ability_name(noteable)
-        "read_#{noteable.class.to_s.underscore.downcase}".to_sym
+        "read_#{noteable.class.to_s.underscore}".to_sym
       end
     end
   end
diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb
new file mode 100644
index 00000000000..d3d6827dc54
--- /dev/null
+++ b/lib/api/sidekiq_metrics.rb
@@ -0,0 +1,90 @@
+require 'sidekiq/api'
+
+module API
+  class SidekiqMetrics < Grape::API
+    before { authenticated_as_admin! }
+
+    helpers do
+      def queue_metrics
+        Sidekiq::Queue.all.each_with_object({}) do |queue, hash|
+          hash[queue.name] = {
+            backlog: queue.size,
+            latency: queue.latency.to_i
+          }
+        end
+      end
+
+      def process_metrics
+        Sidekiq::ProcessSet.new.map do |process|
+          {
+            hostname:    process['hostname'],
+            pid:         process['pid'],
+            tag:         process['tag'],
+            started_at:  Time.at(process['started_at']),
+            queues:      process['queues'],
+            labels:      process['labels'],
+            concurrency: process['concurrency'],
+            busy:        process['busy']
+          }
+        end
+      end
+
+      def job_stats
+        stats = Sidekiq::Stats.new
+        {
+          processed: stats.processed,
+          failed: stats.failed,
+          enqueued: stats.enqueued
+        }
+      end
+    end
+
+    # Get Sidekiq Queue metrics
+    #
+    # Parameters:
+    #   None
+    #
+    # Example:
+    #   GET /sidekiq/queue_metrics
+    #
+    get 'sidekiq/queue_metrics' do
+      { queues: queue_metrics }
+    end
+
+    # Get Sidekiq Process metrics
+    #
+    # Parameters:
+    #   None
+    #
+    # Example:
+    #   GET /sidekiq/process_metrics
+    #
+    get 'sidekiq/process_metrics' do
+      { processes: process_metrics }
+    end
+
+    # Get Sidekiq Job statistics
+    #
+    # Parameters:
+    #   None
+    #
+    # Example:
+    #   GET /sidekiq/job_stats
+    #
+    get 'sidekiq/job_stats' do
+      { jobs: job_stats }
+    end
+
+    # Get Sidekiq Compound metrics. Includes all previous metrics
+    #
+    # Parameters:
+    #   None
+    #
+    # Example:
+    #   GET /sidekiq/compound_metrics
+    #
+    get 'sidekiq/compound_metrics' do
+      { queues: queue_metrics, processes: process_metrics, jobs: job_stats }
+    end
+  end
+end
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index 4815bafe238..81d66271136 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -218,8 +218,9 @@ module Banzai
           nodes.each do |node|
             node.to_html.scan(regex) do
               project = $~[:project] || current_project_path
+              symbol = $~[object_sym]
 
-              refs[project] << $~[object_sym]
+              refs[project] << symbol if object_class.reference_valid?(symbol)
             end
           end
 
diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb
index 37a2779d453..1bb6d6bba87 100644
--- a/lib/banzai/filter/wiki_link_filter.rb
+++ b/lib/banzai/filter/wiki_link_filter.rb
@@ -29,7 +29,7 @@ module Banzai
         return if html_attr.blank?
 
         html_attr.value = apply_rewrite_rules(html_attr.value)
-      rescue URI::Error
+      rescue URI::Error, Addressable::URI::InvalidURIError
         # noop
       end
 
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index a66602f9194..ed86de819eb 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -54,7 +54,7 @@ module Ci
       job = @jobs[name.to_sym]
       return [] unless job
 
-      job.fetch(:variables, [])
+      job[:variables] || []
     end
 
     private
@@ -204,12 +204,12 @@ module Ci
         raise ValidationError, "#{name} job: tags parameter should be an array of strings"
       end
 
-      if job[:only] && !validate_array_of_strings(job[:only])
-        raise ValidationError, "#{name} job: only parameter should be an array of strings"
+      if job[:only] && !validate_array_of_strings_or_regexps(job[:only])
+        raise ValidationError, "#{name} job: only parameter should be an array of strings or regexps"
       end
 
-      if job[:except] && !validate_array_of_strings(job[:except])
-        raise ValidationError, "#{name} job: except parameter should be an array of strings"
+      if job[:except] && !validate_array_of_strings_or_regexps(job[:except])
+        raise ValidationError, "#{name} job: except parameter should be an array of strings or regexps"
       end
 
       if job[:allow_failure] && !validate_boolean(job[:allow_failure])
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
index e0b3f14d384..42232b7129d 100644
--- a/lib/container_registry/client.rb
+++ b/lib/container_registry/client.rb
@@ -15,11 +15,11 @@ module ContainerRegistry
     end
 
     def repository_tags(name)
-      @faraday.get("/v2/#{name}/tags/list").body
+      response_body @faraday.get("/v2/#{name}/tags/list")
     end
 
     def repository_manifest(name, reference)
-      @faraday.get("/v2/#{name}/manifests/#{reference}").body
+      response_body @faraday.get("/v2/#{name}/manifests/#{reference}")
     end
 
     def repository_tag_digest(name, reference)
@@ -34,7 +34,7 @@ module ContainerRegistry
     def blob(name, digest, type = nil)
       headers = {}
       headers['Accept'] = type if type
-      @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers).body
+      response_body @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers)
     end
 
     def delete_blob(name, digest)
@@ -47,6 +47,7 @@ module ContainerRegistry
       conn.request :json
       conn.headers['Accept'] = MANIFEST_VERSION
 
+      conn.response :json, content_type: 'application/json'
       conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws'
       conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json'
       conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json'
@@ -59,5 +60,9 @@ module ContainerRegistry
 
       conn.adapter :net_http
     end
+
+    def response_body(response)
+      response.body if response.success?
+    end
   end
 end
diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/validation_helpers.rb
index 3900fc89391..72f648975dc 100644
--- a/lib/gitlab/ci/config/node/validation_helpers.rb
+++ b/lib/gitlab/ci/config/node/validation_helpers.rb
@@ -15,6 +15,10 @@ module Gitlab
             values.is_a?(Array) && values.all? { |value| validate_string(value) }
           end
 
+          def validate_array_of_strings_or_regexps(values)
+            values.is_a?(Array) && values.all? { |value| validate_string_or_regexp(value) }
+          end
+
           def validate_variables(variables)
             variables.is_a?(Hash) &&
               variables.all? { |key, value| validate_string(key) && validate_string(value) }
@@ -24,6 +28,19 @@ module Gitlab
             value.is_a?(String) || value.is_a?(Symbol)
           end
 
+          def validate_string_or_regexp(value)
+            return true if value.is_a?(Symbol)
+            return false unless value.is_a?(String)
+
+            if value.first == '/' && value.last == '/'
+              Regexp.new(value[1...-1])
+            else
+              true
+            end
+          rescue RegexpError
+            false
+          end
+
           def validate_environment(value)
             value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex
           end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 5e7532f57ae..28c34429c1f 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -36,7 +36,7 @@ module Gitlab
         default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
-        import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
+        import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project],
         shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
         max_artifacts_size: Settings.artifacts['max_size'],
         require_two_factor_authentication: false,
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index d76ecb54017..078609c86f1 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -1,5 +1,10 @@
 module Gitlab
   module Database
+    # The max value of INTEGER type is the same between MySQL and PostgreSQL:
+    # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html
+    # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html
+    MAX_INT_VALUE = 2147483647
+
     def self.adapter_name
       connection.adapter_name
     end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index dd3ff0ab18b..dec20d8659b 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -28,65 +28,79 @@ module Gitlab
       # Updates the value of a column in batches.
       #
       # This method updates the table in batches of 5% of the total row count.
-      # Any data inserted while running this method (or after it has finished
-      # running) is _not_ updated automatically.
+      # This method will continue updating rows until no rows remain.
+      #
+      # When given a block this method will yield two values to the block:
+      #
+      # 1. An instance of `Arel::Table` for the table that is being updated.
+      # 2. The query to run as an Arel object.
+      #
+      # By supplying a block one can add extra conditions to the queries being
+      # executed. Note that the same block is used for _all_ queries.
+      #
+      # Example:
+      #
+      #     update_column_in_batches(:projects, :foo, 10) do |table, query|
+      #       query.where(table[:some_column].eq('hello'))
+      #     end
+      #
+      # This would result in this method updating only rows where
+      # `projects.some_column` equals "hello".
       #
       # table - The name of the table.
       # column - The name of the column to update.
       # value - The value for the column.
+      #
+      # Rubocop's Metrics/AbcSize metric is disabled for this method as Rubocop
+      # determines this method to be too complex while there's no way to make it
+      # less "complex" without introducing extra methods (which actually will
+      # make things _more_ complex).
+      #
+      # rubocop: disable Metrics/AbcSize
       def update_column_in_batches(table, column, value)
-        quoted_table = quote_table_name(table)
-        quoted_column = quote_column_name(column)
-
-        ##
-        # Workaround for #17711
-        #
-        # It looks like for MySQL `ActiveRecord::Base.conntection.quote(true)`
-        # returns correct value (1), but `ActiveRecord::Migration.new.quote`
-        # returns incorrect value ('true'), which causes migrations to fail.
-        #
-        quoted_value = connection.quote(value)
-        processed = 0
-
-        total = exec_query("SELECT COUNT(*) AS count FROM #{quoted_table}").
-          to_hash.
-          first['count'].
-          to_i
+        table = Arel::Table.new(table)
+
+        count_arel = table.project(Arel.star.count.as('count'))
+        count_arel = yield table, count_arel if block_given?
+
+        total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i
+
+        return if total == 0
 
         # Update in batches of 5% until we run out of any rows to update.
         batch_size = ((total / 100.0) * 5.0).ceil
 
+        start_arel = table.project(table[:id]).order(table[:id].asc).take(1)
+        start_arel = yield table, start_arel if block_given?
+        start_id = exec_query(start_arel.to_sql).to_hash.first['id'].to_i
+
         loop do
-          start_row = exec_query(%Q{
-            SELECT id
-            FROM #{quoted_table}
-            ORDER BY id ASC
-            LIMIT 1 OFFSET #{processed}
-          }).to_hash.first
-
-          # There are no more rows to process
-          break unless start_row
-
-          stop_row = exec_query(%Q{
-            SELECT id
-            FROM #{quoted_table}
-            ORDER BY id ASC
-            LIMIT 1 OFFSET #{processed + batch_size}
-          }).to_hash.first
-
-          query = %Q{
-            UPDATE #{quoted_table}
-            SET #{quoted_column} = #{quoted_value}
-            WHERE id >= #{start_row['id']}
-          }
+          stop_arel = table.project(table[:id]).
+            where(table[:id].gteq(start_id)).
+            order(table[:id].asc).
+            take(1).
+            skip(batch_size)
+
+          stop_arel = yield table, stop_arel if block_given?
+          stop_row = exec_query(stop_arel.to_sql).to_hash.first
+
+          update_arel = Arel::UpdateManager.new(ActiveRecord::Base).
+            table(table).
+            set([[table[column], value]]).
+            where(table[:id].gteq(start_id))
 
           if stop_row
-            query += " AND id < #{stop_row['id']}"
+            stop_id = stop_row['id'].to_i
+            start_id = stop_id
+            update_arel = update_arel.where(table[:id].lt(stop_id))
           end
 
-          execute(query)
+          update_arel = yield table, update_arel if block_given?
+
+          execute(update_arel.to_sql)
 
-          processed += batch_size
+          # There are no more rows left to update.
+          break unless stop_row
         end
       end
 
@@ -95,9 +109,9 @@ module Gitlab
       # This method runs the following steps:
       #
       # 1. Add the column with a default value of NULL.
-      # 2. Update all existing rows in batches.
-      # 3. Change the default value of the column to the specified value.
-      # 4. Update any remaining rows.
+      # 2. Change the default value of the column to the specified value.
+      # 3. Update all existing rows in batches.
+      # 4. Set a `NOT NULL` constraint on the column if desired (the default).
       #
       # These steps ensure a column can be added to a large and commonly used
       # table without locking the entire table for the duration of the table
@@ -109,7 +123,10 @@ module Gitlab
       # default - The default value for the column.
       # allow_null - When set to `true` the column will allow NULL values, the
       #              default is to not allow NULL values.
-      def add_column_with_default(table, column, type, default:, allow_null: false)
+      #
+      # This method can also take a block which is passed directly to the
+      # `update_column_in_batches` method.
+      def add_column_with_default(table, column, type, default:, allow_null: false, &block)
         if transaction_open?
           raise 'add_column_with_default can not be run inside a transaction, ' \
             'you can disable transactions by calling disable_ddl_transaction! ' \
@@ -125,11 +142,9 @@ module Gitlab
         end
 
         begin
-          transaction do
-            update_column_in_batches(table, column, default)
+          update_column_in_batches(table, column, default, &block)
 
-            change_column_null(table, column, false) unless allow_null
-          end
+          change_column_null(table, column, false) unless allow_null
         # We want to rescue _all_ exceptions here, even those that don't inherit
         # from StandardError.
         rescue Exception => error # rubocop: disable all
diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb
index e2fee6b9f3e..047c77c6fc2 100644
--- a/lib/gitlab/email/message/repository_push.rb
+++ b/lib/gitlab/email/message/repository_push.rb
@@ -37,7 +37,7 @@ module Gitlab
         end
 
         def diffs
-          @diffs ||= (safe_diff_files(compare.diffs, diff_refs) if compare)
+          @diffs ||= (safe_diff_files(compare.diffs(max_files: 30), diff_refs) if compare)
         end
 
         def diffs_count
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index e5cf66a0371..2286ac8829c 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -66,8 +66,7 @@ module Gitlab
       end
 
       def import_pull_requests
-        hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?)
-        disable_webhooks(hooks)
+        disable_webhooks
 
         pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
         pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
@@ -90,14 +89,14 @@ module Gitlab
         raise Projects::ImportService::Error, e.message
       ensure
         clean_up_restored_branches(branches_removed)
-        clean_up_disabled_webhooks(hooks)
+        clean_up_disabled_webhooks
       end
 
-      def disable_webhooks(hooks)
+      def disable_webhooks
         update_webhooks(hooks, active: false)
       end
 
-      def clean_up_disabled_webhooks(hooks)
+      def clean_up_disabled_webhooks
         update_webhooks(hooks, active: true)
       end
 
@@ -107,6 +106,20 @@ module Gitlab
         end
       end
 
+      def hooks
+        @hooks ||=
+          begin
+            client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?)
+
+          # The GitHub Repository Webhooks API returns 404 for users
+          # without admin access to the repository when listing hooks.
+          # In this case we just want to return gracefully instead of
+          # spitting out an error and stop the import process.
+          rescue Octokit::NotFound
+            []
+          end
+      end
+
       def restore_branches(branches)
         branches.each do |name, sha|
           client.create_ref(repo, "refs/heads/#{name}", sha)
diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb
index 77c33db4b59..3d0418261bb 100644
--- a/lib/gitlab/gitlab_import/project_creator.rb
+++ b/lib/gitlab/gitlab_import/project_creator.rb
@@ -11,7 +11,7 @@ module Gitlab
       end
 
       def execute
-        project = ::Projects::CreateService.new(
+        ::Projects::CreateService.new(
           current_user,
           name: repo["name"],
           path: repo["path"],
@@ -22,8 +22,6 @@ module Gitlab
           import_source: repo["path_with_namespace"],
           import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@")
         ).execute
-
-        project
       end
     end
   end
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
new file mode 100644
index 00000000000..99cf85d9a3b
--- /dev/null
+++ b/lib/gitlab/import_export.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module ImportExport
+    extend self
+
+    VERSION = '0.1.0'
+
+    def export_path(relative_path:)
+      File.join(storage_path, relative_path)
+    end
+
+    def storage_path
+      File.join(Settings.shared['path'], 'tmp/project_exports')
+    end
+
+    def project_filename
+      "project.json"
+    end
+
+    def project_bundle_filename
+      "project.bundle"
+    end
+
+    def config_file
+      Rails.root.join('lib/gitlab/import_export/import_export.yml')
+    end
+
+    def version_filename
+      'VERSION'
+    end
+
+    def version
+      VERSION
+    end
+
+    def reset_tokens?
+      true
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb
new file mode 100644
index 00000000000..d230de781d5
--- /dev/null
+++ b/lib/gitlab/import_export/attributes_finder.rb
@@ -0,0 +1,47 @@
+module Gitlab
+  module ImportExport
+    class AttributesFinder
+    
+      def initialize(included_attributes:, excluded_attributes:, methods:)
+        @included_attributes = included_attributes || {}
+        @excluded_attributes = excluded_attributes || {}
+        @methods = methods || {}
+      end
+
+      def find(model_object)
+        parsed_hash = find_attributes_only(model_object)
+        parsed_hash.empty? ? model_object : { model_object => parsed_hash }
+      end
+
+      def parse(model_object)
+        parsed_hash = find_attributes_only(model_object)
+        yield parsed_hash unless parsed_hash.empty?
+      end
+
+      def find_included(value)
+        key = key_from_hash(value)
+        @included_attributes[key].nil? ? {} : { only: @included_attributes[key] }
+      end
+
+      def find_excluded(value)
+        key = key_from_hash(value)
+        @excluded_attributes[key].nil? ? {} : { except: @excluded_attributes[key] }
+      end
+
+      def find_method(value)
+        key = key_from_hash(value)
+        @methods[key].nil? ? {} : { methods: @methods[key] }
+      end
+
+      private
+
+      def find_attributes_only(value)
+        find_included(value).merge(find_excluded(value)).merge(find_method(value))
+      end
+
+      def key_from_hash(value)
+        value.is_a?(Hash) ? value.keys.first : value
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
new file mode 100644
index 00000000000..78664f076eb
--- /dev/null
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -0,0 +1,40 @@
+module Gitlab
+  module ImportExport
+    module CommandLineUtil
+      def tar_czf(archive:, dir:)
+        tar_with_options(archive: archive, dir: dir, options: 'czf')
+      end
+
+      def untar_zxf(archive:, dir:)
+        untar_with_options(archive: archive, dir: dir, options: 'zxf')
+      end
+
+      def git_bundle(repo_path:, bundle_path:)
+        execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all))
+      end
+
+      def git_unbundle(repo_path:, bundle_path:)
+        execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path}))
+      end
+
+      private
+
+      def tar_with_options(archive:, dir:, options:)
+        execute(%W(tar -#{options} #{archive} -C #{dir} .))
+      end
+
+      def untar_with_options(archive:, dir:, options:)
+        execute(%W(tar -#{options} #{archive} -C #{dir}))
+      end
+
+      def execute(cmd)
+        _output, status = Gitlab::Popen.popen(cmd)
+        status.zero?
+      end
+
+      def git_bin_path
+        Gitlab.config.git.bin_path
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb
new file mode 100644
index 00000000000..e341c4d9cf8
--- /dev/null
+++ b/lib/gitlab/import_export/error.rb
@@ -0,0 +1,5 @@
+module Gitlab
+  module ImportExport
+    class Error < StandardError; end
+  end
+end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
new file mode 100644
index 00000000000..0e70d9282d5
--- /dev/null
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module ImportExport
+    class FileImporter
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def self.import(*args)
+        new(*args).import
+      end
+
+      def initialize(archive_file:, shared:)
+        @archive_file = archive_file
+        @shared = shared
+      end
+
+      def import
+        FileUtils.mkdir_p(@shared.export_path)
+        decompress_archive
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def decompress_archive
+        untar_zxf(archive: @archive_file, dir: @shared.export_path)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
new file mode 100644
index 00000000000..164ab6238c4
--- /dev/null
+++ b/lib/gitlab/import_export/import_export.yml
@@ -0,0 +1,54 @@
+# Model relationships to be included in the project import/export
+project_tree:
+  - issues:
+    - notes:
+        :author
+  - :labels
+  - :milestones
+  - snippets:
+    - notes:
+        :author
+  - :releases
+  - :events
+  - project_members:
+    - :user
+  - merge_requests:
+    - notes:
+        :author
+    - :merge_request_diff
+  - pipelines:
+    - notes:
+        :author
+    - :statuses
+  - :variables
+  - :triggers
+  - :deploy_keys
+  - :services
+  - :hooks
+  - :protected_branches
+
+# Only include the following attributes for the models specified.
+included_attributes:
+  project:
+    - :description
+    - :issues_enabled
+    - :merge_requests_enabled
+    - :wiki_enabled
+    - :snippets_enabled
+    - :visibility_level
+    - :archived
+  user:
+    - :id
+    - :email
+    - :username
+  author:
+    - :name
+
+# Do not include the following attributes for the models specified.
+excluded_attributes:
+  snippets:
+    - :expired_at
+
+methods:
+  statuses:
+    - :type
\ No newline at end of file
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
new file mode 100644
index 00000000000..d209e04f7be
--- /dev/null
+++ b/lib/gitlab/import_export/importer.rb
@@ -0,0 +1,64 @@
+module Gitlab
+  module ImportExport
+    class Importer
+
+      def initialize(project)
+        @archive_file = project.import_source
+        @current_user = project.creator
+        @project = project
+        @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace)
+      end
+
+      def execute
+        Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
+                                                  shared: @shared)
+        if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
+          project_tree.restored_project
+        else
+          raise Projects::ImportService::Error.new(@shared.errors.join(', '))
+        end
+      end
+
+      private
+
+      def check_version!
+        Gitlab::ImportExport::VersionChecker.check!(shared: @shared)
+      end
+
+      def project_tree
+        @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user,
+                                                                        shared: @shared,
+                                                                        project: @project)
+      end
+
+      def repo_restorer
+        Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path,
+                                               shared: @shared,
+                                               project: project_tree.restored_project)
+      end
+
+      def wiki_restorer
+        Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path,
+                                               shared: @shared,
+                                               project: ProjectWiki.new(project_tree.restored_project),
+                                               wiki: true)
+      end
+
+      def uploads_restorer
+        Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared)
+      end
+
+      def path_with_namespace
+        File.join(@project.namespace.path, @project.path)
+      end
+
+      def repo_path
+        File.join(@shared.export_path, 'project.bundle')
+      end
+
+      def wiki_repo_path
+        File.join(@shared.export_path, 'project.wiki.bundle')
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb
new file mode 100644
index 00000000000..c569a35a48b
--- /dev/null
+++ b/lib/gitlab/import_export/members_mapper.rb
@@ -0,0 +1,68 @@
+module Gitlab
+  module ImportExport
+    class MembersMapper
+
+      attr_reader :missing_author_ids
+
+      def initialize(exported_members:, user:, project:)
+        @exported_members = exported_members
+        @user = user
+        @project = project
+        @missing_author_ids = []
+
+        # This needs to run first, as second call would be from #map
+        # which means project members already exist.
+        ensure_default_member!
+      end
+
+      def map
+        @map ||=
+          begin
+            @exported_members.inject(missing_keys_tracking_hash) do |hash, member|
+              existing_user = User.where(find_project_user_query(member)).first
+              old_user_id = member['user']['id']
+              if existing_user && add_user_as_team_member(existing_user, member)
+                hash[old_user_id] = existing_user.id
+              end
+              hash
+            end
+          end
+      end
+
+      def default_user_id
+        @user.id
+      end
+
+      private
+
+      def missing_keys_tracking_hash
+        Hash.new do |_, key|
+          @missing_author_ids << key
+          default_user_id
+        end
+      end
+
+      def ensure_default_member!
+        ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true)
+      end
+
+      def add_user_as_team_member(existing_user, member)
+        member['user'] = existing_user
+
+        ProjectMember.create(member_hash(member)).persisted?
+      end
+
+      def member_hash(member)
+        member.except('id').merge(source_id: @project.id, importing: true)
+      end
+
+      def find_project_user_query(member)
+        user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email']))
+      end
+
+      def user_arel
+        @user_arel ||= User.arel_table
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb
new file mode 100644
index 00000000000..89388d1984b
--- /dev/null
+++ b/lib/gitlab/import_export/project_creator.rb
@@ -0,0 +1,24 @@
+module Gitlab
+  module ImportExport
+    class ProjectCreator
+
+      def initialize(namespace_id, current_user, file, project_path)
+        @namespace_id = namespace_id
+        @current_user = current_user
+        @file = file
+        @project_path = project_path
+      end
+
+      def execute
+        ::Projects::CreateService.new(
+          @current_user,
+          name: @project_path,
+          path: @project_path,
+          namespace_id: @namespace_id,
+          import_type: "gitlab_project",
+          import_source: @file
+        ).execute
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
new file mode 100644
index 00000000000..dd71b92c522
--- /dev/null
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -0,0 +1,105 @@
+module Gitlab
+  module ImportExport
+    class ProjectTreeRestorer
+
+      def initialize(user:, shared:, project:)
+        @path = File.join(shared.export_path, 'project.json')
+        @user = user
+        @shared = shared
+        @project = project
+      end
+
+      def restore
+        json = IO.read(@path)
+        @tree_hash = ActiveSupport::JSON.decode(json)
+        @project_members = @tree_hash.delete('project_members')
+        create_relations
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      def restored_project
+        @restored_project ||= restore_project
+      end
+
+      private
+
+      def members_mapper
+        @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
+                                                                    user: @user,
+                                                                    project: restored_project)
+      end
+
+      # Loops through the tree of models defined in import_export.yml and
+      # finds them in the imported JSON so they can be instantiated and saved
+      # in the DB. The structure and relationships between models are guessed from
+      # the configuration yaml file too.
+      # Finally, it updates each attribute in the newly imported project.
+      def create_relations
+        saved = []
+        default_relation_list.each do |relation|
+          next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present?
+
+          create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash)
+
+          relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
+          relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s])
+          saved << restored_project.update_attribute(relation_key, relation_hash)
+        end
+        saved.all?
+      end
+
+      def default_relation_list
+        Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model|
+          model.is_a?(Hash) && model[:project_members]
+        end
+      end
+
+      def restore_project
+        return @project unless @tree_hash
+
+        project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) }
+        @project.update(project_params)
+        @project
+      end
+
+      # Given a relation hash containing one or more models and its relationships,
+      # loops through each model and each object from a model type and
+      # and assigns its correspondent attributes hash from +tree_hash+
+      # Example:
+      # +relation_key+ issues, loops through the list of *issues* and for each individual
+      # issue, finds any subrelations such as notes, creates them and assign them back to the hash
+      def create_sub_relations(relation, tree_hash)
+        relation_key = relation.keys.first.to_s
+        tree_hash[relation_key].each do |relation_item|
+          relation.values.flatten.each do |sub_relation|
+            relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
+            relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
+          end
+        end
+      end
+
+      def assign_relation_hash(relation_item, sub_relation)
+        if sub_relation.is_a?(Hash)
+          relation_hash = relation_item[sub_relation.keys.first.to_s]
+          sub_relation = sub_relation.keys.first
+        else
+          relation_hash = relation_item[sub_relation.to_s]
+        end
+        [relation_hash, sub_relation]
+      end
+
+      def create_relation(relation, relation_hash_list)
+        relation_array = [relation_hash_list].flatten.map do |relation_hash|
+          Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
+                                                       relation_hash: relation_hash.merge('project_id' => restored_project.id),
+                                                       members_mapper: members_mapper,
+                                                       user: @user)
+        end
+
+        relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb
new file mode 100644
index 00000000000..9153088e966
--- /dev/null
+++ b/lib/gitlab/import_export/project_tree_saver.rb
@@ -0,0 +1,29 @@
+module Gitlab
+  module ImportExport
+    class ProjectTreeSaver
+      attr_reader :full_path
+
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+        @full_path = File.join(@shared.export_path, ImportExport.project_filename)
+      end
+
+      def save
+        FileUtils.mkdir_p(@shared.export_path)
+
+        File.write(full_path, project_json_tree)
+        true
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def project_json_tree
+        @project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb
new file mode 100644
index 00000000000..19defd8f03a
--- /dev/null
+++ b/lib/gitlab/import_export/reader.rb
@@ -0,0 +1,117 @@
+module Gitlab
+  module ImportExport
+    class Reader
+
+      attr_reader :tree
+
+      def initialize(shared:)
+        @shared = shared
+        config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys
+        @tree = config_hash[:project_tree]
+        @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes],
+                                                                        excluded_attributes: config_hash[:excluded_attributes],
+                                                                        methods: config_hash[:methods])
+      end
+
+      # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
+      # for outputting a project in JSON format, including its relations and sub relations.
+      def project_tree
+        @attributes_finder.find_included(:project).merge(include: build_hash(@tree))
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
+      #
+      # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file
+      def build_hash(model_list)
+        model_list.map do |model_objects|
+          if model_objects.is_a?(Hash)
+            build_json_config_hash(model_objects)
+          else
+            @attributes_finder.find(model_objects)
+          end
+        end
+      end
+
+      # Called when the model is actually a hash containing other relations (more models)
+      # Returns the config in the right format for calling +to_json+
+      # +model_object_hash+ - A model relationship such as:
+      #   {:merge_requests=>[:merge_request_diff, :notes]}
+      def build_json_config_hash(model_object_hash)
+        @json_config_hash = {}
+
+        model_object_hash.values.flatten.each do |model_object|
+          current_key = model_object_hash.keys.first
+
+          @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash }
+
+          handle_model_object(current_key, model_object)
+          process_sub_model(current_key, model_object) if model_object.is_a?(Hash)
+        end
+        @json_config_hash
+      end
+
+
+      # If the model is a hash, process the sub_models, which could also be hashes
+      # If there is a list, add to an existing array, otherwise use hash syntax
+      # +current_key+ main model that will be a key in the hash
+      # +model_object+ model or list of models to include in the hash
+      def process_sub_model(current_key, model_object)
+        sub_model_json = build_json_config_hash(model_object).dup
+        @json_config_hash.slice!(current_key)
+
+        if @json_config_hash[current_key] && @json_config_hash[current_key][:include]
+          @json_config_hash[current_key][:include] << sub_model_json
+        else
+          @json_config_hash[current_key] = { include: sub_model_json }
+        end
+      end
+
+      # Creates or adds to an existing hash an individual model or list
+      # +current_key+ main model that will be a key in the hash
+      # +model_object+ model or list of models to include in the hash
+      def handle_model_object(current_key, model_object)
+        if @json_config_hash[current_key]
+          add_model_value(current_key, model_object)
+        else
+          create_model_value(current_key, model_object)
+        end
+      end
+
+      # Constructs a new hash that will hold the configuration for that particular object
+      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
+      # +current_key+ main model that will be a key in the hash
+      # +value+ existing model to be included in the hash
+      def create_model_value(current_key, value)
+        parsed_hash = { include: value }
+
+        @attributes_finder.parse(value) do |hash|
+          parsed_hash = { include: hash_or_merge(value, hash) }
+        end
+        @json_config_hash[current_key] = parsed_hash
+      end
+
+      # Adds new model configuration to an existing hash with key +current_key+
+      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
+      # +current_key+ main model that will be a key in the hash
+      # +value+ existing model to be included in the hash
+      def add_model_value(current_key, value)
+        @attributes_finder.parse(value) { |hash| value = { value => hash } }
+        old_values = @json_config_hash[current_key][:include]
+        @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten
+      end
+
+      # Construct a new hash or merge with an existing one a model configuration
+      # This is to fulfil +to_json+ requirements.
+      # +value+ existing model to be included in the hash
+      # +hash+ hash containing configuration generated mainly from +@attributes_finder+
+      def hash_or_merge(value, hash)
+        value.is_a?(Hash) ? value.merge(hash) : { value => hash }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
new file mode 100644
index 00000000000..b872780f20a
--- /dev/null
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -0,0 +1,128 @@
+module Gitlab
+  module ImportExport
+    class RelationFactory
+
+      OVERRIDES = { snippets: :project_snippets,
+                    pipelines: 'Ci::Pipeline',
+                    statuses: 'commit_status',
+                    variables: 'Ci::Variable',
+                    triggers: 'Ci::Trigger',
+                    builds: 'Ci::Build',
+                    hooks: 'ProjectHook' }.freeze
+
+      USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze
+
+      def self.create(*args)
+        new(*args).create
+      end
+
+      def initialize(relation_sym:, relation_hash:, members_mapper:, user:)
+        @relation_name = OVERRIDES[relation_sym] || relation_sym
+        @relation_hash = relation_hash.except('id', 'noteable_id')
+        @members_mapper = members_mapper
+        @user = user
+      end
+
+      # Creates an object from an actual model with name "relation_sym" with params from
+      # the relation_hash, updating references with new object IDs, mapping users using
+      # the "members_mapper" object, also updating notes if required.
+      def create
+        set_note_author if @relation_name == :notes
+        update_user_references
+        update_project_references
+        reset_ci_tokens if @relation_name == 'Ci::Trigger'
+
+        generate_imported_object
+      end
+
+      private
+
+      def update_user_references
+        USER_REFERENCES.each do |reference|
+          if @relation_hash[reference]
+            @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]]
+          end
+        end
+      end
+
+      # Sets the author for a note. If the user importing the project
+      # has admin access, an actual mapping with new project members
+      # will be used. Otherwise, a note stating the original author name
+      # is left.
+      def set_note_author
+        old_author_id = @relation_hash['author_id']
+
+        # Users with admin access can map users
+        @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id
+
+        author = @relation_hash.delete('author')
+
+        update_note_for_missing_author(author['name']) if missing_author?(old_author_id)
+      end
+
+      def missing_author?(old_author_id)
+        !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id)
+      end
+
+      def missing_author_note(updated_at, author_name)
+        timestamp = updated_at.split('.').first
+        "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*"
+      end
+
+      def generate_imported_object
+        if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes
+          trace = @relation_hash.delete('trace')
+          imported_object do |object|
+            object.trace = trace
+            object.commit_id = nil
+          end
+        else
+          imported_object
+        end
+      end
+
+      def update_project_references
+        project_id = @relation_hash.delete('project_id')
+
+        # project_id may not be part of the export, but we always need to populate it if required.
+        @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id')
+        @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id']
+        @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id']
+        @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id']
+
+        # If source and target are the same, populate them with the new project ID.
+        if @relation_hash['source_project_id'] && @relation_hash['target_project_id'] &&
+          @relation_hash['target_project_id'] == @relation_hash['source_project_id']
+          @relation_hash['source_project_id'] = project_id
+        end
+      end
+
+      def reset_ci_tokens
+        return unless Gitlab::ImportExport.reset_tokens?
+
+        # If we import/export a project to the same instance, tokens will have to be reset.
+        @relation_hash['token'] = nil
+      end
+
+      def relation_class
+        @relation_class ||= @relation_name.to_s.classify.constantize
+      end
+
+      def imported_object
+        imported_object = relation_class.new(@relation_hash)
+        yield(imported_object) if block_given?
+        imported_object.importing = true if imported_object.respond_to?(:importing)
+        imported_object
+      end
+
+      def update_note_for_missing_author(author_name)
+        @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank?
+        @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name)
+      end
+
+      def admin_user?
+        @user.is_admin?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
new file mode 100644
index 00000000000..546dae4d122
--- /dev/null
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module ImportExport
+    class RepoRestorer
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def initialize(project:, shared:, path_to_bundle:, wiki: false)
+        @project = project
+        @path_to_bundle = path_to_bundle
+        @shared = shared
+        @wiki = wiki
+      end
+
+      def restore
+        return wiki? unless File.exist?(@path_to_bundle)
+
+        FileUtils.mkdir_p(path_to_repo)
+
+        git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def repos_path
+        Gitlab.config.gitlab_shell.repos_path
+      end
+
+      def path_to_repo
+        @project.repository.path_to_repo
+      end
+
+      def wiki?
+        @wiki
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
new file mode 100644
index 00000000000..cce43fe994b
--- /dev/null
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module ImportExport
+    class RepoSaver
+      include Gitlab::ImportExport::CommandLineUtil
+
+      attr_reader :full_path
+
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+      end
+
+      def save
+        return false if @project.empty_repo?
+
+        @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename)
+        bundle_to_disk
+      end
+
+      private
+
+      def bundle_to_disk
+        FileUtils.mkdir_p(@shared.export_path)
+        git_bundle(repo_path: path_to_repo, bundle_path: @full_path)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      def path_to_repo
+        @project.repository.path_to_repo
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
new file mode 100644
index 00000000000..f38229c6c59
--- /dev/null
+++ b/lib/gitlab/import_export/saver.rb
@@ -0,0 +1,42 @@
+module Gitlab
+  module ImportExport
+    class Saver
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def self.save(*args)
+        new(*args).save
+      end
+
+      def initialize(shared:)
+        @shared = shared
+      end
+
+      def save
+        if compress_and_save
+          remove_export_path
+          Rails.logger.info("Saved project export #{archive_file}")
+          archive_file
+        else
+          false
+        end
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def compress_and_save
+        tar_czf(archive: archive_file, dir: @shared.export_path)
+      end
+
+      def remove_export_path
+        FileUtils.rm_rf(@shared.export_path)
+      end
+
+      def archive_file
+        @archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
new file mode 100644
index 00000000000..6aff05b886a
--- /dev/null
+++ b/lib/gitlab/import_export/shared.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module ImportExport
+    class Shared
+
+      attr_reader :errors, :opts
+
+      def initialize(opts)
+        @opts = opts
+        @errors = []
+      end
+
+      def export_path
+        @export_path ||= Gitlab::ImportExport.export_path(relative_path: opts[:relative_path])
+      end
+
+      def error(error)
+        error_out(error.message, caller[0].dup)
+        @errors << error.message
+        # Debug:
+        Rails.logger.error(error.backtrace)
+      end
+
+      private
+
+      def error_out(message, caller)
+        Rails.logger.error("Import/Export error raised on #{caller}: #{message}")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb
new file mode 100644
index 00000000000..df19354b76e
--- /dev/null
+++ b/lib/gitlab/import_export/uploads_restorer.rb
@@ -0,0 +1,14 @@
+module Gitlab
+  module ImportExport
+    class UploadsRestorer < UploadsSaver
+      def restore
+        return true unless File.directory?(uploads_export_path)
+
+        copy_files(uploads_export_path, uploads_path)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb
new file mode 100644
index 00000000000..7292e9d9712
--- /dev/null
+++ b/lib/gitlab/import_export/uploads_saver.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module ImportExport
+    class UploadsSaver
+
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+      end
+
+      def save
+        return true unless File.directory?(uploads_path)
+
+        copy_files(uploads_path, uploads_export_path)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def copy_files(source, destination)
+        FileUtils.mkdir_p(destination)
+        FileUtils.copy_entry(source, destination)
+        true
+      end
+
+      def uploads_export_path
+        File.join(@shared.export_path, 'uploads')
+      end
+
+      def uploads_path
+        File.join(Rails.root.join('public/uploads'), @project.path_with_namespace)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb
new file mode 100644
index 00000000000..cf5c62c5e3c
--- /dev/null
+++ b/lib/gitlab/import_export/version_checker.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module ImportExport
+    class VersionChecker
+
+      def self.check!(*args)
+        new(*args).check!
+      end
+
+      def initialize(shared:)
+        @shared = shared
+      end
+
+      def check!
+        version = File.open(version_file, &:readline)
+        verify_version!(version)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def version_file
+        File.join(@shared.export_path, Gitlab::ImportExport.version_filename)
+      end
+
+      def verify_version!(version)
+        if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version)
+          raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}")
+        else
+          true
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb
new file mode 100644
index 00000000000..f7f73dc9343
--- /dev/null
+++ b/lib/gitlab/import_export/version_saver.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module ImportExport
+    class VersionSaver
+
+      def initialize(shared:)
+        @shared = shared
+      end
+
+      def save
+        FileUtils.mkdir_p(@shared.export_path)
+
+        File.write(version_file, Gitlab::ImportExport.version, mode: 'w')
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def version_file
+        File.join(@shared.export_path, Gitlab::ImportExport.version_filename)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
new file mode 100644
index 00000000000..1eedae39f8a
--- /dev/null
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -0,0 +1,33 @@
+module Gitlab
+  module ImportExport
+    class WikiRepoSaver < RepoSaver
+      def save
+        @wiki = ProjectWiki.new(@project)
+        return true unless wiki_repository_exists? # it's okay to have no Wiki
+        bundle_to_disk(File.join(@shared.export_path, project_filename))
+      end
+
+      def bundle_to_disk(full_path)
+        FileUtils.mkdir_p(@shared.export_path)
+        git_bundle(repo_path: path_to_repo, bundle_path: full_path)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def project_filename
+        "project.wiki.bundle"
+      end
+
+      def path_to_repo
+        @wiki.repository.path_to_repo
+      end
+
+      def wiki_repository_exists?
+        File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb
index ccfdfbe73e8..948d43582cf 100644
--- a/lib/gitlab/import_sources.rb
+++ b/lib/gitlab/import_sources.rb
@@ -20,7 +20,8 @@ module Gitlab
           'Gitorious.org'   => 'gitorious',
           'Google Code'     => 'google_code',
           'FogBugz'         => 'fogbugz',
-          'Any repo by URL' => 'git',
+          'Repo by URL'     => 'git',
+          'GitLab export' => 'gitlab_project'
         }
       end
 
diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index d81d26754fe..dcec7543c13 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -148,23 +148,8 @@ module Gitlab
 
         proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1
           def #{name}(#{args_signature})
-            trans = Gitlab::Metrics::Instrumentation.transaction
-
-            if trans
-              start     = Time.now
-              cpu_start = Gitlab::Metrics::System.cpu_time
-              retval    = super
-              duration  = (Time.now - start) * 1000.0
-
-              if duration >= Gitlab::Metrics.method_call_threshold
-                cpu_duration = Gitlab::Metrics::System.cpu_time - cpu_start
-
-                trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES,
-                                 { duration: duration, cpu_duration: cpu_duration },
-                                 method: #{label.inspect})
-              end
-
-              retval
+            if trans = Gitlab::Metrics::Instrumentation.transaction
+              trans.measure_method(#{label.inspect}) { super }
             else
               super
             end
diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb
new file mode 100644
index 00000000000..faf0d9b6318
--- /dev/null
+++ b/lib/gitlab/metrics/method_call.rb
@@ -0,0 +1,52 @@
+module Gitlab
+  module Metrics
+    # Class for tracking timing information about method calls
+    class MethodCall
+      attr_reader :real_time, :cpu_time, :call_count
+
+      # name - The full name of the method (including namespace) such as
+      #        `User#sign_in`.
+      #
+      # series - The series to use for storing the data.
+      def initialize(name, series)
+        @name = name
+        @series = series
+        @real_time = 0.0
+        @cpu_time = 0.0
+        @call_count = 0
+      end
+
+      # Measures the real and CPU execution time of the supplied block.
+      def measure
+        start_real = Time.now
+        start_cpu = System.cpu_time
+        retval = yield
+
+        @real_time += (Time.now - start_real) * 1000.0
+        @cpu_time += System.cpu_time.to_f - start_cpu
+        @call_count += 1
+
+        retval
+      end
+
+      # Returns a Metric instance of the current method call.
+      def to_metric
+        Metric.new(
+          @series,
+          {
+            duration:     real_time,
+            cpu_duration: cpu_time,
+            call_count:   call_count
+          },
+          method: @name
+        )
+      end
+
+      # Returns true if the total runtime of this method exceeds the method call
+      # threshold.
+      def above_threshold?
+        real_time >= Metrics.method_call_threshold
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index 3fe27779d03..e61670f491c 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -35,7 +35,7 @@ module Gitlab
       def transaction_from_env(env)
         trans = Transaction.new
 
-        trans.set(:request_uri, env['REQUEST_URI'])
+        trans.set(:request_uri, filtered_path(env))
         trans.set(:request_method, env['REQUEST_METHOD'])
 
         trans
@@ -54,6 +54,10 @@ module Gitlab
 
       private
 
+      def filtered_path(env)
+        ActionDispatch::Request.new(env).filtered_path.presence || env['REQUEST_URI']
+      end
+
       def endpoint_paths_cache
         @endpoint_paths_cache ||= Hash.new do |hash, http_method|
           hash[http_method] = Hash.new do |inner_hash, raw_path|
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 2578ddc49f4..4bc5081aa03 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -4,7 +4,7 @@ module Gitlab
     class Transaction
       THREAD_KEY = :_gitlab_metrics_transaction
 
-      attr_reader :tags, :values
+      attr_reader :tags, :values, :methods
 
       attr_accessor :action
 
@@ -16,6 +16,7 @@ module Gitlab
       #          plus method name.
       def initialize(action = nil)
         @metrics = []
+        @methods = {}
 
         @started_at  = nil
         @finished_at = nil
@@ -51,9 +52,23 @@ module Gitlab
       end
 
       def add_metric(series, values, tags = {})
-        prefix = sidekiq? ? 'sidekiq_' : 'rails_'
+        @metrics << Metric.new("#{series_prefix}#{series}", values, tags)
+      end
+
+      # Measures the time it takes to execute a method.
+      #
+      # Multiple calls to the same method add up to the total runtime of the
+      # method.
+      #
+      # name - The full name of the method to measure (e.g. `User#sign_in`).
+      def measure_method(name, &block)
+        unless @methods[name]
+          series = "#{series_prefix}#{Instrumentation::SERIES}"
+
+          @methods[name] = MethodCall.new(name, series)
+        end
 
-        @metrics << Metric.new("#{prefix}#{series}", values, tags)
+        @methods[name].measure(&block)
       end
 
       def increment(name, value)
@@ -84,7 +99,13 @@ module Gitlab
       end
 
       def submit
-        metrics = @metrics.map do |metric|
+        submit = @metrics.dup
+
+        @methods.each do |name, method|
+          submit << method.to_metric if method.above_threshold?
+        end
+
+        submit_hashes = submit.map do |metric|
           hash = metric.to_hash
 
           hash[:tags][:action] ||= @action if @action
@@ -92,12 +113,16 @@ module Gitlab
           hash
         end
 
-        Metrics.submit_metrics(metrics)
+        Metrics.submit_metrics(submit_hashes)
       end
 
       def sidekiq?
         Sidekiq.server?
       end
+
+      def series_prefix
+        sidekiq? ? 'sidekiq_' : 'rails_'
+      end
     end
   end
 end