Merge remote-tracking branch 'origin/master' into ci-lfs-fetch

72 files changed, 2136 insertions, 260 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 6cd909f6115..0e7a1cc2623 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -26,38 +26,40 @@ module API
     # Ensure the namespace is right, otherwise we might load Grape::API::Helpers
     helpers ::API::Helpers
 
-    mount ::API::Groups
+    mount ::API::AwardEmoji
+    mount ::API::Branches
+    mount ::API::Builds
+    mount ::API::CommitStatuses
+    mount ::API::Commits
+    mount ::API::DeployKeys
+    mount ::API::Files
+    mount ::API::Gitignores
     mount ::API::GroupMembers
-    mount ::API::Users
-    mount ::API::Projects
-    mount ::API::Repositories
+    mount ::API::Groups
+    mount ::API::Internal
     mount ::API::Issues
-    mount ::API::Milestones
-    mount ::API::Session
+    mount ::API::Keys
+    mount ::API::Labels
+    mount ::API::Licenses
     mount ::API::MergeRequests
+    mount ::API::Milestones
+    mount ::API::Namespaces
     mount ::API::Notes
-    mount ::API::Internal
-    mount ::API::SystemHooks
-    mount ::API::ProjectSnippets
-    mount ::API::ProjectMembers
-    mount ::API::DeployKeys
     mount ::API::ProjectHooks
+    mount ::API::ProjectMembers
+    mount ::API::ProjectSnippets
+    mount ::API::Projects
+    mount ::API::Repositories
+    mount ::API::Runners
     mount ::API::Services
-    mount ::API::Files
-    mount ::API::Commits
-    mount ::API::CommitStatuses
-    mount ::API::Namespaces
-    mount ::API::Branches
-    mount ::API::Labels
+    mount ::API::Session
     mount ::API::Settings
-    mount ::API::Keys
+    mount ::API::SidekiqMetrics
+    mount ::API::Subscriptions
+    mount ::API::SystemHooks
     mount ::API::Tags
     mount ::API::Triggers
-    mount ::API::Builds
+    mount ::API::Users
     mount ::API::Variables
-    mount ::API::Runners
-    mount ::API::Licenses
-    mount ::API::Subscriptions
-    mount ::API::Gitignores
   end
 end
diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb
new file mode 100644
index 00000000000..985590312e3
--- /dev/null
+++ b/lib/api/award_emoji.rb
@@ -0,0 +1,116 @@
+module API
+  class AwardEmoji < Grape::API
+    before { authenticate! }
+    AWARDABLES = [Issue, MergeRequest]
+
+    resource :projects do
+      AWARDABLES.each do |awardable_type|
+        awardable_string = awardable_type.to_s.underscore.pluralize
+        awardable_id_string = "#{awardable_type.to_s.underscore}_id"
+
+        [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji",
+          ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji"
+        ].each do |endpoint|
+
+          # Get a list of project +awardable+ award emoji
+          #
+          # Parameters:
+          #   id (required)           - The ID of a project
+          #   awardable_id (required) - The ID of an issue or MR
+          # Example Request:
+          #   GET /projects/:id/issues/:awardable_id/award_emoji
+          get endpoint do
+            if can_read_awardable?
+              awards = paginate(awardable.award_emoji)
+              present awards, with: Entities::AwardEmoji
+            else
+              not_found!("Award Emoji")
+            end
+          end
+
+          # Get a specific award emoji
+          #
+          # Parameters:
+          #   id (required)           - The ID of a project
+          #   awardable_id (required) - The ID of an issue or MR
+          #   award_id (required)     - The ID of the award
+          # Example Request:
+          #   GET /projects/:id/issues/:awardable_id/award_emoji/:award_id
+          get "#{endpoint}/:award_id" do
+            if can_read_awardable?
+              present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji
+            else
+              not_found!("Award Emoji")
+            end
+          end
+
+          # Award a new Emoji
+          #
+          # Parameters:
+          #   id (required) - The ID of a project
+          #   awardable_id (required) - The ID of an issue or mr
+          #   name (required) - The name of a award_emoji (without colons)
+          # Example Request:
+          #   POST /projects/:id/issues/:awardable_id/award_emoji
+          post endpoint do
+            required_attributes! [:name]
+
+            not_found!('Award Emoji') unless can_read_awardable?
+
+            award = awardable.award_emoji.new(name: params[:name], user: current_user)
+
+            if award.save
+              present award, with: Entities::AwardEmoji
+            else
+              not_found!("Award Emoji #{award.errors.messages}")
+            end
+          end
+
+          # Delete a +awardables+ award emoji
+          #
+          # Parameters:
+          #   id (required) - The ID of a project
+          #   awardable_id (required) - The ID of an issue or MR
+          #   award_emoji_id (required) - The ID of an award emoji
+          # Example Request:
+          #   DELETE /projects/:id/issues/:issue_id/notes/:note_id/award_emoji/:award_id
+          delete "#{endpoint}/:award_id" do
+            award = awardable.award_emoji.find(params[:award_id])
+
+            unauthorized! unless award.user == current_user || current_user.admin?
+
+            award.destroy
+            present award, with: Entities::AwardEmoji
+          end
+        end
+      end
+    end
+
+    helpers do
+      def can_read_awardable?
+        ability = "read_#{awardable.class.to_s.underscore}".to_sym
+
+        can?(current_user, ability, awardable)
+      end
+
+      def awardable
+        @awardable ||=
+          begin
+            if params.include?(:note_id)
+              noteable.notes.find(params[:note_id])
+            else
+              noteable
+            end
+          end
+      end
+
+      def noteable
+        if params.include?(:issue_id)
+          user_project.issues.find(params[:issue_id])
+        else
+          user_project.merge_requests.find(params[:merge_request_id])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/builds.rb b/lib/api/builds.rb
index 0ff8fa74a84..979328efe0e 100644
--- a/lib/api/builds.rb
+++ b/lib/api/builds.rb
@@ -142,7 +142,7 @@ module API
         return not_found!(build) unless build
         return forbidden!('Build is not retryable') unless build.retryable?
 
-        build = Ci::Build.retry(build)
+        build = Ci::Build.retry(build, current_user)
 
         present build, with: Entities::Build,
                        user_can_download_artifacts: can?(current_user, :read_build, user_project)
@@ -166,6 +166,26 @@ module API
         present build, with: Entities::Build,
                        user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project)
       end
+
+      # Keep the artifacts to prevent them from being deleted
+      #
+      # Parameters:
+      #   id (required) - the id of a project
+      #   build_id (required) - The ID of a build
+      # Example Request:
+      #   POST /projects/:id/builds/:build_id/artifacts/keep
+      post ':id/builds/:build_id/artifacts/keep' do
+        authorize_update_builds!
+
+        build = get_build(params[:build_id])
+        return not_found!(build) unless build && build.artifacts?
+
+        build.keep_artifacts!
+
+        status 200
+        present build, with: Entities::Build,
+                       user_can_download_artifacts: can?(current_user, :read_build, user_project)
+      end
     end
 
     helpers do
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 50d69274b2e..2e397643ed1 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -88,10 +88,7 @@ module API
     class Group < Grape::Entity
       expose :id, :name, :path, :description, :visibility_level
       expose :avatar_url
-
-      expose :web_url do |group, options|
-        Gitlab::Routing.url_helpers.group_url(group)
-      end
+      expose :web_url
     end
 
     class GroupDetail < Group
@@ -179,6 +176,11 @@ module API
       expose :upvotes, :downvotes
     end
 
+    class ExternalIssue < Grape::Entity
+      expose :title
+      expose :id
+    end
+
     class MergeRequest < ProjectEntity
       expose :target_branch, :source_branch
       expose :upvotes, :downvotes
@@ -223,6 +225,14 @@ module API
       expose(:downvote?)  { |note| false }
     end
 
+    class AwardEmoji < Grape::Entity
+      expose :id
+      expose :name
+      expose :user, using: Entities::UserBasic
+      expose :created_at, :updated_at
+      expose :awardable_id, :awardable_type
+    end
+
     class MRNote < Grape::Entity
       expose :note
       expose :author, using: Entities::UserBasic
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index e1d3bbcc02d..77e407b54c5 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -9,9 +9,13 @@ module API
       [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value)
     end
 
+    def find_user_by_private_token
+      token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s
+      User.find_by_authentication_token(token_string) || User.find_by_personal_access_token(token_string)
+    end
+
     def current_user
-      private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s
-      @current_user ||= (User.find_by(authentication_token: private_token) || doorkeeper_guard)
+      @current_user ||= (find_user_by_private_token || doorkeeper_guard)
 
       unless @current_user && Gitlab::UserAccess.allowed?(@current_user)
         return nil
@@ -33,7 +37,7 @@ module API
       identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER]
 
       # Regex for integers
-      if !!(identifier =~ /^[0-9]+$/)
+      if !!(identifier =~ /\A[0-9]+\z/)
         identifier.to_i
       else
         identifier
@@ -418,5 +422,13 @@ module API
     def send_git_archive(repository, ref:, format:)
       header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format))
     end
+
+    def issue_entity(project)
+      if project.has_external_issue_tracker?
+        Entities::ExternalIssue
+      else
+        Entities::Issue
+      end
+    end
   end
 end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 24df3e397e0..0e94efd4acd 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -329,7 +329,7 @@ module API
         get "#{path}/closes_issues" do
           merge_request = user_project.merge_requests.find(params[:merge_request_id])
           issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user))
-          present paginate(issues), with: Entities::Issue, current_user: current_user
+          present paginate(issues), with: issue_entity(user_project), current_user: current_user
         end
       end
     end
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index d4fcfd3d4d3..8bfa998dc53 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -144,7 +144,7 @@ module API
 
     helpers do
       def noteable_read_ability_name(noteable)
-        "read_#{noteable.class.to_s.underscore.downcase}".to_sym
+        "read_#{noteable.class.to_s.underscore}".to_sym
       end
     end
   end
diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb
index 4aefdf319c6..b703da0557a 100644
--- a/lib/api/project_members.rb
+++ b/lib/api/project_members.rb
@@ -46,7 +46,7 @@ module API
         required_attributes! [:user_id, :access_level]
 
         # either the user is already a team member or a new one
-        project_member = user_project.project_member_by_id(params[:user_id])
+        project_member = user_project.project_member(params[:user_id])
         if project_member.nil?
           project_member = user_project.project_members.new(
             user_id: params[:user_id],
diff --git a/lib/api/session.rb b/lib/api/session.rb
index 56e69b2366f..56c202f1294 100644
--- a/lib/api/session.rb
+++ b/lib/api/session.rb
@@ -11,7 +11,7 @@ module API
     # Example Request:
     #  POST /session
     post "/session" do
-      user = Gitlab::Auth.find_in_gitlab_or_ldap(params[:email] || params[:login], params[:password])
+      user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password])
 
       return unauthorized! unless user
       present user, with: Entities::UserLogin
diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb
new file mode 100644
index 00000000000..d3d6827dc54
--- /dev/null
+++ b/lib/api/sidekiq_metrics.rb
@@ -0,0 +1,90 @@
+require 'sidekiq/api'
+
+module API
+  class SidekiqMetrics < Grape::API
+    before { authenticated_as_admin! }
+
+    helpers do
+      def queue_metrics
+        Sidekiq::Queue.all.each_with_object({}) do |queue, hash|
+          hash[queue.name] = {
+            backlog: queue.size,
+            latency: queue.latency.to_i
+          }
+        end
+      end
+
+      def process_metrics
+        Sidekiq::ProcessSet.new.map do |process|
+          {
+            hostname:    process['hostname'],
+            pid:         process['pid'],
+            tag:         process['tag'],
+            started_at:  Time.at(process['started_at']),
+            queues:      process['queues'],
+            labels:      process['labels'],
+            concurrency: process['concurrency'],
+            busy:        process['busy']
+          }
+        end
+      end
+
+      def job_stats
+        stats = Sidekiq::Stats.new
+        {
+          processed: stats.processed,
+          failed: stats.failed,
+          enqueued: stats.enqueued
+        }
+      end
+    end
+
+    # Get Sidekiq Queue metrics
+    #
+    # Parameters:
+    #   None
+    #
+    # Example:
+    #   GET /sidekiq/queue_metrics
+    #
+    get 'sidekiq/queue_metrics' do
+      { queues: queue_metrics }
+    end
+
+    # Get Sidekiq Process metrics
+    #
+    # Parameters:
+    #   None
+    #
+    # Example:
+    #   GET /sidekiq/process_metrics
+    #
+    get 'sidekiq/process_metrics' do
+      { processes: process_metrics }
+    end
+
+    # Get Sidekiq Job statistics
+    #
+    # Parameters:
+    #   None
+    #
+    # Example:
+    #   GET /sidekiq/job_stats
+    #
+    get 'sidekiq/job_stats' do
+      { jobs: job_stats }
+    end
+
+    # Get Sidekiq Compound metrics. Includes all previous metrics
+    #
+    # Parameters:
+    #   None
+    #
+    # Example:
+    #   GET /sidekiq/compound_metrics
+    #
+    get 'sidekiq/compound_metrics' do
+      { queues: queue_metrics, processes: process_metrics, jobs: job_stats }
+    end
+  end
+end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index 9dd665441a0..2ff3e3bdfb0 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -38,7 +38,6 @@ module Backup
     end
 
     def upload(tar_file)
-      remote_directory = Gitlab.config.backup.upload.remote_directory
       $progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
 
       connection_settings = Gitlab.config.backup.upload.connection
@@ -47,8 +46,7 @@ module Backup
         return
       end
 
-      connection = ::Fog::Storage.new(connection_settings)
-      directory = connection.directories.create(key: remote_directory)
+      directory = connect_to_remote_directory(connection_settings)
 
       if directory.files.create(key: tar_file, body: File.open(tar_file), public: false,
           multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size,
@@ -155,6 +153,23 @@ module Backup
 
     private
 
+    def connect_to_remote_directory(connection_settings)
+      connection = ::Fog::Storage.new(connection_settings)
+
+      # We only attempt to create the directory for local backups. For AWS
+      # and other cloud providers, we cannot guarantee the user will have
+      # permission to create the bucket.
+      if connection.service == ::Fog::Storage::Local
+        connection.directories.create(key: remote_directory)
+      else
+        connection.directories.get(remote_directory)
+      end
+    end
+
+    def remote_directory
+      Gitlab.config.backup.upload.remote_directory
+    end
+
     def backup_contents
       folders_to_backup + archives_to_backup + ["backup_information.yml"]
     end
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index db95d7c908b..81d66271136 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -103,7 +103,7 @@ module Banzai
         ref_pattern = object_class.reference_pattern
         link_pattern = object_class.link_reference_pattern
 
-        each_node do |node|
+        nodes.each do |node|
           if text_node?(node) && ref_pattern
             replace_text_when_pattern_matches(node, ref_pattern) do |content|
               object_link_filter(content, ref_pattern)
@@ -206,6 +206,56 @@ module Banzai
         text
       end
 
+      # Returns a Hash containing all object references (e.g. issue IDs) per the
+      # project they belong to.
+      def references_per_project
+        @references_per_project ||= begin
+          refs = Hash.new { |hash, key| hash[key] = Set.new }
+
+          regex = Regexp.union(object_class.reference_pattern,
+                               object_class.link_reference_pattern)
+
+          nodes.each do |node|
+            node.to_html.scan(regex) do
+              project = $~[:project] || current_project_path
+              symbol = $~[object_sym]
+
+              refs[project] << symbol if object_class.reference_valid?(symbol)
+            end
+          end
+
+          refs
+        end
+      end
+
+      # Returns a Hash containing referenced projects grouped per their full
+      # path.
+      def projects_per_reference
+        @projects_per_reference ||= begin
+          hash = {}
+          refs = Set.new
+
+          references_per_project.each do |project_ref, _|
+            refs << project_ref
+          end
+
+          find_projects_for_paths(refs.to_a).each do |project|
+            hash[project.path_with_namespace] = project
+          end
+
+          hash
+        end
+      end
+
+      # Returns the projects for the given paths.
+      def find_projects_for_paths(paths)
+        Project.where_paths_in(paths).includes(:namespace)
+      end
+
+      def current_project_path
+        @current_project_path ||= project.path_with_namespace
+      end
+
       private
 
       def project_refs_cache
diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb
index f73ecfc9418..0a29c547a4d 100644
--- a/lib/banzai/filter/external_link_filter.rb
+++ b/lib/banzai/filter/external_link_filter.rb
@@ -3,17 +3,8 @@ module Banzai
     # HTML Filter to modify the attributes of external links
     class ExternalLinkFilter < HTML::Pipeline::Filter
       def call
-        doc.search('a').each do |node|
-          link = node.attr('href')
-
-          next unless link
-
-          # Skip non-HTTP(S) links
-          next unless link.start_with?('http')
-
-          # Skip internal links
-          next if link.start_with?(internal_url)
-
+        # Skip non-HTTP(S) links and internal links
+        doc.xpath("descendant-or-self::a[starts-with(@href, 'http') and not(starts-with(@href, '#{internal_url}'))]").each do |node|
           node.set_attribute('rel', 'nofollow noreferrer')
           node.set_attribute('target', '_blank')
         end
diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb
index 2496e704002..2614261f9eb 100644
--- a/lib/banzai/filter/issue_reference_filter.rb
+++ b/lib/banzai/filter/issue_reference_filter.rb
@@ -11,13 +11,40 @@ module Banzai
         Issue
       end
 
-      def find_object(project, id)
-        project.get_issue(id)
+      def find_object(project, iid)
+        issues_per_project[project][iid]
       end
 
       def url_for_object(issue, project)
         IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path])
       end
+
+      def project_from_ref(ref)
+        projects_per_reference[ref || current_project_path]
+      end
+
+      # Returns a Hash containing the issues per Project instance.
+      def issues_per_project
+        @issues_per_project ||= begin
+          hash = Hash.new { |h, k| h[k] = {} }
+
+          projects_per_reference.each do |path, project|
+            issue_ids = references_per_project[path]
+
+            next unless project.default_issues_tracker?
+
+            project.issues.where(iid: issue_ids.to_a).each do |issue|
+              hash[project][issue.iid] = issue
+            end
+          end
+
+          hash
+        end
+      end
+
+      def find_projects_for_paths(paths)
+        super(paths).includes(:gitlab_issue_tracker_service)
+      end
     end
   end
 end
diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb
index c0f503c9af3..45bb66dc99f 100644
--- a/lib/banzai/filter/upload_link_filter.rb
+++ b/lib/banzai/filter/upload_link_filter.rb
@@ -10,11 +10,11 @@ module Banzai
       def call
         return doc unless project
 
-        doc.search('a').each do |el|
+        doc.xpath('descendant-or-self::a[starts-with(@href, "/uploads/")]').each do |el|
           process_link_attr el.attribute('href')
         end
 
-        doc.search('img').each do |el|
+        doc.xpath('descendant-or-self::img[starts-with(@src, "/uploads/")]').each do |el|
           process_link_attr el.attribute('src')
         end
 
@@ -24,12 +24,7 @@ module Banzai
       protected
 
       def process_link_attr(html_attr)
-        return if html_attr.blank?
-
-        uri = html_attr.value
-        if uri.starts_with?("/uploads/")
-          html_attr.value = build_url(uri).to_s
-        end
+        html_attr.value = build_url(html_attr.value).to_s
       end
 
       def build_url(uri)
diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb
index 37a2779d453..1bb6d6bba87 100644
--- a/lib/banzai/filter/wiki_link_filter.rb
+++ b/lib/banzai/filter/wiki_link_filter.rb
@@ -29,7 +29,7 @@ module Banzai
         return if html_attr.blank?
 
         html_attr.value = apply_rewrite_rules(html_attr.value)
-      rescue URI::Error
+      rescue URI::Error, Addressable::URI::InvalidURIError
         # noop
       end
 
diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb
index f2395867658..042fb2e6e14 100644
--- a/lib/banzai/pipeline/description_pipeline.rb
+++ b/lib/banzai/pipeline/description_pipeline.rb
@@ -1,23 +1,16 @@
 module Banzai
   module Pipeline
     class DescriptionPipeline < FullPipeline
+      WHITELIST = Banzai::Filter::SanitizationFilter::LIMITED.deep_dup.merge(
+        elements: Banzai::Filter::SanitizationFilter::LIMITED[:elements] - %w(pre code img ol ul li)
+      )
+
       def self.transform_context(context)
         super(context).merge(
           # SanitizationFilter
-          whitelist: whitelist
+          whitelist: WHITELIST
         )
       end
-
-      private
-
-      def self.whitelist
-        # Descriptions are more heavily sanitized, allowing only a few elements.
-        # See http://git.io/vkuAN
-        whitelist = Banzai::Filter::SanitizationFilter::LIMITED
-        whitelist[:elements] -= %w(pre code img ol ul li)
-
-        whitelist
-      end
     end
   end
 end
diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb
index 24076e3d9ec..f306079d833 100644
--- a/lib/banzai/reference_parser/issue_parser.rb
+++ b/lib/banzai/reference_parser/issue_parser.rb
@@ -25,7 +25,21 @@ module Banzai
       def issues_for_nodes(nodes)
         @issues_for_nodes ||= grouped_objects_for_nodes(
           nodes,
-          Issue.all.includes(:author, :assignee, :project),
+          Issue.all.includes(
+            :author,
+            :assignee,
+            {
+              # These associations are primarily used for checking permissions.
+              # Eager loading these ensures we don't end up running dozens of
+              # queries in this process.
+              project: [
+                { namespace: :owner },
+                { group: [:owners, :group_members] },
+                :invited_groups,
+                :project_members
+              ]
+            }
+          ),
           self.class.data_attribute
         )
       end
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 607359769d1..9f270f7b387 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -114,6 +114,7 @@ module Ci
         #   id (required) - The ID of a build
         #   token (required) - The build authorization token
         #   file (required) - Artifacts file
+        #   expire_in (optional) - Specify when artifacts should expire (ex. 7d)
         # Parameters (accelerated by GitLab Workhorse):
         #   file.path - path to locally stored body (generated by Workhorse)
         #   file.name - real filename as send in Content-Disposition
@@ -145,6 +146,7 @@ module Ci
 
           build.artifacts_file = artifacts
           build.artifacts_metadata = metadata
+          build.artifacts_expire_in = params['expire_in']
 
           if build.save
             present(build, with: Entities::BuildDetails)
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
index a902ced35d7..3f5bdaba3f5 100644
--- a/lib/ci/api/entities.rb
+++ b/lib/ci/api/entities.rb
@@ -20,7 +20,7 @@ module Ci
         expose :name, :token, :stage
         expose :project_id
         expose :project_name
-        expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts? }
+        expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? }
       end
 
       class BuildDetails < Build
@@ -29,6 +29,7 @@ module Ci
         expose :before_sha
         expose :allow_git_fetch
         expose :token
+        expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? }
 
         expose :options do |model|
           model.options
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 130f5b0892e..ed86de819eb 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -2,17 +2,24 @@ module Ci
   class GitlabCiYamlProcessor
     class ValidationError < StandardError; end
 
+    include Gitlab::Ci::Config::Node::ValidationHelpers
+
     DEFAULT_STAGES = %w(build test deploy)
     DEFAULT_STAGE = 'test'
     ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache]
     ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services,
                         :allow_failure, :type, :stage, :when, :artifacts, :cache,
-                        :dependencies, :before_script, :after_script, :variables]
+                        :dependencies, :before_script, :after_script, :variables,
+                        :environment]
+    ALLOWED_CACHE_KEYS = [:key, :untracked, :paths]
+    ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in]
 
-    attr_reader :before_script, :after_script, :image, :services, :path, :cache
+    attr_reader :after_script, :image, :services, :path, :cache
 
     def initialize(config, path = nil)
-      @config = Gitlab::Ci::Config.new(config).to_hash
+      @ci_config = Gitlab::Ci::Config.new(config)
+      @config = @ci_config.to_hash
+
       @path = path
 
       initial_parsing
@@ -23,7 +30,10 @@ module Ci
     end
 
     def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil)
-      builds.select{|build| build[:stage] == stage && process?(build[:only], build[:except], ref, tag, trigger_request)}
+      builds.select do |build|
+        build[:stage] == stage &&
+          process?(build[:only], build[:except], ref, tag, trigger_request)
+      end
     end
 
     def builds
@@ -44,13 +54,12 @@ module Ci
       job = @jobs[name.to_sym]
       return [] unless job
 
-      job.fetch(:variables, [])
+      job[:variables] || []
     end
 
     private
 
     def initial_parsing
-      @before_script = @config[:before_script] || []
       @after_script = @config[:after_script]
       @image = @config[:image]
       @services = @config[:services]
@@ -78,13 +87,14 @@ module Ci
       {
         stage_idx: stages.index(job[:stage]),
         stage: job[:stage],
-        commands: [job[:before_script] || @before_script, job[:script]].flatten.join("\n"),
+        commands: [job[:before_script] || [@ci_config.before_script], job[:script]].flatten.compact.join("\n"),
         tag_list: job[:tags] || [],
         name: name,
         only: job[:only],
         except: job[:except],
         allow_failure: job[:allow_failure] || false,
         when: job[:when] || 'on_success',
+        environment: job[:environment],
         options: {
           image: job[:image] || @image,
           services: job[:services] || @services,
@@ -97,6 +107,10 @@ module Ci
     end
 
     def validate!
+      unless @ci_config.valid?
+        raise ValidationError, @ci_config.errors.first
+      end
+
       validate_global!
 
       @jobs.each do |name, job|
@@ -107,10 +121,6 @@ module Ci
     end
 
     def validate_global!
-      unless validate_array_of_strings(@before_script)
-        raise ValidationError, "before_script should be an array of strings"
-      end
-
       unless @after_script.nil? || validate_array_of_strings(@after_script)
         raise ValidationError, "after_script should be an array of strings"
       end
@@ -135,6 +145,12 @@ module Ci
     end
 
     def validate_global_cache!
+      @cache.keys.each do |key|
+        unless ALLOWED_CACHE_KEYS.include? key
+          raise ValidationError, "#{name} cache unknown parameter #{key}"
+        end
+      end
+
       if @cache[:key] && !validate_string(@cache[:key])
         raise ValidationError, "cache:key parameter should be a string"
       end
@@ -188,21 +204,25 @@ module Ci
         raise ValidationError, "#{name} job: tags parameter should be an array of strings"
       end
 
-      if job[:only] && !validate_array_of_strings(job[:only])
-        raise ValidationError, "#{name} job: only parameter should be an array of strings"
+      if job[:only] && !validate_array_of_strings_or_regexps(job[:only])
+        raise ValidationError, "#{name} job: only parameter should be an array of strings or regexps"
       end
 
-      if job[:except] && !validate_array_of_strings(job[:except])
-        raise ValidationError, "#{name} job: except parameter should be an array of strings"
+      if job[:except] && !validate_array_of_strings_or_regexps(job[:except])
+        raise ValidationError, "#{name} job: except parameter should be an array of strings or regexps"
       end
 
       if job[:allow_failure] && !validate_boolean(job[:allow_failure])
         raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
       end
 
-      if job[:when] && !job[:when].in?(%w(on_success on_failure always))
+      if job[:when] && !job[:when].in?(%w[on_success on_failure always])
         raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always"
       end
+
+      if job[:environment] && !validate_environment(job[:environment])
+        raise ValidationError, "#{name} job: environment parameter #{Gitlab::Regex.environment_name_regex_message}"
+      end
     end
 
     def validate_job_script!(name, job)
@@ -233,6 +253,12 @@ module Ci
     end
 
     def validate_job_cache!(name, job)
+      job[:cache].keys.each do |key|
+        unless ALLOWED_CACHE_KEYS.include? key
+          raise ValidationError, "#{name} job: cache unknown parameter #{key}"
+        end
+      end
+
       if job[:cache][:key] && !validate_string(job[:cache][:key])
         raise ValidationError, "#{name} job: cache:key parameter should be a string"
       end
@@ -247,6 +273,12 @@ module Ci
     end
 
     def validate_job_artifacts!(name, job)
+      job[:artifacts].keys.each do |key|
+        unless ALLOWED_ARTIFACTS_KEYS.include? key
+          raise ValidationError, "#{name} job: artifacts unknown parameter #{key}"
+        end
+      end
+
       if job[:artifacts][:name] && !validate_string(job[:artifacts][:name])
         raise ValidationError, "#{name} job: artifacts:name parameter should be a string"
       end
@@ -258,6 +290,14 @@ module Ci
       if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
         raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
       end
+
+      if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always])
+        raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always"
+      end
+
+      if job[:artifacts][:expire_in] && !validate_duration(job[:artifacts][:expire_in])
+        raise ValidationError, "#{name} job: artifacts:expire_in parameter should be a duration"
+      end
     end
 
     def validate_job_dependencies!(name, job)
@@ -276,22 +316,6 @@ module Ci
       end
     end
 
-    def validate_array_of_strings(values)
-      values.is_a?(Array) && values.all? { |value| validate_string(value) }
-    end
-
-    def validate_variables(variables)
-      variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) }
-    end
-
-    def validate_string(value)
-      value.is_a?(String) || value.is_a?(Symbol)
-    end
-
-    def validate_boolean(value)
-      value.in?([true, false])
-    end
-
     def process?(only_params, except_params, ref, tag, trigger_request)
       if only_params.present?
         return false unless matching?(only_params, ref, tag, trigger_request)
diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb
index 4e20dc4f875..eb5a2596177 100644
--- a/lib/container_registry/blob.rb
+++ b/lib/container_registry/blob.rb
@@ -18,7 +18,7 @@ module ContainerRegistry
     end
 
     def digest
-      config['digest']
+      config['digest'] || config['blobSum']
     end
 
     def type
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
index 4d726692f45..42232b7129d 100644
--- a/lib/container_registry/client.rb
+++ b/lib/container_registry/client.rb
@@ -15,11 +15,11 @@ module ContainerRegistry
     end
 
     def repository_tags(name)
-      @faraday.get("/v2/#{name}/tags/list").body
+      response_body @faraday.get("/v2/#{name}/tags/list")
     end
 
     def repository_manifest(name, reference)
-      @faraday.get("/v2/#{name}/manifests/#{reference}").body
+      response_body @faraday.get("/v2/#{name}/manifests/#{reference}")
     end
 
     def repository_tag_digest(name, reference)
@@ -34,7 +34,7 @@ module ContainerRegistry
     def blob(name, digest, type = nil)
       headers = {}
       headers['Accept'] = type if type
-      @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers).body
+      response_body @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers)
     end
 
     def delete_blob(name, digest)
@@ -47,7 +47,10 @@ module ContainerRegistry
       conn.request :json
       conn.headers['Accept'] = MANIFEST_VERSION
 
-      conn.response :json, content_type: /\bjson$/
+      conn.response :json, content_type: 'application/json'
+      conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws'
+      conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json'
+      conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json'
 
       if options[:user] && options[:password]
         conn.request(:basic_auth, options[:user].to_s, options[:password].to_s)
@@ -57,5 +60,9 @@ module ContainerRegistry
 
       conn.adapter :net_http
     end
+
+    def response_body(response)
+      response.body if response.success?
+    end
   end
 end
diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb
index 43f8d6dc8c2..7a0929d774e 100644
--- a/lib/container_registry/tag.rb
+++ b/lib/container_registry/tag.rb
@@ -12,6 +12,14 @@ module ContainerRegistry
       manifest.present?
     end
 
+    def v1?
+      manifest && manifest['schemaVersion'] == 1
+    end
+
+    def v2?
+      manifest && manifest['schemaVersion'] == 2
+    end
+
     def manifest
       return @manifest if defined?(@manifest)
 
@@ -57,7 +65,9 @@ module ContainerRegistry
       return @layers if defined?(@layers)
       return unless manifest
 
-      @layers = manifest['layers'].map do |layer|
+      layers = manifest['layers'] || manifest['fsLayers']
+
+      @layers = layers.map do |layer|
         repository.blob(layer)
       end
     end
@@ -65,7 +75,7 @@ module ContainerRegistry
     def total_size
       return unless layers
 
-      layers.map(&:size).sum
+      layers.map(&:size).sum if v2?
     end
 
     def delete
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 076e2af7d38..db1704af75e 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -3,14 +3,14 @@ module Gitlab
     Result = Struct.new(:user, :type)
 
     class << self
-      def find(login, password, project:, ip:)
+      def find_for_git_client(login, password, project:, ip:)
         raise "Must provide an IP for rate limiting" if ip.nil?
 
         result = Result.new
 
         if valid_ci_request?(login, password, project)
           result.type = :ci
-        elsif result.user = find_in_gitlab_or_ldap(login, password)
+        elsif result.user = find_with_user_password(login, password)
           result.type = :gitlab_or_ldap
         elsif result.user = oauth_access_token_check(login, password)
           result.type = :oauth
@@ -20,7 +20,7 @@ module Gitlab
         result
       end
 
-      def find_in_gitlab_or_ldap(login, password)
+      def find_with_user_password(login, password)
         user = User.by_login(login)
 
         # If no user is found, or it's an LDAP server, try LDAP.
diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb
index f07ff1daf55..ab7b811c5d8 100644
--- a/lib/gitlab/backend/grack_auth.rb
+++ b/lib/gitlab/backend/grack_auth.rb
@@ -1,5 +1,3 @@
-require_relative 'shell_env'
-
 module Grack
   class AuthSpawner
     def self.call(env)
@@ -61,11 +59,6 @@ module Grack
       end
 
       @user = authenticate_user(login, password)
-
-      if @user
-        Gitlab::ShellEnv.set_env(@user)
-        @env['REMOTE_USER'] = @auth.username
-      end
     end
 
     def ci_request?(login, password)
@@ -95,7 +88,7 @@ module Grack
     end
 
     def authenticate_user(login, password)
-      user = Gitlab::Auth.find_in_gitlab_or_ldap(login, password)
+      user = Gitlab::Auth.find_with_user_password(login, password)
 
       unless user
         user = oauth_access_token_check(login, password)
diff --git a/lib/gitlab/backend/shell_env.rb b/lib/gitlab/backend/shell_env.rb
deleted file mode 100644
index 9f5adee594a..00000000000
--- a/lib/gitlab/backend/shell_env.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-module Gitlab
-  # This module provide 2 methods
-  # to set specific ENV variables for GitLab Shell
-  module ShellEnv
-    extend self
-
-    def set_env(user)
-      # Set GL_ID env variable
-      if user
-        ENV['GL_ID'] = gl_id(user)
-      end
-    end
-
-    def reset_env
-      # Reset GL_ID env variable
-      ENV['GL_ID'] = nil
-    end
-
-    def gl_id(user)
-      if user.present?
-        "user-#{user.id}"
-      else
-        # This empty string is used in the render_grack_auth_ok method
-        ""
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index ffe633d4b63..b48d3592f16 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -1,11 +1,21 @@
 module Gitlab
   module Ci
+    ##
+    # Base GitLab CI Configuration facade
+    #
     class Config
-      class LoaderError < StandardError; end
+      delegate :valid?, :errors, to: :@global
+
+      ##
+      # Temporary delegations that should be removed after refactoring
+      #
+      delegate :before_script, to: :@global
 
       def initialize(config)
-        loader = Loader.new(config)
-        @config = loader.load!
+        @config = Loader.new(config).load!
+
+        @global = Node::Global.new(@config)
+        @global.process!
       end
 
       def to_hash
diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb
new file mode 100644
index 00000000000..d60f87f3f94
--- /dev/null
+++ b/lib/gitlab/ci/config/node/configurable.rb
@@ -0,0 +1,61 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # This mixin is responsible for adding DSL, which purpose is to
+        # simplifly process of adding child nodes.
+        #
+        # This can be used only if parent node is a configuration entry that
+        # holds a hash as a configuration value, for example:
+        #
+        # job:
+        #   script: ...
+        #   artifacts: ...
+        #
+        module Configurable
+          extend ActiveSupport::Concern
+
+          def allowed_nodes
+            self.class.allowed_nodes || {}
+          end
+
+          private
+
+          def prevalidate!
+            unless @value.is_a?(Hash)
+              @errors << 'should be a configuration entry with hash value'
+            end
+          end
+
+          def create_node(key, factory)
+            factory.with(value: @value[key])
+            factory.nullify! unless @value.has_key?(key)
+            factory.create!
+          end
+
+          class_methods do
+            def allowed_nodes
+              Hash[@allowed_nodes.map { |key, factory| [key, factory.dup] }]
+            end
+
+            private
+
+            def allow_node(symbol, entry_class, metadata)
+              factory = Node::Factory.new(entry_class)
+                .with(description: metadata[:description])
+
+              define_method(symbol) do
+                raise Entry::InvalidError unless valid?
+
+                @nodes[symbol].try(:value)
+              end
+
+              (@allowed_nodes ||= {}).merge!(symbol => factory)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb
new file mode 100644
index 00000000000..52758a962f3
--- /dev/null
+++ b/lib/gitlab/ci/config/node/entry.rb
@@ -0,0 +1,77 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Base abstract class for each configuration entry node.
+        #
+        class Entry
+          class InvalidError < StandardError; end
+
+          attr_accessor :description
+
+          def initialize(value)
+            @value = value
+            @nodes = {}
+            @errors = []
+
+            prevalidate!
+          end
+
+          def process!
+            return if leaf?
+            return unless valid?
+
+            compose!
+
+            nodes.each(&:process!)
+            nodes.each(&:validate!)
+          end
+
+          def nodes
+            @nodes.values
+          end
+
+          def valid?
+            errors.none?
+          end
+
+          def leaf?
+            allowed_nodes.none?
+          end
+
+          def errors
+            @errors + nodes.map(&:errors).flatten
+          end
+
+          def allowed_nodes
+            {}
+          end
+
+          def validate!
+            raise NotImplementedError
+          end
+
+          def value
+            raise NotImplementedError
+          end
+
+          private
+
+          def prevalidate!
+          end
+
+          def compose!
+            allowed_nodes.each do |key, essence|
+              @nodes[key] = create_node(key, essence)
+            end
+          end
+
+          def create_node(key, essence)
+            raise NotImplementedError
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb
new file mode 100644
index 00000000000..787ca006f5a
--- /dev/null
+++ b/lib/gitlab/ci/config/node/factory.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Factory class responsible for fabricating node entry objects.
+        #
+        # It uses Fluent Interface pattern to set all necessary attributes.
+        #
+        class Factory
+          class InvalidFactory < StandardError; end
+
+          def initialize(entry_class)
+            @entry_class = entry_class
+            @attributes = {}
+          end
+
+          def with(attributes)
+            @attributes.merge!(attributes)
+            self
+          end
+
+          def nullify!
+            @entry_class = Node::Null
+            self
+          end
+
+          def create!
+            raise InvalidFactory unless @attributes.has_key?(:value)
+
+            @entry_class.new(@attributes[:value]).tap do |entry|
+              entry.description = @attributes[:description]
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb
new file mode 100644
index 00000000000..044603423d5
--- /dev/null
+++ b/lib/gitlab/ci/config/node/global.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # This class represents a global entry - root node for entire
+        # GitLab CI Configuration file.
+        #
+        class Global < Entry
+          include Configurable
+
+          allow_node :before_script, Script,
+            description: 'Script that will be executed before each job.'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb
new file mode 100644
index 00000000000..4f590f6bec8
--- /dev/null
+++ b/lib/gitlab/ci/config/node/null.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # This class represents a configuration entry that is not being used
+        # in configuration file.
+        #
+        # This implements Null Object pattern.
+        #
+        class Null < Entry
+          def value
+            nil
+          end
+
+          def validate!
+            nil
+          end
+
+          def method_missing(*)
+            nil
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/node/script.rb
new file mode 100644
index 00000000000..5072bf0db7d
--- /dev/null
+++ b/lib/gitlab/ci/config/node/script.rb
@@ -0,0 +1,29 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a script.
+        #
+        # Each element in the value array is a command that will be executed
+        # by GitLab Runner. Currently we concatenate these commands with
+        # new line character as a separator, what is compatible with
+        # implementation in Runner.
+        #
+        class Script < Entry
+          include ValidationHelpers
+
+          def value
+            @value.join("\n")
+          end
+
+          def validate!
+            unless validate_array_of_strings(@value)
+              @errors << 'before_script should be an array of strings'
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/validation_helpers.rb
new file mode 100644
index 00000000000..72f648975dc
--- /dev/null
+++ b/lib/gitlab/ci/config/node/validation_helpers.rb
@@ -0,0 +1,55 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        module ValidationHelpers
+          private
+
+          def validate_duration(value)
+            value.is_a?(String) && ChronicDuration.parse(value)
+          rescue ChronicDuration::DurationParseError
+            false
+          end
+
+          def validate_array_of_strings(values)
+            values.is_a?(Array) && values.all? { |value| validate_string(value) }
+          end
+
+          def validate_array_of_strings_or_regexps(values)
+            values.is_a?(Array) && values.all? { |value| validate_string_or_regexp(value) }
+          end
+
+          def validate_variables(variables)
+            variables.is_a?(Hash) &&
+              variables.all? { |key, value| validate_string(key) && validate_string(value) }
+          end
+
+          def validate_string(value)
+            value.is_a?(String) || value.is_a?(Symbol)
+          end
+
+          def validate_string_or_regexp(value)
+            return true if value.is_a?(Symbol)
+            return false unless value.is_a?(String)
+
+            if value.first == '/' && value.last == '/'
+              Regexp.new(value[1...-1])
+            else
+              true
+            end
+          rescue RegexpError
+            false
+          end
+
+          def validate_environment(value)
+            value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex
+          end
+
+          def validate_boolean(value)
+            value.in?([true, false])
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 5e7532f57ae..28c34429c1f 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -36,7 +36,7 @@ module Gitlab
         default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
-        import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
+        import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project],
         shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
         max_artifacts_size: Settings.artifacts['max_size'],
         require_two_factor_authentication: false,
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 04fa6a3a5de..078609c86f1 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -1,5 +1,10 @@
 module Gitlab
   module Database
+    # The max value of INTEGER type is the same between MySQL and PostgreSQL:
+    # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html
+    # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html
+    MAX_INT_VALUE = 2147483647
+
     def self.adapter_name
       connection.adapter_name
     end
@@ -30,6 +35,10 @@ module Gitlab
       order
     end
 
+    def self.random
+      Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()"
+    end
+
     def true_value
       if Gitlab::Database.postgresql?
         "'t'"
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 978c3f7896d..dec20d8659b 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -28,64 +28,79 @@ module Gitlab
       # Updates the value of a column in batches.
       #
       # This method updates the table in batches of 5% of the total row count.
-      # Any data inserted while running this method (or after it has finished
-      # running) is _not_ updated automatically.
+      # This method will continue updating rows until no rows remain.
       #
-      # This method _only_ updates rows where the column's value is set to NULL.
+      # When given a block this method will yield two values to the block:
+      #
+      # 1. An instance of `Arel::Table` for the table that is being updated.
+      # 2. The query to run as an Arel object.
+      #
+      # By supplying a block one can add extra conditions to the queries being
+      # executed. Note that the same block is used for _all_ queries.
+      #
+      # Example:
+      #
+      #     update_column_in_batches(:projects, :foo, 10) do |table, query|
+      #       query.where(table[:some_column].eq('hello'))
+      #     end
+      #
+      # This would result in this method updating only rows where
+      # `projects.some_column` equals "hello".
       #
       # table - The name of the table.
       # column - The name of the column to update.
       # value - The value for the column.
+      #
+      # Rubocop's Metrics/AbcSize metric is disabled for this method as Rubocop
+      # determines this method to be too complex while there's no way to make it
+      # less "complex" without introducing extra methods (which actually will
+      # make things _more_ complex).
+      #
+      # rubocop: disable Metrics/AbcSize
       def update_column_in_batches(table, column, value)
-        quoted_table = quote_table_name(table)
-        quoted_column = quote_column_name(column)
-
-        ##
-        # Workaround for #17711
-        #
-        # It looks like for MySQL `ActiveRecord::Base.conntection.quote(true)`
-        # returns correct value (1), but `ActiveRecord::Migration.new.quote`
-        # returns incorrect value ('true'), which causes migrations to fail.
-        #
-        quoted_value = connection.quote(value)
-        processed = 0
-
-        total = exec_query("SELECT COUNT(*) AS count FROM #{quoted_table}").
-          to_hash.
-          first['count'].
-          to_i
-
-        # Update in batches of 5%
+        table = Arel::Table.new(table)
+
+        count_arel = table.project(Arel.star.count.as('count'))
+        count_arel = yield table, count_arel if block_given?
+
+        total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i
+
+        return if total == 0
+
+        # Update in batches of 5% until we run out of any rows to update.
         batch_size = ((total / 100.0) * 5.0).ceil
 
-        while processed < total
-          start_row = exec_query(%Q{
-            SELECT id
-            FROM #{quoted_table}
-            ORDER BY id ASC
-            LIMIT 1 OFFSET #{processed}
-          }).to_hash.first
-
-          stop_row = exec_query(%Q{
-            SELECT id
-            FROM #{quoted_table}
-            ORDER BY id ASC
-            LIMIT 1 OFFSET #{processed + batch_size}
-          }).to_hash.first
-
-          query = %Q{
-            UPDATE #{quoted_table}
-            SET #{quoted_column} = #{quoted_value}
-            WHERE id >= #{start_row['id']}
-          }
+        start_arel = table.project(table[:id]).order(table[:id].asc).take(1)
+        start_arel = yield table, start_arel if block_given?
+        start_id = exec_query(start_arel.to_sql).to_hash.first['id'].to_i
+
+        loop do
+          stop_arel = table.project(table[:id]).
+            where(table[:id].gteq(start_id)).
+            order(table[:id].asc).
+            take(1).
+            skip(batch_size)
+
+          stop_arel = yield table, stop_arel if block_given?
+          stop_row = exec_query(stop_arel.to_sql).to_hash.first
+
+          update_arel = Arel::UpdateManager.new(ActiveRecord::Base).
+            table(table).
+            set([[table[column], value]]).
+            where(table[:id].gteq(start_id))
 
           if stop_row
-            query += " AND id < #{stop_row['id']}"
+            stop_id = stop_row['id'].to_i
+            start_id = stop_id
+            update_arel = update_arel.where(table[:id].lt(stop_id))
           end
 
-          execute(query)
+          update_arel = yield table, update_arel if block_given?
 
-          processed += batch_size
+          execute(update_arel.to_sql)
+
+          # There are no more rows left to update.
+          break unless stop_row
         end
       end
 
@@ -94,9 +109,9 @@ module Gitlab
       # This method runs the following steps:
       #
       # 1. Add the column with a default value of NULL.
-      # 2. Update all existing rows in batches.
-      # 3. Change the default value of the column to the specified value.
-      # 4. Update any remaining rows.
+      # 2. Change the default value of the column to the specified value.
+      # 3. Update all existing rows in batches.
+      # 4. Set a `NOT NULL` constraint on the column if desired (the default).
       #
       # These steps ensure a column can be added to a large and commonly used
       # table without locking the entire table for the duration of the table
@@ -108,7 +123,10 @@ module Gitlab
       # default - The default value for the column.
       # allow_null - When set to `true` the column will allow NULL values, the
       #              default is to not allow NULL values.
-      def add_column_with_default(table, column, type, default:, allow_null: false)
+      #
+      # This method can also take a block which is passed directly to the
+      # `update_column_in_batches` method.
+      def add_column_with_default(table, column, type, default:, allow_null: false, &block)
         if transaction_open?
           raise 'add_column_with_default can not be run inside a transaction, ' \
             'you can disable transactions by calling disable_ddl_transaction! ' \
@@ -124,9 +142,9 @@ module Gitlab
         end
 
         begin
-          transaction do
-            update_column_in_batches(table, column, default)
-          end
+          update_column_in_batches(table, column, default, &block)
+
+          change_column_null(table, column, false) unless allow_null
         # We want to rescue _all_ exceptions here, even those that don't inherit
         # from StandardError.
         rescue Exception => error # rubocop: disable all
@@ -134,8 +152,6 @@ module Gitlab
 
           raise error
         end
-
-        change_column_null(table, column, false) unless allow_null
       end
     end
   end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index e5cf66a0371..2286ac8829c 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -66,8 +66,7 @@ module Gitlab
       end
 
       def import_pull_requests
-        hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?)
-        disable_webhooks(hooks)
+        disable_webhooks
 
         pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
         pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
@@ -90,14 +89,14 @@ module Gitlab
         raise Projects::ImportService::Error, e.message
       ensure
         clean_up_restored_branches(branches_removed)
-        clean_up_disabled_webhooks(hooks)
+        clean_up_disabled_webhooks
       end
 
-      def disable_webhooks(hooks)
+      def disable_webhooks
         update_webhooks(hooks, active: false)
       end
 
-      def clean_up_disabled_webhooks(hooks)
+      def clean_up_disabled_webhooks
         update_webhooks(hooks, active: true)
       end
 
@@ -107,6 +106,20 @@ module Gitlab
         end
       end
 
+      def hooks
+        @hooks ||=
+          begin
+            client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?)
+
+          # The GitHub Repository Webhooks API returns 404 for users
+          # without admin access to the repository when listing hooks.
+          # In this case we just want to return gracefully instead of
+          # spitting out an error and stop the import process.
+          rescue Octokit::NotFound
+            []
+          end
+      end
+
       def restore_branches(branches)
         branches.each do |name, sha|
           client.create_ref(repo, "refs/heads/#{name}", sha)
diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb
index 77c33db4b59..3d0418261bb 100644
--- a/lib/gitlab/gitlab_import/project_creator.rb
+++ b/lib/gitlab/gitlab_import/project_creator.rb
@@ -11,7 +11,7 @@ module Gitlab
       end
 
       def execute
-        project = ::Projects::CreateService.new(
+        ::Projects::CreateService.new(
           current_user,
           name: repo["name"],
           path: repo["path"],
@@ -22,8 +22,6 @@ module Gitlab
           import_source: repo["path_with_namespace"],
           import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@")
         ).execute
-
-        project
       end
     end
   end
diff --git a/lib/gitlab/gl_id.rb b/lib/gitlab/gl_id.rb
new file mode 100644
index 00000000000..624fd00367e
--- /dev/null
+++ b/lib/gitlab/gl_id.rb
@@ -0,0 +1,11 @@
+module Gitlab
+  module GlId
+    def self.gl_id(user)
+      if user.present?
+        "user-#{user.id}"
+      else
+        ""
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
new file mode 100644
index 00000000000..99cf85d9a3b
--- /dev/null
+++ b/lib/gitlab/import_export.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module ImportExport
+    extend self
+
+    VERSION = '0.1.0'
+
+    def export_path(relative_path:)
+      File.join(storage_path, relative_path)
+    end
+
+    def storage_path
+      File.join(Settings.shared['path'], 'tmp/project_exports')
+    end
+
+    def project_filename
+      "project.json"
+    end
+
+    def project_bundle_filename
+      "project.bundle"
+    end
+
+    def config_file
+      Rails.root.join('lib/gitlab/import_export/import_export.yml')
+    end
+
+    def version_filename
+      'VERSION'
+    end
+
+    def version
+      VERSION
+    end
+
+    def reset_tokens?
+      true
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb
new file mode 100644
index 00000000000..d230de781d5
--- /dev/null
+++ b/lib/gitlab/import_export/attributes_finder.rb
@@ -0,0 +1,47 @@
+module Gitlab
+  module ImportExport
+    class AttributesFinder
+    
+      def initialize(included_attributes:, excluded_attributes:, methods:)
+        @included_attributes = included_attributes || {}
+        @excluded_attributes = excluded_attributes || {}
+        @methods = methods || {}
+      end
+
+      def find(model_object)
+        parsed_hash = find_attributes_only(model_object)
+        parsed_hash.empty? ? model_object : { model_object => parsed_hash }
+      end
+
+      def parse(model_object)
+        parsed_hash = find_attributes_only(model_object)
+        yield parsed_hash unless parsed_hash.empty?
+      end
+
+      def find_included(value)
+        key = key_from_hash(value)
+        @included_attributes[key].nil? ? {} : { only: @included_attributes[key] }
+      end
+
+      def find_excluded(value)
+        key = key_from_hash(value)
+        @excluded_attributes[key].nil? ? {} : { except: @excluded_attributes[key] }
+      end
+
+      def find_method(value)
+        key = key_from_hash(value)
+        @methods[key].nil? ? {} : { methods: @methods[key] }
+      end
+
+      private
+
+      def find_attributes_only(value)
+        find_included(value).merge(find_excluded(value)).merge(find_method(value))
+      end
+
+      def key_from_hash(value)
+        value.is_a?(Hash) ? value.keys.first : value
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
new file mode 100644
index 00000000000..78664f076eb
--- /dev/null
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -0,0 +1,40 @@
+module Gitlab
+  module ImportExport
+    module CommandLineUtil
+      def tar_czf(archive:, dir:)
+        tar_with_options(archive: archive, dir: dir, options: 'czf')
+      end
+
+      def untar_zxf(archive:, dir:)
+        untar_with_options(archive: archive, dir: dir, options: 'zxf')
+      end
+
+      def git_bundle(repo_path:, bundle_path:)
+        execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all))
+      end
+
+      def git_unbundle(repo_path:, bundle_path:)
+        execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path}))
+      end
+
+      private
+
+      def tar_with_options(archive:, dir:, options:)
+        execute(%W(tar -#{options} #{archive} -C #{dir} .))
+      end
+
+      def untar_with_options(archive:, dir:, options:)
+        execute(%W(tar -#{options} #{archive} -C #{dir}))
+      end
+
+      def execute(cmd)
+        _output, status = Gitlab::Popen.popen(cmd)
+        status.zero?
+      end
+
+      def git_bin_path
+        Gitlab.config.git.bin_path
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb
new file mode 100644
index 00000000000..e341c4d9cf8
--- /dev/null
+++ b/lib/gitlab/import_export/error.rb
@@ -0,0 +1,5 @@
+module Gitlab
+  module ImportExport
+    class Error < StandardError; end
+  end
+end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
new file mode 100644
index 00000000000..0e70d9282d5
--- /dev/null
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module ImportExport
+    class FileImporter
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def self.import(*args)
+        new(*args).import
+      end
+
+      def initialize(archive_file:, shared:)
+        @archive_file = archive_file
+        @shared = shared
+      end
+
+      def import
+        FileUtils.mkdir_p(@shared.export_path)
+        decompress_archive
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def decompress_archive
+        untar_zxf(archive: @archive_file, dir: @shared.export_path)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
new file mode 100644
index 00000000000..164ab6238c4
--- /dev/null
+++ b/lib/gitlab/import_export/import_export.yml
@@ -0,0 +1,54 @@
+# Model relationships to be included in the project import/export
+project_tree:
+  - issues:
+    - notes:
+        :author
+  - :labels
+  - :milestones
+  - snippets:
+    - notes:
+        :author
+  - :releases
+  - :events
+  - project_members:
+    - :user
+  - merge_requests:
+    - notes:
+        :author
+    - :merge_request_diff
+  - pipelines:
+    - notes:
+        :author
+    - :statuses
+  - :variables
+  - :triggers
+  - :deploy_keys
+  - :services
+  - :hooks
+  - :protected_branches
+
+# Only include the following attributes for the models specified.
+included_attributes:
+  project:
+    - :description
+    - :issues_enabled
+    - :merge_requests_enabled
+    - :wiki_enabled
+    - :snippets_enabled
+    - :visibility_level
+    - :archived
+  user:
+    - :id
+    - :email
+    - :username
+  author:
+    - :name
+
+# Do not include the following attributes for the models specified.
+excluded_attributes:
+  snippets:
+    - :expired_at
+
+methods:
+  statuses:
+    - :type
\ No newline at end of file
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
new file mode 100644
index 00000000000..d209e04f7be
--- /dev/null
+++ b/lib/gitlab/import_export/importer.rb
@@ -0,0 +1,64 @@
+module Gitlab
+  module ImportExport
+    class Importer
+
+      def initialize(project)
+        @archive_file = project.import_source
+        @current_user = project.creator
+        @project = project
+        @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace)
+      end
+
+      def execute
+        Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
+                                                  shared: @shared)
+        if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
+          project_tree.restored_project
+        else
+          raise Projects::ImportService::Error.new(@shared.errors.join(', '))
+        end
+      end
+
+      private
+
+      def check_version!
+        Gitlab::ImportExport::VersionChecker.check!(shared: @shared)
+      end
+
+      def project_tree
+        @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user,
+                                                                        shared: @shared,
+                                                                        project: @project)
+      end
+
+      def repo_restorer
+        Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path,
+                                               shared: @shared,
+                                               project: project_tree.restored_project)
+      end
+
+      def wiki_restorer
+        Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path,
+                                               shared: @shared,
+                                               project: ProjectWiki.new(project_tree.restored_project),
+                                               wiki: true)
+      end
+
+      def uploads_restorer
+        Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared)
+      end
+
+      def path_with_namespace
+        File.join(@project.namespace.path, @project.path)
+      end
+
+      def repo_path
+        File.join(@shared.export_path, 'project.bundle')
+      end
+
+      def wiki_repo_path
+        File.join(@shared.export_path, 'project.wiki.bundle')
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb
new file mode 100644
index 00000000000..c569a35a48b
--- /dev/null
+++ b/lib/gitlab/import_export/members_mapper.rb
@@ -0,0 +1,68 @@
+module Gitlab
+  module ImportExport
+    class MembersMapper
+
+      attr_reader :missing_author_ids
+
+      def initialize(exported_members:, user:, project:)
+        @exported_members = exported_members
+        @user = user
+        @project = project
+        @missing_author_ids = []
+
+        # This needs to run first, as second call would be from #map
+        # which means project members already exist.
+        ensure_default_member!
+      end
+
+      def map
+        @map ||=
+          begin
+            @exported_members.inject(missing_keys_tracking_hash) do |hash, member|
+              existing_user = User.where(find_project_user_query(member)).first
+              old_user_id = member['user']['id']
+              if existing_user && add_user_as_team_member(existing_user, member)
+                hash[old_user_id] = existing_user.id
+              end
+              hash
+            end
+          end
+      end
+
+      def default_user_id
+        @user.id
+      end
+
+      private
+
+      def missing_keys_tracking_hash
+        Hash.new do |_, key|
+          @missing_author_ids << key
+          default_user_id
+        end
+      end
+
+      def ensure_default_member!
+        ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true)
+      end
+
+      def add_user_as_team_member(existing_user, member)
+        member['user'] = existing_user
+
+        ProjectMember.create(member_hash(member)).persisted?
+      end
+
+      def member_hash(member)
+        member.except('id').merge(source_id: @project.id, importing: true)
+      end
+
+      def find_project_user_query(member)
+        user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email']))
+      end
+
+      def user_arel
+        @user_arel ||= User.arel_table
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb
new file mode 100644
index 00000000000..89388d1984b
--- /dev/null
+++ b/lib/gitlab/import_export/project_creator.rb
@@ -0,0 +1,24 @@
+module Gitlab
+  module ImportExport
+    class ProjectCreator
+
+      def initialize(namespace_id, current_user, file, project_path)
+        @namespace_id = namespace_id
+        @current_user = current_user
+        @file = file
+        @project_path = project_path
+      end
+
+      def execute
+        ::Projects::CreateService.new(
+          @current_user,
+          name: @project_path,
+          path: @project_path,
+          namespace_id: @namespace_id,
+          import_type: "gitlab_project",
+          import_source: @file
+        ).execute
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
new file mode 100644
index 00000000000..dd71b92c522
--- /dev/null
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -0,0 +1,105 @@
+module Gitlab
+  module ImportExport
+    class ProjectTreeRestorer
+
+      def initialize(user:, shared:, project:)
+        @path = File.join(shared.export_path, 'project.json')
+        @user = user
+        @shared = shared
+        @project = project
+      end
+
+      def restore
+        json = IO.read(@path)
+        @tree_hash = ActiveSupport::JSON.decode(json)
+        @project_members = @tree_hash.delete('project_members')
+        create_relations
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      def restored_project
+        @restored_project ||= restore_project
+      end
+
+      private
+
+      def members_mapper
+        @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
+                                                                    user: @user,
+                                                                    project: restored_project)
+      end
+
+      # Loops through the tree of models defined in import_export.yml and
+      # finds them in the imported JSON so they can be instantiated and saved
+      # in the DB. The structure and relationships between models are guessed from
+      # the configuration yaml file too.
+      # Finally, it updates each attribute in the newly imported project.
+      def create_relations
+        saved = []
+        default_relation_list.each do |relation|
+          next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present?
+
+          create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash)
+
+          relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
+          relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s])
+          saved << restored_project.update_attribute(relation_key, relation_hash)
+        end
+        saved.all?
+      end
+
+      def default_relation_list
+        Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model|
+          model.is_a?(Hash) && model[:project_members]
+        end
+      end
+
+      def restore_project
+        return @project unless @tree_hash
+
+        project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) }
+        @project.update(project_params)
+        @project
+      end
+
+      # Given a relation hash containing one or more models and its relationships,
+      # loops through each model and each object from a model type and
+      # and assigns its correspondent attributes hash from +tree_hash+
+      # Example:
+      # +relation_key+ issues, loops through the list of *issues* and for each individual
+      # issue, finds any subrelations such as notes, creates them and assign them back to the hash
+      def create_sub_relations(relation, tree_hash)
+        relation_key = relation.keys.first.to_s
+        tree_hash[relation_key].each do |relation_item|
+          relation.values.flatten.each do |sub_relation|
+            relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
+            relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
+          end
+        end
+      end
+
+      def assign_relation_hash(relation_item, sub_relation)
+        if sub_relation.is_a?(Hash)
+          relation_hash = relation_item[sub_relation.keys.first.to_s]
+          sub_relation = sub_relation.keys.first
+        else
+          relation_hash = relation_item[sub_relation.to_s]
+        end
+        [relation_hash, sub_relation]
+      end
+
+      def create_relation(relation, relation_hash_list)
+        relation_array = [relation_hash_list].flatten.map do |relation_hash|
+          Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
+                                                       relation_hash: relation_hash.merge('project_id' => restored_project.id),
+                                                       members_mapper: members_mapper,
+                                                       user: @user)
+        end
+
+        relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb
new file mode 100644
index 00000000000..9153088e966
--- /dev/null
+++ b/lib/gitlab/import_export/project_tree_saver.rb
@@ -0,0 +1,29 @@
+module Gitlab
+  module ImportExport
+    class ProjectTreeSaver
+      attr_reader :full_path
+
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+        @full_path = File.join(@shared.export_path, ImportExport.project_filename)
+      end
+
+      def save
+        FileUtils.mkdir_p(@shared.export_path)
+
+        File.write(full_path, project_json_tree)
+        true
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def project_json_tree
+        @project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb
new file mode 100644
index 00000000000..19defd8f03a
--- /dev/null
+++ b/lib/gitlab/import_export/reader.rb
@@ -0,0 +1,117 @@
+module Gitlab
+  module ImportExport
+    class Reader
+
+      attr_reader :tree
+
+      def initialize(shared:)
+        @shared = shared
+        config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys
+        @tree = config_hash[:project_tree]
+        @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes],
+                                                                        excluded_attributes: config_hash[:excluded_attributes],
+                                                                        methods: config_hash[:methods])
+      end
+
+      # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
+      # for outputting a project in JSON format, including its relations and sub relations.
+      def project_tree
+        @attributes_finder.find_included(:project).merge(include: build_hash(@tree))
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
+      #
+      # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file
+      def build_hash(model_list)
+        model_list.map do |model_objects|
+          if model_objects.is_a?(Hash)
+            build_json_config_hash(model_objects)
+          else
+            @attributes_finder.find(model_objects)
+          end
+        end
+      end
+
+      # Called when the model is actually a hash containing other relations (more models)
+      # Returns the config in the right format for calling +to_json+
+      # +model_object_hash+ - A model relationship such as:
+      #   {:merge_requests=>[:merge_request_diff, :notes]}
+      def build_json_config_hash(model_object_hash)
+        @json_config_hash = {}
+
+        model_object_hash.values.flatten.each do |model_object|
+          current_key = model_object_hash.keys.first
+
+          @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash }
+
+          handle_model_object(current_key, model_object)
+          process_sub_model(current_key, model_object) if model_object.is_a?(Hash)
+        end
+        @json_config_hash
+      end
+
+
+      # If the model is a hash, process the sub_models, which could also be hashes
+      # If there is a list, add to an existing array, otherwise use hash syntax
+      # +current_key+ main model that will be a key in the hash
+      # +model_object+ model or list of models to include in the hash
+      def process_sub_model(current_key, model_object)
+        sub_model_json = build_json_config_hash(model_object).dup
+        @json_config_hash.slice!(current_key)
+
+        if @json_config_hash[current_key] && @json_config_hash[current_key][:include]
+          @json_config_hash[current_key][:include] << sub_model_json
+        else
+          @json_config_hash[current_key] = { include: sub_model_json }
+        end
+      end
+
+      # Creates or adds to an existing hash an individual model or list
+      # +current_key+ main model that will be a key in the hash
+      # +model_object+ model or list of models to include in the hash
+      def handle_model_object(current_key, model_object)
+        if @json_config_hash[current_key]
+          add_model_value(current_key, model_object)
+        else
+          create_model_value(current_key, model_object)
+        end
+      end
+
+      # Constructs a new hash that will hold the configuration for that particular object
+      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
+      # +current_key+ main model that will be a key in the hash
+      # +value+ existing model to be included in the hash
+      def create_model_value(current_key, value)
+        parsed_hash = { include: value }
+
+        @attributes_finder.parse(value) do |hash|
+          parsed_hash = { include: hash_or_merge(value, hash) }
+        end
+        @json_config_hash[current_key] = parsed_hash
+      end
+
+      # Adds new model configuration to an existing hash with key +current_key+
+      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
+      # +current_key+ main model that will be a key in the hash
+      # +value+ existing model to be included in the hash
+      def add_model_value(current_key, value)
+        @attributes_finder.parse(value) { |hash| value = { value => hash } }
+        old_values = @json_config_hash[current_key][:include]
+        @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten
+      end
+
+      # Construct a new hash or merge with an existing one a model configuration
+      # This is to fulfil +to_json+ requirements.
+      # +value+ existing model to be included in the hash
+      # +hash+ hash containing configuration generated mainly from +@attributes_finder+
+      def hash_or_merge(value, hash)
+        value.is_a?(Hash) ? value.merge(hash) : { value => hash }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
new file mode 100644
index 00000000000..b872780f20a
--- /dev/null
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -0,0 +1,128 @@
+module Gitlab
+  module ImportExport
+    class RelationFactory
+
+      OVERRIDES = { snippets: :project_snippets,
+                    pipelines: 'Ci::Pipeline',
+                    statuses: 'commit_status',
+                    variables: 'Ci::Variable',
+                    triggers: 'Ci::Trigger',
+                    builds: 'Ci::Build',
+                    hooks: 'ProjectHook' }.freeze
+
+      USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze
+
+      def self.create(*args)
+        new(*args).create
+      end
+
+      def initialize(relation_sym:, relation_hash:, members_mapper:, user:)
+        @relation_name = OVERRIDES[relation_sym] || relation_sym
+        @relation_hash = relation_hash.except('id', 'noteable_id')
+        @members_mapper = members_mapper
+        @user = user
+      end
+
+      # Creates an object from an actual model with name "relation_sym" with params from
+      # the relation_hash, updating references with new object IDs, mapping users using
+      # the "members_mapper" object, also updating notes if required.
+      def create
+        set_note_author if @relation_name == :notes
+        update_user_references
+        update_project_references
+        reset_ci_tokens if @relation_name == 'Ci::Trigger'
+
+        generate_imported_object
+      end
+
+      private
+
+      def update_user_references
+        USER_REFERENCES.each do |reference|
+          if @relation_hash[reference]
+            @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]]
+          end
+        end
+      end
+
+      # Sets the author for a note. If the user importing the project
+      # has admin access, an actual mapping with new project members
+      # will be used. Otherwise, a note stating the original author name
+      # is left.
+      def set_note_author
+        old_author_id = @relation_hash['author_id']
+
+        # Users with admin access can map users
+        @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id
+
+        author = @relation_hash.delete('author')
+
+        update_note_for_missing_author(author['name']) if missing_author?(old_author_id)
+      end
+
+      def missing_author?(old_author_id)
+        !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id)
+      end
+
+      def missing_author_note(updated_at, author_name)
+        timestamp = updated_at.split('.').first
+        "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*"
+      end
+
+      def generate_imported_object
+        if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes
+          trace = @relation_hash.delete('trace')
+          imported_object do |object|
+            object.trace = trace
+            object.commit_id = nil
+          end
+        else
+          imported_object
+        end
+      end
+
+      def update_project_references
+        project_id = @relation_hash.delete('project_id')
+
+        # project_id may not be part of the export, but we always need to populate it if required.
+        @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id')
+        @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id']
+        @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id']
+        @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id']
+
+        # If source and target are the same, populate them with the new project ID.
+        if @relation_hash['source_project_id'] && @relation_hash['target_project_id'] &&
+          @relation_hash['target_project_id'] == @relation_hash['source_project_id']
+          @relation_hash['source_project_id'] = project_id
+        end
+      end
+
+      def reset_ci_tokens
+        return unless Gitlab::ImportExport.reset_tokens?
+
+        # If we import/export a project to the same instance, tokens will have to be reset.
+        @relation_hash['token'] = nil
+      end
+
+      def relation_class
+        @relation_class ||= @relation_name.to_s.classify.constantize
+      end
+
+      def imported_object
+        imported_object = relation_class.new(@relation_hash)
+        yield(imported_object) if block_given?
+        imported_object.importing = true if imported_object.respond_to?(:importing)
+        imported_object
+      end
+
+      def update_note_for_missing_author(author_name)
+        @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank?
+        @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name)
+      end
+
+      def admin_user?
+        @user.is_admin?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
new file mode 100644
index 00000000000..546dae4d122
--- /dev/null
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module ImportExport
+    class RepoRestorer
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def initialize(project:, shared:, path_to_bundle:, wiki: false)
+        @project = project
+        @path_to_bundle = path_to_bundle
+        @shared = shared
+        @wiki = wiki
+      end
+
+      def restore
+        return wiki? unless File.exist?(@path_to_bundle)
+
+        FileUtils.mkdir_p(path_to_repo)
+
+        git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def repos_path
+        Gitlab.config.gitlab_shell.repos_path
+      end
+
+      def path_to_repo
+        @project.repository.path_to_repo
+      end
+
+      def wiki?
+        @wiki
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
new file mode 100644
index 00000000000..cce43fe994b
--- /dev/null
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module ImportExport
+    class RepoSaver
+      include Gitlab::ImportExport::CommandLineUtil
+
+      attr_reader :full_path
+
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+      end
+
+      def save
+        return false if @project.empty_repo?
+
+        @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename)
+        bundle_to_disk
+      end
+
+      private
+
+      def bundle_to_disk
+        FileUtils.mkdir_p(@shared.export_path)
+        git_bundle(repo_path: path_to_repo, bundle_path: @full_path)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      def path_to_repo
+        @project.repository.path_to_repo
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
new file mode 100644
index 00000000000..f38229c6c59
--- /dev/null
+++ b/lib/gitlab/import_export/saver.rb
@@ -0,0 +1,42 @@
+module Gitlab
+  module ImportExport
+    class Saver
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def self.save(*args)
+        new(*args).save
+      end
+
+      def initialize(shared:)
+        @shared = shared
+      end
+
+      def save
+        if compress_and_save
+          remove_export_path
+          Rails.logger.info("Saved project export #{archive_file}")
+          archive_file
+        else
+          false
+        end
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def compress_and_save
+        tar_czf(archive: archive_file, dir: @shared.export_path)
+      end
+
+      def remove_export_path
+        FileUtils.rm_rf(@shared.export_path)
+      end
+
+      def archive_file
+        @archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
new file mode 100644
index 00000000000..6aff05b886a
--- /dev/null
+++ b/lib/gitlab/import_export/shared.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module ImportExport
+    class Shared
+
+      attr_reader :errors, :opts
+
+      def initialize(opts)
+        @opts = opts
+        @errors = []
+      end
+
+      def export_path
+        @export_path ||= Gitlab::ImportExport.export_path(relative_path: opts[:relative_path])
+      end
+
+      def error(error)
+        error_out(error.message, caller[0].dup)
+        @errors << error.message
+        # Debug:
+        Rails.logger.error(error.backtrace)
+      end
+
+      private
+
+      def error_out(message, caller)
+        Rails.logger.error("Import/Export error raised on #{caller}: #{message}")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb
new file mode 100644
index 00000000000..df19354b76e
--- /dev/null
+++ b/lib/gitlab/import_export/uploads_restorer.rb
@@ -0,0 +1,14 @@
+module Gitlab
+  module ImportExport
+    class UploadsRestorer < UploadsSaver
+      def restore
+        return true unless File.directory?(uploads_export_path)
+
+        copy_files(uploads_export_path, uploads_path)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb
new file mode 100644
index 00000000000..7292e9d9712
--- /dev/null
+++ b/lib/gitlab/import_export/uploads_saver.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module ImportExport
+    class UploadsSaver
+
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+      end
+
+      def save
+        return true unless File.directory?(uploads_path)
+
+        copy_files(uploads_path, uploads_export_path)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def copy_files(source, destination)
+        FileUtils.mkdir_p(destination)
+        FileUtils.copy_entry(source, destination)
+        true
+      end
+
+      def uploads_export_path
+        File.join(@shared.export_path, 'uploads')
+      end
+
+      def uploads_path
+        File.join(Rails.root.join('public/uploads'), @project.path_with_namespace)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb
new file mode 100644
index 00000000000..cf5c62c5e3c
--- /dev/null
+++ b/lib/gitlab/import_export/version_checker.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module ImportExport
+    class VersionChecker
+
+      def self.check!(*args)
+        new(*args).check!
+      end
+
+      def initialize(shared:)
+        @shared = shared
+      end
+
+      def check!
+        version = File.open(version_file, &:readline)
+        verify_version!(version)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def version_file
+        File.join(@shared.export_path, Gitlab::ImportExport.version_filename)
+      end
+
+      def verify_version!(version)
+        if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version)
+          raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}")
+        else
+          true
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb
new file mode 100644
index 00000000000..f7f73dc9343
--- /dev/null
+++ b/lib/gitlab/import_export/version_saver.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module ImportExport
+    class VersionSaver
+
+      def initialize(shared:)
+        @shared = shared
+      end
+
+      def save
+        FileUtils.mkdir_p(@shared.export_path)
+
+        File.write(version_file, Gitlab::ImportExport.version, mode: 'w')
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def version_file
+        File.join(@shared.export_path, Gitlab::ImportExport.version_filename)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
new file mode 100644
index 00000000000..1eedae39f8a
--- /dev/null
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -0,0 +1,33 @@
+module Gitlab
+  module ImportExport
+    class WikiRepoSaver < RepoSaver
+      def save
+        @wiki = ProjectWiki.new(@project)
+        return true unless wiki_repository_exists? # it's okay to have no Wiki
+        bundle_to_disk(File.join(@shared.export_path, project_filename))
+      end
+
+      def bundle_to_disk(full_path)
+        FileUtils.mkdir_p(@shared.export_path)
+        git_bundle(repo_path: path_to_repo, bundle_path: full_path)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def project_filename
+        "project.wiki.bundle"
+      end
+
+      def path_to_repo
+        @wiki.repository.path_to_repo
+      end
+
+      def wiki_repository_exists?
+        File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb
index ccfdfbe73e8..948d43582cf 100644
--- a/lib/gitlab/import_sources.rb
+++ b/lib/gitlab/import_sources.rb
@@ -20,7 +20,8 @@ module Gitlab
           'Gitorious.org'   => 'gitorious',
           'Google Code'     => 'google_code',
           'FogBugz'         => 'fogbugz',
-          'Any repo by URL' => 'git',
+          'Repo by URL'     => 'git',
+          'GitLab export' => 'gitlab_project'
         }
       end
 
diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index 0f115893a15..dcec7543c13 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -56,7 +56,7 @@ module Gitlab
         end
       end
 
-      # Instruments all public methods of a module.
+      # Instruments all public and private methods of a module.
       #
       # This method optionally takes a block that can be used to determine if a
       # method should be instrumented or not. The block is passed the receiving
@@ -65,7 +65,8 @@ module Gitlab
       #
       # mod - The module to instrument.
       def self.instrument_methods(mod)
-        mod.public_methods(false).each do |name|
+        methods = mod.methods(false) + mod.private_methods(false)
+        methods.each do |name|
           method = mod.method(name)
 
           if method.owner == mod.singleton_class
@@ -76,13 +77,14 @@ module Gitlab
         end
       end
 
-      # Instruments all public instance methods of a module.
+      # Instruments all public and private instance methods of a module.
       #
       # See `instrument_methods` for more information.
       #
       # mod - The module to instrument.
       def self.instrument_instance_methods(mod)
-        mod.public_instance_methods(false).each do |name|
+        methods = mod.instance_methods(false) + mod.private_instance_methods(false)
+        methods.each do |name|
           method = mod.instance_method(name)
 
           if method.owner == mod
@@ -146,20 +148,8 @@ module Gitlab
 
         proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1
           def #{name}(#{args_signature})
-            trans = Gitlab::Metrics::Instrumentation.transaction
-
-            if trans
-              start    = Time.now
-              retval   = super
-              duration = (Time.now - start) * 1000.0
-
-              if duration >= Gitlab::Metrics.method_call_threshold
-                trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES,
-                                 { duration: duration },
-                                 method: #{label.inspect})
-              end
-
-              retval
+            if trans = Gitlab::Metrics::Instrumentation.transaction
+              trans.measure_method(#{label.inspect}) { super }
             else
               super
             end
diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb
new file mode 100644
index 00000000000..faf0d9b6318
--- /dev/null
+++ b/lib/gitlab/metrics/method_call.rb
@@ -0,0 +1,52 @@
+module Gitlab
+  module Metrics
+    # Class for tracking timing information about method calls
+    class MethodCall
+      attr_reader :real_time, :cpu_time, :call_count
+
+      # name - The full name of the method (including namespace) such as
+      #        `User#sign_in`.
+      #
+      # series - The series to use for storing the data.
+      def initialize(name, series)
+        @name = name
+        @series = series
+        @real_time = 0.0
+        @cpu_time = 0.0
+        @call_count = 0
+      end
+
+      # Measures the real and CPU execution time of the supplied block.
+      def measure
+        start_real = Time.now
+        start_cpu = System.cpu_time
+        retval = yield
+
+        @real_time += (Time.now - start_real) * 1000.0
+        @cpu_time += System.cpu_time.to_f - start_cpu
+        @call_count += 1
+
+        retval
+      end
+
+      # Returns a Metric instance of the current method call.
+      def to_metric
+        Metric.new(
+          @series,
+          {
+            duration:     real_time,
+            cpu_duration: cpu_time,
+            call_count:   call_count
+          },
+          method: @name
+        )
+      end
+
+      # Returns true if the total runtime of this method exceeds the method call
+      # threshold.
+      def above_threshold?
+        real_time >= Metrics.method_call_threshold
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index 6f179789d3e..e61670f491c 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -1,8 +1,9 @@
 module Gitlab
   module Metrics
-    # Rack middleware for tracking Rails requests.
+    # Rack middleware for tracking Rails and Grape requests.
     class RackMiddleware
       CONTROLLER_KEY = 'action_controller.instance'
+      ENDPOINT_KEY   = 'api.endpoint'
 
       def initialize(app)
         @app = app
@@ -21,6 +22,8 @@ module Gitlab
         ensure
           if env[CONTROLLER_KEY]
             tag_controller(trans, env)
+          elsif env[ENDPOINT_KEY]
+            tag_endpoint(trans, env)
           end
 
           trans.finish
@@ -32,7 +35,7 @@ module Gitlab
       def transaction_from_env(env)
         trans = Transaction.new
 
-        trans.set(:request_uri, env['REQUEST_URI'])
+        trans.set(:request_uri, filtered_path(env))
         trans.set(:request_method, env['REQUEST_METHOD'])
 
         trans
@@ -42,6 +45,30 @@ module Gitlab
         controller   = env[CONTROLLER_KEY]
         trans.action = "#{controller.class.name}##{controller.action_name}"
       end
+
+      def tag_endpoint(trans, env)
+        endpoint = env[ENDPOINT_KEY]
+        path = endpoint_paths_cache[endpoint.route.route_method][endpoint.route.route_path]
+        trans.action = "Grape##{endpoint.route.route_method} #{path}"
+      end
+
+      private
+
+      def filtered_path(env)
+        ActionDispatch::Request.new(env).filtered_path.presence || env['REQUEST_URI']
+      end
+
+      def endpoint_paths_cache
+        @endpoint_paths_cache ||= Hash.new do |hash, http_method|
+          hash[http_method] = Hash.new do |inner_hash, raw_path|
+            inner_hash[raw_path] = endpoint_instrumentable_path(raw_path)
+          end
+        end
+      end
+
+      def endpoint_instrumentable_path(raw_path)
+        raw_path.sub('(.:format)', '').sub('/:version', '')
+      end
     end
   end
 end
diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb
index fc709222a9b..0000450d9bb 100644
--- a/lib/gitlab/metrics/sampler.rb
+++ b/lib/gitlab/metrics/sampler.rb
@@ -66,7 +66,11 @@ module Gitlab
         def sample_objects
           sample = Allocations.to_hash
           counts = sample.each_with_object({}) do |(klass, count), hash|
-            hash[klass.name] = count
+            name = klass.name
+
+            next unless name
+
+            hash[name] = count
           end
 
           # Symbols aren't allocated so we'll need to add those manually.
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 2578ddc49f4..4bc5081aa03 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -4,7 +4,7 @@ module Gitlab
     class Transaction
       THREAD_KEY = :_gitlab_metrics_transaction
 
-      attr_reader :tags, :values
+      attr_reader :tags, :values, :methods
 
       attr_accessor :action
 
@@ -16,6 +16,7 @@ module Gitlab
       #          plus method name.
       def initialize(action = nil)
         @metrics = []
+        @methods = {}
 
         @started_at  = nil
         @finished_at = nil
@@ -51,9 +52,23 @@ module Gitlab
       end
 
       def add_metric(series, values, tags = {})
-        prefix = sidekiq? ? 'sidekiq_' : 'rails_'
+        @metrics << Metric.new("#{series_prefix}#{series}", values, tags)
+      end
+
+      # Measures the time it takes to execute a method.
+      #
+      # Multiple calls to the same method add up to the total runtime of the
+      # method.
+      #
+      # name - The full name of the method to measure (e.g. `User#sign_in`).
+      def measure_method(name, &block)
+        unless @methods[name]
+          series = "#{series_prefix}#{Instrumentation::SERIES}"
+
+          @methods[name] = MethodCall.new(name, series)
+        end
 
-        @metrics << Metric.new("#{prefix}#{series}", values, tags)
+        @methods[name].measure(&block)
       end
 
       def increment(name, value)
@@ -84,7 +99,13 @@ module Gitlab
       end
 
       def submit
-        metrics = @metrics.map do |metric|
+        submit = @metrics.dup
+
+        @methods.each do |name, method|
+          submit << method.to_metric if method.above_threshold?
+        end
+
+        submit_hashes = submit.map do |metric|
           hash = metric.to_hash
 
           hash[:tags][:action] ||= @action if @action
@@ -92,12 +113,16 @@ module Gitlab
           hash
         end
 
-        Metrics.submit_metrics(metrics)
+        Metrics.submit_metrics(submit_hashes)
       end
 
       def sidekiq?
         Sidekiq.server?
       end
+
+      def series_prefix
+        sidekiq? ? 'sidekiq_' : 'rails_'
+      end
     end
   end
 end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 1cbd6d945a0..c84c68f96f6 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -100,5 +100,13 @@ module Gitlab
     def container_registry_reference_regex
       git_reference_regex
     end
+
+    def environment_name_regex
+      @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze
+    end
+
+    def environment_name_regex_message
+      "can contain only letters, digits, '-' and '_'."
+    end
   end
 end
diff --git a/lib/gitlab/sanitizers/svg.rb b/lib/gitlab/sanitizers/svg.rb
index 5e95f6c0529..8304b9a482c 100644
--- a/lib/gitlab/sanitizers/svg.rb
+++ b/lib/gitlab/sanitizers/svg.rb
@@ -12,23 +12,45 @@ module Gitlab
         def scrub(node)
           unless Whitelist::ALLOWED_ELEMENTS.include?(node.name)
             node.unlink
-          else
-            node.attributes.each do |attr_name, attr|
-              valid_attributes = Whitelist::ALLOWED_ATTRIBUTES[node.name]
-
-              unless valid_attributes && valid_attributes.include?(attr_name)
-                if Whitelist::ALLOWED_DATA_ATTRIBUTES_IN_ELEMENTS.include?(node.name) &&
-                    attr_name.start_with?('data-')
-                  # Arbitrary data attributes are allowed. Verify that the attribute
-                  # is a valid data attribute.
-                  attr.unlink unless attr_name =~ DATA_ATTR_PATTERN
-                else
-                  attr.unlink
-                end
+            return
+          end
+
+          valid_attributes = Whitelist::ALLOWED_ATTRIBUTES[node.name]
+          return unless valid_attributes
+
+          node.attribute_nodes.each do |attr|
+            attr_name = attribute_name_with_namespace(attr)
+
+            if valid_attributes.include?(attr_name)
+              attr.unlink if unsafe_href?(attr)
+            else
+              # Arbitrary data attributes are allowed.
+              unless allows_data_attribute?(node) && data_attribute?(attr)
+                attr.unlink
               end
             end
           end
         end
+
+        def attribute_name_with_namespace(attr)
+          if attr.namespace
+            "#{attr.namespace.prefix}:#{attr.name}"
+          else
+            attr.name
+          end
+        end
+
+        def allows_data_attribute?(node)
+          Whitelist::ALLOWED_DATA_ATTRIBUTES_IN_ELEMENTS.include?(node.name)
+        end
+
+        def unsafe_href?(attr)
+          attribute_name_with_namespace(attr) == 'xlink:href' && !attr.value.start_with?('#')
+        end
+
+        def data_attribute?(attr)
+          attr.name.start_with?('data-') && attr.name =~ DATA_ATTR_PATTERN && attr.namespace.nil?
+        end
       end
     end
   end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 388f84dbe0e..40e8299c36b 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -8,7 +8,7 @@ module Gitlab
     class << self
       def git_http_ok(repository, user)
         {
-          'GL_ID' => Gitlab::ShellEnv.gl_id(user),
+          'GL_ID' => Gitlab::GlId.gl_id(user),
           'RepoPath' => repository.path_to_repo,
         }
       end