Merge branch 'master' of https://gitlab.com/gitlab-org/gitlab-ce into remove-grack-lfs

98 files changed, 1746 insertions, 786 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 3d7d67510a8..bd16806892b 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -7,6 +7,10 @@ module API
       rack_response({ 'message' => '404 Not found' }.to_json, 404)
     end
 
+    rescue_from Grape::Exceptions::ValidationErrors do |e|
+      error!({ messages: e.full_messages }, 400)
+    end
+
     rescue_from :all do |exception|
       # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60
       # why is this not wrapped in something reusable?
@@ -32,6 +36,7 @@ module API
     mount ::API::CommitStatuses
     mount ::API::Commits
     mount ::API::DeployKeys
+    mount ::API::Environments
     mount ::API::Files
     mount ::API::GroupMembers
     mount ::API::Groups
diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index 66b853eb342..a77afe634f6 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -35,6 +35,10 @@ module API
 
       # Protect a single branch
       #
+      # Note: The internal data model moved from `developers_can_{merge,push}` to `allowed_to_{merge,push}`
+      # in `gitlab-org/gitlab-ce!5081`. The API interface has not been changed (to maintain compatibility),
+      # but it works with the changed data model to infer `developers_can_merge` and `developers_can_push`.
+      #
       # Parameters:
       #   id (required) - The ID of a project
       #   branch (required) - The name of the branch
@@ -49,17 +53,36 @@ module API
         @branch = user_project.repository.find_branch(params[:branch])
         not_found!('Branch') unless @branch
         protected_branch = user_project.protected_branches.find_by(name: @branch.name)
-        developers_can_push = to_boolean(params[:developers_can_push])
+
         developers_can_merge = to_boolean(params[:developers_can_merge])
+        developers_can_push = to_boolean(params[:developers_can_push])
+
+        protected_branch_params = {
+          name: @branch.name
+        }
+
+        unless developers_can_merge.nil?
+          protected_branch_params.merge!({
+            merge_access_level_attributes: {
+              access_level: developers_can_merge ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER
+            }
+          })
+        end
+
+        unless developers_can_push.nil?
+          protected_branch_params.merge!({
+            push_access_level_attributes: {
+              access_level: developers_can_push ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER
+            }
+          })
+        end
 
         if protected_branch
-          protected_branch.developers_can_push = developers_can_push unless developers_can_push.nil?
-          protected_branch.developers_can_merge = developers_can_merge unless developers_can_merge.nil?
-          protected_branch.save
+          service = ProtectedBranches::UpdateService.new(user_project, current_user, protected_branch_params)
+          service.execute(protected_branch)
         else
-          user_project.protected_branches.create(name: @branch.name,
-                                                 developers_can_push: developers_can_push || false,
-                                                 developers_can_merge: developers_can_merge || false)
+          service = ProtectedBranches::CreateService.new(user_project, current_user, protected_branch_params)
+          service.execute
         end
 
         present @branch, with: Entities::RepoBranch, project: user_project
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 4a11c8e3620..b4eaf1813d4 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -54,7 +54,7 @@ module API
         sha = params[:sha]
         commit = user_project.commit(sha)
         not_found! "Commit" unless commit
-        commit.diffs.to_a
+        commit.raw_diffs.to_a
       end
 
       # Get a commit's comments
@@ -96,7 +96,7 @@ module API
         }
 
         if params[:path] && params[:line] && params[:line_type]
-          commit.diffs(all_diffs: true).each do |diff|
+          commit.raw_diffs(all_diffs: true).each do |diff|
             next unless diff.new_path == params[:path]
             lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line)
 
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb
index 5c570b5e5ca..825e05fbae3 100644
--- a/lib/api/deploy_keys.rb
+++ b/lib/api/deploy_keys.rb
@@ -10,6 +10,9 @@ module API
       present keys, with: Entities::SSHKey
     end
 
+    params do
+      requires :id, type: String, desc: 'The ID of the project'
+    end
     resource :projects do
       before { authorize_admin_project }
 
@@ -17,52 +20,43 @@ module API
       # Use "projects/:id/deploy_keys/..." instead.
       #
       %w(keys deploy_keys).each do |path|
-        # Get a specific project's deploy keys
-        #
-        # Example Request:
-        #   GET /projects/:id/deploy_keys
+        desc "Get a specific project's deploy keys" do
+          success Entities::SSHKey
+        end
         get ":id/#{path}" do
           present user_project.deploy_keys, with: Entities::SSHKey
         end
 
-        # Get single deploy key owned by currently authenticated user
-        #
-        # Example Request:
-        #   GET /projects/:id/deploy_keys/:key_id
+        desc 'Get single deploy key' do
+          success Entities::SSHKey
+        end
+        params do
+          requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+        end
         get ":id/#{path}/:key_id" do
           key = user_project.deploy_keys.find params[:key_id]
           present key, with: Entities::SSHKey
         end
 
-        # Add new deploy key to currently authenticated user
-        # If deploy key already exists - it will be joined to project
-        # but only if original one was accessible by same user
-        #
-        # Parameters:
-        #   key (required) - New deploy Key
-        #   title (required) - New deploy Key's title
-        # Example Request:
-        #   POST /projects/:id/deploy_keys
+        # TODO: for 9.0 we should check if params are there with the params block
+        # grape provides, at this point we'd change behaviour so we can't
+        # Behaviour now if you don't provide all required params: it renders a
+        # validation error or two.
+        desc 'Add new deploy key to currently authenticated user' do
+          success Entities::SSHKey
+        end
         post ":id/#{path}" do
           attrs = attributes_for_keys [:title, :key]
+          attrs[:key].strip! if attrs[:key]
 
-          if attrs[:key].present?
-            attrs[:key].strip!
-
-            # check if key already exist in project
-            key = user_project.deploy_keys.find_by(key: attrs[:key])
-            if key
-              present key, with: Entities::SSHKey
-              next
-            end
+          key = user_project.deploy_keys.find_by(key: attrs[:key])
+          present key, with: Entities::SSHKey if key
 
-            # Check for available deploy keys in other projects
-            key = current_user.accessible_deploy_keys.find_by(key: attrs[:key])
-            if key
-              user_project.deploy_keys << key
-              present key, with: Entities::SSHKey
-              next
-            end
+          # Check for available deploy keys in other projects
+          key = current_user.accessible_deploy_keys.find_by(key: attrs[:key])
+          if key
+            user_project.deploy_keys << key
+            present key, with: Entities::SSHKey
           end
 
           key = DeployKey.new attrs
@@ -74,12 +68,46 @@ module API
           end
         end
 
-        # Delete existing deploy key of currently authenticated user
-        #
-        # Example Request:
-        #   DELETE /projects/:id/deploy_keys/:key_id
+        desc 'Enable a deploy key for a project' do
+          detail 'This feature was added in GitLab 8.11'
+          success Entities::SSHKey
+        end
+        params do
+          requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+        end
+        post ":id/#{path}/:key_id/enable" do
+          key = ::Projects::EnableDeployKeyService.new(user_project,
+                                                        current_user, declared(params)).execute
+
+          if key
+            present key, with: Entities::SSHKey
+          else
+            not_found!('Deploy Key')
+          end
+        end
+
+        desc 'Disable a deploy key for a project' do
+          detail 'This feature was added in GitLab 8.11'
+          success Entities::SSHKey
+        end
+        params do
+          requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+        end
+        delete ":id/#{path}/:key_id/disable" do
+          key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id])
+          key.destroy
+
+          present key.deploy_key, with: Entities::SSHKey
+        end
+
+        desc 'Delete existing deploy key of currently authenticated user' do
+          success Key
+        end
+        params do
+          requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+        end
         delete ":id/#{path}/:key_id" do
-          key = user_project.deploy_keys.find params[:key_id]
+          key = user_project.deploy_keys.find(params[:key_id])
           key.destroy
         end
       end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index fbf0d74663f..e5b00dc45a5 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -126,11 +126,13 @@ module API
       end
 
       expose :developers_can_push do |repo_branch, options|
-        options[:project].developers_can_push_to_protected_branch? repo_branch.name
+        project = options[:project]
+        project.protected_branches.matching(repo_branch.name).any? { |protected_branch| protected_branch.push_access_level.access_level == Gitlab::Access::DEVELOPER }
       end
 
       expose :developers_can_merge do |repo_branch, options|
-        options[:project].developers_can_merge_to_protected_branch? repo_branch.name
+        project = options[:project]
+        project.protected_branches.matching(repo_branch.name).any? { |protected_branch| protected_branch.merge_access_level.access_level == Gitlab::Access::DEVELOPER }
       end
     end
 
@@ -149,8 +151,13 @@ module API
       expose :safe_message, as: :message
     end
 
+    class RepoCommitStats < Grape::Entity
+      expose :additions, :deletions, :total
+    end
+
     class RepoCommitDetail < RepoCommit
       expose :parent_ids, :committed_date, :authored_date
+      expose :stats, using: Entities::RepoCommitStats
       expose :status
     end
 
@@ -217,7 +224,7 @@ module API
 
     class MergeRequestChanges < MergeRequest
       expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _|
-        compare.diffs(all_diffs: true).to_a
+        compare.raw_diffs(all_diffs: true).to_a
       end
     end
 
@@ -489,6 +496,10 @@ module API
       expose :key, :value
     end
 
+    class Environment < Grape::Entity
+      expose :id, :name, :external_url
+    end
+
     class RepoLicense < Grape::Entity
       expose :key, :name, :nickname
       expose :featured, as: :popular
diff --git a/lib/api/environments.rb b/lib/api/environments.rb
new file mode 100644
index 00000000000..819f80d8365
--- /dev/null
+++ b/lib/api/environments.rb
@@ -0,0 +1,83 @@
+module API
+  # Environments RESTfull API endpoints
+  class Environments < Grape::API
+    before { authenticate! }
+
+    params do
+      requires :id, type: String, desc: 'The project ID'
+    end
+    resource :projects do
+      desc 'Get all environments of the project' do
+        detail 'This feature was introduced in GitLab 8.11.'
+        success Entities::Environment
+      end
+      params do
+        optional :page,     type: Integer, desc: 'Page number of the current request'
+        optional :per_page, type: Integer, desc: 'Number of items per page'
+      end
+      get ':id/environments' do
+        authorize! :read_environment, user_project
+
+        present paginate(user_project.environments), with: Entities::Environment
+      end
+
+      desc 'Creates a new environment' do
+        detail 'This feature was introduced in GitLab 8.11.'
+        success Entities::Environment
+      end
+      params do
+        requires :name,           type: String,   desc: 'The name of the environment to be created'
+        optional :external_url,   type: String,   desc: 'URL on which this deployment is viewable'
+      end
+      post ':id/environments' do
+        authorize! :create_environment, user_project
+
+        create_params = declared(params, include_parent_namespaces: false).to_h
+        environment = user_project.environments.create(create_params)
+
+        if environment.persisted?
+          present environment, with: Entities::Environment
+        else
+          render_validation_error!(environment)
+        end
+      end
+
+      desc 'Updates an existing environment' do
+        detail 'This feature was introduced in GitLab 8.11.'
+        success Entities::Environment
+      end
+      params do
+        requires :environment_id, type: Integer,  desc: 'The environment ID'
+        optional :name,           type: String,   desc: 'The new environment name'
+        optional :external_url,   type: String,   desc: 'The new URL on which this deployment is viewable'
+      end
+      put ':id/environments/:environment_id' do
+        authorize! :update_environment, user_project
+
+        environment = user_project.environments.find(params[:environment_id])
+        
+        update_params = declared(params, include_missing: false).extract!(:name, :external_url).to_h
+        if environment.update(update_params)
+          present environment, with: Entities::Environment
+        else
+          render_validation_error!(environment)
+        end
+      end
+
+      desc 'Deletes an existing environment' do
+        detail 'This feature was introduced in GitLab 8.11.'
+        success Entities::Environment
+      end
+      params do
+        requires :environment_id, type: Integer,  desc: 'The environment ID'
+      end
+      delete ':id/environments/:environment_id' do
+        authorize! :update_environment, user_project
+
+        environment = user_project.environments.find(params[:environment_id])
+
+        present environment.destroy, with: Entities::Environment
+      end
+    end
+  end
+end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index c588103e517..c4d3134da6c 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -21,17 +21,6 @@ module API
       def filter_issues_milestone(issues, milestone)
         issues.includes(:milestone).where('milestones.title' => milestone)
       end
-
-      def create_spam_log(project, current_user, attrs)
-        params = attrs.merge({
-          source_ip: client_ip(env),
-          user_agent: user_agent(env),
-          noteable_type: 'Issue',
-          via_api: true
-        })
-
-        ::CreateSpamLogService.new(project, current_user, params).execute
-      end
     end
 
     resource :issues do
@@ -168,15 +157,13 @@ module API
         end
 
         project = user_project
-        text = [attrs[:title], attrs[:description]].reject(&:blank?).join("\n")
 
-        if check_for_spam?(project, current_user) && is_spam?(env, current_user, text)
-          create_spam_log(project, current_user, attrs)
+        issue = ::Issues::CreateService.new(project, current_user, attrs.merge(request: request, api: true)).execute
+
+        if issue.spam?
           render_api_error!({ error: 'Spam detected' }, 400)
         end
 
-        issue = ::Issues::CreateService.new(project, current_user, attrs).execute
-
         if issue.valid?
           # Find or create labels and attach to issue. Labels are valid because
           # we already checked its name, so there can't be an error here
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index b9773f98d75..1f5917b8127 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -54,10 +54,10 @@ module Backup
         # Move repos dir to 'repositories.old' dir
         bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s)
         FileUtils.mv(path, bk_repos_path)
+        # This is expected from gitlab:check
+        FileUtils.mkdir_p(path, mode: 2770)
       end
 
-      FileUtils.mkdir_p(repos_path)
-
       Project.find_each(batch_size: 1000) do |project|
         $progress.print " * #{project.path_with_namespace} ... "
 
diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb
index 9ed45707515..799b83b1069 100644
--- a/lib/banzai/filter/autolink_filter.rb
+++ b/lib/banzai/filter/autolink_filter.rb
@@ -31,6 +31,14 @@ module Banzai
       # Text matching LINK_PATTERN inside these elements will not be linked
       IGNORE_PARENTS = %w(a code kbd pre script style).to_set
 
+      # The XPath query to use for finding text nodes to parse.
+      TEXT_QUERY = %Q(descendant-or-self::text()[
+        not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')})
+        and contains(., '://')
+        and not(starts-with(., 'http'))
+        and not(starts-with(., 'ftp'))
+      ])
+
       def call
         return doc if context[:autolink] == false
 
@@ -66,16 +74,11 @@ module Banzai
       # Autolinks any text matching LINK_PATTERN that Rinku didn't already
       # replace
       def text_parse
-        search_text_nodes(doc).each do |node|
+        doc.xpath(TEXT_QUERY).each do |node|
           content = node.to_html
 
-          next if has_ancestor?(node, IGNORE_PARENTS)
           next unless content.match(LINK_PATTERN)
 
-          # If Rinku didn't link this, there's probably a good reason, so we'll
-          # skip it too
-          next if content.start_with?(*%w(http https ftp))
-
           html = autolink_filter(content)
 
           next if html == content
diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb
index ae7d31cf191..2492b5213ac 100644
--- a/lib/banzai/filter/emoji_filter.rb
+++ b/lib/banzai/filter/emoji_filter.rb
@@ -38,6 +38,11 @@ module Banzai
         end
       end
 
+      # Build a regexp that matches all valid :emoji: names.
+      def self.emoji_pattern
+        @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/
+      end
+
       private
 
       def emoji_url(name)
@@ -59,11 +64,6 @@ module Banzai
         ActionController::Base.helpers.url_to_image(image)
       end
 
-      # Build a regexp that matches all valid :emoji: names.
-      def self.emoji_pattern
-        @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/
-      end
-
       def emoji_pattern
         self.class.emoji_pattern
       end
diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb
index 9b209533a89..ff580ec68f8 100644
--- a/lib/banzai/filter/markdown_filter.rb
+++ b/lib/banzai/filter/markdown_filter.rb
@@ -12,7 +12,12 @@ module Banzai
         html
       end
 
-      private
+      def self.renderer
+        @renderer ||= begin
+          renderer = Redcarpet::Render::HTML.new
+          Redcarpet::Markdown.new(renderer, redcarpet_options)
+        end
+      end
 
       def self.redcarpet_options
         # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use
@@ -28,12 +33,7 @@ module Banzai
         }.freeze
       end
 
-      def self.renderer
-        @renderer ||= begin
-          renderer = Redcarpet::Render::HTML.new
-          Redcarpet::Markdown.new(renderer, redcarpet_options)
-        end
-      end
+      private_class_method :redcarpet_options
     end
   end
 end
diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb
index 21ed0410f7f..46762d401fb 100644
--- a/lib/banzai/filter/relative_link_filter.rb
+++ b/lib/banzai/filter/relative_link_filter.rb
@@ -20,7 +20,7 @@ module Banzai
           process_link_attr el.attribute('href')
         end
 
-        doc.search('img').each do |el|
+        doc.css('img, video').each do |el|
           process_link_attr el.attribute('src')
         end
 
@@ -35,6 +35,7 @@ module Banzai
 
       def process_link_attr(html_attr)
         return if html_attr.blank?
+        return if html_attr.value.start_with?('//')
 
         uri = URI(html_attr.value)
         if uri.relative? && uri.path.present?
@@ -87,10 +88,13 @@ module Banzai
       def build_relative_path(path, request_path)
         return request_path if path.empty?
         return path unless request_path
+        return path[1..-1] if path.start_with?('/')
 
         parts = request_path.split('/')
         parts.pop if uri_type(request_path) != :tree
 
+        path.sub!(%r{\A\./}, '')
+
         while path.start_with?('../')
           parts.pop
           path.sub!('../', '')
diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb
index 91f0159f9a1..fcdb496aed2 100644
--- a/lib/banzai/filter/syntax_highlight_filter.rb
+++ b/lib/banzai/filter/syntax_highlight_filter.rb
@@ -17,15 +17,12 @@ module Banzai
 
       def highlight_node(node)
         language = node.attr('class')
-        code     = node.text
-
+        code = node.text
         css_classes = "code highlight"
-
-        lexer = Rouge::Lexer.find_fancy(language) || Rouge::Lexers::PlainText
-        formatter = Rouge::Formatters::HTML.new
+        lexer = lexer_for(language)
 
         begin
-          code = formatter.format(lexer.lex(code))
+          code = format(lex(lexer, code))
 
           css_classes << " js-syntax-highlight #{lexer.tag}"
         rescue
@@ -41,14 +38,27 @@ module Banzai
 
       private
 
+      # Separate method so it can be instrumented.
+      def lex(lexer, code)
+        lexer.lex(code)
+      end
+
+      def format(tokens)
+        rouge_formatter.format(tokens)
+      end
+
+      def lexer_for(language)
+        (Rouge::Lexer.find(language) || Rouge::Lexers::PlainText).new
+      end
+
       def replace_parent_pre_element(node, highlighted)
         # Replace the parent `pre` element with the entire highlighted block
         node.parent.replace(highlighted)
       end
 
       # Override Rouge::Plugins::Redcarpet#rouge_formatter
-      def rouge_formatter(lexer)
-        Rouge::Formatters::HTML.new
+      def rouge_formatter(lexer = nil)
+        @rouge_formatter ||= Rouge::Formatters::HTML.new
       end
     end
   end
diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb
index fd8b9a6f0cc..ac7bbcb0d10 100644
--- a/lib/banzai/filter/video_link_filter.rb
+++ b/lib/banzai/filter/video_link_filter.rb
@@ -1,11 +1,9 @@
 module Banzai
   module Filter
-
     # Find every image that isn't already wrapped in an `a` tag, and that has
     # a `src` attribute ending with a video extension, add a new video node and
     # a "Download" link in the case the video cannot be played.
     class VideoLinkFilter < HTML::Pipeline::Filter
-
       def call
         doc.xpath(query).each do |el|
           el.replace(video_node(doc, el))
@@ -54,6 +52,5 @@ module Banzai
         container
       end
     end
-
   end
 end
diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb
index f306079d833..6c20dec5734 100644
--- a/lib/banzai/reference_parser/issue_parser.rb
+++ b/lib/banzai/reference_parser/issue_parser.rb
@@ -9,10 +9,11 @@ module Banzai
 
         issues = issues_for_nodes(nodes)
 
-        nodes.select do |node|
-          issue = issue_for_node(issues, node)
+        readable_issues = Ability.
+          issues_readable_by_user(issues.values, user).to_set
 
-          issue ? can?(user, :read_issue, issue) : false
+        nodes.select do |node|
+          readable_issues.include?(issue_for_node(issues, node))
         end
       end
 
diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb
index 910687a7b6a..a4ae27eefd8 100644
--- a/lib/banzai/renderer.rb
+++ b/lib/banzai/renderer.rb
@@ -1,5 +1,7 @@
 module Banzai
   module Renderer
+    extend self
+
     # Convert a Markdown String into an HTML-safe String of HTML
     #
     # Note that while the returned HTML will have been sanitized of dangerous
@@ -14,7 +16,7 @@ module Banzai
     # context  - Hash of context options passed to our HTML Pipeline
     #
     # Returns an HTML-safe String
-    def self.render(text, context = {})
+    def render(text, context = {})
       cache_key = context.delete(:cache_key)
       cache_key = full_cache_key(cache_key, context[:pipeline])
 
@@ -52,7 +54,7 @@ module Banzai
     #    texts_and_contexts
     #    => [{ text: '### Hello',
     #          context: { cache_key: [note, :note] } }]
-    def self.cache_collection_render(texts_and_contexts)
+    def cache_collection_render(texts_and_contexts)
       items_collection = texts_and_contexts.each_with_index do |item, index|
         context = item[:context]
         cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline])
@@ -81,7 +83,7 @@ module Banzai
       items_collection.map { |item| item[:rendered] }
     end
 
-    def self.render_result(text, context = {})
+    def render_result(text, context = {})
       text = Pipeline[:pre_process].to_html(text, context) if text
 
       Pipeline[context[:pipeline]].call(text, context)
@@ -100,7 +102,7 @@ module Banzai
     #            :user      - User object
     #
     # Returns an HTML-safe String
-    def self.post_process(html, context)
+    def post_process(html, context)
       context = Pipeline[context[:pipeline]].transform_context(context)
 
       pipeline = Pipeline[:post_process]
@@ -113,7 +115,7 @@ module Banzai
 
     private
 
-    def self.cacheless_render(text, context = {})
+    def cacheless_render(text, context = {})
       Gitlab::Metrics.measure(:banzai_cacheless_render) do
         result = render_result(text, context)
 
@@ -126,7 +128,7 @@ module Banzai
       end
     end
 
-    def self.full_cache_key(cache_key, pipeline_name)
+    def full_cache_key(cache_key, pipeline_name)
       return unless cache_key
       ["banzai", *cache_key, pipeline_name || :full]
     end
@@ -134,7 +136,7 @@ module Banzai
     # To map Rails.cache.read_multi results we need to know the Rails.cache.expanded_key.
     # Other option will be to generate stringified keys on our side and don't delegate to Rails.cache.expanded_key
     # method.
-    def self.full_cache_multi_key(cache_key, pipeline_name)
+    def full_cache_multi_key(cache_key, pipeline_name)
       return unless cache_key
       Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name))
     end
diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb
index 1d7126a432d..3decc3b1a26 100644
--- a/lib/ci/charts.rb
+++ b/lib/ci/charts.rb
@@ -1,5 +1,37 @@
 module Ci
   module Charts
+    module DailyInterval
+      def grouped_count(query)
+        query.
+          group("DATE(#{Ci::Build.table_name}.created_at)").
+          count(:created_at).
+          transform_keys { |date| date.strftime(@format) }
+      end
+
+      def interval_step
+        @interval_step ||= 1.day
+      end
+    end
+
+    module MonthlyInterval
+      def grouped_count(query)
+        if Gitlab::Database.postgresql?
+          query.
+            group("to_char(#{Ci::Build.table_name}.created_at, '01 Month YYYY')").
+            count(:created_at).
+            transform_keys(&:squish)
+        else
+          query.
+            group("DATE_FORMAT(#{Ci::Build.table_name}.created_at, '01 %M %Y')").
+            count(:created_at)
+        end
+      end
+
+      def interval_step
+        @interval_step ||= 1.month
+      end
+    end
+
     class Chart
       attr_reader :labels, :total, :success, :project, :build_times
 
@@ -13,47 +45,59 @@ module Ci
         collect
       end
 
-      def push(from, to, format)
-        @labels << from.strftime(format)
-        @total << project.builds.
-          where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", to, from).
-          count(:all)
-        @success << project.builds.
-          where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", to, from).
-          success.count(:all)
+      def collect
+        query = project.builds.
+          where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", @to, @from)
+
+        totals_count  = grouped_count(query)
+        success_count = grouped_count(query.success)
+
+        current = @from
+        while current < @to
+          label = current.strftime(@format)
+
+          @labels  << label
+          @total   << (totals_count[label] || 0)
+          @success << (success_count[label] || 0)
+
+          current += interval_step
+        end
       end
     end
 
     class YearChart < Chart
-      def collect
-        13.times do |i|
-          start_month = (Date.today.years_ago(1) + i.month).beginning_of_month
-          end_month = start_month.end_of_month
+      include MonthlyInterval
 
-          push(start_month, end_month, "%d %B %Y")
-        end
+      def initialize(*)
+        @to     = Date.today.end_of_month
+        @from   = @to.years_ago(1).beginning_of_month
+        @format = '%d %B %Y'
+
+        super
       end
     end
 
     class MonthChart < Chart
-      def collect
-        30.times do |i|
-          start_day = Date.today - 30.days + i.days
-          end_day = Date.today - 30.days + i.day + 1.day
+      include DailyInterval
 
-          push(start_day, end_day, "%d %B")
-        end
+      def initialize(*)
+        @to     = Date.today
+        @from   = @to - 30.days
+        @format = '%d %B'
+
+        super
       end
     end
 
     class WeekChart < Chart
-      def collect
-        7.times do |i|
-          start_day = Date.today - 7.days + i.days
-          end_day = Date.today - 7.days + i.day + 1.day
+      include DailyInterval
 
-          push(start_day, end_day, "%d %B")
-        end
+      def initialize(*)
+        @to     = Date.today
+        @from   = @to - 7.days
+        @format = '%d %B'
+
+        super
       end
     end
 
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 83afed9f49f..a2e8bd22a52 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -4,21 +4,11 @@ module Ci
 
     include Gitlab::Ci::Config::Node::LegacyValidationHelpers
 
-    DEFAULT_STAGE = 'test'
-    ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache]
-    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services,
-                        :allow_failure, :type, :stage, :when, :artifacts, :cache,
-                        :dependencies, :before_script, :after_script, :variables,
-                        :environment]
-    ALLOWED_CACHE_KEYS = [:key, :untracked, :paths]
-    ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in]
-
     attr_reader :path, :cache, :stages
 
     def initialize(config, path = nil)
       @ci_config = Gitlab::Ci::Config.new(config)
       @config = @ci_config.to_hash
-
       @path = path
 
       unless @ci_config.valid?
@@ -26,7 +16,6 @@ module Ci
       end
 
       initial_parsing
-      validate!
     rescue Gitlab::Ci::Config::Loader::FormatError => e
       raise ValidationError, e.message
     end
@@ -73,7 +62,7 @@ module Ci
         #  - before script should be a concatenated command
         commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"),
         tag_list: job[:tags] || [],
-        name: name,
+        name: job[:name],
         allow_failure: job[:allow_failure] || false,
         when: job[:when] || 'on_success',
         environment: job[:environment],
@@ -92,6 +81,9 @@ module Ci
     private
 
     def initial_parsing
+      ##
+      # Global config
+      #
       @before_script = @ci_config.before_script
       @image = @ci_config.image
       @after_script = @ci_config.after_script
@@ -100,34 +92,28 @@ module Ci
       @stages = @ci_config.stages
       @cache = @ci_config.cache
 
-      @jobs = {}
-
-      @config.except!(*ALLOWED_YAML_KEYS)
-      @config.each { |name, param| add_job(name, param) }
-
-      raise ValidationError, "Please define at least one job" if @jobs.none?
-    end
-
-    def add_job(name, job)
-      return if name.to_s.start_with?('.')
+      ##
+      # Jobs
+      #
+      @jobs = @ci_config.jobs
 
-      raise ValidationError, "Unknown parameter: #{name}" unless job.is_a?(Hash) && job.has_key?(:script)
+      @jobs.each do |name, job|
+        # logical validation for job
 
-      stage = job[:stage] || job[:type] || DEFAULT_STAGE
-      @jobs[name] = { stage: stage }.merge(job)
+        validate_job_stage!(name, job)
+        validate_job_dependencies!(name, job)
+      end
     end
 
     def yaml_variables(name)
-      variables = global_variables.merge(job_variables(name))
+      variables = (@variables || {})
+        .merge(job_variables(name))
+
       variables.map do |key, value|
         { key: key, value: value, public: true }
       end
     end
 
-    def global_variables
-      @variables || {}
-    end
-
     def job_variables(name)
       job = @jobs[name.to_sym]
       return {} unless job
@@ -135,154 +121,16 @@ module Ci
       job[:variables] || {}
     end
 
-    def validate!
-      @jobs.each do |name, job|
-        validate_job!(name, job)
-      end
-
-      true
-    end
-
-    def validate_job!(name, job)
-      validate_job_name!(name)
-      validate_job_keys!(name, job)
-      validate_job_types!(name, job)
-      validate_job_script!(name, job)
-
-      validate_job_stage!(name, job) if job[:stage]
-      validate_job_variables!(name, job) if job[:variables]
-      validate_job_cache!(name, job) if job[:cache]
-      validate_job_artifacts!(name, job) if job[:artifacts]
-      validate_job_dependencies!(name, job) if job[:dependencies]
-    end
-
-    def validate_job_name!(name)
-      if name.blank? || !validate_string(name)
-        raise ValidationError, "job name should be non-empty string"
-      end
-    end
-
-    def validate_job_keys!(name, job)
-      job.keys.each do |key|
-        unless ALLOWED_JOB_KEYS.include? key
-          raise ValidationError, "#{name} job: unknown parameter #{key}"
-        end
-      end
-    end
-
-    def validate_job_types!(name, job)
-      if job[:image] && !validate_string(job[:image])
-        raise ValidationError, "#{name} job: image should be a string"
-      end
-
-      if job[:services] && !validate_array_of_strings(job[:services])
-        raise ValidationError, "#{name} job: services should be an array of strings"
-      end
-
-      if job[:tags] && !validate_array_of_strings(job[:tags])
-        raise ValidationError, "#{name} job: tags parameter should be an array of strings"
-      end
-
-      if job[:only] && !validate_array_of_strings_or_regexps(job[:only])
-        raise ValidationError, "#{name} job: only parameter should be an array of strings or regexps"
-      end
-
-      if job[:except] && !validate_array_of_strings_or_regexps(job[:except])
-        raise ValidationError, "#{name} job: except parameter should be an array of strings or regexps"
-      end
-
-      if job[:allow_failure] && !validate_boolean(job[:allow_failure])
-        raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
-      end
-
-      if job[:when] && !job[:when].in?(%w[on_success on_failure always manual])
-        raise ValidationError, "#{name} job: when parameter should be on_success, on_failure, always or manual"
-      end
-
-      if job[:environment] && !validate_environment(job[:environment])
-        raise ValidationError, "#{name} job: environment parameter #{Gitlab::Regex.environment_name_regex_message}"
-      end
-    end
-
-    def validate_job_script!(name, job)
-      if !validate_string(job[:script]) && !validate_array_of_strings(job[:script])
-        raise ValidationError, "#{name} job: script should be a string or an array of a strings"
-      end
-
-      if job[:before_script] && !validate_array_of_strings(job[:before_script])
-        raise ValidationError, "#{name} job: before_script should be an array of strings"
-      end
-
-      if job[:after_script] && !validate_array_of_strings(job[:after_script])
-        raise ValidationError, "#{name} job: after_script should be an array of strings"
-      end
-    end
-
     def validate_job_stage!(name, job)
+      return unless job[:stage]
+
       unless job[:stage].is_a?(String) && job[:stage].in?(@stages)
         raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}"
       end
     end
 
-    def validate_job_variables!(name, job)
-      unless validate_variables(job[:variables])
-        raise ValidationError,
-          "#{name} job: variables should be a map of key-value strings"
-      end
-    end
-
-    def validate_job_cache!(name, job)
-      job[:cache].keys.each do |key|
-        unless ALLOWED_CACHE_KEYS.include? key
-          raise ValidationError, "#{name} job: cache unknown parameter #{key}"
-        end
-      end
-
-      if job[:cache][:key] && !validate_string(job[:cache][:key])
-        raise ValidationError, "#{name} job: cache:key parameter should be a string"
-      end
-
-      if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked])
-        raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean"
-      end
-
-      if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths])
-        raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings"
-      end
-    end
-
-    def validate_job_artifacts!(name, job)
-      job[:artifacts].keys.each do |key|
-        unless ALLOWED_ARTIFACTS_KEYS.include? key
-          raise ValidationError, "#{name} job: artifacts unknown parameter #{key}"
-        end
-      end
-
-      if job[:artifacts][:name] && !validate_string(job[:artifacts][:name])
-        raise ValidationError, "#{name} job: artifacts:name parameter should be a string"
-      end
-
-      if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
-        raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
-      end
-
-      if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
-        raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
-      end
-
-      if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always])
-        raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always"
-      end
-
-      if job[:artifacts][:expire_in] && !validate_duration(job[:artifacts][:expire_in])
-        raise ValidationError, "#{name} job: artifacts:expire_in parameter should be a duration"
-      end
-    end
-
     def validate_job_dependencies!(name, job)
-      unless validate_array_of_strings(job[:dependencies])
-        raise ValidationError, "#{name} job: dependencies parameter should be an array of strings"
-      end
+      return unless job[:dependencies]
 
       stage_index = @stages.index(job[:stage])
 
diff --git a/lib/ci/static_model.rb b/lib/ci/static_model.rb
deleted file mode 100644
index bb2bdbed495..00000000000
--- a/lib/ci/static_model.rb
+++ /dev/null
@@ -1,49 +0,0 @@
-# Provides an ActiveRecord-like interface to a model whose data is not persisted to a database.
-module Ci
-  module StaticModel
-    extend ActiveSupport::Concern
-
-    module ClassMethods
-      # Used by ActiveRecord's polymorphic association to set object_id
-      def primary_key
-        'id'
-      end
-
-      # Used by ActiveRecord's polymorphic association to set object_type
-      def base_class
-        self
-      end
-    end
-
-    # Used by AR for fetching attributes
-    #
-    # Pass it along if we respond to it.
-    def [](key)
-      send(key) if respond_to?(key)
-    end
-
-    def to_param
-      id
-    end
-
-    def new_record?
-      false
-    end
-
-    def persisted?
-      false
-    end
-
-    def destroyed?
-      false
-    end
-
-    def ==(other)
-      if other.is_a? ::Ci::StaticModel
-        id == other.id
-      else
-        super
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb
index de41ea415a6..a533bac2692 100644
--- a/lib/gitlab/access.rb
+++ b/lib/gitlab/access.rb
@@ -7,6 +7,7 @@ module Gitlab
   module Access
     class AccessDeniedError < StandardError; end
 
+    NO_ACCESS = 0
     GUEST     = 10
     REPORTER  = 20
     DEVELOPER = 30
diff --git a/lib/gitlab/akismet_helper.rb b/lib/gitlab/akismet_helper.rb
index 04676fdb748..207736b59db 100644
--- a/lib/gitlab/akismet_helper.rb
+++ b/lib/gitlab/akismet_helper.rb
@@ -17,8 +17,8 @@ module Gitlab
       env['HTTP_USER_AGENT']
     end
 
-    def check_for_spam?(project, user)
-      akismet_enabled? && !project.team.member?(user)
+    def check_for_spam?(project)
+      akismet_enabled? && project.public?
     end
 
     def is_spam?(environment, user, text)
diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb
index 34e0143a82e..839a4fa30d5 100644
--- a/lib/gitlab/backend/shell.rb
+++ b/lib/gitlab/backend/shell.rb
@@ -60,16 +60,18 @@ module Gitlab
     end
 
     # Fork repository to new namespace
-    # storage - project's storage path
+    # forked_from_storage - forked-from project's storage path
     # path - project path with namespace
+    # forked_to_storage - forked-to project's storage path
     # fork_namespace - namespace for forked project
     #
     # Ex.
-    #  fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx")
+    #  fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx")
     #
-    def fork_repository(storage, path, fork_namespace)
+    def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace)
       Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project',
-                                   storage, "#{path}.git", fork_namespace])
+                                   forked_from_storage, "#{path}.git", forked_to_storage,
+                                   fork_namespace])
     end
 
     # Remove repository from file system
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index e6cc1529760..ae82c0db3f1 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -8,7 +8,7 @@ module Gitlab
       # Temporary delegations that should be removed after refactoring
       #
       delegate :before_script, :image, :services, :after_script, :variables,
-               :stages, :cache, to: :@global
+               :stages, :cache, :jobs, to: :@global
 
       def initialize(config)
         @config = Loader.new(config).load!
diff --git a/lib/gitlab/ci/config/node/artifacts.rb b/lib/gitlab/ci/config/node/artifacts.rb
new file mode 100644
index 00000000000..844bd2fe998
--- /dev/null
+++ b/lib/gitlab/ci/config/node/artifacts.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a configuration of job artifacts.
+        #
+        class Artifacts < Entry
+          include Validatable
+          include Attributable
+
+          ALLOWED_KEYS = %i[name untracked paths when expire_in]
+
+          attributes ALLOWED_KEYS
+
+          validations do
+            validates :config, type: Hash
+            validates :config, allowed_keys: ALLOWED_KEYS
+
+            with_options allow_nil: true do
+              validates :name, type: String
+              validates :untracked, boolean: true
+              validates :paths, array_of_strings: true
+              validates :when,
+                inclusion: { in: %w[on_success on_failure always],
+                             message: 'should be on_success, on_failure ' \
+                                      'or always' }
+              validates :expire_in, duration: true
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/attributable.rb b/lib/gitlab/ci/config/node/attributable.rb
new file mode 100644
index 00000000000..221b666f9f6
--- /dev/null
+++ b/lib/gitlab/ci/config/node/attributable.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        module Attributable
+          extend ActiveSupport::Concern
+
+          class_methods do
+            def attributes(*attributes)
+              attributes.flatten.each do |attribute|
+                define_method(attribute) do
+                  return unless config.is_a?(Hash)
+
+                  config[attribute]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/cache.rb b/lib/gitlab/ci/config/node/cache.rb
index cdf8ba2e35d..b4bda2841ac 100644
--- a/lib/gitlab/ci/config/node/cache.rb
+++ b/lib/gitlab/ci/config/node/cache.rb
@@ -8,6 +8,12 @@ module Gitlab
         class Cache < Entry
           include Configurable
 
+          ALLOWED_KEYS = %i[key untracked paths]
+
+          validations do
+            validates :config, allowed_keys: ALLOWED_KEYS
+          end
+
           node :key, Node::Key,
             description: 'Cache key used to define a cache affinity.'
 
@@ -16,10 +22,6 @@ module Gitlab
 
           node :paths, Node::Paths,
             description: 'Specify which paths should be cached across builds.'
-
-          validations do
-            validates :config, allowed_keys: true
-          end
         end
       end
     end
diff --git a/lib/gitlab/ci/config/node/commands.rb b/lib/gitlab/ci/config/node/commands.rb
new file mode 100644
index 00000000000..d7657ae314b
--- /dev/null
+++ b/lib/gitlab/ci/config/node/commands.rb
@@ -0,0 +1,33 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a job script.
+        #
+        class Commands < Entry
+          include Validatable
+
+          validations do
+            include LegacyValidationHelpers
+
+            validate do
+              unless string_or_array_of_strings?(config)
+                errors.add(:config,
+                           'should be a string or an array of strings')
+              end
+            end
+
+            def string_or_array_of_strings?(field)
+              validate_string(field) || validate_array_of_strings(field)
+            end
+          end
+
+          def value
+            Array(@config)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb
index 37936fc8242..2de82d40c9d 100644
--- a/lib/gitlab/ci/config/node/configurable.rb
+++ b/lib/gitlab/ci/config/node/configurable.rb
@@ -25,10 +25,14 @@ module Gitlab
 
           private
 
-          def create_node(key, factory)
-            factory.with(value: @config[key], key: key, parent: self)
+          def compose!
+            self.class.nodes.each do |key, factory|
+              factory
+                .value(@config[key])
+                .with(key: key, parent: self)
 
-            factory.create!
+              @entries[key] = factory.create!
+            end
           end
 
           class_methods do
@@ -36,24 +40,25 @@ module Gitlab
               Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }]
             end
 
-            private
+            private # rubocop:disable Lint/UselessAccessModifier
 
-            def node(symbol, entry_class, metadata)
-              factory = Node::Factory.new(entry_class)
+            def node(key, node, metadata)
+              factory = Node::Factory.new(node)
                 .with(description: metadata[:description])
 
-              (@nodes ||= {}).merge!(symbol.to_sym => factory)
+              (@nodes ||= {}).merge!(key.to_sym => factory)
             end
 
             def helpers(*nodes)
               nodes.each do |symbol|
                 define_method("#{symbol}_defined?") do
-                  @nodes[symbol].try(:defined?)
+                  @entries[symbol].specified? if @entries[symbol]
                 end
 
                 define_method("#{symbol}_value") do
-                  raise Entry::InvalidError unless valid?
-                  @nodes[symbol].try(:value)
+                  return unless @entries[symbol] && @entries[symbol].valid?
+
+                  @entries[symbol].value
                 end
 
                 alias_method symbol.to_sym, "#{symbol}_value".to_sym
diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb
index 9e79e170a4f..0c782c422b5 100644
--- a/lib/gitlab/ci/config/node/entry.rb
+++ b/lib/gitlab/ci/config/node/entry.rb
@@ -8,30 +8,31 @@ module Gitlab
         class Entry
           class InvalidError < StandardError; end
 
-          attr_reader :config
+          attr_reader :config, :metadata
           attr_accessor :key, :parent, :description
 
-          def initialize(config)
+          def initialize(config, **metadata)
             @config = config
-            @nodes = {}
+            @metadata = metadata
+            @entries = {}
+
             @validator = self.class.validator.new(self)
-            @validator.validate
+            @validator.validate(:new)
           end
 
           def process!
-            return if leaf?
             return unless valid?
 
             compose!
-            process_nodes!
+            descendants.each(&:process!)
           end
 
-          def nodes
-            @nodes.values
+          def leaf?
+            @entries.none?
           end
 
-          def leaf?
-            self.class.nodes.none?
+          def descendants
+            @entries.values
           end
 
           def ancestors
@@ -43,27 +44,30 @@ module Gitlab
           end
 
           def errors
-            @validator.messages + nodes.flat_map(&:errors)
+            @validator.messages + descendants.flat_map(&:errors)
           end
 
           def value
             if leaf?
               @config
             else
-              defined = @nodes.select { |_key, value| value.defined? }
-              Hash[defined.map { |key, node| [key, node.value] }]
+              meaningful = @entries.select do |_key, value|
+                value.specified? && value.relevant?
+              end
+
+              Hash[meaningful.map { |key, entry| [key, entry.value] }]
             end
           end
 
-          def defined?
+          def specified?
             true
           end
 
-          def self.default
+          def relevant?
+            true
           end
 
-          def self.nodes
-            {}
+          def self.default
           end
 
           def self.validator
@@ -73,17 +77,6 @@ module Gitlab
           private
 
           def compose!
-            self.class.nodes.each do |key, essence|
-              @nodes[key] = create_node(key, essence)
-            end
-          end
-
-          def process_nodes!
-            nodes.each(&:process!)
-          end
-
-          def create_node(key, essence)
-            raise NotImplementedError
           end
         end
       end
diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb
index 5919a283283..707b052e6a8 100644
--- a/lib/gitlab/ci/config/node/factory.rb
+++ b/lib/gitlab/ci/config/node/factory.rb
@@ -10,35 +10,60 @@ module Gitlab
 
           def initialize(node)
             @node = node
+            @metadata = {}
             @attributes = {}
           end
 
+          def value(value)
+            @value = value
+            self
+          end
+
+          def metadata(metadata)
+            @metadata.merge!(metadata)
+            self
+          end
+
           def with(attributes)
             @attributes.merge!(attributes)
             self
           end
 
           def create!
-            raise InvalidFactory unless @attributes.has_key?(:value)
+            raise InvalidFactory unless defined?(@value)
 
-            fabricate.tap do |entry|
-              entry.key = @attributes[:key]
-              entry.parent = @attributes[:parent]
-              entry.description = @attributes[:description]
+            ##
+            # We assume that unspecified entry is undefined.
+            # See issue #18775.
+            #
+            if @value.nil?
+              Node::Undefined.new(
+                fabricate_undefined
+              )
+            else
+              fabricate(@node, @value)
             end
           end
 
           private
 
-          def fabricate
+          def fabricate_undefined
             ##
-            # We assume that unspecified entry is undefined.
-            # See issue #18775.
+            # If node has a default value we fabricate concrete node
+            # with default value.
             #
-            if @attributes[:value].nil?
-              Node::Undefined.new(@node)
+            if @node.default.nil?
+              fabricate(Node::Null)
             else
-              @node.new(@attributes[:value])
+              fabricate(@node, @node.default)
+            end
+          end
+
+          def fabricate(node, value = nil)
+            node.new(value, @metadata).tap do |entry|
+              entry.key = @attributes[:key]
+              entry.parent = @attributes[:parent]
+              entry.description = @attributes[:description]
             end
           end
         end
diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb
index f92e1eccbcf..ccd539fb003 100644
--- a/lib/gitlab/ci/config/node/global.rb
+++ b/lib/gitlab/ci/config/node/global.rb
@@ -34,10 +34,36 @@ module Gitlab
             description: 'Configure caching between build jobs.'
 
           helpers :before_script, :image, :services, :after_script,
-                  :variables, :stages, :types, :cache
+                  :variables, :stages, :types, :cache, :jobs
 
-          def stages
-            stages_defined? ? stages_value : types_value
+          private
+
+          def compose!
+            super
+
+            compose_jobs!
+            compose_deprecated_entries!
+          end
+
+          def compose_jobs!
+            factory = Node::Factory.new(Node::Jobs)
+              .value(@config.except(*self.class.nodes.keys))
+              .with(key: :jobs, parent: self,
+                    description: 'Jobs definition for this pipeline')
+
+            @entries[:jobs] = factory.create!
+          end
+
+          def compose_deprecated_entries!
+            ##
+            # Deprecated `:types` key workaround - if types are defined and
+            # stages are not defined we use types definition as stages.
+            #
+            if types_defined? && !stages_defined?
+              @entries[:stages] = @entries[:types]
+            end
+
+            @entries.delete(:types)
           end
         end
       end
diff --git a/lib/gitlab/ci/config/node/hidden_job.rb b/lib/gitlab/ci/config/node/hidden_job.rb
new file mode 100644
index 00000000000..073044b66f8
--- /dev/null
+++ b/lib/gitlab/ci/config/node/hidden_job.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a hidden CI/CD job.
+        #
+        class HiddenJob < Entry
+          include Validatable
+
+          validations do
+            validates :config, type: Hash
+            validates :config, presence: true
+          end
+
+          def relevant?
+            false
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/job.rb b/lib/gitlab/ci/config/node/job.rb
new file mode 100644
index 00000000000..e84737acbb9
--- /dev/null
+++ b/lib/gitlab/ci/config/node/job.rb
@@ -0,0 +1,123 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a concrete CI/CD job.
+        #
+        class Job < Entry
+          include Configurable
+          include Attributable
+
+          ALLOWED_KEYS = %i[tags script only except type image services allow_failure
+                            type stage when artifacts cache dependencies before_script
+                            after_script variables environment]
+
+          attributes :tags, :allow_failure, :when, :environment, :dependencies
+
+          validations do
+            validates :config, allowed_keys: ALLOWED_KEYS
+
+            validates :config, presence: true
+            validates :name, presence: true
+            validates :name, type: Symbol
+
+            with_options allow_nil: true do
+              validates :tags, array_of_strings: true
+              validates :allow_failure, boolean: true
+              validates :when,
+                inclusion: { in: %w[on_success on_failure always manual],
+                             message: 'should be on_success, on_failure, ' \
+                                      'always or manual' }
+              validates :environment,
+                type: {
+                  with: String,
+                  message: Gitlab::Regex.environment_name_regex_message }
+              validates :environment,
+                format: {
+                  with: Gitlab::Regex.environment_name_regex,
+                  message: Gitlab::Regex.environment_name_regex_message }
+
+              validates :dependencies, array_of_strings: true
+            end
+          end
+
+          node :before_script, Script,
+            description: 'Global before script overridden in this job.'
+
+          node :script, Commands,
+            description: 'Commands that will be executed in this job.'
+
+          node :stage, Stage,
+            description: 'Pipeline stage this job will be executed into.'
+
+          node :type, Stage,
+            description: 'Deprecated: stage this job will be executed into.'
+
+          node :after_script, Script,
+            description: 'Commands that will be executed when finishing job.'
+
+          node :cache, Cache,
+            description: 'Cache definition for this job.'
+
+          node :image, Image,
+            description: 'Image that will be used to execute this job.'
+
+          node :services, Services,
+            description: 'Services that will be used to execute this job.'
+
+          node :only, Trigger,
+            description: 'Refs policy this job will be executed for.'
+
+          node :except, Trigger,
+            description: 'Refs policy this job will be executed for.'
+
+          node :variables, Variables,
+            description: 'Environment variables available for this job.'
+
+          node :artifacts, Artifacts,
+            description: 'Artifacts configuration for this job.'
+
+          helpers :before_script, :script, :stage, :type, :after_script,
+                  :cache, :image, :services, :only, :except, :variables,
+                  :artifacts
+
+          def name
+            @metadata[:name]
+          end
+
+          def value
+            @config.merge(to_hash.compact)
+          end
+
+          private
+
+          def to_hash
+            { name: name,
+              before_script: before_script,
+              script: script,
+              image: image,
+              services: services,
+              stage: stage,
+              cache: cache,
+              only: only,
+              except: except,
+              variables: variables_defined? ? variables : nil,
+              artifacts: artifacts,
+              after_script: after_script }
+          end
+
+          def compose!
+            super
+
+            if type_defined? && !stage_defined?
+              @entries[:stage] = @entries[:type]
+            end
+
+            @entries.delete(:type)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/jobs.rb b/lib/gitlab/ci/config/node/jobs.rb
new file mode 100644
index 00000000000..51683c82ceb
--- /dev/null
+++ b/lib/gitlab/ci/config/node/jobs.rb
@@ -0,0 +1,48 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a set of jobs.
+        #
+        class Jobs < Entry
+          include Validatable
+
+          validations do
+            validates :config, type: Hash
+
+            validate do
+              unless has_visible_job?
+                errors.add(:config, 'should contain at least one visible job')
+              end
+            end
+
+            def has_visible_job?
+              config.any? { |name, _| !hidden?(name) }
+            end
+          end
+
+          def hidden?(name)
+            name.to_s.start_with?('.')
+          end
+
+          private
+
+          def compose!
+            @config.each do |name, config|
+              node = hidden?(name) ? Node::HiddenJob : Node::Job
+
+              factory = Node::Factory.new(node)
+                .value(config || {})
+                .metadata(name: name)
+                .with(key: name, parent: self,
+                      description: "#{name} job definition.")
+
+              @entries[name] = factory.create!
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb
index 4d9a508796a..0c291efe6a5 100644
--- a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb
+++ b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb
@@ -41,10 +41,6 @@ module Gitlab
             false
           end
 
-          def validate_environment(value)
-            value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex
-          end
-
           def validate_boolean(value)
             value.in?([true, false])
           end
diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb
new file mode 100644
index 00000000000..88a5f53f13c
--- /dev/null
+++ b/lib/gitlab/ci/config/node/null.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # This class represents an undefined node.
+        #
+        # Implements the Null Object pattern.
+        #
+        class Null < Entry
+          def value
+            nil
+          end
+
+          def valid?
+            true
+          end
+
+          def errors
+            []
+          end
+
+          def specified?
+            false
+          end
+
+          def relevant?
+            false
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/stage.rb b/lib/gitlab/ci/config/node/stage.rb
new file mode 100644
index 00000000000..cbc97641f5a
--- /dev/null
+++ b/lib/gitlab/ci/config/node/stage.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a stage for a job.
+        #
+        class Stage < Entry
+          include Validatable
+
+          validations do
+            validates :config, type: String
+          end
+
+          def self.default
+            'test'
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/trigger.rb b/lib/gitlab/ci/config/node/trigger.rb
new file mode 100644
index 00000000000..d8b31975088
--- /dev/null
+++ b/lib/gitlab/ci/config/node/trigger.rb
@@ -0,0 +1,26 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a trigger policy for the job.
+        #
+        class Trigger < Entry
+          include Validatable
+
+          validations do
+            include LegacyValidationHelpers
+
+            validate :array_of_strings_or_regexps
+
+            def array_of_strings_or_regexps
+              unless validate_array_of_strings_or_regexps(config)
+                errors.add(:config, 'should be an array of strings or regexps')
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/undefined.rb b/lib/gitlab/ci/config/node/undefined.rb
index 699605e1e3a..45fef8c3ae5 100644
--- a/lib/gitlab/ci/config/node/undefined.rb
+++ b/lib/gitlab/ci/config/node/undefined.rb
@@ -3,24 +3,13 @@ module Gitlab
     class Config
       module Node
         ##
-        # This class represents an undefined entry node.
+        # This class represents an unspecified entry node.
         #
-        # It takes original entry class as configuration and returns default
-        # value of original entry as self value.
+        # It decorates original entry adding method that indicates it is
+        # unspecified.
         #
-        #
-        class Undefined < Entry
-          include Validatable
-
-          validations do
-            validates :config, type: Class
-          end
-
-          def value
-            @config.default
-          end
-
-          def defined?
+        class Undefined < SimpleDelegator
+          def specified?
             false
           end
         end
diff --git a/lib/gitlab/ci/config/node/validator.rb b/lib/gitlab/ci/config/node/validator.rb
index 758a6cf4356..43c7e102b50 100644
--- a/lib/gitlab/ci/config/node/validator.rb
+++ b/lib/gitlab/ci/config/node/validator.rb
@@ -21,18 +21,19 @@ module Gitlab
             'Validator'
           end
 
-          def unknown_keys
-            return [] unless config.is_a?(Hash)
-
-            config.keys - @node.class.nodes.keys
-          end
-
           private
 
           def location
             predecessors = ancestors.map(&:key).compact
-            current = key || @node.class.name.demodulize.underscore
-            predecessors.append(current).join(':')
+            predecessors.append(key_name).join(':')
+          end
+
+          def key_name
+            if key.blank?
+              @node.class.name.demodulize.underscore.humanize
+            else
+              key
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/config/node/validators.rb b/lib/gitlab/ci/config/node/validators.rb
index 7b2f57990b5..e20908ad3cb 100644
--- a/lib/gitlab/ci/config/node/validators.rb
+++ b/lib/gitlab/ci/config/node/validators.rb
@@ -5,10 +5,11 @@ module Gitlab
         module Validators
           class AllowedKeysValidator < ActiveModel::EachValidator
             def validate_each(record, attribute, value)
-              if record.unknown_keys.any?
-                unknown_list = record.unknown_keys.join(', ')
-                record.errors.add(:config,
-                                  "contains unknown keys: #{unknown_list}")
+              unknown_keys = record.config.try(:keys).to_a - options[:in]
+
+              if unknown_keys.any?
+                record.errors.add(:config, 'contains unknown keys: ' +
+                                            unknown_keys.join(', '))
               end
             end
           end
@@ -33,6 +34,16 @@ module Gitlab
             end
           end
 
+          class DurationValidator < ActiveModel::EachValidator
+            include LegacyValidationHelpers
+
+            def validate_each(record, attribute, value)
+              unless validate_duration(value)
+                record.errors.add(attribute, 'should be a duration')
+              end
+            end
+          end
+
           class KeyValidator < ActiveModel::EachValidator
             include LegacyValidationHelpers
 
@@ -49,7 +60,8 @@ module Gitlab
               raise unless type.is_a?(Class)
 
               unless value.is_a?(type)
-                record.errors.add(attribute, "should be a #{type.name}")
+                message = options[:message] || "should be a #{type.name}"
+                record.errors.add(attribute, message)
               end
             end
           end
diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb
index 9bef9037ad6..58f86abc5c4 100644
--- a/lib/gitlab/closing_issue_extractor.rb
+++ b/lib/gitlab/closing_issue_extractor.rb
@@ -22,7 +22,9 @@ module Gitlab
 
       @extractor.analyze(closing_statements.join(" "))
 
-      @extractor.issues
+      @extractor.issues.reject do |issue|
+        @extractor.project.forked_from?(issue.project) # Don't extract issues on original project
+      end
     end
   end
 end
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 078609c86f1..55b8f888d53 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -55,12 +55,12 @@ module Gitlab
       end
     end
 
-    private
-
     def self.connection
       ActiveRecord::Base.connection
     end
 
+    private_class_method :connection
+
     def self.database_version
       row = connection.execute("SELECT VERSION()").first
 
@@ -70,5 +70,7 @@ module Gitlab
         row.first
       end
     end
+
+    private_class_method :database_version
   end
 end
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index b09ca1fb8b0..e47df508ca2 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -63,15 +63,18 @@ module Gitlab
         diff_refs.try(:head_sha)
       end
 
+      attr_writer :highlighted_diff_lines
+
       # Array of Gitlab::Diff::Line objects
       def diff_lines
-        @lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a
+        @diff_lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a
       end
 
       def highlighted_diff_lines
         @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight
       end
 
+      # Array[<Hash>] with right/left keys that contains Gitlab::Diff::Line objects which text is hightlighted
       def parallel_diff_lines
         @parallel_diff_lines ||= Gitlab::Diff::ParallelDiff.new(self).parallelize
       end
diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb
new file mode 100644
index 00000000000..2b9fc65b985
--- /dev/null
+++ b/lib/gitlab/diff/file_collection/base.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module Diff
+    module FileCollection
+      class Base
+        attr_reader :project, :diff_options, :diff_view, :diff_refs
+
+        delegate :count, :size, :real_size, to: :diff_files
+
+        def self.default_options
+          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, no_collapse: false)
+        end
+
+        def initialize(diffable, project:, diff_options: nil, diff_refs: nil)
+          diff_options = self.class.default_options.merge(diff_options || {})
+
+          @diffable     = diffable
+          @diffs        = diffable.raw_diffs(diff_options)
+          @project      = project
+          @diff_options = diff_options
+          @diff_refs    = diff_refs
+        end
+
+        def diff_files
+          @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) }
+        end
+
+        private
+
+        def decorate_diff!(diff)
+          Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/file_collection/commit.rb b/lib/gitlab/diff/file_collection/commit.rb
new file mode 100644
index 00000000000..4dc297ec036
--- /dev/null
+++ b/lib/gitlab/diff/file_collection/commit.rb
@@ -0,0 +1,14 @@
+module Gitlab
+  module Diff
+    module FileCollection
+      class Commit < Base
+        def initialize(commit, diff_options:)
+          super(commit,
+            project: commit.project,
+            diff_options: diff_options,
+            diff_refs: commit.diff_refs)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/file_collection/compare.rb b/lib/gitlab/diff/file_collection/compare.rb
new file mode 100644
index 00000000000..20d8f891cc3
--- /dev/null
+++ b/lib/gitlab/diff/file_collection/compare.rb
@@ -0,0 +1,14 @@
+module Gitlab
+  module Diff
+    module FileCollection
+      class Compare < Base
+        def initialize(compare, project:, diff_options:, diff_refs: nil)
+          super(compare,
+            project:      project,
+            diff_options: diff_options,
+            diff_refs:    diff_refs)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/file_collection/merge_request.rb b/lib/gitlab/diff/file_collection/merge_request.rb
new file mode 100644
index 00000000000..4f946908e2f
--- /dev/null
+++ b/lib/gitlab/diff/file_collection/merge_request.rb
@@ -0,0 +1,73 @@
+module Gitlab
+  module Diff
+    module FileCollection
+      class MergeRequest < Base
+        def initialize(merge_request, diff_options:)
+          @merge_request = merge_request
+
+          super(merge_request,
+            project: merge_request.project,
+            diff_options: diff_options,
+            diff_refs: merge_request.diff_refs)
+        end
+
+        def diff_files
+          super.tap { |_| store_highlight_cache }
+        end
+
+        private
+
+        # Extracted method to highlight in the same iteration to the diff_collection.
+        def decorate_diff!(diff)
+          diff_file = super
+          cache_highlight!(diff_file) if cacheable?
+          diff_file
+        end
+
+        def highlight_diff_file_from_cache!(diff_file, cache_diff_lines)
+          diff_file.highlighted_diff_lines = cache_diff_lines.map do |line|
+            Gitlab::Diff::Line.init_from_hash(line)
+          end
+        end
+
+        #
+        # If we find the highlighted diff files lines on the cache we replace existing diff_files lines (no highlighted)
+        # for the highlighted ones, so we just skip their execution.
+        # If the highlighted diff files lines are not cached we calculate and cache them.
+        #
+        # The content of the cache is a Hash where the key correspond to the file_path and the values are Arrays of
+        # hashes that represent serialized diff lines.
+        #
+        def cache_highlight!(diff_file)
+          file_path = diff_file.file_path
+
+          if highlight_cache[file_path]
+            highlight_diff_file_from_cache!(diff_file, highlight_cache[file_path])
+          else
+            highlight_cache[file_path] = diff_file.highlighted_diff_lines.map(&:to_hash)
+          end
+        end
+
+        def highlight_cache
+          return @highlight_cache if defined?(@highlight_cache)
+
+          @highlight_cache = Rails.cache.read(cache_key) || {}
+          @highlight_cache_was_empty = @highlight_cache.empty?
+          @highlight_cache
+        end
+
+        def store_highlight_cache
+          Rails.cache.write(cache_key, highlight_cache) if @highlight_cache_was_empty
+        end
+
+        def cacheable?
+          @merge_request.merge_request_diff.present?
+        end
+
+        def cache_key
+          [@merge_request.merge_request_diff, 'highlighted-diff-files', diff_options]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb
index 649a265a02c..9ea976e18fa 100644
--- a/lib/gitlab/diff/highlight.rb
+++ b/lib/gitlab/diff/highlight.rb
@@ -40,8 +40,6 @@ module Gitlab
       def highlight_line(diff_line)
         return unless diff_file && diff_file.diff_refs
 
-        line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' '
-
         rich_line =
           if diff_line.unchanged? || diff_line.added?
             new_lines[diff_line.new_pos - 1]
@@ -51,7 +49,10 @@ module Gitlab
 
         # Only update text if line is found. This will prevent
         # issues with submodules given the line only exists in diff content.
-        "#{line_prefix}#{rich_line}".html_safe if rich_line
+        if rich_line
+          line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' '
+          "#{line_prefix}#{rich_line}".html_safe
+        end
       end
 
       def inline_diffs
diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb
index 28ad637fda4..55708d42161 100644
--- a/lib/gitlab/diff/inline_diff.rb
+++ b/lib/gitlab/diff/inline_diff.rb
@@ -19,24 +19,6 @@ module Gitlab
 
       attr_accessor :old_line, :new_line, :offset
 
-      def self.for_lines(lines)
-        changed_line_pairs = self.find_changed_line_pairs(lines)
-
-        inline_diffs = []
-
-        changed_line_pairs.each do |old_index, new_index|
-          old_line = lines[old_index]
-          new_line = lines[new_index]
-
-          old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs
-
-          inline_diffs[old_index] = old_diffs
-          inline_diffs[new_index] = new_diffs
-        end
-
-        inline_diffs
-      end
-
       def initialize(old_line, new_line, offset: 0)
         @old_line = old_line[offset..-1]
         @new_line = new_line[offset..-1]
@@ -63,32 +45,54 @@ module Gitlab
         [old_diffs, new_diffs]
       end
 
-      private
+      class << self
+        def for_lines(lines)
+          changed_line_pairs = find_changed_line_pairs(lines)
 
-      # Finds pairs of old/new line pairs that represent the same line that changed
-      def self.find_changed_line_pairs(lines)
-        # Prefixes of all diff lines, indicating their types
-        # For example: `" - +  -+  ---+++ --+  -++"`
-        line_prefixes = lines.each_with_object("") { |line, s| s << line[0] }.gsub(/[^ +-]/, ' ')
+          inline_diffs = []
 
-        changed_line_pairs = []
-        line_prefixes.scan(LINE_PAIRS_PATTERN) do
-          # For `"---+++"`, `begin_index == 0`, `end_index == 6`
-          begin_index, end_index = Regexp.last_match.offset(:del_ins)
+          changed_line_pairs.each do |old_index, new_index|
+            old_line = lines[old_index]
+            new_line = lines[new_index]
 
-          # For `"---+++"`, `changed_line_count == 3`
-          changed_line_count = (end_index - begin_index) / 2
+            old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs
 
-          halfway_index = begin_index + changed_line_count
-          (begin_index...halfway_index).each do |i|
-            # For `"---+++"`, index 1 maps to 1 + 3 = 4
-            changed_line_pairs << [i, i + changed_line_count]
+            inline_diffs[old_index] = old_diffs
+            inline_diffs[new_index] = new_diffs
           end
+
+          inline_diffs
         end
 
-        changed_line_pairs
+        private
+
+        # Finds pairs of old/new line pairs that represent the same line that changed
+        def find_changed_line_pairs(lines)
+          # Prefixes of all diff lines, indicating their types
+          # For example: `" - +  -+  ---+++ --+  -++"`
+          line_prefixes = lines.each_with_object("") { |line, s| s << line[0] }.gsub(/[^ +-]/, ' ')
+
+          changed_line_pairs = []
+          line_prefixes.scan(LINE_PAIRS_PATTERN) do
+            # For `"---+++"`, `begin_index == 0`, `end_index == 6`
+            begin_index, end_index = Regexp.last_match.offset(:del_ins)
+
+            # For `"---+++"`, `changed_line_count == 3`
+            changed_line_count = (end_index - begin_index) / 2
+
+            halfway_index = begin_index + changed_line_count
+            (begin_index...halfway_index).each do |i|
+              # For `"---+++"`, index 1 maps to 1 + 3 = 4
+              changed_line_pairs << [i, i + changed_line_count]
+            end
+          end
+
+          changed_line_pairs
+        end
       end
 
+      private
+
       def longest_common_prefix(a, b)
         max_length = [a.length, b.length].max
 
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index c6189d660c2..cf097e0d0de 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -9,6 +9,20 @@ module Gitlab
         @old_pos, @new_pos = old_pos, new_pos
       end
 
+      def self.init_from_hash(hash)
+        new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos])
+      end
+
+      def serialize_keys
+        @serialize_keys ||= %i(text type index old_pos new_pos)
+      end
+
+      def to_hash
+        hash = {}
+        serialize_keys.each { |key| hash[key] = send(key) }
+        hash
+      end
+
       def old_line
         old_pos unless added? || meta?
       end
diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb
new file mode 100644
index 00000000000..bd3267e2a80
--- /dev/null
+++ b/lib/gitlab/email/handler.rb
@@ -0,0 +1,17 @@
+require 'gitlab/email/handler/create_note_handler'
+require 'gitlab/email/handler/create_issue_handler'
+
+module Gitlab
+  module Email
+    module Handler
+      HANDLERS = [CreateNoteHandler, CreateIssueHandler]
+
+      def self.for(mail, mail_key)
+        HANDLERS.find do |klass|
+          handler = klass.new(mail, mail_key)
+          break handler if handler.can_handle?
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb
new file mode 100644
index 00000000000..b7ed11cb638
--- /dev/null
+++ b/lib/gitlab/email/handler/base_handler.rb
@@ -0,0 +1,60 @@
+module Gitlab
+  module Email
+    module Handler
+      class BaseHandler
+        attr_reader :mail, :mail_key
+
+        def initialize(mail, mail_key)
+          @mail = mail
+          @mail_key = mail_key
+        end
+
+        def message
+          @message ||= process_message
+        end
+
+        def author
+          raise NotImplementedError
+        end
+
+        def project
+          raise NotImplementedError
+        end
+
+        private
+
+        def validate_permission!(permission)
+          raise UserNotFoundError unless author
+          raise UserBlockedError if author.blocked?
+          raise ProjectNotFound unless author.can?(:read_project, project)
+          raise UserNotAuthorizedError unless author.can?(permission, project)
+        end
+
+        def process_message
+          message = ReplyParser.new(mail).execute.strip
+          add_attachments(message)
+        end
+
+        def add_attachments(reply)
+          attachments = Email::AttachmentUploader.new(mail).execute(project)
+
+          reply + attachments.map do |link|
+            "\n\n#{link[:markdown]}"
+          end.join
+        end
+
+        def verify_record!(record:, invalid_exception:, record_name:)
+          return if record.persisted?
+
+          error_title = "The #{record_name} could not be created for the following reasons:"
+
+          msg = error_title + record.errors.full_messages.map do |error|
+            "\n\n- #{error}"
+          end.join
+
+          raise invalid_exception, msg
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb
new file mode 100644
index 00000000000..4e6566af8ab
--- /dev/null
+++ b/lib/gitlab/email/handler/create_issue_handler.rb
@@ -0,0 +1,52 @@
+
+require 'gitlab/email/handler/base_handler'
+
+module Gitlab
+  module Email
+    module Handler
+      class CreateIssueHandler < BaseHandler
+        attr_reader :project_path, :authentication_token
+
+        def initialize(mail, mail_key)
+          super(mail, mail_key)
+          @project_path, @authentication_token =
+            mail_key && mail_key.split('+', 2)
+        end
+
+        def can_handle?
+          !authentication_token.nil?
+        end
+
+        def execute
+          raise ProjectNotFound unless project
+
+          validate_permission!(:create_issue)
+
+          verify_record!(
+            record: create_issue,
+            invalid_exception: InvalidIssueError,
+            record_name: 'issue')
+        end
+
+        def author
+          @author ||= User.find_by(authentication_token: authentication_token)
+        end
+
+        def project
+          @project ||= Project.find_with_namespace(project_path)
+        end
+
+        private
+
+        def create_issue
+          Issues::CreateService.new(
+            project,
+            author,
+            title:       mail.subject,
+            description: message
+          ).execute
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb
new file mode 100644
index 00000000000..06dae31cc27
--- /dev/null
+++ b/lib/gitlab/email/handler/create_note_handler.rb
@@ -0,0 +1,55 @@
+
+require 'gitlab/email/handler/base_handler'
+
+module Gitlab
+  module Email
+    module Handler
+      class CreateNoteHandler < BaseHandler
+        def can_handle?
+          mail_key =~ /\A\w+\z/
+        end
+
+        def execute
+          raise SentNotificationNotFoundError unless sent_notification
+          raise AutoGeneratedEmailError if mail.header.to_s =~ /auto-(generated|replied)/
+
+          validate_permission!(:create_note)
+
+          raise NoteableNotFoundError unless sent_notification.noteable
+          raise EmptyEmailError if message.blank?
+
+          verify_record!(
+            record: create_note,
+            invalid_exception: InvalidNoteError,
+            record_name: 'comment')
+        end
+
+        def author
+          sent_notification.recipient
+        end
+
+        def project
+          sent_notification.project
+        end
+
+        def sent_notification
+          @sent_notification ||= SentNotification.for(mail_key)
+        end
+
+        private
+
+        def create_note
+          Notes::CreateService.new(
+            project,
+            author,
+            note:           message,
+            noteable_type:  sent_notification.noteable_type,
+            noteable_id:    sent_notification.noteable_id,
+            commit_id:      sent_notification.commit_id,
+            line_code:      sent_notification.line_code
+          ).execute
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb
index 97701b0cd42..0e3b65fceb4 100644
--- a/lib/gitlab/email/message/repository_push.rb
+++ b/lib/gitlab/email/message/repository_push.rb
@@ -35,21 +35,22 @@ module Gitlab
         def commits
           return unless compare
 
-          @commits ||= Commit.decorate(compare.commits, project)
+          @commits ||= compare.commits
         end
 
         def diffs
           return unless compare
-          
-          @diffs ||= safe_diff_files(compare.diffs(max_files: 30), diff_refs: diff_refs, repository: project.repository)
+
+          # This diff is more moderated in number of files and lines
+          @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, no_collapse: true).diff_files
         end
 
         def diffs_count
-          diffs.count if diffs
+          diffs.size if diffs
         end
 
         def compare
-          @opts[:compare]
+          @opts[:compare] if @opts[:compare]
         end
 
         def diff_refs
@@ -97,16 +98,18 @@ module Gitlab
             if commits.length > 1
               namespace_project_compare_url(project_namespace,
                                             project,
-                                            from: Commit.new(compare.base, project),
-                                            to:   Commit.new(compare.head, project))
+                                            from: compare.start_commit,
+                                            to:   compare.head_commit)
             else
               namespace_project_commit_url(project_namespace,
-                                           project, commits.first)
+                                           project,
+                                           commits.first)
             end
           else
             unless @action == :delete
               namespace_project_tree_url(project_namespace,
-                                         project, ref_name)
+                                         project,
+                                         ref_name)
             end
           end
         end
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index 1c671a7487b..9213cfb51e8 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -1,18 +1,24 @@
+
+require 'gitlab/email/handler'
+
 # Inspired in great part by Discourse's Email::Receiver
 module Gitlab
   module Email
-    class Receiver
-      class ProcessingError < StandardError; end
-      class EmailUnparsableError < ProcessingError; end
-      class SentNotificationNotFoundError < ProcessingError; end
-      class EmptyEmailError < ProcessingError; end
-      class AutoGeneratedEmailError < ProcessingError; end
-      class UserNotFoundError < ProcessingError; end
-      class UserBlockedError < ProcessingError; end
-      class UserNotAuthorizedError < ProcessingError; end
-      class NoteableNotFoundError < ProcessingError; end
-      class InvalidNoteError < ProcessingError; end
+    class ProcessingError < StandardError; end
+    class EmailUnparsableError < ProcessingError; end
+    class SentNotificationNotFoundError < ProcessingError; end
+    class ProjectNotFound < ProcessingError; end
+    class EmptyEmailError < ProcessingError; end
+    class AutoGeneratedEmailError < ProcessingError; end
+    class UserNotFoundError < ProcessingError; end
+    class UserBlockedError < ProcessingError; end
+    class UserNotAuthorizedError < ProcessingError; end
+    class NoteableNotFoundError < ProcessingError; end
+    class InvalidNoteError < ProcessingError; end
+    class InvalidIssueError < ProcessingError; end
+    class UnknownIncomingEmail < ProcessingError; end
 
+    class Receiver
       def initialize(raw)
         @raw = raw
       end
@@ -20,91 +26,38 @@ module Gitlab
       def execute
         raise EmptyEmailError if @raw.blank?
 
-        raise SentNotificationNotFoundError unless sent_notification
-
-        raise AutoGeneratedEmailError if message.header.to_s =~ /auto-(generated|replied)/
-
-        author = sent_notification.recipient
-
-        raise UserNotFoundError unless author
-
-        raise UserBlockedError if author.blocked?
-
-        project = sent_notification.project
-
-        raise UserNotAuthorizedError unless project && author.can?(:create_note, project)
-
-        raise NoteableNotFoundError unless sent_notification.noteable
-
-        reply = ReplyParser.new(message).execute.strip
-
-        raise EmptyEmailError if reply.blank?
-
-        reply = add_attachments(reply)
-
-        note = create_note(reply)
+        mail = build_mail
+        mail_key = extract_mail_key(mail)
+        handler = Handler.for(mail, mail_key)
 
-        unless note.persisted?
-          msg = "The comment could not be created for the following reasons:"
-          note.errors.full_messages.each do |error|
-            msg << "\n\n- #{error}"
-          end
+        raise UnknownIncomingEmail unless handler
 
-          raise InvalidNoteError, msg
-        end
+        handler.execute
       end
 
-      private
-
-      def message
-        @message ||= Mail::Message.new(@raw)
-      rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e
+      def build_mail
+        Mail::Message.new(@raw)
+      rescue Encoding::UndefinedConversionError,
+             Encoding::InvalidByteSequenceError => e
         raise EmailUnparsableError, e
       end
 
-      def reply_key
-        key_from_to_header || key_from_additional_headers
+      def extract_mail_key(mail)
+        key_from_to_header(mail) || key_from_additional_headers(mail)
       end
 
-      def key_from_to_header
-        key = nil
-        message.to.each do |address|
+      def key_from_to_header(mail)
+        mail.to.find do |address|
           key = Gitlab::IncomingEmail.key_from_address(address)
-          break if key
+          break key if key
         end
-
-        key
       end
 
-      def key_from_additional_headers
-        reply_key = nil
-
-        Array(message.references).each do |message_id|
-          reply_key = Gitlab::IncomingEmail.key_from_fallback_reply_message_id(message_id)
-          break if reply_key
+      def key_from_additional_headers(mail)
+        Array(mail.references).find do |mail_id|
+          key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id)
+          break key if key
         end
-
-        reply_key
-      end
-
-      def sent_notification
-        return nil unless reply_key
-
-        SentNotification.for(reply_key)
-      end
-
-      def add_attachments(reply)
-        attachments = Email::AttachmentUploader.new(message).execute(sent_notification.project)
-
-        attachments.each do |link|
-          reply << "\n\n#{link[:markdown]}"
-        end
-
-        reply
-      end
-
-      def create_note(reply)
-        sent_notification.create_note(reply)
       end
     end
   end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 8e8f39d9cb2..69943e22353 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -14,7 +14,7 @@ module Gitlab
       @user_access = UserAccess.new(user, project: project)
     end
 
-    def check(cmd, changes = nil)
+    def check(cmd, changes)
       return build_status_object(false, "Git access over #{protocol.upcase} is not allowed") unless protocol_allowed?
 
       unless actor
diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb
index a088e19d1e7..d32bdd86427 100644
--- a/lib/gitlab/git_post_receive.rb
+++ b/lib/gitlab/git_post_receive.rb
@@ -39,7 +39,6 @@ module Gitlab
     end
 
     def deserialize_changes(changes)
-      changes = Base64.decode64(changes) unless changes.include?(' ')
       changes = utf8_encode_changes(changes)
       changes.lines
     end
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index d6d14bd98a0..bb562bdcd2c 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -2,7 +2,7 @@ module Gitlab
   module ImportExport
     extend self
 
-    VERSION = '0.1.2'
+    VERSION = '0.1.3'
     FILENAME_LIMIT = 50
 
     def export_path(relative_path:)
@@ -13,6 +13,10 @@ module Gitlab
       File.join(Settings.shared['path'], 'tmp/project_exports')
     end
 
+    def import_upload_path(filename:)
+      File.join(storage_path, 'uploads', filename)
+    end
+
     def project_filename
       "project.json"
     end
diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb
index 352539eb594..cfa595629f4 100644
--- a/lib/gitlab/import_export/avatar_restorer.rb
+++ b/lib/gitlab/import_export/avatar_restorer.rb
@@ -1,7 +1,6 @@
 module Gitlab
   module ImportExport
     class AvatarRestorer
-
       def initialize(project:, shared:)
         @project = project
         @shared = shared
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index 5dd0e34c18e..e522a0fc8f6 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -17,6 +17,10 @@ module Gitlab
         execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path}))
       end
 
+      def git_restore_hooks
+        execute(%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args)
+      end
+
       private
 
       def tar_with_options(archive:, dir:, options:)
@@ -45,6 +49,10 @@ module Gitlab
         FileUtils.copy_entry(source, destination)
         true
       end
+
+      def repository_storage_paths_args
+        Gitlab.config.repositories.storages.values
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 82d1e1805c5..eca6e5b6d51 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -3,6 +3,8 @@ module Gitlab
     class FileImporter
       include Gitlab::ImportExport::CommandLineUtil
 
+      MAX_RETRIES = 8
+
       def self.import(*args)
         new(*args).import
       end
@@ -14,7 +16,10 @@ module Gitlab
 
       def import
         FileUtils.mkdir_p(@shared.export_path)
-        decompress_archive
+
+        wait_for_archived_file do
+          decompress_archive
+        end
       rescue => e
         @shared.error(e)
         false
@@ -22,6 +27,17 @@ module Gitlab
 
       private
 
+      # Exponentially sleep until I/O finishes copying the file
+      def wait_for_archived_file
+        MAX_RETRIES.times do |retry_number|
+          break if File.exist?(@archive_file)
+
+          sleep(2**retry_number)
+        end
+
+        yield
+      end
+
       def decompress_archive
         result = untar_zxf(archive: @archive_file, dir: @shared.export_path)
 
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 15afe8174a4..1da51043611 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -3,11 +3,12 @@ project_tree:
   - issues:
     - :events
     - notes:
-        - :author
-        - :events
-  - :labels
-  - milestones:
-    - :events
+      - :author
+      - :events
+    - label_links:
+      - :label
+    - milestone:
+      - :events
   - snippets:
     - notes:
         :author
@@ -20,6 +21,10 @@ project_tree:
       - :events
     - :merge_request_diff
     - :events
+    - label_links:
+      - :label
+    - milestone:
+      - :events
   - pipelines:
     - notes:
       - :author
@@ -31,6 +36,9 @@ project_tree:
   - :services
   - :hooks
   - :protected_branches
+  - :labels
+  - milestones:
+    - :events
 
 # Only include the following attributes for the models specified.
 included_attributes:
@@ -55,6 +63,10 @@ excluded_attributes:
     - :expired_at
   merge_request_diff:
     - :st_diffs
+  issues:
+    - :milestone_id
+  merge_requests:
+    - :milestone_id
 
 methods:
   statuses:
diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb
new file mode 100644
index 00000000000..008300bde45
--- /dev/null
+++ b/lib/gitlab/import_export/json_hash_builder.rb
@@ -0,0 +1,110 @@
+module Gitlab
+  module ImportExport
+    # Generates a hash that conforms with http://apidock.com/rails/Hash/to_json
+    # and its peculiar options.
+    class JsonHashBuilder
+      def self.build(model_objects, attributes_finder)
+        new(model_objects, attributes_finder).build
+      end
+
+      def initialize(model_objects, attributes_finder)
+        @model_objects = model_objects
+        @attributes_finder = attributes_finder
+      end
+
+      def build
+        process_model_objects(@model_objects)
+      end
+
+      private
+
+      # Called when the model is actually a hash containing other relations (more models)
+      # Returns the config in the right format for calling +to_json+
+      #
+      # +model_object_hash+ - A model relationship such as:
+      #   {:merge_requests=>[:merge_request_diff, :notes]}
+      def process_model_objects(model_object_hash)
+        json_config_hash = {}
+        current_key = model_object_hash.keys.first
+
+        model_object_hash.values.flatten.each do |model_object|
+          @attributes_finder.parse(current_key) { |hash| json_config_hash[current_key] ||= hash }
+          handle_model_object(current_key, model_object, json_config_hash)
+        end
+
+        json_config_hash
+      end
+
+      # Creates or adds to an existing hash an individual model or list
+      #
+      # +current_key+ main model that will be a key in the hash
+      # +model_object+ model or list of models to include in the hash
+      # +json_config_hash+ the original hash containing the root model
+      def handle_model_object(current_key, model_object, json_config_hash)
+        model_or_sub_model = model_object.is_a?(Hash) ? process_model_objects(model_object) : model_object
+
+        if json_config_hash[current_key]
+          add_model_value(current_key, model_or_sub_model, json_config_hash)
+        else
+          create_model_value(current_key, model_or_sub_model, json_config_hash)
+        end
+      end
+
+      # Constructs a new hash that will hold the configuration for that particular object
+      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
+      #
+      # +current_key+ main model that will be a key in the hash
+      # +value+ existing model to be included in the hash
+      # +json_config_hash+ the original hash containing the root model
+      def create_model_value(current_key, value, json_config_hash)
+        parsed_hash = { include: value }
+        parse_hash(value, parsed_hash)
+
+        json_config_hash[current_key] = parsed_hash
+      end
+
+      # Calls attributes finder to parse the hash and add any attributes to it
+      #
+      # +value+ existing model to be included in the hash
+      # +parsed_hash+ the original hash
+      def parse_hash(value, parsed_hash)
+        @attributes_finder.parse(value) do |hash|
+          parsed_hash = { include: hash_or_merge(value, hash) }
+        end
+      end
+
+      # Adds new model configuration to an existing hash with key +current_key+
+      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
+      #
+      # +current_key+ main model that will be a key in the hash
+      # +value+ existing model to be included in the hash
+      # +json_config_hash+ the original hash containing the root model
+      def add_model_value(current_key, value, json_config_hash)
+        @attributes_finder.parse(value) { |hash| value = { value => hash } }
+
+        add_to_array(current_key, json_config_hash, value)
+      end
+
+      # Adds new model configuration to an existing hash with key +current_key+
+      # it creates a new array if it was previously a single value
+      #
+      # +current_key+ main model that will be a key in the hash
+      # +value+ existing model to be included in the hash
+      # +json_config_hash+ the original hash containing the root model
+      def add_to_array(current_key, json_config_hash, value)
+        old_values = json_config_hash[current_key][:include]
+
+        json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten
+      end
+
+      # Construct a new hash or merge with an existing one a model configuration
+      # This is to fulfil +to_json+ requirements.
+      #
+      # +hash+ hash containing configuration generated mainly from +@attributes_finder+
+      # +value+ existing model to be included in the hash
+      def hash_or_merge(value, hash)
+        value.is_a?(Hash) ? value.merge(hash) : { value => hash }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb
index b459054c198..36c4cf6efa0 100644
--- a/lib/gitlab/import_export/members_mapper.rb
+++ b/lib/gitlab/import_export/members_mapper.rb
@@ -18,11 +18,14 @@ module Gitlab
         @map ||=
           begin
             @exported_members.inject(missing_keys_tracking_hash) do |hash, member|
-              existing_user = User.where(find_project_user_query(member)).first
-              old_user_id = member['user']['id']
-              if existing_user && add_user_as_team_member(existing_user, member)
-                hash[old_user_id] = existing_user.id
+              if member['user']
+                old_user_id = member['user']['id']
+                existing_user = User.where(find_project_user_query(member)).first
+                hash[old_user_id] = existing_user.id if existing_user && add_team_member(member, existing_user)
+              else
+                add_team_member(member)
               end
+
               hash
             end
           end
@@ -45,7 +48,7 @@ module Gitlab
         ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true)
       end
 
-      def add_user_as_team_member(existing_user, member)
+      def add_team_member(member, existing_user = nil)
         member['user'] = existing_user
 
         ProjectMember.create(member_hash(member)).persisted?
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 051110c23cf..c7b3551b84c 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -47,7 +47,7 @@ module Gitlab
 
           relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
           relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s])
-          saved << restored_project.update_attribute(relation_key, relation_hash)
+          saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
         end
         saved.all?
       end
@@ -78,7 +78,7 @@ module Gitlab
         relation_key = relation.keys.first.to_s
         return if tree_hash[relation_key].blank?
 
-        tree_hash[relation_key].each do |relation_item|
+        [tree_hash[relation_key]].flatten.each do |relation_item|
           relation.values.flatten.each do |sub_relation|
             # We just use author to get the user ID, do not attempt to create an instance.
             next if sub_relation == :author
diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb
index 15f5dd31035..5021a1a14ce 100644
--- a/lib/gitlab/import_export/reader.rb
+++ b/lib/gitlab/import_export/reader.rb
@@ -29,87 +29,12 @@ module Gitlab
       def build_hash(model_list)
         model_list.map do |model_objects|
           if model_objects.is_a?(Hash)
-            build_json_config_hash(model_objects)
+            Gitlab::ImportExport::JsonHashBuilder.build(model_objects, @attributes_finder)
           else
             @attributes_finder.find(model_objects)
           end
         end
       end
-
-      # Called when the model is actually a hash containing other relations (more models)
-      # Returns the config in the right format for calling +to_json+
-      # +model_object_hash+ - A model relationship such as:
-      #   {:merge_requests=>[:merge_request_diff, :notes]}
-      def build_json_config_hash(model_object_hash)
-        @json_config_hash = {}
-
-        model_object_hash.values.flatten.each do |model_object|
-          current_key = model_object_hash.keys.first
-
-          @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash }
-
-          handle_model_object(current_key, model_object)
-          process_sub_model(current_key, model_object) if model_object.is_a?(Hash)
-        end
-        @json_config_hash
-      end
-
-      # If the model is a hash, process the sub_models, which could also be hashes
-      # If there is a list, add to an existing array, otherwise use hash syntax
-      # +current_key+ main model that will be a key in the hash
-      # +model_object+ model or list of models to include in the hash
-      def process_sub_model(current_key, model_object)
-        sub_model_json = build_json_config_hash(model_object).dup
-        @json_config_hash.slice!(current_key)
-
-        if @json_config_hash[current_key] && @json_config_hash[current_key][:include]
-          @json_config_hash[current_key][:include] << sub_model_json
-        else
-          @json_config_hash[current_key] = { include: sub_model_json }
-        end
-      end
-
-      # Creates or adds to an existing hash an individual model or list
-      # +current_key+ main model that will be a key in the hash
-      # +model_object+ model or list of models to include in the hash
-      def handle_model_object(current_key, model_object)
-        if @json_config_hash[current_key]
-          add_model_value(current_key, model_object)
-        else
-          create_model_value(current_key, model_object)
-        end
-      end
-
-      # Constructs a new hash that will hold the configuration for that particular object
-      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
-      # +current_key+ main model that will be a key in the hash
-      # +value+ existing model to be included in the hash
-      def create_model_value(current_key, value)
-        parsed_hash = { include: value }
-
-        @attributes_finder.parse(value) do |hash|
-          parsed_hash = { include: hash_or_merge(value, hash) }
-        end
-        @json_config_hash[current_key] = parsed_hash
-      end
-
-      # Adds new model configuration to an existing hash with key +current_key+
-      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
-      # +current_key+ main model that will be a key in the hash
-      # +value+ existing model to be included in the hash
-      def add_model_value(current_key, value)
-        @attributes_finder.parse(value) { |hash| value = { value => hash } }
-        old_values = @json_config_hash[current_key][:include]
-        @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten
-      end
-
-      # Construct a new hash or merge with an existing one a model configuration
-      # This is to fulfil +to_json+ requirements.
-      # +value+ existing model to be included in the hash
-      # +hash+ hash containing configuration generated mainly from +@attributes_finder+
-      def hash_or_merge(value, hash)
-        value.is_a?(Hash) ? value.merge(hash) : { value => hash }
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index e41c7e6bf4f..5e56b3d1aa7 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -13,6 +13,10 @@ module Gitlab
 
       BUILD_MODELS = %w[Ci::Build commit_status].freeze
 
+      IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
+
+      EXISTING_OBJECT_CHECK = %i[milestone milestones label labels].freeze
+
       def self.create(*args)
         new(*args).create
       end
@@ -22,24 +26,35 @@ module Gitlab
         @relation_hash = relation_hash.except('id', 'noteable_id')
         @members_mapper = members_mapper
         @user = user
+        @imported_object_retries = 0
       end
 
       # Creates an object from an actual model with name "relation_sym" with params from
       # the relation_hash, updating references with new object IDs, mapping users using
       # the "members_mapper" object, also updating notes if required.
       def create
-        set_note_author if @relation_name == :notes
+        setup_models
+
+        generate_imported_object
+      end
+
+      private
+
+      def setup_models
+        if @relation_name == :notes
+          set_note_author
+
+          # attachment is deprecated and note uploads are handled by Markdown uploader
+          @relation_hash['attachment'] = nil
+        end
+
         update_user_references
         update_project_references
         reset_ci_tokens if @relation_name == 'Ci::Trigger'
         @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data']
         set_st_diffs if @relation_name == :merge_request_diff
-
-        generate_imported_object
       end
 
-      private
-
       def update_user_references
         USER_REFERENCES.each do |reference|
           if @relation_hash[reference]
@@ -112,10 +127,14 @@ module Gitlab
       end
 
       def imported_object
-        imported_object = relation_class.new(parsed_relation_hash)
-        yield(imported_object) if block_given?
-        imported_object.importing = true if imported_object.respond_to?(:importing)
-        imported_object
+        yield(existing_or_new_object) if block_given?
+        existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing)
+        existing_or_new_object
+      rescue ActiveRecord::RecordNotUnique
+        # as the operation is not atomic, retry in the unlikely scenario an INSERT is
+        # performed on the same object between the SELECT and the INSERT
+        @imported_object_retries += 1
+        retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES
       end
 
       def update_note_for_missing_author(author_name)
@@ -134,6 +153,20 @@ module Gitlab
       def set_st_diffs
         @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs')
       end
+
+      def existing_or_new_object
+        # Only find existing records to avoid mapping tables such as milestones
+        # Otherwise always create the record, skipping the extra SELECT clause.
+        @existing_or_new_object ||= begin
+          if EXISTING_OBJECT_CHECK.include?(@relation_name)
+            existing_object = relation_class.find_or_initialize_by(parsed_relation_hash.slice('title', 'project_id'))
+            existing_object.assign_attributes(parsed_relation_hash)
+            existing_object
+          else
+            relation_class.new(parsed_relation_hash)
+          end
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index f84de652a57..6d9379acf25 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -14,7 +14,7 @@ module Gitlab
 
         FileUtils.mkdir_p(path_to_repo)
 
-        git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle)
+        git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) && repo_restore_hooks
       rescue => e
         @shared.error(e)
         false
@@ -29,6 +29,16 @@ module Gitlab
       def path_to_repo
         @project.repository.path_to_repo
       end
+
+      def repo_restore_hooks
+        return true if wiki?
+
+        git_restore_hooks
+      end
+
+      def wiki?
+        @project.class.name == 'ProjectWiki'
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb
index abfc694b879..de3fe6d822e 100644
--- a/lib/gitlab/import_export/version_checker.rb
+++ b/lib/gitlab/import_export/version_checker.rb
@@ -25,7 +25,7 @@ module Gitlab
 
       def verify_version!(version)
         if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version)
-          raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}")
+          raise Gitlab::ImportExport::Error.new("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}")
         else
           true
         end
diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb
index 8ce9d32abe0..d7be50bd437 100644
--- a/lib/gitlab/incoming_email.rb
+++ b/lib/gitlab/incoming_email.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module IncomingEmail
     class << self
-      FALLBACK_REPLY_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze
+      FALLBACK_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze
 
       def enabled?
         config.enabled && config.address
@@ -21,8 +21,8 @@ module Gitlab
         match[1]
       end
 
-      def key_from_fallback_reply_message_id(message_id)
-        match = message_id.match(FALLBACK_REPLY_MESSAGE_ID_REGEX)
+      def key_from_fallback_message_id(mail_id)
+        match = mail_id.match(FALLBACK_MESSAGE_ID_REGEX)
         return unless match
 
         match[1]
diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb
index f2b649e50a2..2f326d00a2f 100644
--- a/lib/gitlab/ldap/access.rb
+++ b/lib/gitlab/ldap/access.rb
@@ -25,7 +25,7 @@ module Gitlab
         end
       end
 
-      def initialize(user, adapter=nil)
+      def initialize(user, adapter = nil)
         @adapter = adapter
         @user = user
         @provider = user.ldap_identity.provider
diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb
index df65179bfea..9a5bcfb5c9b 100644
--- a/lib/gitlab/ldap/adapter.rb
+++ b/lib/gitlab/ldap/adapter.rb
@@ -13,7 +13,7 @@ module Gitlab
         Gitlab::LDAP::Config.new(provider)
       end
 
-      def initialize(provider, ldap=nil)
+      def initialize(provider, ldap = nil)
         @provider = provider
         @ldap = ldap || Net::LDAP.new(config.adapter_options)
       end
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 49f702f91f6..41fcd971c22 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -124,6 +124,11 @@ module Gitlab
       trans.action = action if trans
     end
 
+    # Returns the prefix to use for the name of a series.
+    def self.series_prefix
+      @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
+    end
+
     # When enabled this should be set before being used as the usual pattern
     # "@foo ||= bar" is _not_ thread-safe.
     if enabled?
@@ -136,8 +141,7 @@ module Gitlab
       end
     end
 
-    private
-
+    # Allow access from other metrics related middlewares
     def self.current_transaction
       Transaction.current
     end
diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index dcec7543c13..4b7a791e497 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -9,14 +9,17 @@ module Gitlab
     #
     #     Gitlab::Metrics::Instrumentation.instrument_method(User, :by_login)
     module Instrumentation
-      SERIES = 'method_calls'
-
       PROXY_IVAR = :@__gitlab_instrumentation_proxy
 
       def self.configure
         yield self
       end
 
+      # Returns the name of the series to use for storing method calls.
+      def self.series
+        @series ||= "#{Metrics.series_prefix}method_calls"
+      end
+
       # Instruments a class method.
       #
       # mod  - The module to instrument as a Module/Class.
@@ -141,15 +144,15 @@ module Gitlab
         # generated method _only_ accepts regular arguments if the underlying
         # method also accepts them.
         if method.arity == 0
-          args_signature = '&block'
+          args_signature = ''
         else
-          args_signature = '*args, &block'
+          args_signature = '*args'
         end
 
         proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1
           def #{name}(#{args_signature})
             if trans = Gitlab::Metrics::Instrumentation.transaction
-              trans.measure_method(#{label.inspect}) { super }
+              trans.method_call_for(#{label.to_sym.inspect}).measure { super }
             else
               super
             end
diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb
index c048fe20ba7..d3465e5ec19 100644
--- a/lib/gitlab/metrics/method_call.rb
+++ b/lib/gitlab/metrics/method_call.rb
@@ -11,8 +11,8 @@ module Gitlab
       def initialize(name, series)
         @name = name
         @series = series
-        @real_time = 0.0
-        @cpu_time = 0.0
+        @real_time = 0
+        @cpu_time = 0
         @call_count = 0
       end
 
diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb
index 82c18bb108b..287b7a83547 100644
--- a/lib/gitlab/metrics/system.rb
+++ b/lib/gitlab/metrics/system.rb
@@ -35,12 +35,12 @@ module Gitlab
       if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID)
         def self.cpu_time
           Process.
-            clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond).to_f
+            clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond)
         end
       else
         def self.cpu_time
           Process.
-            clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond).to_f
+            clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond)
         end
       end
 
@@ -48,14 +48,14 @@ module Gitlab
       #
       # Returns the time as a Float.
       def self.real_time(precision = :millisecond)
-        Process.clock_gettime(Process::CLOCK_REALTIME, precision).to_f
+        Process.clock_gettime(Process::CLOCK_REALTIME, precision)
       end
 
       # Returns the current monotonic clock time in a given precision.
       #
       # Returns the time as a Float.
       def self.monotonic_time(precision = :millisecond)
-        Process.clock_gettime(Process::CLOCK_MONOTONIC, precision).to_f
+        Process.clock_gettime(Process::CLOCK_MONOTONIC, precision)
       end
     end
   end
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index bded245da43..968f3218950 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -52,23 +52,16 @@ module Gitlab
       end
 
       def add_metric(series, values, tags = {})
-        @metrics << Metric.new("#{series_prefix}#{series}", values, tags)
+        @metrics << Metric.new("#{Metrics.series_prefix}#{series}", values, tags)
       end
 
-      # Measures the time it takes to execute a method.
-      #
-      # Multiple calls to the same method add up to the total runtime of the
-      # method.
-      #
-      # name - The full name of the method to measure (e.g. `User#sign_in`).
-      def measure_method(name, &block)
-        unless @methods[name]
-          series = "#{series_prefix}#{Instrumentation::SERIES}"
-
-          @methods[name] = MethodCall.new(name, series)
+      # Returns a MethodCall object for the given name.
+      def method_call_for(name)
+        unless method = @methods[name]
+          @methods[name] = method = MethodCall.new(name, Instrumentation.series)
         end
 
-        @methods[name].measure(&block)
+        method
       end
 
       def increment(name, value)
@@ -115,14 +108,6 @@ module Gitlab
 
         Metrics.submit_metrics(submit_hashes)
       end
-
-      def sidekiq?
-        Sidekiq.server?
-      end
-
-      def series_prefix
-        sidekiq? ? 'sidekiq_' : 'rails_'
-      end
     end
   end
 end
diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb
index 43e07e09160..ca23ccef25b 100644
--- a/lib/gitlab/popen.rb
+++ b/lib/gitlab/popen.rb
@@ -5,7 +5,7 @@ module Gitlab
   module Popen
     extend self
 
-    def popen(cmd, path=nil)
+    def popen(cmd, path = nil)
       unless cmd.is_a?(Array)
         raise "System commands must be given as an array of strings"
       end
diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb
index 40766f35f77..1f92986ec9a 100644
--- a/lib/gitlab/redis.rb
+++ b/lib/gitlab/redis.rb
@@ -37,7 +37,7 @@ module Gitlab
       redis_config_hash
     end
 
-    def initialize(rails_env=nil)
+    def initialize(rails_env = nil)
       rails_env ||= Rails.env
       config_file = File.expand_path('../../../config/resque.yml', __FILE__)
 
diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb
new file mode 100644
index 00000000000..8130e55351e
--- /dev/null
+++ b/lib/gitlab/request_profiler.rb
@@ -0,0 +1,19 @@
+require 'fileutils'
+
+module Gitlab
+  module RequestProfiler
+    PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles"
+
+    def profile_token
+      Rails.cache.fetch('profile-token') do
+        Devise.friendly_token
+      end
+    end
+    module_function :profile_token
+
+    def remove_all_profiles
+      FileUtils.rm_rf(PROFILES_DIR)
+    end
+    module_function :remove_all_profiles
+  end
+end
diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb
new file mode 100644
index 00000000000..4e787dc0656
--- /dev/null
+++ b/lib/gitlab/request_profiler/middleware.rb
@@ -0,0 +1,54 @@
+require 'ruby-prof'
+require 'gitlab/request_profiler'
+
+module Gitlab
+  module RequestProfiler
+    class Middleware
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        if profile?(env)
+          call_with_profiling(env)
+        else
+          @app.call(env)
+        end
+      end
+
+      def profile?(env)
+        header_token = env['HTTP_X_PROFILE_TOKEN']
+        return unless header_token.present?
+
+        profile_token = RequestProfiler.profile_token
+        return unless profile_token.present?
+
+        header_token == profile_token
+      end
+
+      def call_with_profiling(env)
+        ret = nil
+        result = RubyProf::Profile.profile do
+          ret = catch(:warden) do
+            @app.call(env)
+          end
+        end
+
+        printer   = RubyProf::CallStackPrinter.new(result)
+        file_name = "#{env['PATH_INFO'].tr('/', '|')}_#{Time.current.to_i}.html"
+        file_path = "#{PROFILES_DIR}/#{file_name}"
+
+        FileUtils.mkdir_p(PROFILES_DIR)
+        File.open(file_path, 'wb') do |file|
+          printer.print(file)
+        end
+
+        if ret.is_a?(Array)
+          ret
+        else
+          throw(:warden, ret)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/request_profiler/profile.rb b/lib/gitlab/request_profiler/profile.rb
new file mode 100644
index 00000000000..f89d56903ef
--- /dev/null
+++ b/lib/gitlab/request_profiler/profile.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module RequestProfiler
+    class Profile
+      attr_reader :name, :time, :request_path
+
+      alias_method :to_param, :name
+
+      def self.all
+        Dir["#{PROFILES_DIR}/*.html"].map do |path|
+          new(File.basename(path))
+        end
+      end
+
+      def self.find(name)
+        name_dup = name.dup
+        name_dup << '.html' unless name.end_with?('.html')
+
+        file_path = "#{PROFILES_DIR}/#{name_dup}"
+        return unless File.exist?(file_path)
+
+        new(name_dup)
+      end
+
+      def initialize(name)
+        @name = name
+
+        set_attributes
+      end
+
+      def content
+        File.read("#{PROFILES_DIR}/#{name}")
+      end
+
+      private
+
+      def set_attributes
+        _, path, timestamp = name.split(/(.*)_(\d+)\.html$/)
+        @request_path      = path.tr('|', '/')
+        @time              = Time.at(timestamp.to_i).utc
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/request_store_middleware.rb b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb
new file mode 100644
index 00000000000..b1fa0e3cb4e
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  module SidekiqMiddleware
+    class RequestStoreMiddleware
+      def call(worker, job, queue)
+        RequestStore.begin!
+        yield
+      ensure
+        RequestStore.end!
+        RequestStore.clear!
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
index 83f91de810c..d4020af76f9 100644
--- a/lib/gitlab/themes.rb
+++ b/lib/gitlab/themes.rb
@@ -2,6 +2,8 @@ module Gitlab
   # Module containing GitLab's application theme definitions and helper methods
   # for accessing them.
   module Themes
+    extend self
+
     # Theme ID used when no `default_theme` configuration setting is provided.
     APPLICATION_DEFAULT = 2
 
@@ -22,7 +24,7 @@ module Gitlab
     # classes that might be applied to the `body` element
     #
     # Returns a String
-    def self.body_classes
+    def body_classes
       THEMES.collect(&:css_class).uniq.join(' ')
     end
 
@@ -33,26 +35,26 @@ module Gitlab
     # id - Integer ID
     #
     # Returns a Theme
-    def self.by_id(id)
+    def by_id(id)
       THEMES.detect { |t| t.id == id } || default
     end
 
     # Returns the number of defined Themes
-    def self.count
+    def count
       THEMES.size
     end
 
     # Get the default Theme
     #
     # Returns a Theme
-    def self.default
+    def default
       by_id(default_id)
     end
 
     # Iterate through each Theme
     #
     # Yields the Theme object
-    def self.each(&block)
+    def each(&block)
       THEMES.each(&block)
     end
 
@@ -61,7 +63,7 @@ module Gitlab
     # user - User record
     #
     # Returns a Theme
-    def self.for_user(user)
+    def for_user(user)
       if user
         by_id(user.theme_id)
       else
@@ -71,7 +73,7 @@ module Gitlab
 
     private
 
-    def self.default_id
+    def default_id
       id = Gitlab.config.gitlab.default_theme.to_i
 
       # Prevent an invalid configuration setting from causing an infinite loop
diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb
index c0f85e9b3a8..c55a7fc4d3d 100644
--- a/lib/gitlab/user_access.rb
+++ b/lib/gitlab/user_access.rb
@@ -29,8 +29,11 @@ module Gitlab
     def can_push_to_branch?(ref)
       return false unless user
 
-      if project.protected_branch?(ref) && !project.developers_can_push_to_protected_branch?(ref)
-        user.can?(:push_code_to_protected_branches, project)
+      if project.protected_branch?(ref)
+        return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user)
+
+        access_levels = project.protected_branches.matching(ref).map(&:push_access_level)
+        access_levels.any? { |access_level| access_level.check_access(user) }
       else
         user.can?(:push_code, project)
       end
@@ -39,8 +42,9 @@ module Gitlab
     def can_merge_to_branch?(ref)
       return false unless user
 
-      if project.protected_branch?(ref) && !project.developers_can_merge_to_protected_branch?(ref)
-        user.can?(:push_code_to_protected_branches, project)
+      if project.protected_branch?(ref)
+        access_levels = project.protected_branches.matching(ref).map(&:merge_access_level)
+        access_levels.any? { |access_level| access_level.check_access(user) }
       else
         user.can?(:push_code, project)
       end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 6aeb49c0219..c6826a09bd2 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -4,6 +4,7 @@ require 'json'
 module Gitlab
   class Workhorse
     SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data'
+    VERSION_FILE = 'GITLAB_WORKHORSE_VERSION'
 
     class << self
       def git_http_ok(repository, user)
@@ -75,6 +76,11 @@ module Gitlab
         ]
       end
 
+      def version
+        path = Rails.root.join(VERSION_FILE)
+        path.readable? ? path.read.chomp : 'unknown'
+      end
+
       protected
 
       def encode(hash)
diff --git a/lib/repository_cache.rb b/lib/repository_cache.rb
index 8ddc3511293..068a95790c0 100644
--- a/lib/repository_cache.rb
+++ b/lib/repository_cache.rb
@@ -1,14 +1,15 @@
 # Interface to the Redis-backed cache store used by the Repository model
 class RepositoryCache
-  attr_reader :namespace, :backend
+  attr_reader :namespace, :backend, :project_id
 
-  def initialize(namespace, backend = Rails.cache)
+  def initialize(namespace, project_id, backend = Rails.cache)
     @namespace = namespace
     @backend = backend
+    @project_id = project_id
   end
 
   def cache_key(type)
-    "#{type}:#{namespace}"
+    "#{type}:#{namespace}:#{project_id}"
   end
 
   def expire(key)
diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb
index f818dc78d34..4edfd015074 100644
--- a/lib/rouge/formatters/html_gitlab.rb
+++ b/lib/rouge/formatters/html_gitlab.rb
@@ -18,7 +18,7 @@ module Rouge
           is_first = false
 
           yield %(<span id="LC#{@line_number}" class="line">)
-          line.each { |token, value| yield span(token, value) }
+          line.each { |token, value| yield span(token, value.chomp) }
           yield %(</span>)
 
           @line_number += 1
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index 4a4892a2e07..d521de28e8a 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -49,12 +49,7 @@ server {
 
     proxy_http_version 1.1;
 
-    ## By overwriting Host and clearing X-Forwarded-Host we ensure that
-    ## internal HTTP redirects generated by GitLab always send users to
-    ## YOUR_SERVER_FQDN.
-    proxy_set_header    Host                YOUR_SERVER_FQDN;
-    proxy_set_header    X-Forwarded-Host    "";
-
+    proxy_set_header    Host                $http_host;
     proxy_set_header    X-Real-IP           $remote_addr;
     proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
     proxy_set_header    X-Forwarded-Proto   $scheme;
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 0b93d7f292f..bf014b56cf6 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -93,12 +93,7 @@ server {
 
     proxy_http_version 1.1;
 
-    ## By overwriting Host and clearing X-Forwarded-Host we ensure that
-    ## internal HTTP redirects generated by GitLab always send users to
-    ## YOUR_SERVER_FQDN.
-    proxy_set_header    Host                YOUR_SERVER_FQDN;
-    proxy_set_header    X-Forwarded-Host    "";
-
+    proxy_set_header    Host                $http_host;
     proxy_set_header    X-Real-IP           $remote_addr;
     proxy_set_header    X-Forwarded-Ssl     on;
     proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
diff --git a/lib/tasks/downtime_check.rake b/lib/tasks/downtime_check.rake
index 30a2e9be5ce..afe5d42910c 100644
--- a/lib/tasks/downtime_check.rake
+++ b/lib/tasks/downtime_check.rake
@@ -1,26 +1,12 @@
 desc 'Checks if migrations in a branch require downtime'
 task downtime_check: :environment do
-  # First we'll want to make sure we're comparing with the right upstream
-  # repository/branch.
-  current_branch = `git rev-parse --abbrev-ref HEAD`.strip
-
-  # Either the developer ran this task directly on the master branch, or they're
-  # making changes directly on the master branch.
-  if current_branch == 'master'
-    if defined?(Gitlab::License)
-      repo = 'gitlab-ee'
-    else
-      repo = 'gitlab-ce'
-    end
-
-    `git fetch https://gitlab.com/gitlab-org/#{repo}.git --depth 1`
-
-    compare_with = 'FETCH_HEAD'
-  # The developer is working on a different branch, in this case we can just
-  # compare with the master branch.
+  if defined?(Gitlab::License)
+    repo = 'gitlab-ee'
   else
-    compare_with = 'master'
+    repo = 'gitlab-ce'
   end
 
-  Rake::Task['gitlab:db:downtime_check'].invoke(compare_with)
+  `git fetch https://gitlab.com/gitlab-org/#{repo}.git --depth 1`
+
+  Rake::Task['gitlab:db:downtime_check'].invoke('FETCH_HEAD')
 end
diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake
index 5dbf7d61e06..83dd870fa31 100644
--- a/lib/tasks/gitlab/bulk_add_permission.rake
+++ b/lib/tasks/gitlab/bulk_add_permission.rake
@@ -4,13 +4,13 @@ namespace :gitlab do
     task all_users_to_all_projects: :environment  do |t, args|
       user_ids = User.where(admin: false).pluck(:id)
       admin_ids = User.where(admin: true).pluck(:id)
-      projects_ids = Project.pluck(:id)
+      project_ids = Project.pluck(:id)
 
-      puts "Importing #{user_ids.size} users into #{projects_ids.size} projects"
-      ProjectMember.add_users_into_projects(projects_ids, user_ids, ProjectMember::DEVELOPER)
+      puts "Importing #{user_ids.size} users into #{project_ids.size} projects"
+      ProjectMember.add_users_to_projects(project_ids, user_ids, ProjectMember::DEVELOPER)
 
-      puts "Importing #{admin_ids.size} admins into #{projects_ids.size} projects"
-      ProjectMember.add_users_into_projects(projects_ids, admin_ids, ProjectMember::MASTER)
+      puts "Importing #{admin_ids.size} admins into #{project_ids.size} projects"
+      ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MASTER)
     end
 
     desc "GitLab | Add a specific user to all projects (as a developer)"
@@ -18,7 +18,7 @@ namespace :gitlab do
       user = User.find_by(email: args.email)
       project_ids = Project.pluck(:id)
       puts "Importing #{user.email} users into #{project_ids.size} projects"
-      ProjectMember.add_users_into_projects(project_ids, Array.wrap(user.id), ProjectMember::DEVELOPER)
+      ProjectMember.add_users_to_projects(project_ids, Array.wrap(user.id), ProjectMember::DEVELOPER)
     end
 
     desc "GitLab | Add all users to all groups (admin users are added as owners)"
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 0ec19e1a625..7c96bc864ce 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -25,6 +25,10 @@ namespace :gitlab do
     desc 'Drop all tables'
     task :drop_tables => :environment do
       connection = ActiveRecord::Base.connection
+
+      # If MySQL, turn off foreign key checks
+      connection.execute('SET FOREIGN_KEY_CHECKS=0') if Gitlab::Database.mysql?
+
       tables = connection.tables
       tables.delete 'schema_migrations'
       # Truncate schema_migrations to ensure migrations re-run
@@ -35,6 +39,9 @@ namespace :gitlab do
       # MySQL: http://dev.mysql.com/doc/refman/5.7/en/drop-table.html
       # Add `IF EXISTS` because cascade could have already deleted a table.
       tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") }
+
+      # If MySQL, re-enable foreign key checks
+      connection.execute('SET FOREIGN_KEY_CHECKS=1') if Gitlab::Database.mysql?
     end
 
     desc 'Configures the database by running migrate, or by loading the schema and seeding if needed'
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index c85ebdf8619..ba93945bd03 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -5,7 +5,8 @@ namespace :gitlab do
       warn_user_is_not_gitlab
 
       default_version = Gitlab::Shell.version_required
-      args.with_defaults(tag: 'v' + default_version, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git")
+      default_version_tag = 'v' + default_version
+      args.with_defaults(tag: default_version_tag, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git")
 
       user = Gitlab.config.gitlab.user
       home_dir = Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home
@@ -15,7 +16,12 @@ namespace :gitlab do
       target_dir = Gitlab.config.gitlab_shell.path
 
       # Clone if needed
-      unless File.directory?(target_dir)
+      if File.directory?(target_dir)
+        Dir.chdir(target_dir) do
+          system(*%W(Gitlab.config.git.bin_path} fetch --tags --quiet))
+          system(*%W(Gitlab.config.git.bin_path} checkout --quiet #{default_version_tag}))
+        end
+      else
         system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir}))
       end
 
diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake
index f467cc0ee29..49530e7a372 100644
--- a/lib/tasks/gitlab/web_hook.rake
+++ b/lib/tasks/gitlab/web_hook.rake
@@ -26,10 +26,10 @@ namespace :gitlab do
       namespace_path = ENV['NAMESPACE']
 
       projects = find_projects(namespace_path)
-      projects_ids = projects.pluck(:id)
+      project_ids = projects.pluck(:id)
 
       puts "Removing webhooks with the url '#{web_hook_url}' ... "
-      count = WebHook.where(url: web_hook_url, project_id: projects_ids, type: 'ProjectHook').delete_all
+      count = WebHook.where(url: web_hook_url, project_id: project_ids, type: 'ProjectHook').delete_all
       puts "#{count} webhooks were removed."
     end
 
diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake
index 21c0e5f1d41..d3dcbd2c29b 100644
--- a/lib/tasks/test.rake
+++ b/lib/tasks/test.rake
@@ -7,5 +7,5 @@ end
 
 unless Rails.env.production?
   desc "GitLab | Run all tests on CI with simplecov"
-  task test_ci: [:rubocop, :brakeman, 'teaspoon', :spinach, :spec]
+  task test_ci: [:rubocop, :brakeman, :teaspoon, :spinach, :spec]
 end