Merge branch 'master' into refactor/ci-config-move-global-entries

* master: (352 commits)
  Display last commit of deleted branch in push events (!4699)
  add changelog
  add missing attribute to attr_encrypted so it is fully backwards-compatible
  Add "GitLab team members only" to diagram link
  doc: note that .gitattributes uses default branch
  use the conf lexer so we have highlighted comments
  first draft of docs
  support cgi style options, such as erb?parent=json
  move the path alias to a more appropriate location
  make #custom_language private
  appease rubocop
  add an alias for Snippet#path
  appease rubocop
  check the tag so that an instance will pass too
  fix the spec, using project.change_head
  Revert "bump the master sha for gitlab-test!9"
  bump the master sha for gitlab-test!9
  add custom highlighting via .gitattributes
  Rename Licenses API to License Templates API
  Check for conflict with wiki projects when creating a new project.
  ...

40 files changed, 557 insertions, 253 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 0e7a1cc2623..c3fff8b2f8f 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -33,14 +33,13 @@ module API
     mount ::API::Commits
     mount ::API::DeployKeys
     mount ::API::Files
-    mount ::API::Gitignores
     mount ::API::GroupMembers
     mount ::API::Groups
     mount ::API::Internal
     mount ::API::Issues
     mount ::API::Keys
     mount ::API::Labels
-    mount ::API::Licenses
+    mount ::API::LicenseTemplates
     mount ::API::MergeRequests
     mount ::API::Milestones
     mount ::API::Namespaces
@@ -58,6 +57,7 @@ module API
     mount ::API::Subscriptions
     mount ::API::SystemHooks
     mount ::API::Tags
+    mount ::API::Templates
     mount ::API::Triggers
     mount ::API::Users
     mount ::API::Variables
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 2e397643ed1..5a23a18fe9c 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -423,6 +423,7 @@ module API
     class RunnerDetails < Runner
       expose :tag_list
       expose :run_untagged
+      expose :locked
       expose :version, :revision, :platform, :architecture
       expose :contacted_at
       expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? }
@@ -444,11 +445,7 @@ module API
       expose :created_at, :started_at, :finished_at
       expose :user, with: User
       expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? }
-      expose :commit, with: RepoCommit do |repo_obj, _options|
-        if repo_obj.respond_to?(:commit)
-          repo_obj.commit.commit_data
-        end
-      end
+      expose :commit, with: RepoCommit
       expose :runner, with: Runner
     end
 
@@ -472,11 +469,11 @@ module API
       expose :content
     end
 
-    class GitignoresList < Grape::Entity
+    class TemplatesList < Grape::Entity
       expose :name
     end
 
-    class Gitignore < Grape::Entity
+    class Template < Grape::Entity
       expose :name, :content
     end
   end
diff --git a/lib/api/gitignores.rb b/lib/api/gitignores.rb
deleted file mode 100644
index 270c9501dd2..00000000000
--- a/lib/api/gitignores.rb
+++ /dev/null
@@ -1,29 +0,0 @@
-module API
-  class Gitignores < Grape::API
-
-    # Get the list of the available gitignore templates
-    #
-    # Example Request:
-    #   GET /gitignores
-    get 'gitignores' do
-      present Gitlab::Gitignore.all, with: Entities::GitignoresList
-    end
-
-    # Get the text for a specific gitignore
-    #
-    # Parameters:
-    #   name (required) - The name of a license
-    #
-    # Example Request:
-    #   GET /gitignores/Elixir
-    #
-    get 'gitignores/:name' do
-      required_attributes! [:name]
-
-      gitignore = Gitlab::Gitignore.find(params[:name])
-      not_found!('.gitignore') unless gitignore
-
-      present gitignore, with: Entities::Gitignore
-    end
-  end
-end
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 3ac7b50c4ce..1d361569d59 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -23,8 +23,6 @@ module API
       end
 
       post "/allowed" do
-        Gitlab::Metrics.action = 'Grape#/internal/allowed'
-
         status 200
 
         actor =
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 4c43257c48a..8a03a41e9c5 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -59,6 +59,41 @@ module API
       end
     end
 
+    resource :groups do
+      # Get a list of group issues
+      #
+      # Parameters:
+      #   id (required) - The ID of a group
+      #   state (optional) - Return "opened" or "closed" issues
+      #   labels (optional) - Comma-separated list of label names
+      #   milestone (optional) - Milestone title
+      #   order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at`
+      #   sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc`
+      #
+      # Example Requests:
+      #   GET /groups/:id/issues
+      #   GET /groups/:id/issues?state=opened
+      #   GET /groups/:id/issues?state=closed
+      #   GET /groups/:id/issues?labels=foo
+      #   GET /groups/:id/issues?labels=foo,bar
+      #   GET /groups/:id/issues?labels=foo,bar&state=opened
+      #   GET /groups/:id/issues?milestone=1.0.0
+      #   GET /groups/:id/issues?milestone=1.0.0&state=closed
+      get ":id/issues" do
+        group = find_group(params[:id])
+
+        params[:state] ||= 'opened'
+        params[:group_id] = group.id
+        params[:milestone_title] = params.delete(:milestone)
+        params[:label_name] = params.delete(:labels)
+        params[:sort] = "#{params.delete(:order_by)}_#{params.delete(:sort)}" if params[:order_by] && params[:sort]
+
+        issues = IssuesFinder.new(current_user, params).execute
+
+        present paginate(issues), with: Entities::Issue, current_user: current_user
+      end
+    end
+
     resource :projects do
       # Get a list of project issues
       #
diff --git a/lib/api/licenses.rb b/lib/api/license_templates.rb
index be0e113fbcb..d0552299ed0 100644
--- a/lib/api/licenses.rb
+++ b/lib/api/license_templates.rb
@@ -1,6 +1,6 @@
 module API
-  # Licenses API
-  class Licenses < Grape::API
+  # License Templates API
+  class LicenseTemplates < Grape::API
     PROJECT_TEMPLATE_REGEX =
       /[\<\{\[]
         (project|description|
diff --git a/lib/api/runners.rb b/lib/api/runners.rb
index 4faba9dc87b..ecc8f2fc5a2 100644
--- a/lib/api/runners.rb
+++ b/lib/api/runners.rb
@@ -49,7 +49,7 @@ module API
         runner = get_runner(params[:id])
         authenticate_update_runner!(runner)
 
-        attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged]
+        attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged, :locked]
         if runner.update(attrs)
           present runner, with: Entities::RunnerDetails, current_user: current_user
         else
@@ -96,9 +96,14 @@ module API
 
         runner = get_runner(params[:runner_id])
         authenticate_enable_runner!(runner)
-        Ci::RunnerProject.create(runner: runner, project: user_project)
 
-        present runner, with: Entities::Runner
+        runner_project = runner.assign_to(user_project)
+
+        if runner_project.persisted?
+          present runner, with: Entities::Runner
+        else
+          conflict!("Runner was already enabled for this project")
+        end
       end
 
       # Disable project's runner
@@ -163,6 +168,7 @@ module API
 
       def authenticate_enable_runner!(runner)
         forbidden!("Runner is shared") if runner.is_shared?
+        forbidden!("Runner is locked") if runner.locked?
         return if current_user.is_admin?
         forbidden!("No access granted") unless user_can_access_runner?(runner)
       end
diff --git a/lib/api/templates.rb b/lib/api/templates.rb
new file mode 100644
index 00000000000..18408797756
--- /dev/null
+++ b/lib/api/templates.rb
@@ -0,0 +1,36 @@
+module API
+  class Templates < Grape::API
+    TEMPLATE_TYPES = {
+      gitignores:     Gitlab::Template::Gitignore,
+      gitlab_ci_ymls: Gitlab::Template::GitlabCiYml
+    }.freeze
+
+    TEMPLATE_TYPES.each do |template, klass|
+      # Get the list of the available template
+      #
+      # Example Request:
+      #   GET /gitignores
+      #   GET /gitlab_ci_ymls
+      get template.to_s do
+        present klass.all, with: Entities::TemplatesList
+      end
+
+      # Get the text for a specific template
+      #
+      # Parameters:
+      #   name (required) - The name of a template
+      #
+      # Example Request:
+      #   GET /gitignores/Elixir
+      #   GET /gitlab_ci_ymls/Ruby
+      get "#{template}/:name" do
+        required_attributes! [:name]
+
+        new_template = klass.find(params[:name])
+        not_found!(template.to_s.singularize) unless new_template
+
+        present new_template, with: Entities::Template
+      end
+    end
+  end
+end
diff --git a/lib/banzai.rb b/lib/banzai.rb
index b467413a7dd..093382261ae 100644
--- a/lib/banzai.rb
+++ b/lib/banzai.rb
@@ -7,10 +7,6 @@ module Banzai
     Renderer.render_result(text, context)
   end
 
-  def self.pre_process(text, context)
-    Renderer.pre_process(text, context)
-  end
-
   def self.post_process(html, context)
     Renderer.post_process(html, context)
   end
diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb
index ccd106860bd..8aa6f8f124a 100644
--- a/lib/banzai/filter/image_link_filter.rb
+++ b/lib/banzai/filter/image_link_filter.rb
@@ -9,6 +9,11 @@ module Banzai
       def call
         doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |img|
 
+          div = doc.document.create_element(
+            'div',
+            class: 'image-container'
+          )
+
           link = doc.document.create_element(
             'a',
             class: 'no-attachment-icon',
@@ -17,7 +22,10 @@ module Banzai
           )
 
           link.children = img.clone
-          img.replace(link)
+
+          div.children = link
+
+          img.replace(div)
         end
 
         doc
diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb
index 2614261f9eb..5351272f42d 100644
--- a/lib/banzai/filter/issue_reference_filter.rb
+++ b/lib/banzai/filter/issue_reference_filter.rb
@@ -31,10 +31,14 @@ module Banzai
           projects_per_reference.each do |path, project|
             issue_ids = references_per_project[path]
 
-            next unless project.default_issues_tracker?
+            if project.default_issues_tracker?
+              issues = project.issues.where(iid: issue_ids.to_a)
+            else
+              issues = issue_ids.map { |id| ExternalIssue.new(id, project) }
+            end
 
-            project.issues.where(iid: issue_ids.to_a).each do |issue|
-              hash[project][issue.iid] = issue
+            issues.each do |issue|
+              hash[project][issue.iid.to_i] = issue
             end
           end
 
diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb
index c753a84a20d..c59a80dd1c7 100644
--- a/lib/banzai/filter/redactor_filter.rb
+++ b/lib/banzai/filter/redactor_filter.rb
@@ -7,40 +7,13 @@ module Banzai
     #
     class RedactorFilter < HTML::Pipeline::Filter
       def call
-        nodes = Querying.css(doc, 'a.gfm[data-reference-type]')
-        visible = nodes_visible_to_user(nodes)
-
-        nodes.each do |node|
-          unless visible.include?(node)
-            # The reference should be replaced by the original text,
-            # which is not always the same as the rendered text.
-            text = node.attr('data-original') || node.text
-            node.replace(text)
-          end
-        end
+        Redactor.new(project, current_user).redact([doc])
 
         doc
       end
 
       private
 
-      def nodes_visible_to_user(nodes)
-        per_type = Hash.new { |h, k| h[k] = [] }
-        visible = Set.new
-
-        nodes.each do |node|
-          per_type[node.attr('data-reference-type')] << node
-        end
-
-        per_type.each do |type, nodes|
-          parser = Banzai::ReferenceParser[type].new(project, current_user)
-
-          visible.merge(parser.nodes_visible_to_user(current_user, nodes))
-        end
-
-        visible
-      end
-
       def current_user
         context[:current_user]
       end
diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb
index ea21c7b041c..c78da404607 100644
--- a/lib/banzai/filter/relative_link_filter.rb
+++ b/lib/banzai/filter/relative_link_filter.rb
@@ -14,6 +14,8 @@ module Banzai
       def call
         return doc unless linkable_files?
 
+        @uri_types = {}
+
         doc.search('a:not(.gfm)').each do |el|
           process_link_attr el.attribute('href')
         end
@@ -48,7 +50,7 @@ module Banzai
         uri.path = [
           relative_url_root,
           context[:project].path_with_namespace,
-          path_type(file_path),
+          uri_type(file_path),
           ref || context[:project].default_branch,  # if no ref exists, point to the default branch
           file_path
         ].compact.join('/').squeeze('/').chomp('/')
@@ -87,7 +89,7 @@ module Banzai
         return path unless request_path
 
         parts = request_path.split('/')
-        parts.pop if path_type(request_path) != 'tree'
+        parts.pop if uri_type(request_path) != :tree
 
         while path.start_with?('../')
           parts.pop
@@ -98,45 +100,20 @@ module Banzai
       end
 
       def file_exists?(path)
-        return false if path.nil?
-        repository.blob_at(current_sha, path).present? ||
-          repository.tree(current_sha, path).entries.any?
-      end
-
-      # Get the type of the given path
-      #
-      # path - String path to check
-      #
-      # Examples:
-      #
-      #   path_type('doc/README.md') # => 'blob'
-      #   path_type('doc/logo.png')  # => 'raw'
-      #   path_type('doc/api')       # => 'tree'
-      #
-      # Returns a String
-      def path_type(path)
-        unescaped_path = Addressable::URI.unescape(path)
-
-        if tree?(unescaped_path)
-          'tree'
-        elsif image?(unescaped_path)
-          'raw'
-        else
-          'blob'
-        end
+        path.present? && !!uri_type(path)
       end
 
-      def tree?(path)
-        repository.tree(current_sha, path).entries.any?
-      end
+      def uri_type(path)
+        @uri_types[path] ||= begin
+          unescaped_path = Addressable::URI.unescape(path)
 
-      def image?(path)
-        repository.blob_at(current_sha, path).try(:image?)
+          current_commit.uri_type(unescaped_path)
+        end
       end
 
-      def current_sha
-        context[:commit].try(:id) ||
-          ref ? repository.commit(ref).try(:sha) : repository.head_commit.sha
+      def current_commit
+        @current_commit ||= context[:commit] ||
+          ref ? repository.commit(ref) : repository.head_commit
       end
 
       def relative_url_root
@@ -148,7 +125,7 @@ module Banzai
       end
 
       def repository
-        context[:project].try(:repository)
+        @repository ||= context[:project].try(:repository)
       end
     end
   end
diff --git a/lib/banzai/note_renderer.rb b/lib/banzai/note_renderer.rb
new file mode 100644
index 00000000000..bab6a9934d1
--- /dev/null
+++ b/lib/banzai/note_renderer.rb
@@ -0,0 +1,22 @@
+module Banzai
+  module NoteRenderer
+    # Renders a collection of Note instances.
+    #
+    # notes - The notes to render.
+    # project - The project to use for rendering/redacting.
+    # user - The user viewing the notes.
+    # path - The request path.
+    # wiki - The project's wiki.
+    # git_ref - The current Git reference.
+    def self.render(notes, project, user = nil, path = nil, wiki = nil, git_ref = nil)
+      renderer = ObjectRenderer.new(project,
+                                    user,
+                                    requested_path: path,
+                                    project_wiki: wiki,
+                                    ref: git_ref,
+                                    pipeline: :note)
+
+      renderer.render(notes, :note)
+    end
+  end
+end
diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb
new file mode 100644
index 00000000000..f0e4f28bf12
--- /dev/null
+++ b/lib/banzai/object_renderer.rb
@@ -0,0 +1,85 @@
+module Banzai
+  # Class for rendering multiple objects (e.g. Note instances) in a single pass.
+  #
+  # Rendered Markdown is stored in an attribute in every object based on the
+  # name of the attribute containing the Markdown. For example, when the
+  # attribute `note` is rendered the HTML is stored in `note_html`.
+  class ObjectRenderer
+    attr_reader :project, :user
+
+    # Make sure to set the appropriate pipeline in the `raw_context` attribute
+    # (e.g. `:note` for Note instances).
+    #
+    # project - A Project to use for rendering and redacting Markdown.
+    # user - The user viewing the Markdown/HTML documents, if any.
+    # context - A Hash containing extra attributes to use in the rendering
+    #           pipeline.
+    def initialize(project, user = nil, raw_context = {})
+      @project = project
+      @user = user
+      @raw_context = raw_context
+    end
+
+    # Renders and redacts an Array of objects.
+    #
+    # objects - The objects to render
+    # attribute - The attribute containing the raw Markdown to render.
+    #
+    # Returns the same input objects.
+    def render(objects, attribute)
+      documents = render_objects(objects, attribute)
+      redacted = redact_documents(documents)
+
+      objects.each_with_index do |object, index|
+        object.__send__("#{attribute}_html=", redacted.fetch(index))
+      end
+
+      objects
+    end
+
+    # Renders the attribute of every given object.
+    def render_objects(objects, attribute)
+      objects.map do |object|
+        render_attribute(object, attribute)
+      end
+    end
+
+    # Redacts the list of documents.
+    #
+    # Returns an Array containing the redacted documents.
+    def redact_documents(documents)
+      redactor = Redactor.new(project, user)
+
+      redactor.redact(documents).map do |document|
+        document.to_html.html_safe
+      end
+    end
+
+    # Returns a Banzai context for the given object and attribute.
+    def context_for(object, attribute)
+      context = base_context.merge(cache_key: [object, attribute])
+
+      if object.respond_to?(:author)
+        context[:author] = object.author
+      end
+
+      context
+    end
+
+    # Renders the attribute of an object.
+    #
+    # Returns a `Nokogiri::HTML::Document`.
+    def render_attribute(object, attribute)
+      context = context_for(object, attribute)
+
+      string = object.__send__(attribute)
+      html = Banzai.render(string, context)
+
+      Banzai::Pipeline[:relative_link].to_document(html, context)
+    end
+
+    def base_context
+      @base_context ||= @raw_context.merge(current_user: user, project: project)
+    end
+  end
+end
diff --git a/lib/banzai/pipeline/relative_link_pipeline.rb b/lib/banzai/pipeline/relative_link_pipeline.rb
new file mode 100644
index 00000000000..270990e7ab4
--- /dev/null
+++ b/lib/banzai/pipeline/relative_link_pipeline.rb
@@ -0,0 +1,11 @@
+module Banzai
+  module Pipeline
+    class RelativeLinkPipeline < BasePipeline
+      def self.filters
+        FilterArray[
+          Filter::RelativeLinkFilter
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb
new file mode 100644
index 00000000000..ffd267d5e9a
--- /dev/null
+++ b/lib/banzai/redactor.rb
@@ -0,0 +1,69 @@
+module Banzai
+  # Class for removing Markdown references a certain user is not allowed to
+  # view.
+  class Redactor
+    attr_reader :user, :project
+
+    # project - A Project to use for redacting links.
+    # user - The currently logged in user (if any).
+    def initialize(project, user = nil)
+      @project = project
+      @user = user
+    end
+
+    # Redacts the references in the given Array of documents.
+    #
+    # This method modifies the given documents in-place.
+    #
+    # documents - A list of HTML documents containing references to redact.
+    #
+    # Returns the documents passed as the first argument.
+    def redact(documents)
+      nodes = documents.flat_map do |document|
+        Querying.css(document, 'a.gfm[data-reference-type]')
+      end
+
+      redact_nodes(nodes)
+
+      documents
+    end
+
+    # Redacts the given nodes
+    #
+    # nodes - An Array of HTML nodes to redact.
+    def redact_nodes(nodes)
+      visible = nodes_visible_to_user(nodes)
+
+      nodes.each do |node|
+        unless visible.include?(node)
+          # The reference should be replaced by the original text,
+          # which is not always the same as the rendered text.
+          text = node.attr('data-original') || node.text
+          node.replace(text)
+        end
+      end
+    end
+
+    # Returns the nodes visible to the current user.
+    #
+    # nodes - The input nodes to check.
+    #
+    # Returns a new Array containing the visible nodes.
+    def nodes_visible_to_user(nodes)
+      per_type = Hash.new { |h, k| h[k] = [] }
+      visible = Set.new
+
+      nodes.each do |node|
+        per_type[node.attr('data-reference-type')] << node
+      end
+
+      per_type.each do |type, nodes|
+        parser = Banzai::ReferenceParser[type].new(project, user)
+
+        visible.merge(parser.nodes_visible_to_user(user, nodes))
+      end
+
+      visible
+    end
+  end
+end
diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb
index c14a9c4c722..6718acdef7e 100644
--- a/lib/banzai/renderer.rb
+++ b/lib/banzai/renderer.rb
@@ -30,13 +30,9 @@ module Banzai
     end
 
     def self.render_result(text, context = {})
-      Pipeline[context[:pipeline]].call(text, context)
-    end
+      text = Pipeline[:pre_process].to_html(text, context) if text
 
-    def self.pre_process(text, context)
-      pipeline = Pipeline[:pre_process]
-
-      pipeline.to_html(text, context)
+      Pipeline[context[:pipeline]].call(text, context)
     end
 
     # Perform post-processing on an HTML String
diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb
index 0c41f22c7c5..bcc82969eb3 100644
--- a/lib/ci/api/runners.rb
+++ b/lib/ci/api/runners.rb
@@ -28,12 +28,9 @@ module Ci
         post "register" do
           required_attributes! [:token]
 
-          attributes = { description: params[:description],
-                         tag_list: params[:tag_list] }
-
-          unless params[:run_untagged].nil?
-            attributes[:run_untagged] = params[:run_untagged]
-          end
+          attributes = attributes_for_keys(
+            [:description, :tag_list, :run_untagged, :locked]
+          )
 
           runner =
             if runner_registration_token_valid?
diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb
index 7a0929d774e..708d01b95a1 100644
--- a/lib/container_registry/tag.rb
+++ b/lib/container_registry/tag.rb
@@ -3,6 +3,7 @@ module ContainerRegistry
     attr_reader :repository, :name
 
     delegate :registry, :client, to: :repository
+    delegate :revision, :short_revision, to: :config_blob, allow_nil: true
 
     def initialize(repository, name)
       @repository, @name = repository, name
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb
index 6d0e30e916f..831f1e635ba 100644
--- a/lib/gitlab/access.rb
+++ b/lib/gitlab/access.rb
@@ -5,6 +5,8 @@
 #
 module Gitlab
   module Access
+    class AccessDeniedError < StandardError; end
+
     GUEST     = 10
     REPORTER  = 20
     DEVELOPER = 30
diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb
index 7e3f5abba62..ab7b811c5d8 100644
--- a/lib/gitlab/backend/grack_auth.rb
+++ b/lib/gitlab/backend/grack_auth.rb
@@ -31,7 +31,7 @@ module Grack
 
       auth!
 
-      lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call
+      lfs_response = Gitlab::Lfs::Router.new(project, @user, @ci, @request).try_call
       return lfs_response unless lfs_response.nil?
 
       if @user.nil? && !@ci
diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb
index 997a22779a0..d62bc50ce78 100644
--- a/lib/gitlab/blame.rb
+++ b/lib/gitlab/blame.rb
@@ -41,7 +41,8 @@ module Gitlab
 
     def highlighted_lines
       @blob.load_all_data!(repository)
-      @highlighted_lines ||= Gitlab::Highlight.highlight(@blob.name, @blob.data).lines
+      @highlighted_lines ||=
+        Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: repository).lines
     end
 
     def project
diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb
index e2fee6b9f3e..047c77c6fc2 100644
--- a/lib/gitlab/email/message/repository_push.rb
+++ b/lib/gitlab/email/message/repository_push.rb
@@ -37,7 +37,7 @@ module Gitlab
         end
 
         def diffs
-          @diffs ||= (safe_diff_files(compare.diffs, diff_refs) if compare)
+          @diffs ||= (safe_diff_files(compare.diffs(max_files: 30), diff_refs) if compare)
         end
 
         def diffs_count
diff --git a/lib/gitlab/gitignore.rb b/lib/gitlab/gitignore.rb
deleted file mode 100644
index f46b43b61a4..00000000000
--- a/lib/gitlab/gitignore.rb
+++ /dev/null
@@ -1,56 +0,0 @@
-module Gitlab
-  class Gitignore
-    FILTER_REGEX = /\.gitignore\z/.freeze
-
-    def initialize(path)
-      @path = path
-    end
-
-    def name
-      File.basename(@path, '.gitignore')
-    end
-
-    def content
-      File.read(@path)
-    end
-
-    class << self
-      def all
-        languages_frameworks + global
-      end
-
-      def find(key)
-        file_name = "#{key}.gitignore"
-
-        directory = select_directory(file_name)
-        directory ? new(File.join(directory, file_name)) : nil
-      end
-
-      def global
-        files_for_folder(global_dir).map { |file| new(File.join(global_dir, file)) }
-      end
-
-      def languages_frameworks
-        files_for_folder(gitignore_dir).map { |file| new(File.join(gitignore_dir, file)) }
-      end
-
-      private
-
-      def select_directory(file_name)
-        [gitignore_dir, global_dir].find { |dir| File.exist?(File.join(dir, file_name)) }
-      end
-
-      def global_dir
-        File.join(gitignore_dir, 'Global')
-      end
-
-      def gitignore_dir
-        Rails.root.join('vendor/gitignore')
-      end
-
-      def files_for_folder(dir)
-        Dir.glob("#{dir.to_s}/*.gitignore").map { |file| file.gsub(FILTER_REGEX, '') }
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb
index 280120b0f9e..41296415e35 100644
--- a/lib/gitlab/highlight.rb
+++ b/lib/gitlab/highlight.rb
@@ -1,7 +1,7 @@
 module Gitlab
   class Highlight
-    def self.highlight(blob_name, blob_content, nowrap: true, plain: false)
-      new(blob_name, blob_content, nowrap: nowrap).
+    def self.highlight(blob_name, blob_content, repository: nil, nowrap: true, plain: false)
+      new(blob_name, blob_content, nowrap: nowrap, repository: repository).
         highlight(blob_content, continue: false, plain: plain)
     end
 
@@ -10,12 +10,21 @@ module Gitlab
       return [] unless blob
 
       blob.load_all_data!(repository)
-      highlight(file_name, blob.data).lines.map!(&:html_safe)
+      highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe)
     end
 
-    def initialize(blob_name, blob_content, nowrap: true)
+    attr_reader :lexer
+    def initialize(blob_name, blob_content, repository: nil, nowrap: true)
+      @blob_name = blob_name
+      @blob_content = blob_content
+      @repository = repository
       @formatter = rouge_formatter(nowrap: nowrap)
-      @lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText
+
+      @lexer = custom_language || begin
+        Rouge::Lexer.guess(filename: blob_name, source: blob_content).new
+      rescue Rouge::Lexer::AmbiguousGuess => e
+        e.alternatives.sort_by(&:tag).first
+      end
     end
 
     def highlight(text, continue: true, plain: false)
@@ -30,6 +39,14 @@ module Gitlab
 
     private
 
+    def custom_language
+      language_name = @repository && @repository.gitattribute(@blob_name, 'gitlab-language')
+
+      return nil unless language_name
+
+      Rouge::Lexer.find_fancy(language_name)
+    end
+
     def rouge_formatter(options = {})
       options = options.reverse_merge(
         nowrap: true,
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 0e70d9282d5..82d1e1805c5 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -23,7 +23,11 @@ module Gitlab
       private
 
       def decompress_archive
-        untar_zxf(archive: @archive_file, dir: @shared.export_path)
+        result = untar_zxf(archive: @archive_file, dir: @shared.export_path)
+
+        raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result
+
+        true
       end
     end
   end
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index d209e04f7be..595b20a09bd 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -10,17 +10,22 @@ module Gitlab
       end
 
       def execute
-        Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
-                                                  shared: @shared)
-        if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
+        if import_file && check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
           project_tree.restored_project
         else
           raise Projects::ImportService::Error.new(@shared.errors.join(', '))
         end
+
+        remove_import_file
       end
 
       private
 
+      def import_file
+        Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
+                                                  shared: @shared)
+      end
+
       def check_version!
         Gitlab::ImportExport::VersionChecker.check!(shared: @shared)
       end
@@ -59,6 +64,10 @@ module Gitlab
       def wiki_repo_path
         File.join(@shared.export_path, 'project.wiki.bundle')
       end
+
+      def remove_import_file
+        FileUtils.rm_rf(@archive_file)
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index b872780f20a..92bf7e0a2fc 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -12,6 +12,8 @@ module Gitlab
 
       USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze
 
+      BUILD_MODELS = %w[Ci::Build commit_status].freeze
+
       def self.create(*args)
         new(*args).create
       end
@@ -70,7 +72,7 @@ module Gitlab
       end
 
       def generate_imported_object
-        if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes
+        if BUILD_MODELS.include?(@relation_name) # call #trace= method after assigning the other attributes
           trace = @relation_hash.delete('trace')
           imported_object do |object|
             object.trace = trace
diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb
index 9d9617761b3..e3ed2f6791d 100644
--- a/lib/gitlab/lfs/response.rb
+++ b/lib/gitlab/lfs/response.rb
@@ -2,10 +2,11 @@ module Gitlab
   module Lfs
     class Response
 
-      def initialize(project, user, request)
+      def initialize(project, user, ci, request)
         @origin_project = project
         @project = storage_project(project)
         @user = user
+        @ci = ci
         @env = request.env
         @request = request
       end
@@ -189,7 +190,7 @@ module Gitlab
         return render_not_enabled unless Gitlab.config.lfs.enabled
 
         unless @project.public?
-          return render_unauthorized unless @user
+          return render_unauthorized unless @user || @ci
           return render_forbidden unless user_can_fetch?
         end
 
@@ -210,7 +211,7 @@ module Gitlab
 
       def user_can_fetch?
         # Check user access against the project they used to initiate the pull
-        @user.can?(:download_code, @origin_project)
+        @ci || @user.can?(:download_code, @origin_project)
       end
 
       def user_can_push?
diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb
index 78d02891102..69bd5e62305 100644
--- a/lib/gitlab/lfs/router.rb
+++ b/lib/gitlab/lfs/router.rb
@@ -1,9 +1,12 @@
 module Gitlab
   module Lfs
     class Router
-      def initialize(project, user, request)
+      attr_reader :project, :user, :ci, :request
+
+      def initialize(project, user, ci, request)
         @project = project
         @user = user
+        @ci = ci
         @env = request.env
         @request = request
       end
@@ -80,7 +83,7 @@ module Gitlab
       def lfs
         return unless @project
 
-        Gitlab::Lfs::Response.new(@project, @user, @request)
+        Gitlab::Lfs::Response.new(@project, @user, @ci, @request)
       end
 
       def sanitize_tmp_filename(name)
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index fd98aa3412e..a1240fd33ee 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -8,6 +8,8 @@ module Gitlab
         trans = Transaction.new("#{worker.class.name}#perform")
 
         begin
+          # Old gitlad-shell messages don't provide enqueued_at/created_at attributes
+          trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0))
           trans.run { yield }
         ensure
           trans.finish
diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb
index 78f3ecb4cb4..7af75a9cc4c 100644
--- a/lib/gitlab/o_auth/user.rb
+++ b/lib/gitlab/o_auth/user.rb
@@ -74,7 +74,7 @@ module Gitlab
         if user
           # Case when a LDAP user already exists in Gitlab. Add the OAuth identity to existing account.
           log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity."
-          user.identities.build(extern_uid: auth_hash.uid, provider: auth_hash.provider)
+          user.identities.find_or_initialize_by(extern_uid: auth_hash.uid, provider: auth_hash.provider)
         else
           log.info "No existing LDAP account was found in GitLab. Checking for #{auth_hash.provider} account."
           user = find_by_uid_and_provider
diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb
index ae85b294d31..4831c46c4be 100644
--- a/lib/gitlab/sidekiq_middleware/memory_killer.rb
+++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb
@@ -25,7 +25,7 @@ module Gitlab
 
           Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\
             "#{MAX_RSS}"
-          Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} "\
+          Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"\
             "in #{GRACE_TIME} seconds"
           sleep(GRACE_TIME)
 
@@ -36,7 +36,7 @@ module Gitlab
             "#{SHUTDOWN_SIGNAL} to PID #{Process.pid}"
           sleep(SHUTDOWN_WAIT)
 
-          Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid}"
+          Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"
           Process.kill(SHUTDOWN_SIGNAL, Process.pid)
         end
       end
diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb
new file mode 100644
index 00000000000..760ff3e614a
--- /dev/null
+++ b/lib/gitlab/template/base_template.rb
@@ -0,0 +1,67 @@
+module Gitlab
+  module Template
+    class BaseTemplate
+      def initialize(path)
+        @path = path
+      end
+
+      def name
+        File.basename(@path, self.class.extension)
+      end
+
+      def content
+        File.read(@path)
+      end
+
+      class << self
+        def all
+          self.categories.keys.flat_map { |cat| by_category(cat) }
+        end
+
+        def find(key)
+          file_name = "#{key}#{self.extension}"
+
+          directory = select_directory(file_name)
+          directory ? new(File.join(category_directory(directory), file_name)) : nil
+        end
+
+        def categories
+          raise NotImplementedError
+        end
+
+        def extension
+          raise NotImplementedError
+        end
+
+        def base_dir
+          raise NotImplementedError
+        end
+
+        def by_category(category)
+          templates_for_directory(category_directory(category))
+        end
+
+        def category_directory(category)
+          File.join(base_dir, categories[category])
+        end
+
+        private
+
+        def select_directory(file_name)
+          categories.keys.find do |category|
+            File.exist?(File.join(category_directory(category), file_name))
+          end
+        end
+
+        def templates_for_directory(dir)
+          dir << '/' unless dir.end_with?('/')
+          Dir.glob(File.join(dir, "*#{self.extension}")).select { |f| f =~ filter_regex }.map { |f| new(f) }
+        end
+
+        def filter_regex
+          @filter_reges ||= /#{Regexp.escape(extension)}\z/
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/template/gitignore.rb b/lib/gitlab/template/gitignore.rb
new file mode 100644
index 00000000000..964fbfd4de3
--- /dev/null
+++ b/lib/gitlab/template/gitignore.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Template
+    class Gitignore < BaseTemplate
+      class << self
+        def extension
+          '.gitignore'
+        end
+
+        def categories
+          {
+            "Languages" => '',
+            "Global"    => 'Global'
+          }
+        end
+
+        def base_dir
+          Rails.root.join('vendor/gitignore')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/template/gitlab_ci_yml.rb b/lib/gitlab/template/gitlab_ci_yml.rb
new file mode 100644
index 00000000000..7f480fe33c0
--- /dev/null
+++ b/lib/gitlab/template/gitlab_ci_yml.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Template
+    class GitlabCiYml < BaseTemplate
+      def content
+        explanation = "# This file is a template, and might need editing before it works on your project."
+        [explanation, super].join("\n")
+      end
+
+      class << self
+        def extension
+          '.gitlab-ci.yml'
+        end
+
+        def categories
+          {
+            "General" => '',
+            "Pages" => 'Pages'
+          }
+        end
+
+        def base_dir
+          Rails.root.join('vendor/gitlab-ci-yml')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake
new file mode 100644
index 00000000000..c2c6031db67
--- /dev/null
+++ b/lib/tasks/gitlab/import_export.rake
@@ -0,0 +1,13 @@
+namespace :gitlab do
+  namespace :import_export do
+    desc "GitLab | Show Import/Export version"
+    task version: :environment do
+      puts "Import/Export v#{Gitlab::ImportExport.version}"
+    end
+
+    desc "GitLab | Display exported DB structure"
+    task data: :environment do
+      puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(:SortKeys => true)
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/update_gitignore.rake b/lib/tasks/gitlab/update_gitignore.rake
deleted file mode 100644
index 4fd48cccb1d..00000000000
--- a/lib/tasks/gitlab/update_gitignore.rake
+++ /dev/null
@@ -1,46 +0,0 @@
-namespace :gitlab do
-  desc "GitLab | Update gitignore"
-  task :update_gitignore do
-    unless clone_gitignores
-      puts "Cloning the gitignores failed".color(:red)
-      return
-    end
-
-    remove_unneeded_files(gitignore_directory)
-    remove_unneeded_files(global_directory)
-
-    puts "Done".color(:green)
-  end
-
-  def clone_gitignores
-    FileUtils.rm_rf(gitignore_directory) if Dir.exist?(gitignore_directory)
-    FileUtils.cd vendor_directory
-
-    system('git clone --depth=1 --branch=master https://github.com/github/gitignore.git')
-  end
-
-  # Retain only certain files:
-  # - The LICENSE, because we have to
-  # - The sub dir global
-  # - The gitignores themself
-  # - Dir.entires returns also the entries '.' and '..'
-  def remove_unneeded_files(path)
-    Dir.foreach(path) do |file|
-      FileUtils.rm_rf(File.join(path, file)) unless file =~ /(\.{1,2}|LICENSE|Global|\.gitignore)\z/
-    end
-  end
-
-  private
-
-  def vendor_directory
-    Rails.root.join('vendor')
-  end
-
-  def gitignore_directory
-    File.join(vendor_directory, 'gitignore')
-  end
-
-  def global_directory
-    File.join(gitignore_directory, 'Global')
-  end
-end
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
new file mode 100644
index 00000000000..4f76dad7286
--- /dev/null
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -0,0 +1,54 @@
+namespace :gitlab do
+  desc "GitLab | Update templates"
+  task :update_templates do
+    TEMPLATE_DATA.each { |template| update(template) }
+  end
+
+  def update(template)
+    sub_dir = template.repo_url.match(/([a-z-]+)\.git\z/)[1]
+    dir = File.join(vendor_directory, sub_dir)
+
+    unless clone_repository(template.repo_url, dir)
+      puts "Cloning the #{sub_dir} templates failed".red
+      return
+    end
+
+    remove_unneeded_files(dir, template.cleanup_regex)
+    puts "Done".green
+  end
+
+  def clone_repository(url, directory)
+    FileUtils.rm_rf(directory) if Dir.exist?(directory)
+
+    system("git clone #{url} --depth=1 --branch=master #{directory}")
+  end
+
+  # Retain only certain files:
+  # - The LICENSE, because we have to
+  # - The sub dirs so we can organise the file by category
+  # - The templates themself
+  # - Dir.entries returns also the entries '.' and '..'
+  def remove_unneeded_files(directory, regex)
+    Dir.foreach(directory) do |file|
+      FileUtils.rm_rf(File.join(directory, file)) unless file =~ regex
+    end
+  end
+
+  private
+
+  Template = Struct.new(:repo_url, :cleanup_regex)
+  TEMPLATE_DATA = [
+    Template.new(
+      "https://github.com/github/gitignore.git",
+      /(\.{1,2}|LICENSE|Global|\.gitignore)\z/
+    ),
+    Template.new(
+      "https://gitlab.com/gitlab-org/gitlab-ci-yml.git",
+      /(\.{1,2}|LICENSE|Pages|\.gitlab-ci.yml)\z/
+    )
+  ]
+
+  def vendor_directory
+    Rails.root.join('vendor')
+  end
+end