Merge branch 'master' into auto-pipelines-vue

17 files changed, 264 insertions, 12 deletions

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 01c0f5072ba..dfbb3ab86dd 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -629,7 +629,7 @@ module API
     end
 
     class EnvironmentBasic < Grape::Entity
-      expose :id, :name, :external_url
+      expose :id, :name, :slug, :external_url
     end
 
     class Environment < EnvironmentBasic
diff --git a/lib/api/environments.rb b/lib/api/environments.rb
index 80bbd9bb6e4..1a7e68f0528 100644
--- a/lib/api/environments.rb
+++ b/lib/api/environments.rb
@@ -1,6 +1,7 @@
 module API
   # Environments RESTfull API endpoints
   class Environments < Grape::API
+    include ::API::Helpers::CustomValidators
     include PaginationParams
 
     before { authenticate! }
@@ -29,6 +30,7 @@ module API
       params do
         requires :name,           type: String,   desc: 'The name of the environment to be created'
         optional :external_url,   type: String,   desc: 'URL on which this deployment is viewable'
+        optional :slug, absence: { message: "is automatically generated and cannot be changed" }
       end
       post ':id/environments' do
         authorize! :create_environment, user_project
@@ -50,6 +52,7 @@ module API
         requires :environment_id, type: Integer,  desc: 'The environment ID'
         optional :name,           type: String,   desc: 'The new environment name'
         optional :external_url,   type: String,   desc: 'The new URL on which this deployment is viewable'
+        optional :slug, absence: { message: "is automatically generated and cannot be changed" }
       end
       put ':id/environments/:environment_id' do
         authorize! :update_environment, user_project
diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb
new file mode 100644
index 00000000000..0a8f3073a50
--- /dev/null
+++ b/lib/api/helpers/custom_validators.rb
@@ -0,0 +1,14 @@
+module API
+  module Helpers
+    module CustomValidators
+      class Absence < Grape::Validations::Base
+        def validate_param!(attr_name, params)
+          return if params.respond_to?(:key?) && !params.key?(attr_name)
+          raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence)
+        end
+      end
+    end
+  end
+end
+
+Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence)
diff --git a/lib/api/services.rb b/lib/api/services.rb
index b1e072b4f47..59232c84c24 100644
--- a/lib/api/services.rb
+++ b/lib/api/services.rb
@@ -473,7 +473,7 @@ module API
           desc: 'The description of the tracker'
         }
       ],
-      'slack' => [
+      'slack-notification' => [
         {
           required: true,
           name: :webhook,
@@ -493,6 +493,14 @@ module API
           desc: 'The channel name'
         }
       ],
+      'mattermost-notification' => [
+        {
+          required: true,
+          name: :webhook,
+          type: String,
+          desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...'
+        }
+      ],
       'teamcity' => [
         {
           required: true,
diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb
new file mode 100644
index 00000000000..cb037f89337
--- /dev/null
+++ b/lib/banzai/filter/math_filter.rb
@@ -0,0 +1,51 @@
+require 'uri'
+
+module Banzai
+  module Filter
+    # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$.
+    #
+    class MathFilter < HTML::Pipeline::Filter
+      # This picks out <code>...</code>.
+      INLINE_MATH = 'descendant-or-self::code'.freeze
+
+      # Pick out a code block which is declared math
+      DISPLAY_MATH = "descendant-or-self::pre[contains(@class, 'math') and contains(@class, 'code')]".freeze
+
+      # Attribute indicating inline or display math.
+      STYLE_ATTRIBUTE = 'data-math-style'.freeze
+
+      # Class used for tagging elements that should be rendered
+      TAG_CLASS = 'js-render-math'.freeze
+
+      INLINE_CLASSES = "code math #{TAG_CLASS}".freeze
+
+      DOLLAR_SIGN = '$'.freeze
+
+      def call
+        doc.xpath(INLINE_MATH).each do |code|
+          closing = code.next
+          opening = code.previous
+
+          # We need a sibling before and after.
+          # They should end and start with $ respectively.
+          if closing && opening &&
+              closing.content.first == DOLLAR_SIGN &&
+              opening.content.last == DOLLAR_SIGN
+
+            code[:class] = INLINE_CLASSES
+            code[STYLE_ATTRIBUTE] = 'inline'
+            closing.content = closing.content[1..-1]
+            opening.content = opening.content[0..-2]
+          end
+        end
+
+        doc.xpath(DISPLAY_MATH).each do |el|
+          el[STYLE_ATTRIBUTE] = 'display'
+          el[:class] += " #{TAG_CLASS}"
+        end
+
+        doc
+      end
+    end
+  end
+end
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index 5da2d0b008c..5a1f873496c 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -6,6 +6,7 @@ module Banzai
           Filter::SyntaxHighlightFilter,
           Filter::SanitizationFilter,
 
+          Filter::MathFilter,
           Filter::UploadLinkFilter,
           Filter::VideoLinkFilter,
           Filter::ImageLinkFilter,
diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb
index 9667df4ffb8..fa234284361 100644
--- a/lib/gitlab/asciidoc.rb
+++ b/lib/gitlab/asciidoc.rb
@@ -1,4 +1,5 @@
 require 'asciidoctor'
+require 'asciidoctor/converter/html5'
 
 module Gitlab
   # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters
@@ -23,7 +24,7 @@ module Gitlab
     def self.render(input, context, asciidoc_opts = {})
       asciidoc_opts.reverse_merge!(
         safe: :secure,
-        backend: :html5,
+        backend: :gitlab_html5,
         attributes: []
       )
       asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS)
@@ -34,5 +35,29 @@ module Gitlab
 
       html.html_safe
     end
+
+    class Html5Converter < Asciidoctor::Converter::Html5Converter
+      extend Asciidoctor::Converter::Config
+
+      register_for 'gitlab_html5'
+
+      def stem(node)
+        return super unless node.style.to_sym == :latexmath
+
+        %(<pre#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="display"><code>#{node.content}</code></pre>)
+      end
+
+      def inline_quoted(node)
+        return super unless node.type.to_sym == :latexmath
+
+        %(<code#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="inline">#{node.text}</code>)
+      end
+
+      private
+
+      def id_attribute(node)
+        node.id ? %( id="#{node.id}") : nil
+      end
+    end
   end
 end
diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/build/status.rb
index 50aa45e5406..b762d85b6e5 100644
--- a/lib/gitlab/badge/build/status.rb
+++ b/lib/gitlab/badge/build/status.rb
@@ -20,8 +20,8 @@ module Gitlab
 
         def status
           @project.pipelines
-            .where(sha: @sha, ref: @ref)
-            .status || 'unknown'
+            .where(sha: @sha)
+            .latest_status(@ref) || 'unknown'
         end
 
         def metadata
diff --git a/lib/gitlab/chat_commands/issue_create.rb b/lib/gitlab/chat_commands/issue_create.rb
index 1dba85c1b51..cefb6775db8 100644
--- a/lib/gitlab/chat_commands/issue_create.rb
+++ b/lib/gitlab/chat_commands/issue_create.rb
@@ -8,7 +8,7 @@ module Gitlab
       end
 
       def self.help_message
-        'issue new <title>\n<description>'
+        'issue new <title> *`⇧ Shift`*+*`↵ Enter`* <description>'
       end
 
       def self.allowed?(project, user)
diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb
index abc8c8c55e6..8fab5489616 100644
--- a/lib/gitlab/gfm/uploads_rewriter.rb
+++ b/lib/gitlab/gfm/uploads_rewriter.rb
@@ -1,3 +1,5 @@
+require 'fileutils'
+
 module Gitlab
   module Gfm
     ##
@@ -22,7 +24,9 @@ module Gitlab
           return markdown unless file.try(:exists?)
 
           new_uploader = FileUploader.new(target_project)
-          new_uploader.store!(file)
+          with_link_in_tmp_dir(file.file) do |open_tmp_file|
+            new_uploader.store!(open_tmp_file)
+          end
           new_uploader.to_markdown
         end
       end
@@ -46,6 +50,19 @@ module Gitlab
         uploader.retrieve_from_store!(file)
         uploader.file
       end
+
+      # Because the uploaders use 'move_to_store' we must have a temporary
+      # file that is allowed to be (re)moved.
+      def with_link_in_tmp_dir(file)
+        dir = Dir.mktmpdir('UploadsRewriter', File.dirname(file))
+        # The filename matters to Carrierwave so we make sure to preserve it
+        tmp_file = File.join(dir, File.basename(file))
+        File.link(file, tmp_file)
+        # Open the file to placate Carrierwave
+        File.open(tmp_file) { |open_file| yield open_file }
+      ensure
+        FileUtils.rm_rf(dir)
+      end
     end
   end
 end
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 2c21804fe7a..4d4e04e9e35 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -8,6 +8,8 @@ module Gitlab
       gon.shortcuts_path         = help_page_path('shortcuts')
       gon.user_color_scheme      = Gitlab::ColorSchemes.for_user(current_user).css_class
       gon.award_menu_url         = emojis_path
+      gon.katex_css_url          = ActionController::Base.helpers.asset_path('katex.css')
+      gon.katex_js_url           = ActionController::Base.helpers.asset_path('katex.js')
 
       if current_user
         gon.current_user_id = current_user.id
diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb
new file mode 100644
index 00000000000..65713e73a59
--- /dev/null
+++ b/lib/gitlab/middleware/multipart.rb
@@ -0,0 +1,99 @@
+# Gitlab::Middleware::Multipart - a Rack::Multipart replacement
+#
+# Rack::Multipart leaves behind tempfiles in /tmp and uses valuable Ruby
+# process time to copy files around. This alternative solution uses
+# gitlab-workhorse to clean up the tempfiles and puts the tempfiles in a
+# location where copying should not be needed.
+#
+# When gitlab-workhorse finds files in a multipart MIME body it sends
+# a signed message via a request header. This message lists the names of
+# the multipart entries that gitlab-workhorse filtered out of the
+# multipart structure and saved to tempfiles. Workhorse adds new entries
+# in the multipart structure with paths to the tempfiles.
+#
+# The job of this Rack middleware is to detect and decode the message
+# from workhorse. If present, it walks the Rack 'params' hash for the
+# current request, opens the respective tempfiles, and inserts the open
+# Ruby File objects in the params hash where Rack::Multipart would have
+# put them. The goal is that application code deeper down can keep
+# working the way it did with Rack::Multipart without changes.
+#
+# CAVEAT: the code that modifies the params hash is a bit complex. It is
+# conceivable that certain Rack params structures will not be modified
+# correctly. We are not aware of such bugs at this time though.
+#
+
+module Gitlab
+  module Middleware
+    class Multipart
+      RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS'
+
+      class Handler
+        def initialize(env, message)
+          @request = Rack::Request.new(env)
+          @rewritten_fields = message['rewritten_fields']
+          @open_files = []
+        end
+
+        def with_open_files
+          @rewritten_fields.each do |field, tmp_path|
+            parsed_field = Rack::Utils.parse_nested_query(field)
+            raise "unexpected field: #{field.inspect}" unless parsed_field.count == 1
+
+            key, value = parsed_field.first
+            if value.nil?
+              value = File.open(tmp_path)
+              @open_files << value
+            else
+              value = decorate_params_value(value, @request.params[key], tmp_path)
+            end
+            @request.update_param(key, value)
+          end
+
+          yield
+        ensure
+          @open_files.each(&:close)
+        end
+
+        # This function calls itself recursively
+        def decorate_params_value(path_hash, value_hash, tmp_path)
+          unless path_hash.is_a?(Hash) && path_hash.count == 1
+            raise "invalid path: #{path_hash.inspect}"
+          end
+          path_key, path_value = path_hash.first
+
+          unless value_hash.is_a?(Hash) && value_hash[path_key]
+            raise "invalid value hash: #{value_hash.inspect}"
+          end
+
+          case path_value
+          when nil
+            value_hash[path_key] = File.open(tmp_path)
+            @open_files << value_hash[path_key]
+            value_hash
+          when Hash
+            decorate_params_value(path_value, value_hash[path_key], tmp_path)
+            value_hash
+          else
+            raise "unexpected path value: #{path_value.inspect}"
+          end
+        end
+      end
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        encoded_message = env.delete(RACK_ENV_KEY)
+        return @app.call(env) if encoded_message.blank?
+
+        message = Gitlab::Workhorse.decode_jwt(encoded_message)[0]
+
+        Handler.new(env, message).with_open_files do
+          @app.call(env)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 66e6b29e798..6bdf3db9cb8 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -110,7 +110,7 @@ module Gitlab
     end
 
     def notes
-      @notes ||= project.notes.user.search(query, as_user: @current_user).order('updated_at DESC')
+      @notes ||= NotesFinder.new(project, @current_user, search: query).execute.user.order('updated_at DESC')
     end
 
     def commits
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 7c711d581e8..9e0b0e5ea98 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -131,5 +131,14 @@ module Gitlab
     def kubernetes_namespace_regex_message
       "can contain only letters, digits or '-', and cannot start or end with '-'"
     end
+
+    def environment_slug_regex
+      @environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/.freeze
+    end
+
+    def environment_slug_regex_message
+      "can contain only lowercase letters, digits, and '-'. " \
+      "Must start with a letter, and cannot end with '-'"
+    end
   end
 end
diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb
index 1cd89b3a9c4..222021e8802 100644
--- a/lib/gitlab/sql/union.rb
+++ b/lib/gitlab/sql/union.rb
@@ -22,9 +22,7 @@ module Gitlab
         # By using "unprepared_statements" we remove the usage of placeholders
         # (thus fixing this problem), at a slight performance cost.
         fragments = ActiveRecord::Base.connection.unprepared_statement do
-          @relations.map do |rel|
-            rel.reorder(nil).to_sql
-          end
+          @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?)
         end
 
         fragments.join("\nUNION\n")
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 594439a5d4b..aeb1a26e1ba 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -117,8 +117,12 @@ module Gitlab
       end
 
       def verify_api_request!(request_headers)
+        decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER])
+      end
+
+      def decode_jwt(encoded_message)
         JWT.decode(
-          request_headers[INTERNAL_API_REQUEST_HEADER],
+          encoded_message,
           secret,
           true,
           { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' },
diff --git a/lib/rouge/lexers/math.rb b/lib/rouge/lexers/math.rb
new file mode 100644
index 00000000000..80784adfd76
--- /dev/null
+++ b/lib/rouge/lexers/math.rb
@@ -0,0 +1,21 @@
+module Rouge
+  module Lexers
+    class Math < Lexer
+      title "A passthrough lexer used for LaTeX input"
+      desc "A boring lexer that doesn't highlight anything"
+
+      tag 'math'
+      mimetypes 'text/plain'
+
+      default_options token: 'Text'
+
+      def token
+        @token ||= Token[option :token]
+      end
+
+      def stream_tokens(string, &b)
+        yield self.token, string
+      end
+    end
+  end
+end