Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ce into store-variables-and-when-in-builds-tablestore-variables-and-when-in-builds-table

# Conflicts:
#	db/schema.rb

16 files changed, 116 insertions, 227 deletions

diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb
index c4fa1838b5a..2efe7e3adf3 100644
--- a/lib/api/award_emoji.rb
+++ b/lib/api/award_emoji.rb
@@ -56,9 +56,9 @@ module API
 
             not_found!('Award Emoji') unless can_read_awardable?
 
-            award = awardable.award_emoji.new(name: params[:name], user: current_user)
+            award = awardable.create_award_emoji(params[:name], current_user)
 
-            if award.save
+            if award.persisted?
               present award, with: Entities::AwardEmoji
             else
               not_found!("Award Emoji #{award.errors.messages}")
diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb
index 323a7086890..acb4812b5cf 100644
--- a/lib/api/commit_statuses.rb
+++ b/lib/api/commit_statuses.rb
@@ -64,7 +64,7 @@ module API
           ref = branches.first
         end
 
-        pipeline = @project.ensure_pipeline(commit.sha, ref)
+        pipeline = @project.ensure_pipeline(commit.sha, ref, current_user)
 
         name = params[:name] || params[:context]
         status = GenericCommitStatus.running_or_pending.find_by(pipeline: pipeline, name: name, ref: params[:ref])
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index d5dfba5e0cc..959b700de78 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -63,7 +63,12 @@ module API
         if access_status.status
           # Return the repository full path so that gitlab-shell has it when
           # handling ssh commands
-          response[:repository_path] = project.repository.path_to_repo
+          response[:repository_path] =
+            if wiki?
+              project.wiki.repository.path_to_repo
+            else
+              project.repository.path_to_repo
+            end
         end
 
         response
diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb
index 536b478979f..91f0159f9a1 100644
--- a/lib/banzai/filter/syntax_highlight_filter.rb
+++ b/lib/banzai/filter/syntax_highlight_filter.rb
@@ -19,14 +19,22 @@ module Banzai
         language = node.attr('class')
         code     = node.text
 
+        css_classes = "code highlight"
+
+        lexer = Rouge::Lexer.find_fancy(language) || Rouge::Lexers::PlainText
+        formatter = Rouge::Formatters::HTML.new
+
         begin
-          highlighted = block_code(code, language)
+          code = formatter.format(lexer.lex(code))
+
+          css_classes << " js-syntax-highlight #{lexer.tag}"
         rescue
           # Gracefully handle syntax highlighter bugs/errors to ensure
           # users can still access an issue/comment/etc.
-          highlighted = "<pre>#{code}</pre>"
         end
 
+        highlighted = %(<pre class="#{css_classes}"><code>#{code}</code></pre>)
+
         # Extracted to a method to measure it
         replace_parent_pre_element(node, highlighted)
       end
@@ -40,8 +48,7 @@ module Banzai
 
       # Override Rouge::Plugins::Redcarpet#rouge_formatter
       def rouge_formatter(lexer)
-        Rouge::Formatters::HTMLGitlab.new(
-          cssclass: "code highlight js-syntax-highlight #{lexer.tag}")
+        Rouge::Formatters::HTML.new
       end
     end
   end
diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb
index 5b0a6d8541b..e1ca7f4d24b 100644
--- a/lib/banzai/filter/user_reference_filter.rb
+++ b/lib/banzai/filter/user_reference_filter.rb
@@ -112,7 +112,7 @@ module Banzai
         data = data_attribute(project: project.id, author: author.try(:id))
         text = link_text || User.reference_prefix + 'all'
 
-        link_tag(url, data, text)
+        link_tag(url, data, text, 'All Project and Group Members')
       end
 
       def link_to_namespace(namespace, link_text: nil)
@@ -128,7 +128,7 @@ module Banzai
         data = data_attribute(group: namespace.id)
         text = link_text || Group.reference_prefix + group
 
-        link_tag(url, data, text)
+        link_tag(url, data, text, namespace.name)
       end
 
       def link_to_user(user, namespace, link_text: nil)
@@ -136,11 +136,11 @@ module Banzai
         data = data_attribute(user: namespace.owner_id)
         text = link_text || User.reference_prefix + user
 
-        link_tag(url, data, text)
+        link_tag(url, data, text, namespace.owner_name)
       end
 
-      def link_tag(url, data, text)
-        %(<a href="#{url}" #{data} class="#{link_class}">#{escape_once(text)}</a>)
+      def link_tag(url, data, text, title)
+        %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{escape_once(text)}</a>)
       end
     end
   end
diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb
index 478f145bfed..ab94abeda77 100644
--- a/lib/gitlab/backend/grack_auth.rb
+++ b/lib/gitlab/backend/grack_auth.rb
@@ -63,7 +63,7 @@ module Grack
     def ci_request?(login, password)
       matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login)
 
-      if project && matched_login.present? && git_cmd == 'git-upload-pack'
+      if project && matched_login.present?
         underscored_service = matched_login['s'].underscore
 
         if underscored_service == 'gitlab_ci'
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index dec20d8659b..927f9dad20b 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -20,11 +20,19 @@ module Gitlab
 
         if Database.postgresql?
           options = options.merge({ algorithm: :concurrently })
+          disable_statement_timeout
         end
 
         add_index(table_name, column_name, options)
       end
 
+      # Long-running migrations may take more than the timeout allowed by
+      # the database. Disable the session's statement timeout to ensure
+      # migrations don't get killed prematurely. (PostgreSQL only)
+      def disable_statement_timeout
+        ActiveRecord::Base.connection.execute('SET statement_timeout TO 0') if Database.postgresql?
+      end
+
       # Updates the value of a column in batches.
       #
       # This method updates the table in batches of 5% of the total row count.
@@ -133,6 +141,8 @@ module Gitlab
             'in the body of your migration class'
         end
 
+        disable_statement_timeout
+
         transaction do
           add_column(table, column, type, default: nil)
 
diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb
index e6d31ea04c0..46d40f75be6 100644
--- a/lib/gitlab/gitlab_import/importer.rb
+++ b/lib/gitlab/gitlab_import/importer.rb
@@ -15,32 +15,35 @@ module Gitlab
       end
 
       def execute
-        project_identifier = CGI.escape(project.import_source)
-
-        # Issues && Comments
-        issues = client.issues(project_identifier)
-
-        issues.each do |issue|
-          body = @formatter.author_line(issue["author"]["name"])
-          body += issue["description"]
-
-          comments = client.issue_comments(project_identifier, issue["id"])
-
-          if comments.any?
-            body += @formatter.comments_header
+        ActiveRecord::Base.no_touching do
+          project_identifier = CGI.escape(project.import_source)
+
+          # Issues && Comments
+          issues = client.issues(project_identifier)
+
+          issues.each do |issue|
+            body = @formatter.author_line(issue["author"]["name"])
+            body += issue["description"]
+
+            comments = client.issue_comments(project_identifier, issue["id"])
+
+            if comments.any?
+              body += @formatter.comments_header
+            end
+
+            comments.each do |comment|
+              body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"])
+            end
+
+            project.issues.create!(
+              iid: issue["iid"],
+              description: body,
+              title: issue["title"],
+              state: issue["state"],
+              updated_at: issue["updated_at"],
+              author_id: gl_user_id(project, issue["author"]["id"])
+            )
           end
-
-          comments.each do |comment|
-            body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"])
-          end
-
-          project.issues.create!(
-            iid: issue["iid"],
-            description: body,
-            title: issue["title"],
-            state: issue["state"],
-            author_id: gl_user_id(project, issue["author"]["id"])
-          )
         end
 
         true
diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb
index 41296415e35..9360afedfcb 100644
--- a/lib/gitlab/highlight.rb
+++ b/lib/gitlab/highlight.rb
@@ -1,7 +1,7 @@
 module Gitlab
   class Highlight
-    def self.highlight(blob_name, blob_content, repository: nil, nowrap: true, plain: false)
-      new(blob_name, blob_content, nowrap: nowrap, repository: repository).
+    def self.highlight(blob_name, blob_content, repository: nil, plain: false)
+      new(blob_name, blob_content, repository: repository).
         highlight(blob_content, continue: false, plain: plain)
     end
 
@@ -13,30 +13,34 @@ module Gitlab
       highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe)
     end
 
-    attr_reader :lexer
-    def initialize(blob_name, blob_content, repository: nil, nowrap: true)
+    def initialize(blob_name, blob_content, repository: nil)
+      @formatter = Rouge::Formatters::HTMLGitlab.new
+      @repository = repository
       @blob_name = blob_name
       @blob_content = blob_content
-      @repository = repository
-      @formatter = rouge_formatter(nowrap: nowrap)
-
-      @lexer = custom_language || begin
-        Rouge::Lexer.guess(filename: blob_name, source: blob_content).new
-      rescue Rouge::Lexer::AmbiguousGuess => e
-        e.alternatives.sort_by(&:tag).first
-      end
     end
 
     def highlight(text, continue: true, plain: false)
       if plain
-        @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe
+        hl_lexer = Rouge::Lexers::PlainText
+        continue = false
       else
-        @formatter.format(@lexer.lex(text, continue: continue)).html_safe
+        hl_lexer = self.lexer
       end
+
+      @formatter.format(hl_lexer.lex(text, continue: continue)).html_safe
     rescue
       @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe
     end
 
+    def lexer
+      @lexer ||= custom_language || begin
+        Rouge::Lexer.guess(filename: @blob_name, source: @blob_content).new
+      rescue Rouge::Guesser::Ambiguous => e
+        e.alternatives.sort_by(&:tag).first
+      end
+    end
+
     private
 
     def custom_language
@@ -46,16 +50,5 @@ module Gitlab
 
       Rouge::Lexer.find_fancy(language_name)
     end
-
-    def rouge_formatter(options = {})
-      options = options.reverse_merge(
-        nowrap: true,
-        cssclass: 'code highlight',
-        lineanchors: true,
-        lineanchorsid: 'LC'
-      )
-
-      Rouge::Formatters::HTMLGitlab.new(options)
-    end
   end
 end
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index 588647e5adb..bab2ea73c4f 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -3,6 +3,7 @@ module Gitlab
     extend self
 
     VERSION = '0.1.1'
+    FILENAME_LIMIT = 50
 
     def export_path(relative_path:)
       File.join(storage_path, relative_path)
@@ -28,6 +29,12 @@ module Gitlab
       'VERSION'
     end
 
+    def export_filename(project:)
+      basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.namespace.path}_#{project.path}"
+
+      "#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
+    end
+
     def version
       VERSION
     end
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 025ecc12f9f..051110c23cf 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -12,7 +12,10 @@ module Gitlab
         json = IO.read(@path)
         @tree_hash = ActiveSupport::JSON.decode(json)
         @project_members = @tree_hash.delete('project_members')
-        create_relations
+
+        ActiveRecord::Base.no_touching do
+          create_relations
+        end
       rescue => e
         @shared.error(e)
         false
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 9824df3f274..6ba25a31641 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -87,7 +87,7 @@ module Gitlab
         project_id = @relation_hash.delete('project_id')
 
         # project_id may not be part of the export, but we always need to populate it if required.
-        @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id')
+        @relation_hash['project_id'] = project_id
         @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id']
         @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id']
         @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id']
@@ -111,7 +111,7 @@ module Gitlab
       end
 
       def imported_object
-        imported_object = relation_class.new(@relation_hash)
+        imported_object = relation_class.new(parsed_relation_hash)
         yield(imported_object) if block_given?
         imported_object.importing = true if imported_object.respond_to?(:importing)
         imported_object
@@ -125,6 +125,10 @@ module Gitlab
       def admin_user?
         @user.is_admin?
       end
+
+      def parsed_relation_hash
+        @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) }
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index 6a60b65071f..6130c124dd1 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -7,7 +7,8 @@ module Gitlab
         new(*args).save
       end
 
-      def initialize(shared:)
+      def initialize(project:, shared:)
+        @project = project
         @shared = shared
       end
 
@@ -36,7 +37,7 @@ module Gitlab
       end
 
       def archive_file
-        @archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz")
+        @archive_file ||= File.join(@shared.export_path, '..', Gitlab::ImportExport.export_filename(project: @project))
       end
     end
   end
diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb
index 811363405a8..a1ee1aa81ff 100644
--- a/lib/gitlab/lfs/response.rb
+++ b/lib/gitlab/lfs/response.rb
@@ -47,6 +47,8 @@ module Gitlab
       end
 
       def render_storage_upload_store_response(oid, size, tmp_file_name)
+        return render_forbidden unless tmp_file_name
+
         render_response_to_push do
           render_lfs_upload_ok(oid, size, tmp_file_name)
         end
diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb
index 69bd5e62305..f2a76a56b8f 100644
--- a/lib/gitlab/lfs/router.rb
+++ b/lib/gitlab/lfs/router.rb
@@ -74,8 +74,6 @@ module Gitlab
           lfs.render_storage_upload_authorize_response(oid, size)
         else
           tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP'])
-          return nil unless tmp_file_name
-
           lfs.render_storage_upload_store_response(oid, size, tmp_file_name)
         end
       end
diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb
index 3358ed6773e..f818dc78d34 100644
--- a/lib/rouge/formatters/html_gitlab.rb
+++ b/lib/rouge/formatters/html_gitlab.rb
@@ -1,171 +1,27 @@
-require 'cgi'
-
 module Rouge
   module Formatters
-    class HTMLGitlab < Rouge::Formatter
+    class HTMLGitlab < Rouge::Formatters::HTML
       tag 'html_gitlab'
 
       # Creates a new <tt>Rouge::Formatter::HTMLGitlab</tt> instance.
       #
-      # [+nowrap+]          If set to True, don't wrap the output at all, not
-      #                     even inside a <tt><pre></tt> tag (default: false).
-      # [+cssclass+]        CSS class for the wrapping <tt><div></tt> tag
-      #                     (default: 'highlight').
-      # [+linenos+]         If set to 'table', output line numbers as a table
-      #                     with two cells, one containing the line numbers,
-      #                     the other the whole code. This is copy paste friendly,
-      #                     but may cause alignment problems with some browsers
-      #                     or fonts. If set to 'inline', the line numbers will
-      #                     be integrated in the <tt><pre></tt> tag that contains
-      #                     the code (default: nil).
       # [+linenostart+]     The line number for the first line (default: 1).
-      # [+lineanchors+]     If set to true the formatter will wrap each output
-      #                     line in an anchor tag with a name of L-linenumber.
-      #                     This allows easy linking to certain lines
-      #                     (default: false).
-      # [+lineanchorsid+]   If lineanchors is true the name of the anchors can
-      #                     be changed with lineanchorsid to e.g. foo-linenumber
-      #                     (default: 'L').
-      # [+anchorlinenos+]   If set to true, will wrap line numbers in <tt><a></tt>
-      #                     tags. Used in combination with linenos and lineanchors
-      #                     (default: false).
-      # [+inline_theme+]    Inline CSS styles for the <pre> tag (default: false).
-      def initialize(
-          nowrap: false,
-          cssclass: 'highlight',
-          linenos: nil,
-          linenostart: 1,
-          lineanchors: false,
-          lineanchorsid: 'L',
-          anchorlinenos: false,
-          inline_theme: nil
-      )
-        @nowrap = nowrap
-        @cssclass = cssclass
-        @linenos = linenos
+      def initialize(linenostart: 1)
         @linenostart = linenostart
-        @lineanchors = lineanchors
-        @lineanchorsid = lineanchorsid
-        @anchorlinenos = anchorlinenos
-        @inline_theme = Theme.find(inline_theme).new if inline_theme.is_a?(String)
-      end
-
-      def render(tokens)
-        case @linenos
-        when 'table'
-          render_tableized(tokens)
-        when 'inline'
-          render_untableized(tokens)
-        else
-          render_untableized(tokens)
-        end
-      end
-
-      alias_method :format, :render
-
-      private
-
-      def render_untableized(tokens)
-        data = process_tokens(tokens)
-
-        html = ''
-        html << "<pre class=\"#{@cssclass}\"><code>" unless @nowrap
-        html << wrap_lines(data[:code])
-        html << "</code></pre>\n" unless @nowrap
-        html
+        @line_number = linenostart
       end
 
-      def render_tableized(tokens)
-        data = process_tokens(tokens)
-
-        html = ''
-        html << "<div class=\"#{@cssclass}\">" unless @nowrap
-        html << '<table><tbody>'
-        html << "<td class=\"linenos\"><pre>"
-        html << wrap_linenos(data[:numbers])
-        html << '</pre></td>'
-        html << "<td class=\"lines\"><pre><code>"
-        html << wrap_lines(data[:code])
-        html << '</code></pre></td>'
-        html << '</tbody></table>'
-        html << '</div>' unless @nowrap
-        html
-      end
-
-      def process_tokens(tokens)
-        rendered = []
-        current_line = ''
-
-        tokens.each do |tok, val|
-          # In the case of multi-line values (e.g. comments), we need to apply
-          # styling to each line since span elements are inline.
-          val.lines.each do |line|
-            stripped = line.chomp
-            current_line << span(tok, stripped)
-
-            if line.end_with?("\n")
-              rendered << current_line
-              current_line = ''
-            end
-          end
-        end
-
-        # Add leftover text
-        rendered << current_line if current_line.present?
-
-        num_lines = rendered.size
-        numbers = (@linenostart..num_lines + @linenostart - 1).to_a
-
-        { numbers: numbers, code: rendered }
-      end
-
-      def wrap_linenos(numbers)
-        if @anchorlinenos
-          numbers.map! do |number|
-            "<a href=\"##{@lineanchorsid}#{number}\">#{number}</a>"
-          end
-        end
-        numbers.join("\n")
-      end
-
-      def wrap_lines(lines)
-        if @lineanchors
-          lines = lines.each_with_index.map do |line, index|
-            number = index + @linenostart
-
-            if @linenos == 'inline'
-              "<a name=\"L#{number}\"></a>" \
-              "<span class=\"linenos\">#{number}</span>" \
-              "<span id=\"#{@lineanchorsid}#{number}\" class=\"line\">#{line}" \
-              '</span>'
-            else
-              "<span id=\"#{@lineanchorsid}#{number}\" class=\"line\">#{line}" \
-              '</span>'
-            end
-          end
-        elsif @linenos == 'inline'
-          lines = lines.each_with_index.map do |line, index|
-            number = index + @linenostart
-            "<span class=\"linenos\">#{number}</span>#{line}"
-          end
-        end
-
-        lines.join("\n")
-      end
+      def stream(tokens, &b)
+        is_first = true
+        token_lines(tokens) do |line|
+          yield "\n" unless is_first
+          is_first = false
 
-      def span(tok, val)
-        # http://stackoverflow.com/a/1600584/2587286
-        val = CGI.escapeHTML(val)
+          yield %(<span id="LC#{@line_number}" class="line">)
+          line.each { |token, value| yield span(token, value) }
+          yield %(</span>)
 
-        if tok.shortname.empty?
-          val
-        else
-          if @inline_theme
-            rules = @inline_theme.style_for(tok).rendered_rules
-            "<span style=\"#{rules.to_a.join(';')}\"#{val}</span>"
-          else
-            "<span class=\"#{tok.shortname}\">#{val}</span>"
-          end
+          @line_number += 1
         end
       end
     end