Merge remote-tracking branch 'origin/master' into ci-permissions

# Conflicts:
#	app/views/projects/builds/index.html.haml

12 files changed, 124 insertions, 25 deletions

diff --git a/lib/api/files.rb b/lib/api/files.rb
index 8ad2c1883c7..c1d86f313b0 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -58,9 +58,11 @@ module API
         commit = user_project.commit(ref)
         not_found! 'Commit' unless commit
 
-        blob = user_project.repository.blob_at(commit.sha, file_path)
+        repo = user_project.repository
+        blob = repo.blob_at(commit.sha, file_path)
 
         if blob
+          blob.load_all_data!(repo)
           status(200)
 
           {
@@ -72,7 +74,7 @@ module API
             ref: ref,
             blob_id: blob.id,
             commit_id: commit.id,
-            last_commit_id: user_project.repository.last_commit_for_path(commit.sha, file_path).id
+            last_commit_id: repo.last_commit_for_path(commit.sha, file_path).id
           }
         else
           not_found! 'File'
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 9dacf7c1e86..a72044e8058 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -30,7 +30,7 @@ module API
     end
 
     def sudo_identifier()
-      identifier ||= params[SUDO_PARAM] ||= env[SUDO_HEADER]
+      identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER]
 
       # Regex for integers
       if !!(identifier =~ /^[0-9]+$/)
@@ -344,12 +344,22 @@ module API
 
     def pagination_links(paginated_data)
       request_url = request.url.split('?').first
+      request_params = params.clone
+      request_params[:per_page] = paginated_data.limit_value
 
       links = []
-      links << %(<#{request_url}?page=#{paginated_data.current_page - 1}&per_page=#{paginated_data.limit_value}>; rel="prev") unless paginated_data.first_page?
-      links << %(<#{request_url}?page=#{paginated_data.current_page + 1}&per_page=#{paginated_data.limit_value}>; rel="next") unless paginated_data.last_page?
-      links << %(<#{request_url}?page=1&per_page=#{paginated_data.limit_value}>; rel="first")
-      links << %(<#{request_url}?page=#{paginated_data.total_pages}&per_page=#{paginated_data.limit_value}>; rel="last")
+
+      request_params[:page] = paginated_data.current_page - 1
+      links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page?
+
+      request_params[:page] = paginated_data.current_page + 1
+      links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page?
+
+      request_params[:page] = 1
+      links << %(<#{request_url}?#{request_params.to_query}>; rel="first")
+
+      request_params[:page] = paginated_data.total_pages
+      links << %(<#{request_url}?#{request_params.to_query}>; rel="last")
 
       links.join(', ')
     end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 6e7a7672070..252744515da 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -3,6 +3,8 @@ module API
   class Issues < Grape::API
     before { authenticate! }
 
+    helpers ::Gitlab::AkismetHelper
+
     helpers do
       def filter_issues_state(issues, state)
         case state
@@ -19,6 +21,17 @@ module API
       def filter_issues_milestone(issues, milestone)
         issues.includes(:milestone).where('milestones.title' => milestone)
       end
+
+      def create_spam_log(project, current_user, attrs)
+        params = attrs.merge({
+          source_ip: env['REMOTE_ADDR'],
+          user_agent: env['HTTP_USER_AGENT'],
+          noteable_type: 'Issue',
+          via_api: true
+        })
+
+        ::CreateSpamLogService.new(project, current_user, params).execute
+      end
     end
 
     resource :issues do
@@ -114,7 +127,15 @@ module API
           render_api_error!({ labels: errors }, 400)
         end
 
-        issue = ::Issues::CreateService.new(user_project, current_user, attrs).execute
+        project = user_project
+        text = [attrs[:title], attrs[:description]].reject(&:blank?).join("\n")
+
+        if check_for_spam?(project, current_user) && is_spam?(env, current_user, text)
+          create_spam_log(project, current_user, attrs)
+          render_api_error!({ error: 'Spam detected' }, 400)
+        end
+
+        issue = ::Issues::CreateService.new(project, current_user, attrs).execute
 
         if issue.valid?
           # Find or create labels and attach to issue. Labels are valid because
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index d7c48639eba..c95d2d2001d 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -57,7 +57,7 @@ module API
         not_found! "File" unless blob
 
         content_type 'text/plain'
-        present blob.data
+        header *Gitlab::Workhorse.send_git_blob(repo, blob)
       end
 
       # Get a raw blob contents by blob sha
@@ -83,7 +83,7 @@ module API
         env['api.format'] = :txt
 
         content_type blob.mime_type
-        present blob.data
+        header *Gitlab::Workhorse.send_git_blob(repo, blob)
       end
 
       # Get a an archive of the repository
diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb
index d1e11eedec3..04ddfe53ed6 100644
--- a/lib/banzai/filter/sanitization_filter.rb
+++ b/lib/banzai/filter/sanitization_filter.rb
@@ -8,14 +8,7 @@ module Banzai
     # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist.
     class SanitizationFilter < HTML::Pipeline::SanitizationFilter
       def whitelist
-        # Descriptions are more heavily sanitized, allowing only a few elements.
-        # See http://git.io/vkuAN
-        if context[:inline_sanitization]
-          whitelist = LIMITED
-          whitelist[:elements] -= %w(pre code img ol ul li)
-        else
-          whitelist = super
-        end
+        whitelist = super
 
         customize_whitelist(whitelist)
 
diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb
index 20e24ace352..f2395867658 100644
--- a/lib/banzai/pipeline/description_pipeline.rb
+++ b/lib/banzai/pipeline/description_pipeline.rb
@@ -4,9 +4,20 @@ module Banzai
       def self.transform_context(context)
         super(context).merge(
           # SanitizationFilter
-          inline_sanitization: true
+          whitelist: whitelist
         )
       end
+
+      private
+
+      def self.whitelist
+        # Descriptions are more heavily sanitized, allowing only a few elements.
+        # See http://git.io/vkuAN
+        whitelist = Banzai::Filter::SanitizationFilter::LIMITED
+        whitelist[:elements] -= %w(pre code img ol ul li)
+
+        whitelist
+      end
     end
   end
 end
diff --git a/lib/gitlab/akismet_helper.rb b/lib/gitlab/akismet_helper.rb
new file mode 100644
index 00000000000..b366c89889e
--- /dev/null
+++ b/lib/gitlab/akismet_helper.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module AkismetHelper
+    def akismet_enabled?
+      current_application_settings.akismet_enabled
+    end
+
+    def akismet_client
+      @akismet_client ||= ::Akismet::Client.new(current_application_settings.akismet_api_key,
+        Gitlab.config.gitlab.url)
+    end
+
+    def check_for_spam?(project, user)
+      akismet_enabled? && !project.team.member?(user)
+    end
+
+    def is_spam?(environment, user, text)
+      client = akismet_client
+      ip_address = environment['REMOTE_ADDR']
+      user_agent = environment['HTTP_USER_AGENT']
+
+      params = {
+        type: 'comment',
+        text: text,
+        created_at: DateTime.now,
+        author: user.name,
+        author_email: user.email,
+        referrer: environment['HTTP_REFERER'],
+      }
+
+      begin
+        is_spam, is_blatant = client.check(ip_address, user_agent, params)
+        is_spam || is_blatant
+      rescue => e
+        Rails.logger.error("Unable to connect to Akismet: #{e}, skipping check")
+        false
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index a6b2f14521c..8531c7e87e1 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -34,7 +34,8 @@ module Gitlab
         shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
         max_artifacts_size: Settings.artifacts['max_size'],
         require_two_factor_authentication: false,
-        two_factor_grace_period: 48
+        two_factor_grace_period: 48,
+        akismet_enabled: false
       )
     end
 
diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb
index 4ddb4fea977..cac76442321 100644
--- a/lib/gitlab/highlight.rb
+++ b/lib/gitlab/highlight.rb
@@ -8,6 +8,7 @@ module Gitlab
       blob = repository.blob_at(ref, file_name)
       return [] unless blob
 
+      blob.load_all_data!(repository)
       highlight(file_name, blob.data).lines.map!(&:html_safe)
     end
 
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 53ab2686b43..5c35c5b1450 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -44,19 +44,19 @@ module Gitlab
 
 
     def file_name_regex
-      @file_name_regex ||= /\A[a-zA-Z0-9_\-\.]*\z/.freeze
+      @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze
     end
 
     def file_name_regex_message
-      "can contain only letters, digits, '_', '-' and '.'. "
+      "can contain only letters, digits, '_', '-', '@' and '.'. "
     end
 
     def file_path_regex
-      @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/]*\z/.freeze
+      @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/\@]*\z/.freeze
     end
 
     def file_path_regex_message
-      "can contain only letters, digits, '_', '-' and '.'. Separate directories with a '/'. "
+      "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'. "
     end
 
 
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
new file mode 100644
index 00000000000..a23120a4176
--- /dev/null
+++ b/lib/gitlab/workhorse.rb
@@ -0,0 +1,21 @@
+require 'base64'
+require 'json'
+
+module Gitlab
+  class Workhorse
+    class << self
+      def send_git_blob(repository, blob)
+        params_hash = {
+          'RepoPath' => repository.path_to_repo,
+          'BlobId' => blob.id,
+        }
+        params = Base64.urlsafe_encode64(JSON.dump(params_hash))
+
+        [
+          'Gitlab-Workhorse-Send-Data',
+          "git-blob:#{params}",
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab
index 1633891c8a0..9e90a99f15b 100755
--- a/lib/support/init.d/gitlab
+++ b/lib/support/init.d/gitlab
@@ -219,7 +219,7 @@ start_gitlab() {
     echo "The Unicorn web server already running with pid $wpid, not restarting."
   else
     # Remove old socket if it exists
-    rm -f "$socket_path"/gitlab.socket 2>/dev/null
+    rm -f "$rails_socket" 2>/dev/null
     # Start the web server
     RAILS_ENV=$RAILS_ENV bin/web start
   fi