diff options
31 files changed, 432 insertions, 7 deletions
diff --git a/CHANGELOG b/CHANGELOG index 7a70516173c..1be1f7374df 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -74,6 +74,7 @@ v 8.4.0 - Don't notify users twice if they are both project watchers and subscribers (Stan Hu) - Remove gray background from layout in UI - Fix signup for OAuth providers that don't provide a name + - Support Akismet spam checking for creation of issues via API (Stan Hu) - Implement new UI for group page - Implement search inside emoji picker - Let the CI runner know about builds that this build depends on @@ -36,8 +36,9 @@ gem 'omniauth-twitter', '~> 1.2.0' gem 'omniauth_crowd', '~> 2.2.0' gem 'rack-oauth2', '~> 1.2.1' -# reCAPTCHA protection +# Spam and anti-bot protection gem 'recaptcha', require: 'recaptcha/rails' +gem 'akismet', '~> 2.0' # Two-factor authentication gem 'devise-two-factor', '~> 2.0.0' diff --git a/Gemfile.lock b/Gemfile.lock index b4f7587c419..9570d387f68 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -49,6 +49,7 @@ GEM addressable (2.3.8) after_commit_queue (1.3.0) activerecord (>= 3.0) + akismet (2.0.0) allocations (1.0.3) annotate (2.6.10) activerecord (>= 3.2, <= 4.3) @@ -884,6 +885,7 @@ DEPENDENCIES acts-as-taggable-on (~> 3.4) addressable (~> 2.3.8) after_commit_queue + akismet (~> 2.0) allocations (~> 1.0) annotate (~> 2.6.0) asana (~> 0.4.0) diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb index 9943745208e..1515086b16d 100644 --- a/app/controllers/admin/application_settings_controller.rb +++ b/app/controllers/admin/application_settings_controller.rb @@ -79,6 +79,8 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController :recaptcha_private_key, :sentry_enabled, :sentry_dsn, + :akismet_enabled, + :akismet_api_key, restricted_visibility_levels: [], import_sources: [] ) diff --git a/app/controllers/admin/spam_logs_controller.rb b/app/controllers/admin/spam_logs_controller.rb new file mode 100644 index 00000000000..69f94dd3ba8 --- /dev/null +++ b/app/controllers/admin/spam_logs_controller.rb @@ -0,0 +1,33 @@ +class Admin::SpamLogsController < Admin::ApplicationController + before_action :set_spam_log, only: [:destroy] + + def index + @spam_logs = SpamLog.order(created_at: :desc).page(params[:page]) + end + + def destroy + @spam_log.destroy + message = 'Spam log was successfully destroyed.' + + if params[:remove_user] + username = @spam_log.user.username + @spam_log.user.destroy + message = "User #{username} was successfully destroyed." + end + + respond_to do |format| + format.json { render json: '{}' } + format.html { redirect_to admin_spam_logs_path, notice: message } + end + end + + private + + def set_spam_log + @spam_log = SpamLog.find(params[:id]) + end + + def spam_log_params + params[:spam_log] + end +end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 59563b8823c..9cafc78f761 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -88,6 +88,10 @@ class ApplicationSetting < ActiveRecord::Base presence: true, if: :sentry_enabled + validates :akismet_api_key, + presence: true, + if: :akismet_enabled + validates_each :restricted_visibility_levels do |record, attr, value| unless value.nil? value.each do |level| @@ -143,7 +147,9 @@ class ApplicationSetting < ActiveRecord::Base shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, - two_factor_grace_period: 48 + two_factor_grace_period: 48, + recaptcha_enabled: false, + akismet_enabled: false ) end diff --git a/app/models/spam_log.rb b/app/models/spam_log.rb new file mode 100644 index 00000000000..132c5713e0a --- /dev/null +++ b/app/models/spam_log.rb @@ -0,0 +1,13 @@ +class SpamLog < ActiveRecord::Base + belongs_to :user + + validates :user, presence: true + + def truncated_description + if description.present? && description.length > 100 + return description[0..100] + "..." + end + + description + end +end diff --git a/app/models/spam_report.rb b/app/models/spam_report.rb new file mode 100644 index 00000000000..cdc7321b08e --- /dev/null +++ b/app/models/spam_report.rb @@ -0,0 +1,5 @@ +class SpamReport < ActiveRecord::Base + belongs_to :user + + validates :user, presence: true +end diff --git a/app/models/user.rb b/app/models/user.rb index 4214f01f6a4..63c895683f7 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -138,6 +138,7 @@ class User < ActiveRecord::Base has_many :assigned_merge_requests, dependent: :destroy, foreign_key: :assignee_id, class_name: "MergeRequest" has_many :oauth_applications, class_name: 'Doorkeeper::Application', as: :owner, dependent: :destroy has_one :abuse_report, dependent: :destroy + has_many :spam_log, dependent: :destroy has_many :builds, dependent: :nullify, class_name: 'Ci::Build' diff --git a/app/services/create_spam_log_service.rb b/app/services/create_spam_log_service.rb new file mode 100644 index 00000000000..59a66fde47a --- /dev/null +++ b/app/services/create_spam_log_service.rb @@ -0,0 +1,13 @@ +class CreateSpamLogService < BaseService + def initialize(project, user, params) + super(project, user, params) + end + + def execute + spam_params = params.merge({ user_id: @current_user.id, + project_id: @project.id } ) + spam_log = SpamLog.new(spam_params) + spam_log.save + spam_log + end +end diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index b0f1a34cbec..b4e3d96d405 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -218,20 +218,37 @@ = f.label :recaptcha_enabled do = f.check_box :recaptcha_enabled Enable reCAPTCHA - %span.help-block#recaptcha_help_block Helps preventing bots from creating accounts + %span.help-block#recaptcha_help_block Helps prevent bots from creating accounts .form-group = f.label :recaptcha_site_key, 'reCAPTCHA Site Key', class: 'control-label col-sm-2' .col-sm-10 = f.text_field :recaptcha_site_key, class: 'form-control' .help-block - Generate site and private keys here: - %a{ href: 'http://www.google.com/recaptcha', target: '_blank'} http://www.google.com/recaptcha + Generate site and private keys at + %a{ href: 'http://www.google.com/recaptcha', target: 'blank'} http://www.google.com/recaptcha + .form-group = f.label :recaptcha_private_key, 'reCAPTCHA Private Key', class: 'control-label col-sm-2' .col-sm-10 = f.text_field :recaptcha_private_key, class: 'form-control' + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :akismet_enabled do + = f.check_box :akismet_enabled + Enable Akismet + %span.help-block#akismet_help_block Helps prevent bots from creating issues + + .form-group + = f.label :akismet_api_key, 'Akismet API Key', class: 'control-label col-sm-2' + .col-sm-10 + = f.text_field :akismet_api_key, class: 'form-control' + .help-block + Generate API key at + %a{ href: 'http://www.akismet.com', target: 'blank'} http://www.akismet.com + %fieldset %legend Error Reporting and Logging %p diff --git a/app/views/admin/spam_logs/_spam_log.html.haml b/app/views/admin/spam_logs/_spam_log.html.haml new file mode 100644 index 00000000000..1c793bbf738 --- /dev/null +++ b/app/views/admin/spam_logs/_spam_log.html.haml @@ -0,0 +1,30 @@ +- user = spam_log.user +%tr + %td + = spam_log.created_at.to_s(:short) + %td + - if user + = link_to user.name, user + - else + (removed) + %td + = spam_log.source_ip + %td + = spam_log.via_api ? 'Y' : 'N' + %td + = spam_log.noteable_type + %td + = spam_log.title + %td + = spam_log.truncated_description + %td + - if user + = link_to 'Remove user', admin_spam_log_path(spam_log, remove_user: true), + data: { confirm: "USER #{user.name} WILL BE REMOVED! Are you sure?" }, method: :delete, class: "btn btn-xs btn-remove" + %td + - if user && !user.blocked? + = link_to 'Block user', block_admin_user_path(user), data: {confirm: 'USER WILL BE BLOCKED! Are you sure?'}, method: :put, class: "btn btn-xs" + - else + .btn.btn-xs.disabled + Already Blocked + = link_to 'Remove log', [:admin, spam_log, format: :json], remote: true, method: :delete, class: "btn btn-xs btn-close js-remove-tr" diff --git a/app/views/admin/spam_logs/index.html.haml b/app/views/admin/spam_logs/index.html.haml new file mode 100644 index 00000000000..439468e572b --- /dev/null +++ b/app/views/admin/spam_logs/index.html.haml @@ -0,0 +1,21 @@ +- page_title "Spam Logs" +%h3.page-title Spam Logs +%hr +- if @spam_logs.present? + .table-holder + %table.table + %thead + %tr + %th Date + %th User + %th Source IP + %th Via API? + %th Type + %th Title + %th Description + %th Primary Action + %th + = render @spam_logs + = paginate @spam_logs +- else + %h4 There are no Spam Logs diff --git a/app/views/layouts/nav/_admin.html.haml b/app/views/layouts/nav/_admin.html.haml index cffdb52cc23..8dc20587421 100644 --- a/app/views/layouts/nav/_admin.html.haml +++ b/app/views/layouts/nav/_admin.html.haml @@ -82,6 +82,13 @@ Abuse Reports %span.count= number_with_delimiter(AbuseReport.count(:all)) + = nav_link(controller: :spam_logs) do + = link_to admin_spam_logs_path, title: "Spam Logs" do + = icon('exclamation-triangle fw') + %span + Spam Logs + %span.count= number_with_delimiter(SpamLog.count(:all)) + = nav_link(controller: :application_settings, html_options: { class: 'separate-item'}) do = link_to admin_application_settings_path, title: 'Settings' do = icon('cogs fw') diff --git a/config/routes.rb b/config/routes.rb index 54cc338b605..034bfaf1bcd 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -211,6 +211,8 @@ Rails.application.routes.draw do end resources :abuse_reports, only: [:index, :destroy] + resources :spam_logs, only: [:index, :destroy] + resources :applications resources :groups, constraints: { id: /[^\/]+/ } do diff --git a/db/migrate/20151231152326_add_akismet_to_application_settings.rb b/db/migrate/20151231152326_add_akismet_to_application_settings.rb new file mode 100644 index 00000000000..3f52c758f9a --- /dev/null +++ b/db/migrate/20151231152326_add_akismet_to_application_settings.rb @@ -0,0 +1,8 @@ +class AddAkismetToApplicationSettings < ActiveRecord::Migration + def change + change_table :application_settings do |t| + t.boolean :akismet_enabled, default: false + t.string :akismet_api_key + end + end +end diff --git a/db/migrate/20160109054846_create_spam_logs.rb b/db/migrate/20160109054846_create_spam_logs.rb new file mode 100644 index 00000000000..f12fe9f8f78 --- /dev/null +++ b/db/migrate/20160109054846_create_spam_logs.rb @@ -0,0 +1,16 @@ +class CreateSpamLogs < ActiveRecord::Migration + def change + create_table :spam_logs do |t| + t.integer :user_id + t.string :source_ip + t.string :user_agent + t.boolean :via_api + t.integer :project_id + t.string :noteable_type + t.string :title + t.text :description + + t.timestamps null: false + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 2ad2c23fba5..d546e06cd8a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -64,6 +64,8 @@ ActiveRecord::Schema.define(version: 20160128233227) do t.integer "metrics_sample_interval", default: 15 t.boolean "sentry_enabled", default: false t.string "sentry_dsn" + t.boolean "akismet_enabled", default: false + t.string "akismet_api_key" end create_table "audit_events", force: :cascade do |t| @@ -770,6 +772,19 @@ ActiveRecord::Schema.define(version: 20160128233227) do add_index "snippets", ["project_id"], name: "index_snippets_on_project_id", using: :btree add_index "snippets", ["visibility_level"], name: "index_snippets_on_visibility_level", using: :btree + create_table "spam_logs", force: :cascade do |t| + t.integer "user_id" + t.string "source_ip" + t.string "user_agent" + t.boolean "via_api" + t.integer "project_id" + t.string "noteable_type" + t.string "title" + t.text "description" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "subscriptions", force: :cascade do |t| t.integer "user_id" t.integer "subscribable_id" diff --git a/doc/integration/README.md b/doc/integration/README.md index 83116bc8370..281eea8363d 100644 --- a/doc/integration/README.md +++ b/doc/integration/README.md @@ -15,6 +15,7 @@ See the documentation below for details on how to configure these services. - [OAuth2 provider](oauth_provider.md) OAuth2 application creation - [Gmail actions buttons](gmail_action_buttons_for_gitlab.md) Adds GitLab actions to messages - [reCAPTCHA](recaptcha.md) Configure GitLab to use Google reCAPTCHA for new users +- [Akismet](akismet.md) Configure Akismet to stop spam GitLab Enterprise Edition contains [advanced Jenkins support][jenkins]. diff --git a/doc/integration/akismet.md b/doc/integration/akismet.md new file mode 100644 index 00000000000..5cc09bd536d --- /dev/null +++ b/doc/integration/akismet.md @@ -0,0 +1,30 @@ +# Akismet + +GitLab leverages [Akismet](http://akismet.com) to protect against spam. Currently +GitLab uses Akismet to prevent users who are not members of a project from +creating spam via the GitLab API. Detected spam will be rejected, and +an entry in the "Spam Log" section in the Admin page will be created. + +Privacy note: GitLab submits the user's IP and user agent to Akismet. Note that +adding a user to a project will disable the Akismet check and prevent this +from happening. + +## Configuration + +To use Akismet: + +1. Go to the URL: https://akismet.com/account/ + +2. Sign-in or create a new account. + +3. Click on "Show" to reveal the API key. + +4. Go to Applications Settings on Admin Area (`admin/application_settings`) + +5. Check the `Enable Akismet` checkbox + +6. Fill in the API key from step 3. + +7. Save the configuration. + + diff --git a/doc/integration/img/akismet_settings.png b/doc/integration/img/akismet_settings.png Binary files differnew file mode 100644 index 00000000000..ccdd3adb1c5 --- /dev/null +++ b/doc/integration/img/akismet_settings.png diff --git a/features/admin/spam_logs.feature b/features/admin/spam_logs.feature new file mode 100644 index 00000000000..92a5389e3a4 --- /dev/null +++ b/features/admin/spam_logs.feature @@ -0,0 +1,8 @@ +Feature: Admin spam logs + Background: + Given I sign in as an admin + And spam logs exist + + Scenario: Browse spam logs + When I visit spam logs page + Then I should see list of spam logs diff --git a/features/steps/admin/spam_logs.rb b/features/steps/admin/spam_logs.rb new file mode 100644 index 00000000000..79d18b036b0 --- /dev/null +++ b/features/steps/admin/spam_logs.rb @@ -0,0 +1,18 @@ +class Spinach::Features::AdminSpamLogs < Spinach::FeatureSteps + include SharedAuthentication + include SharedPaths + include SharedAdmin + + step 'I should see list of spam logs' do + page.should have_content("Spam Logs") + spam_log = SpamLog.first + page.should have_content spam_log.title + page.should have_content spam_log.description + page.should have_link("Remove user") + page.should have_link("Block user") + end + + step 'spam logs exist' do + create(:spam_log) + end +end diff --git a/features/steps/shared/paths.rb b/features/steps/shared/paths.rb index 4264c9c6f1a..2c854ac7bf9 100644 --- a/features/steps/shared/paths.rb +++ b/features/steps/shared/paths.rb @@ -191,6 +191,10 @@ module SharedPaths visit admin_application_settings_path end + step 'I visit spam logs page' do + visit admin_spam_logs_path + end + step 'I visit applications page' do visit admin_applications_path end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 6e7a7672070..cdadd13c13a 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -3,6 +3,8 @@ module API class Issues < Grape::API before { authenticate! } + helpers ::Gitlab::AkismetHelper + helpers do def filter_issues_state(issues, state) case state @@ -19,6 +21,15 @@ module API def filter_issues_milestone(issues, milestone) issues.includes(:milestone).where('milestones.title' => milestone) end + + def create_spam_log(project, current_user, attrs) + params = attrs.dup + params[:source_ip] = env['REMOTE_ADDR'] + params[:user_agent] = env['HTTP_USER_AGENT'] + params[:noteable_type] = 'Issue' + params[:via_api] = true + ::CreateSpamLogService.new(project, current_user, params).execute + end end resource :issues do @@ -114,7 +125,16 @@ module API render_api_error!({ labels: errors }, 400) end - issue = ::Issues::CreateService.new(user_project, current_user, attrs).execute + project = user_project + text = attrs[:title] + text += "\n#{attrs[:description]}" if attrs[:description].present? + + if check_for_spam?(project, current_user) && is_spam?(env, current_user, text) + create_spam_log(project, current_user, attrs) + render_api_error!({ error: 'Spam detected' }, 400) + end + + issue = ::Issues::CreateService.new(project, current_user, attrs).execute if issue.valid? # Find or create labels and attach to issue. Labels are valid because diff --git a/lib/gitlab/akismet_helper.rb b/lib/gitlab/akismet_helper.rb new file mode 100644 index 00000000000..71f525309fe --- /dev/null +++ b/lib/gitlab/akismet_helper.rb @@ -0,0 +1,39 @@ +module Gitlab + module AkismetHelper + def akismet_enabled? + current_application_settings.akismet_enabled + end + + def akismet_client + ::Akismet::Client.new(current_application_settings.akismet_api_key, + Gitlab.config.gitlab.url) + end + + def check_for_spam?(project, user) + akismet_enabled? && !project.team.member?(user) + end + + def is_spam?(environment, user, text) + client = akismet_client + ip_address = environment['REMOTE_ADDR'] + user_agent = environment['HTTP_USER_AGENT'] + + params = { + type: 'comment', + text: text, + created_at: DateTime.now, + author: user.name, + author_email: user.email, + referrer: environment['HTTP_REFERER'], + } + + begin + is_spam, is_blatant = client.check(ip_address, user_agent, params) + is_spam || is_blatant + rescue => e + Rails.logger.error("Unable to connect to Akismet: #{e}, skipping check") + false + end + end + end +end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index a6b2f14521c..8531c7e87e1 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -34,7 +34,8 @@ module Gitlab shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, - two_factor_grace_period: 48 + two_factor_grace_period: 48, + akismet_enabled: false ) end diff --git a/spec/controllers/admin/spam_logs_controller_spec.rb b/spec/controllers/admin/spam_logs_controller_spec.rb new file mode 100644 index 00000000000..2486298fc78 --- /dev/null +++ b/spec/controllers/admin/spam_logs_controller_spec.rb @@ -0,0 +1,47 @@ +require 'spec_helper' + +describe Admin::SpamLogsController do + let(:admin) { create(:admin) } + let(:spam_log) { create(:spam_log, user: admin) } + + before do + sign_in(admin) + end + + describe '#index' do + it 'lists all spam logs' do + get :index + expect(response.status).to eq(200) + end + end + + describe '#destroy' do + it 'destroys just spam log' do + user = spam_log.user + delete :destroy, id: spam_log.id + + expect(SpamLog.all.count).to eq(0) + expect(User.find(user.id)).to be_truthy + expect(response.status).to eq(302) + end + + it 'destroys user and his spam logs' do + user = spam_log.user + delete :destroy, id: spam_log.id, remove_user: true + + expect(SpamLog.all.count).to eq(0) + expect { User.find(user.id) }.to raise_error(ActiveRecord::RecordNotFound) + expect(response.status).to eq(302) + end + + it 'destroys user and his spam logs with JSON format' do + user = spam_log.user + delete :destroy, id: spam_log.id, remove_user: true, format: :json + + expect(SpamLog.all.count).to eq(0) + expect { User.find(user.id) }.to raise_error(ActiveRecord::RecordNotFound) + expect(JSON.parse(response.body)).to eq({}) + expect(response.status).to eq(200) + end + end +end diff --git a/spec/factories/spam_logs.rb b/spec/factories/spam_logs.rb new file mode 100644 index 00000000000..9e8686d73c2 --- /dev/null +++ b/spec/factories/spam_logs.rb @@ -0,0 +1,7 @@ +# Read about factories at https://github.com/thoughtbot/factory_girl + +FactoryGirl.define do + factory :spam_log do + user + end +end diff --git a/spec/lib/gitlab/akismet_helper_spec.rb b/spec/lib/gitlab/akismet_helper_spec.rb new file mode 100644 index 00000000000..9858935180a --- /dev/null +++ b/spec/lib/gitlab/akismet_helper_spec.rb @@ -0,0 +1,35 @@ +require 'spec_helper' + +describe Gitlab::AkismetHelper, type: :helper do + let(:project) { create(:project) } + let(:user) { create(:user) } + + before do + allow(Gitlab.config.gitlab).to receive(:url).and_return(Settings.send(:build_gitlab_url)) + current_application_settings.akismet_enabled = true + current_application_settings.akismet_api_key = '12345' + end + + describe '#check_for_spam?' do + it 'returns true for non-member' do + expect(helper.check_for_spam?(project, user)).to eq(true) + end + + it 'returns false for member' do + project.team << [user, :guest] + expect(helper.check_for_spam?(project, user)).to eq(false) + end + end + + describe '#is_spam?' do + it 'returns true for spam' do + environment = { + 'REMOTE_ADDR' => '127.0.0.1', + 'HTTP_USER_AGENT' => 'Test User Agent' + } + + allow_any_instance_of(::Akismet::Client).to receive(:check).and_return([true, true]) + expect(helper.is_spam?(environment, user, 'Is this spam?')).to eq(true) + end + end +end diff --git a/spec/requests/api/issues_spec.rb b/spec/requests/api/issues_spec.rb index 5e65ad18c0e..2e50344c149 100644 --- a/spec/requests/api/issues_spec.rb +++ b/spec/requests/api/issues_spec.rb @@ -241,6 +241,28 @@ describe API::API, api: true do end end + describe 'POST /projects/:id/issues with spam filtering' do + before do + Grape::Endpoint.before_each do |endpoint| + allow(endpoint).to receive(:check_for_spam?).and_return(true) + allow(endpoint).to receive(:is_spam?).and_return(true) + end + end + + it "should create a new project issue" do + post api("/projects/#{project.id}/issues", user), + title: 'new issue', labels: 'label, label2' + expect(response.status).to eq(400) + expect(json_response['message']).to eq({ "error" => "Spam detected" }) + spam_logs = SpamLog.all + expect(spam_logs.count).to eq(1) + expect(spam_logs[0].title).to eq('new issue') + expect(spam_logs[0].user).to eq(user) + expect(spam_logs[0].noteable_type).to eq('Issue') + expect(spam_logs[0].project_id).to eq(project.id) + end + end + describe "PUT /projects/:id/issues/:issue_id to update only title" do it "should update a project issue" do put api("/projects/#{project.id}/issues/#{issue.id}", user), |
