Merge branch 'feature/system_hook_push' into 'master'

Added System Hooks for push and tag_push

Implements `push` and `tag_push` events for System Hooks.

They are based on Webhooks Events but without any deprecated item. We don't send commits too.
We are implementing this, to be used by Geo repository syncing (gitlab-org/gitlab-ee#76).

These hook events will be sent only when the respective configuration flags are set to true.
UI changes to admin screen will be made in another MR.

See merge request !3744

15 files changed, 209 insertions, 49 deletions

diff --git a/CHANGELOG b/CHANGELOG
index 1a4b367e474..3f23b183202 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -87,6 +87,8 @@ v 8.7.0 (unreleased)
   - Use GitHub Issue/PR number as iid to keep references
   - Import GitHub labels
   - Import GitHub milestones
+  - Fix emoji catgories in the emoji picker
+  - Execute system web hooks on push to the project
 
 v 8.6.6
   - Expire the exists cache before deletion to ensure project dir actually exists (Stan Hu). !3413
diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb
index fe923fafbe0..7365e360de2 100644
--- a/app/models/hooks/project_hook.rb
+++ b/app/models/hooks/project_hook.rb
@@ -21,8 +21,6 @@
 class ProjectHook < WebHook
   belongs_to :project
 
-  scope :push_hooks, -> { where(push_events: true) }
-  scope :tag_push_hooks, -> { where(tag_push_events: true) }
   scope :issue_hooks, -> { where(issues_events: true) }
   scope :note_hooks, -> { where(note_events: true) }
   scope :merge_request_hooks, -> { where(merge_requests_events: true) }
diff --git a/app/models/hooks/system_hook.rb b/app/models/hooks/system_hook.rb
index c147d8762a9..15dddcc2447 100644
--- a/app/models/hooks/system_hook.rb
+++ b/app/models/hooks/system_hook.rb
@@ -19,4 +19,7 @@
 #
 
 class SystemHook < WebHook
+  def async_execute(data, hook_name)
+    Sidekiq::Client.enqueue(SystemHookWorker, id, data, hook_name)
+  end
 end
diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb
index 7a13c3f0a39..3a2e4f546f7 100644
--- a/app/models/hooks/web_hook.rb
+++ b/app/models/hooks/web_hook.rb
@@ -30,6 +30,9 @@ class WebHook < ActiveRecord::Base
   default_value_for :build_events, false
   default_value_for :enable_ssl_verification, true
 
+  scope :push_hooks, -> { where(push_events: true) }
+  scope :tag_push_hooks, -> { where(tag_push_events: true) }
+
   # HTTParty timeout
   default_timeout Gitlab.config.gitlab.webhook_timeout
 
diff --git a/app/models/project.rb b/app/models/project.rb
index 8f20922e3c5..e7dac23a122 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -831,8 +831,8 @@ class Project < ActiveRecord::Base
     end
   end
 
-  def hook_attrs
-    {
+  def hook_attrs(backward: true)
+    attrs = {
       name: name,
       description: description,
       web_url: web_url,
@@ -843,12 +843,19 @@ class Project < ActiveRecord::Base
       visibility_level: visibility_level,
       path_with_namespace: path_with_namespace,
       default_branch: default_branch,
-      # Backward compatibility
-      homepage: web_url,
-      url: url_to_repo,
-      ssh_url: ssh_url_to_repo,
-      http_url: http_url_to_repo
     }
+
+    # Backward compatibility
+    if backward
+      attrs.merge!({
+                    homepage: web_url,
+                    url: url_to_repo,
+                    ssh_url: ssh_url_to_repo,
+                    http_url: http_url_to_repo
+                  })
+    end
+
+    attrs
   end
 
   # Reset events cache related to this project
diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb
index dc74c02760b..1e1be8cd04b 100644
--- a/app/services/git_push_service.rb
+++ b/app/services/git_push_service.rb
@@ -73,6 +73,7 @@ class GitPushService < BaseService
     @project.update_merge_requests(params[:oldrev], params[:newrev], params[:ref], current_user)
 
     EventCreateService.new.push(@project, current_user, build_push_data)
+    SystemHooksService.new.execute_hooks(build_push_data_system_hook.dup, :push_hooks)
     @project.execute_hooks(build_push_data.dup, :push_hooks)
     @project.execute_services(build_push_data.dup, :push_hooks)
     CreateCommitBuildsService.new.execute(@project, current_user, build_push_data)
@@ -138,6 +139,11 @@ class GitPushService < BaseService
       build(@project, current_user, params[:oldrev], params[:newrev], params[:ref], push_commits)
   end
 
+  def build_push_data_system_hook
+    @push_data_system ||= Gitlab::PushDataBuilder.
+      build(@project, current_user, params[:oldrev], params[:newrev], params[:ref], [])
+  end
+
   def push_to_existing_branch?
     # Return if this is not a push to a branch (e.g. new commits)
     Gitlab::Git.branch_ref?(params[:ref]) && !Gitlab::Git.blank_ref?(params[:oldrev])
diff --git a/app/services/git_tag_push_service.rb b/app/services/git_tag_push_service.rb
index c88c7672805..64271d8bc5c 100644
--- a/app/services/git_tag_push_service.rb
+++ b/app/services/git_tag_push_service.rb
@@ -1,16 +1,16 @@
-class GitTagPushService
-  attr_accessor :project, :user, :push_data
+class GitTagPushService < BaseService
+  attr_accessor :push_data
 
-  def execute(project, user, oldrev, newrev, ref)
+  def execute
     project.repository.before_push_tag
 
-    @project, @user = project, user
-    @push_data = build_push_data(oldrev, newrev, ref)
+    @push_data = build_push_data
 
-    EventCreateService.new.push(project, user, @push_data)
+    EventCreateService.new.push(project, current_user, @push_data)
+    SystemHooksService.new.execute_hooks(build_system_push_data.dup, :tag_push_hooks)
     project.execute_hooks(@push_data.dup, :tag_push_hooks)
     project.execute_services(@push_data.dup, :tag_push_hooks)
-    CreateCommitBuildsService.new.execute(project, @user, @push_data)
+    CreateCommitBuildsService.new.execute(project, current_user, @push_data)
     ProjectCacheWorker.perform_async(project.id)
 
     true
@@ -18,14 +18,14 @@ class GitTagPushService
 
   private
 
-  def build_push_data(oldrev, newrev, ref)
+  def build_push_data
     commits = []
     message = nil
 
-    if !Gitlab::Git.blank_ref?(newrev)
-      tag_name = Gitlab::Git.ref_name(ref)
+    if !Gitlab::Git.blank_ref?(params[:newrev])
+      tag_name = Gitlab::Git.ref_name(params[:ref])
       tag = project.repository.find_tag(tag_name)
-      if tag && tag.target == newrev
+      if tag && tag.target == params[:newrev]
         commit = project.commit(tag.target)
         commits = [commit].compact
         message = tag.message
@@ -33,6 +33,11 @@ class GitTagPushService
     end
 
     Gitlab::PushDataBuilder.
-      build(project, user, oldrev, newrev, ref, commits, message)
+      build(project, current_user, params[:oldrev], params[:newrev], params[:ref], commits, message)
+  end
+
+  def build_system_push_data
+    Gitlab::PushDataBuilder.
+      build(project, current_user, params[:oldrev], params[:newrev], params[:ref], [], '')
   end
 end
diff --git a/app/services/system_hooks_service.rb b/app/services/system_hooks_service.rb
index f0615ec7420..e43b5b51e5b 100644
--- a/app/services/system_hooks_service.rb
+++ b/app/services/system_hooks_service.rb
@@ -3,17 +3,13 @@ class SystemHooksService
     execute_hooks(build_event_data(model, event))
   end
 
-  private
-
-  def execute_hooks(data)
-    SystemHook.all.each do |sh|
-      async_execute_hook(sh, data, 'system_hooks')
+  def execute_hooks(data, hooks_scope = :all)
+    SystemHook.send(hooks_scope).each do |hook|
+      hook.async_execute(data, 'system_hooks')
     end
   end
 
-  def async_execute_hook(hook, data, hook_name)
-    Sidekiq::Client.enqueue(SystemHookWorker, hook.id, data, hook_name)
-  end
+  private
 
   def build_event_data(model, event)
     data = {
diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb
index 9e1215b21a6..f3327ca9e61 100644
--- a/app/workers/post_receive.rb
+++ b/app/workers/post_receive.rb
@@ -39,7 +39,7 @@ class PostReceive
       end
 
       if Gitlab::Git.tag_ref?(ref)
-        GitTagPushService.new.execute(post_received.project, @user, oldrev, newrev, ref)
+        GitTagPushService.new(post_received.project, @user, oldrev: oldrev, newrev: newrev, ref: ref).execute
       elsif Gitlab::Git.branch_ref?(ref)
         GitPushService.new(post_received.project, @user, oldrev: oldrev, newrev: newrev, ref: ref).execute
       end
@@ -47,7 +47,7 @@ class PostReceive
   end
 
   private
-  
+
   def log(message)
     Gitlab::GitLogger.error("POST-RECEIVE: #{message}")
   end
diff --git a/doc/system_hooks/system_hooks.md b/doc/system_hooks/system_hooks.md
index 612376e3a49..c44930a4ceb 100644
--- a/doc/system_hooks/system_hooks.md
+++ b/doc/system_hooks/system_hooks.md
@@ -4,6 +4,12 @@ Your GitLab instance can perform HTTP POST requests on the following events: `pr
 
 System hooks can be used, e.g. for logging or changing information in a LDAP server.
 
+> **Note:**
+>
+> We follow the same structure from Webhooks for Push and Tag events, but we never display commits.
+>
+> Same deprecations from Webhooks are valid here.
+
 ## Hooks request example
 
 **Request header**:
@@ -240,3 +246,110 @@ X-Gitlab-Event: System Hook
        "user_id": 41
 }
 ```
+
+## Push events
+
+Triggered when you push to the repository except when pushing tags.
+
+**Request header**:
+
+```
+X-Gitlab-Event: System Hook
+```
+
+**Request body:**
+
+```json
+{
+  "event_name": "push",
+  "before": "95790bf891e76fee5e1747ab589903a6a1f80f22",
+  "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+  "ref": "refs/heads/master",
+  "checkout_sha": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+  "user_id": 4,
+  "user_name": "John Smith",
+  "user_email": "john@example.com",
+  "user_avatar": "https://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=8://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=80",
+  "project_id": 15,
+  "project":{
+    "name":"Diaspora",
+    "description":"",
+    "web_url":"http://example.com/mike/diaspora",
+    "avatar_url":null,
+    "git_ssh_url":"git@example.com:mike/diaspora.git",
+    "git_http_url":"http://example.com/mike/diaspora.git",
+    "namespace":"Mike",
+    "visibility_level":0,
+    "path_with_namespace":"mike/diaspora",
+    "default_branch":"master",
+    "homepage":"http://example.com/mike/diaspora",
+    "url":"git@example.com:mike/diaspora.git",
+    "ssh_url":"git@example.com:mike/diaspora.git",
+    "http_url":"http://example.com/mike/diaspora.git"
+  },
+  "repository":{
+    "name": "Diaspora",
+    "url": "git@example.com:mike/diaspora.git",
+    "description": "",
+    "homepage": "http://example.com/mike/diaspora",
+    "git_http_url":"http://example.com/mike/diaspora.git",
+    "git_ssh_url":"git@example.com:mike/diaspora.git",
+    "visibility_level":0
+  },
+  "commits": [],
+  "total_commits_count": 0
+}
+```
+
+## Tag events
+
+Triggered when you create (or delete) tags to the repository.
+
+**Request header**:
+
+```
+X-Gitlab-Event: System Hook
+```
+
+**Request body:**
+
+```json
+{
+  "event_name": "tag_push",
+  "before": "0000000000000000000000000000000000000000",
+  "after": "82b3d5ae55f7080f1e6022629cdb57bfae7cccc7",
+  "ref": "refs/tags/v1.0.0",
+  "checkout_sha": "5937ac0a7beb003549fc5fd26fc247adbce4a52e",
+  "user_id": 1,
+  "user_name": "John Smith",
+  "user_avatar": "https://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=8://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=80",
+  "project_id": 1,
+  "project":{
+    "name":"Example",
+    "description":"",
+    "web_url":"http://example.com/jsmith/example",
+    "avatar_url":null,
+    "git_ssh_url":"git@example.com:jsmith/example.git",
+    "git_http_url":"http://example.com/jsmith/example.git",
+    "namespace":"Jsmith",
+    "visibility_level":0,
+    "path_with_namespace":"jsmith/example",
+    "default_branch":"master",
+    "homepage":"http://example.com/jsmith/example",
+    "url":"git@example.com:jsmith/example.git",
+    "ssh_url":"git@example.com:jsmith/example.git",
+    "http_url":"http://example.com/jsmith/example.git"
+  },
+  "repository":{
+    "name": "Example",
+    "url": "ssh://git@example.com/jsmith/example.git",
+    "description": "",
+    "homepage": "http://example.com/jsmith/example",
+    "git_http_url":"http://example.com/jsmith/example.git",
+    "git_ssh_url":"git@example.com:jsmith/example.git",
+    "visibility_level":0
+  },
+  "commits": [],
+  "total_commits_count": 0
+}
+```
diff --git a/doc/web_hooks/web_hooks.md b/doc/web_hooks/web_hooks.md
index 22e207b6d32..c1c51302e79 100644
--- a/doc/web_hooks/web_hooks.md
+++ b/doc/web_hooks/web_hooks.md
@@ -41,6 +41,7 @@ X-Gitlab-Event: Push Hook
   "before": "95790bf891e76fee5e1747ab589903a6a1f80f22",
   "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
   "ref": "refs/heads/master",
+  "checkout_sha": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
   "user_id": 4,
   "user_name": "John Smith",
   "user_email": "john@example.com",
@@ -118,9 +119,10 @@ X-Gitlab-Event: Tag Push Hook
 ```json
 {
   "object_kind": "tag_push",
-  "ref": "refs/tags/v1.0.0",
   "before": "0000000000000000000000000000000000000000",
   "after": "82b3d5ae55f7080f1e6022629cdb57bfae7cccc7",
+  "ref": "refs/tags/v1.0.0",
+  "checkout_sha": "82b3d5ae55f7080f1e6022629cdb57bfae7cccc7",
   "user_id": 1,
   "user_name": "John Smith",
   "user_avatar": "https://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=8://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=80",
diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb
index 97d1edab9c1..67622f321a6 100644
--- a/lib/gitlab/push_data_builder.rb
+++ b/lib/gitlab/push_data_builder.rb
@@ -36,11 +36,12 @@ module Gitlab
           commit.hook_attrs(with_changed_files: true)
         end
 
-        type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push"
+        type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push'
 
         # Hash to be passed as post_receive_data
         data = {
           object_kind: type,
+          event_name: type,
           before: oldrev,
           after: newrev,
           ref: ref,
diff --git a/spec/lib/gitlab/push_data_builder_spec.rb b/spec/lib/gitlab/push_data_builder_spec.rb
index 961022b9d12..7fc34139eff 100644
--- a/spec/lib/gitlab/push_data_builder_spec.rb
+++ b/spec/lib/gitlab/push_data_builder_spec.rb
@@ -14,11 +14,11 @@ describe Gitlab::PushDataBuilder, lib: true do
     it { expect(data[:ref]).to eq('refs/heads/master') }
     it { expect(data[:commits].size).to eq(3) }
     it { expect(data[:total_commits_count]).to eq(3) }
-    it { expect(data[:commits].first[:added]).to eq(["gitlab-grack"]) }
-    it { expect(data[:commits].first[:modified]).to eq([".gitmodules"]) }
+    it { expect(data[:commits].first[:added]).to eq(['gitlab-grack']) }
+    it { expect(data[:commits].first[:modified]).to eq(['.gitmodules']) }
     it { expect(data[:commits].first[:removed]).to eq([]) }
 
-    include_examples 'project hook data'
+    include_examples 'project hook data with deprecateds'
     include_examples 'deprecated repository hook data'
   end
 
@@ -34,9 +34,18 @@ describe Gitlab::PushDataBuilder, lib: true do
     it { expect(data[:checkout_sha]).to eq('5937ac0a7beb003549fc5fd26fc247adbce4a52e') }
     it { expect(data[:after]).to eq('8a2a6eb295bb170b34c24c76c49ed0e9b2eaf34b') }
     it { expect(data[:ref]).to eq('refs/tags/v1.1.0') }
+    it { expect(data[:user_id]).to eq(user.id) }
+    it { expect(data[:user_name]).to eq(user.name) }
+    it { expect(data[:user_email]).to eq(user.email) }
+    it { expect(data[:user_avatar]).to eq(user.avatar_url) }
+    it { expect(data[:project_id]).to eq(project.id) }
+    it { expect(data[:project]).to be_a(Hash) }
     it { expect(data[:commits]).to be_empty }
     it { expect(data[:total_commits_count]).to be_zero }
 
+    include_examples 'project hook data with deprecateds'
+    include_examples 'deprecated repository hook data'
+
     it 'does not raise an error when given nil commits' do
       expect { described_class.build(spy, spy, spy, spy, spy, nil) }.
         not_to raise_error
diff --git a/spec/services/git_tag_push_service_spec.rb b/spec/services/git_tag_push_service_spec.rb
index cc780587e74..a63656e6268 100644
--- a/spec/services/git_tag_push_service_spec.rb
+++ b/spec/services/git_tag_push_service_spec.rb
@@ -5,19 +5,17 @@ describe GitTagPushService, services: true do
 
   let(:user) { create :user }
   let(:project) { create :project }
-  let(:service) { GitTagPushService.new }
+  let(:service) { GitTagPushService.new(project, user, oldrev: oldrev, newrev: newrev, ref: ref) }
 
-  before do
-    @oldrev = Gitlab::Git::BLANK_SHA
-    @newrev = "8a2a6eb295bb170b34c24c76c49ed0e9b2eaf34b" # gitlab-test: git rev-parse refs/tags/v1.1.0
-    @ref = 'refs/tags/v1.1.0'
-  end
+  let(:oldrev) { Gitlab::Git::BLANK_SHA }
+  let(:newrev) { "8a2a6eb295bb170b34c24c76c49ed0e9b2eaf34b" } # gitlab-test: git rev-parse refs/tags/v1.1.0
+  let(:ref) { 'refs/tags/v1.1.0' }
 
   describe "Git Tag Push Data" do
     before do
-      service.execute(project, user, @oldrev, @newrev, @ref)
+      service.execute
       @push_data = service.push_data
-      @tag_name = Gitlab::Git.ref_name(@ref)
+      @tag_name = Gitlab::Git.ref_name(ref)
       @tag = project.repository.find_tag(@tag_name)
       @commit = project.commit(@tag.target)
     end
@@ -25,9 +23,9 @@ describe GitTagPushService, services: true do
     subject { @push_data }
 
     it { is_expected.to include(object_kind: 'tag_push') }
-    it { is_expected.to include(ref: @ref) }
-    it { is_expected.to include(before: @oldrev) }
-    it { is_expected.to include(after: @newrev) }
+    it { is_expected.to include(ref: ref) }
+    it { is_expected.to include(before: oldrev) }
+    it { is_expected.to include(after: newrev) }
     it { is_expected.to include(message: @tag.message) }
     it { is_expected.to include(user_id: user.id) }
     it { is_expected.to include(user_name: user.name) }
@@ -80,9 +78,11 @@ describe GitTagPushService, services: true do
 
   describe "Webhooks" do
     context "execute webhooks" do
+      let(:service) { GitTagPushService.new(project, user, oldrev: 'oldrev', newrev: 'newrev', ref: 'refs/tags/v1.0.0') }
+
       it "when pushing tags" do
         expect(project).to receive(:execute_hooks)
-        service.execute(project, user, 'oldrev', 'newrev', 'refs/tags/v1.0.0')
+        service.execute
       end
     end
   end
diff --git a/spec/support/project_hook_data_shared_example.rb b/spec/support/project_hook_data_shared_example.rb
index 422083875d7..7dbaa6a6459 100644
--- a/spec/support/project_hook_data_shared_example.rb
+++ b/spec/support/project_hook_data_shared_example.rb
@@ -1,4 +1,4 @@
-RSpec.shared_examples 'project hook data' do |project_key: :project|
+RSpec.shared_examples 'project hook data with deprecateds' do |project_key: :project|
   it 'contains project data' do
     expect(data[project_key][:name]).to eq(project.name)
     expect(data[project_key][:description]).to eq(project.description)
@@ -17,6 +17,21 @@ RSpec.shared_examples 'project hook data' do |project_key: :project|
   end
 end
 
+RSpec.shared_examples 'project hook data' do |project_key: :project|
+  it 'contains project data' do
+    expect(data[project_key][:name]).to eq(project.name)
+    expect(data[project_key][:description]).to eq(project.description)
+    expect(data[project_key][:web_url]).to eq(project.web_url)
+    expect(data[project_key][:avatar_url]).to eq(project.avatar_url)
+    expect(data[project_key][:git_http_url]).to eq(project.http_url_to_repo)
+    expect(data[project_key][:git_ssh_url]).to eq(project.ssh_url_to_repo)
+    expect(data[project_key][:namespace]).to eq(project.namespace.name)
+    expect(data[project_key][:visibility_level]).to eq(project.visibility_level)
+    expect(data[project_key][:path_with_namespace]).to eq(project.path_with_namespace)
+    expect(data[project_key][:default_branch]).to eq(project.default_branch)
+  end
+end
+
 RSpec.shared_examples 'deprecated repository hook data' do |project_key: :project|
   it 'contains deprecated repository data' do
     expect(data[:repository][:name]).to eq(project.name)