Merge branch 'snowplow-backend-ee-to-ce' into 'master'

Migrates Snowplow backend implementation from EE to CE

See merge request gitlab-org/gitlab-ce!31199

16 files changed, 298 insertions, 4 deletions

diff --git a/Gemfile b/Gemfile
index 55143693d5c..ae9ee1cb333 100644
--- a/Gemfile
+++ b/Gemfile
@@ -297,6 +297,9 @@ gem 'batch-loader', '~> 1.4.0'
 # Perf bar
 gem 'peek', '~> 1.0.1'
 
+# Snowplow events tracking
+gem 'snowplow-tracker', '~> 0.6.1'
+
 # Memory benchmarks
 gem 'derailed_benchmarks', require: false
 
diff --git a/Gemfile.lock b/Gemfile.lock
index 6aa96d54abb..918115b3b01 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -152,6 +152,7 @@ GEM
     concurrent-ruby-ext (1.1.5)
       concurrent-ruby (= 1.1.5)
     connection_pool (2.2.2)
+    contracts (0.11.0)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     crass (1.0.4)
@@ -901,6 +902,8 @@ GEM
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
     slack-notifier (1.5.1)
+    snowplow-tracker (0.6.1)
+      contracts (~> 0.7, <= 0.11)
     spring (2.0.2)
       activesupport (>= 4.2)
     spring-commands-rspec (1.0.4)
@@ -1229,6 +1232,7 @@ DEPENDENCIES
   simple_po_parser (~> 1.1.2)
   simplecov (~> 0.16.1)
   slack-notifier (~> 1.5.1)
+  snowplow-tracker (~> 0.6.1)
   spring (~> 2.0.0)
   spring-commands-rspec (~> 1.0.4)
   sprockets (~> 3.7.0)
diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb
index acbcf0ded17..0ab19f1d2d2 100644
--- a/app/helpers/application_settings_helper.rb
+++ b/app/helpers/application_settings_helper.rb
@@ -270,7 +270,11 @@ module ApplicationSettingsHelper
       :diff_max_patch_bytes,
       :commit_email_hostname,
       :protected_ci_variables,
-      :local_markdown_version
+      :local_markdown_version,
+      :snowplow_collector_hostname,
+      :snowplow_cookie_domain,
+      :snowplow_enabled,
+      :snowplow_site_id
     ]
   end
 
diff --git a/app/helpers/tracking_helper.rb b/app/helpers/tracking_helper.rb
index 51ea79d1ddd..221d9692661 100644
--- a/app/helpers/tracking_helper.rb
+++ b/app/helpers/tracking_helper.rb
@@ -2,6 +2,21 @@
 
 module TrackingHelper
   def tracking_attrs(label, event, property)
-    {} # CE has no tracking features
+    return {} unless tracking_enabled?
+
+    {
+      data: {
+        track_label: label,
+        track_event: event,
+        track_property: property
+      }
+    }
+  end
+
+  private
+
+  def tracking_enabled?
+    Rails.env.production? &&
+      ::Gitlab::CurrentSettings.snowplow_enabled?
   end
 end
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index cb6346421ec..2a99c6e5c59 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -99,6 +99,11 @@ class ApplicationSetting < ApplicationRecord
             presence: true,
             if: :plantuml_enabled
 
+  validates :snowplow_collector_hostname,
+            presence: true,
+            hostname: true,
+            if: :snowplow_enabled
+
   validates :max_attachment_size,
             presence: true,
             numericality: { only_integer: true, greater_than: 0 }
diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb
index b7a4d7aa803..55ac1e129cf 100644
--- a/app/models/application_setting_implementation.rb
+++ b/app/models/application_setting_implementation.rb
@@ -97,6 +97,10 @@ module ApplicationSettingImplementation
         usage_stats_set_by_user_id: nil,
         diff_max_patch_bytes: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES,
         commit_email_hostname: default_commit_email_hostname,
+        snowplow_collector_hostname: nil,
+        snowplow_cookie_domain: nil,
+        snowplow_enabled: false,
+        snowplow_site_id: nil,
         protected_ci_variables: false,
         local_markdown_version: 0,
         outbound_local_requests_whitelist: [],
diff --git a/app/views/admin/application_settings/_snowplow.html.haml b/app/views/admin/application_settings/_snowplow.html.haml
new file mode 100644
index 00000000000..b60b5d55a1b
--- /dev/null
+++ b/app/views/admin/application_settings/_snowplow.html.haml
@@ -0,0 +1,30 @@
+- expanded = true if !@application_setting.valid? && @application_setting.errors.any? { |k| k.to_s.start_with?('snowplow_') }
+%section.settings.as-snowplow.no-animate#js-snowplow-settings{ class: ('expanded' if expanded) }
+  .settings-header
+    %h4
+      = _('Snowplow')
+    %button.btn.btn-default.js-settings-toggle{ type: 'button' }
+      = expanded ? _('Collapse') : _('Expand')
+    %p
+      = _('Configure the %{link} integration.').html_safe % { link: link_to('Snowplow', 'https://snowplowanalytics.com/', target: '_blank') }
+  .settings-content
+
+    = form_for @application_setting, url: integrations_admin_application_settings_path, html: { class: 'fieldset-form' } do |f|
+      = form_errors(@application_setting)
+
+      %fieldset
+        .form-group
+          .form-check
+            = f.check_box :snowplow_enabled, class: 'form-check-input'
+            = f.label :snowplow_enabled, _('Enable snowplow tracking'), class: 'form-check-label'
+        .form-group
+          = f.label :snowplow_collector_hostname, _('Collector hostname'), class: 'label-light'
+          = f.text_field :snowplow_collector_hostname, class: 'form-control', placeholder: 'snowplow.example.com'
+        .form-group
+          = f.label :snowplow_site_id, _('Site ID'), class: 'label-light'
+          = f.text_field :snowplow_site_id, class: 'form-control'
+        .form-group
+          = f.label :snowplow_cookie_domain, _('Cookie domain'), class: 'label-light'
+          = f.text_field :snowplow_cookie_domain, class: 'form-control'
+
+      = f.submit _('Save changes'), class: 'btn btn-success'
diff --git a/app/views/layouts/_snowplow.html.haml b/app/views/layouts/_snowplow.html.haml
new file mode 100644
index 00000000000..5f5c5e984c5
--- /dev/null
+++ b/app/views/layouts/_snowplow.html.haml
@@ -0,0 +1,29 @@
+- return unless Gitlab::CurrentSettings.snowplow_enabled?
+
+= javascript_tag nonce: true do
+  :plain
+    ;(function(p,l,o,w,i,n,g){if(!p[i]){p.GlobalSnowplowNamespace=p.GlobalSnowplowNamespace||[];
+    p.GlobalSnowplowNamespace.push(i);p[i]=function(){(p[i].q=p[i].q||[]).push(arguments)
+    };p[i].q=p[i].q||[];n=l.createElement(o);g=l.getElementsByTagName(o)[0];n.async=1;
+    n.src=w;g.parentNode.insertBefore(n,g)}}(window,document,"script","#{asset_url('snowplow/sp.js')}","snowplow"));
+
+    window.snowplow('newTracker', '#{Gitlab::SnowplowTracker::NAMESPACE}', '#{Gitlab::CurrentSettings.snowplow_collector_hostname}', {
+      appId: '#{Gitlab::CurrentSettings.snowplow_site_id}',
+      cookieDomain: '#{Gitlab::CurrentSettings.snowplow_cookie_domain}',
+      userFingerprint: false,
+      respectDoNotTrack: true,
+      forceSecureTracker: true,
+      post: true,
+      contexts: { webPage: true },
+      stateStorageStrategy: "localStorage"
+    });
+
+    window.snowplow('enableActivityTracking', 30, 30);
+    window.snowplow('trackPageView');
+
+- return unless Feature.enabled?(:additional_snowplow_tracking, @group)
+
+= javascript_tag nonce: true do
+  :plain
+    window.snowplow('enableFormTracking');
+    window.snowplow('enableLinkClickTracking');
diff --git a/doc/api/settings.md b/doc/api/settings.md
index 83125aff264..248d19461f6 100644
--- a/doc/api/settings.md
+++ b/doc/api/settings.md
@@ -321,4 +321,8 @@ are listed in the descriptions of the relevant settings.
 | `user_show_add_ssh_key_message`          | boolean          | no                                   | When set to `false` disable the "You won't be able to pull or push project code via SSH" warning shown to users with no uploaded SSH key. |
 | `version_check_enabled`                  | boolean          | no                                   | Let GitLab inform you when an update is available. |
 | `local_markdown_version`                 | integer          | no                                   | Increase this value when any cached markdown should be invalidated. |
+| `snowplow_enabled`                       | boolean          | no                                   | Enable snowplow tracking. |
+| `snowplow_collector_hostname`            | string           | required by: `snowplow_enabled`      | The Snowplow collector hostname. (e.g. `snowplow.trx.gitlab.net`) | 
+| `snowplow_site_id`                       | string           | no                                   | The Snowplow site name / application id. (e.g. `gitlab`) |
+| `snowplow_cookie_domain`                 | string           | no                                   | The Snowplow cookie domain. (e.g. `.gitlab.com`) |
 | `geo_node_allowed_ips`                   | string           | yes                                  | **(PREMIUM)** Comma-separated list of IPs and CIDRs of allowed secondary nodes. For example, `1.1.1.1, 2.2.2.0/24`. |
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index 196ef1fcdfa..c36ee5af63f 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -125,6 +125,12 @@ module API
       optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins'
       optional :local_markdown_version, type: Integer, desc: "Local markdown version, increase this value when any cached markdown should be invalidated"
       optional :allow_local_requests_from_hooks_and_services, type: Boolean, desc: 'Deprecated: Use :allow_local_requests_from_web_hooks_and_services instead. Allow requests to the local network from hooks and services.' # support legacy names, can be removed in v5
+      optional :snowplow_enabled, type: Grape::API::Boolean, desc: 'Enable Snowplow tracking'
+      given snowplow_enabled: ->(val) { val } do
+        requires :snowplow_collector_hostname, type: String, desc: 'The Snowplow collector hostname'
+        optional :snowplow_cookie_domain, type: String, desc: 'The Snowplow cookie domain'
+        optional :snowplow_site_id, type: String, desc: 'The Snowplow site name / application ic'
+      end
 
       ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type|
         optional :"#{type}_key_restriction",
diff --git a/lib/gitlab/snowplow_tracker.rb b/lib/gitlab/snowplow_tracker.rb
new file mode 100644
index 00000000000..9f12513e09e
--- /dev/null
+++ b/lib/gitlab/snowplow_tracker.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'snowplow-tracker'
+
+module Gitlab
+  module SnowplowTracker
+    NAMESPACE = 'cf'
+
+    class << self
+      def track_event(category, action, label: nil, property: nil, value: nil, context: nil)
+        tracker&.track_struct_event(category, action, label, property, value, context, Time.now.to_i)
+      end
+
+      private
+
+      def tracker
+        return unless enabled?
+
+        @tracker ||= ::SnowplowTracker::Tracker.new(emitter, subject, NAMESPACE, Gitlab::CurrentSettings.snowplow_site_id)
+      end
+
+      def subject
+        ::SnowplowTracker::Subject.new
+      end
+
+      def emitter
+        ::SnowplowTracker::Emitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname)
+      end
+
+      def enabled?
+        Gitlab::CurrentSettings.snowplow_enabled?
+      end
+    end
+  end
+end
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 63882d94726..c91c220f696 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -2953,6 +2953,9 @@ msgstr ""
 msgid "Collapse sidebar"
 msgstr ""
 
+msgid "Collector hostname"
+msgstr ""
+
 msgid "ComboSearch is not defined"
 msgstr ""
 
@@ -3120,6 +3123,9 @@ msgstr ""
 msgid "Configure storage path settings."
 msgstr ""
 
+msgid "Configure the %{link} integration."
+msgstr ""
+
 msgid "Configure the way a user creates a new account."
 msgstr ""
 
@@ -3261,6 +3267,9 @@ msgstr ""
 msgid "ConvDev Index"
 msgstr ""
 
+msgid "Cookie domain"
+msgstr ""
+
 msgid "Copied"
 msgstr ""
 
@@ -4253,6 +4262,9 @@ msgstr ""
 msgid "Enable shared Runners"
 msgstr ""
 
+msgid "Enable snowplow tracking"
+msgstr ""
+
 msgid "Enable two-factor authentication"
 msgstr ""
 
@@ -10286,6 +10298,9 @@ msgstr ""
 msgid "Similar issues"
 msgstr ""
 
+msgid "Site ID"
+msgstr ""
+
 msgid "Size and domain settings for static websites"
 msgstr ""
 
@@ -10316,6 +10331,9 @@ msgstr ""
 msgid "SnippetsEmptyState|They can be either public or private."
 msgstr ""
 
+msgid "Snowplow"
+msgstr ""
+
 msgid "Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead."
 msgstr ""
 
diff --git a/spec/helpers/tracking_helper_spec.rb b/spec/helpers/tracking_helper_spec.rb
index 71505e8ea69..b0c98be4130 100644
--- a/spec/helpers/tracking_helper_spec.rb
+++ b/spec/helpers/tracking_helper_spec.rb
@@ -4,8 +4,32 @@ require 'spec_helper'
 
 describe TrackingHelper do
   describe '#tracking_attrs' do
-    it 'returns an empty hash' do
-      expect(helper.tracking_attrs('a', 'b', 'c')).to eq({})
+    using RSpec::Parameterized::TableSyntax
+
+    let(:input) { %w(a b c) }
+    let(:results) do
+      {
+        no_data: {},
+        with_data: { data: { track_label: 'a', track_event: 'b', track_property: 'c' } }
+      }
+    end
+
+    where(:snowplow_enabled, :environment, :result) do
+      true  | 'production'  | :with_data
+      false | 'production'  | :no_data
+      true  | 'development' | :no_data
+      false | 'development' | :no_data
+      true  | 'test'        | :no_data
+      false | 'test'        | :no_data
+    end
+
+    with_them do
+      it 'returns a hash' do
+        stub_application_setting(snowplow_enabled: snowplow_enabled)
+        allow(Rails).to receive(:env).and_return(environment.inquiry)
+
+        expect(helper.tracking_attrs(*input)).to eq(results[result])
+      end
     end
   end
 end
diff --git a/spec/lib/gitlab/snowplow_tracker_spec.rb b/spec/lib/gitlab/snowplow_tracker_spec.rb
new file mode 100644
index 00000000000..073a33e5973
--- /dev/null
+++ b/spec/lib/gitlab/snowplow_tracker_spec.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Gitlab::SnowplowTracker do
+  let(:timestamp) { Time.utc(2017, 3, 22) }
+
+  around do |example|
+    Timecop.freeze(timestamp) { example.run }
+  end
+
+  subject { described_class.track_event('epics', 'action', property: 'what', value: 'doit') }
+
+  context '.track_event' do
+    context 'when Snowplow tracker is disabled' do
+      it 'does not track the event' do
+        expect(SnowplowTracker::Tracker).not_to receive(:new)
+
+        subject
+      end
+    end
+
+    context 'when Snowplow tracker is enabled' do
+      before do
+        stub_application_setting(snowplow_enabled: true)
+        stub_application_setting(snowplow_site_id: 'awesome gitlab')
+        stub_application_setting(snowplow_collector_hostname: 'url.com')
+      end
+
+      it 'tracks the event' do
+        tracker = double
+
+        expect(::SnowplowTracker::Tracker).to receive(:new)
+          .with(
+            an_instance_of(::SnowplowTracker::Emitter),
+            an_instance_of(::SnowplowTracker::Subject),
+            'cf', 'awesome gitlab'
+          ).and_return(tracker)
+        expect(tracker).to receive(:track_struct_event)
+          .with('epics', 'action', nil, 'what', 'doit', nil, timestamp.to_i)
+
+        subject
+      end
+    end
+  end
+end
diff --git a/spec/requests/api/settings_spec.rb b/spec/requests/api/settings_spec.rb
index 184c00a356a..590107d5161 100644
--- a/spec/requests/api/settings_spec.rb
+++ b/spec/requests/api/settings_spec.rb
@@ -144,6 +144,7 @@ describe API::Settings, 'Settings' do
           external_auth_client_key_pass: "5iveL!fe"
         }
       end
+
       let(:attribute_names) { settings.keys.map(&:to_s) }
 
       it 'includes the attributes in the API' do
@@ -165,6 +166,56 @@ describe API::Settings, 'Settings' do
       end
     end
 
+    context "snowplow tracking settings" do
+      let(:settings) do
+        {
+          snowplow_collector_hostname: "snowplow.example.com",
+          snowplow_cookie_domain: ".example.com",
+          snowplow_enabled: true,
+          snowplow_site_id: "site_id"
+        }
+      end
+
+      let(:attribute_names) { settings.keys.map(&:to_s) }
+
+      it "includes the attributes in the API" do
+        get api("/application/settings", admin)
+
+        expect(response).to have_gitlab_http_status(200)
+        attribute_names.each do |attribute|
+          expect(json_response.keys).to include(attribute)
+        end
+      end
+
+      it "allows updating the settings" do
+        put api("/application/settings", admin), params: settings
+
+        expect(response).to have_gitlab_http_status(200)
+        settings.each do |attribute, value|
+          expect(ApplicationSetting.current.public_send(attribute)).to eq(value)
+        end
+      end
+
+      context "missing snowplow_collector_hostname value when snowplow_enabled is true" do
+        it "returns a blank parameter error message" do
+          put api("/application/settings", admin), params: { snowplow_enabled: true }
+
+          expect(response).to have_gitlab_http_status(400)
+          expect(json_response["error"]).to eq("snowplow_collector_hostname is missing")
+        end
+
+        it "handles validation errors" do
+          put api("/application/settings", admin), params: settings.merge({
+            snowplow_collector_hostname: nil
+          })
+
+          expect(response).to have_gitlab_http_status(400)
+          message = json_response["message"]
+          expect(message["snowplow_collector_hostname"]).to include("can't be blank")
+        end
+      end
+    end
+
     context "missing plantuml_url value when plantuml_enabled is true" do
       it "returns a blank parameter error message" do
         put api("/application/settings", admin), params: { plantuml_enabled: true }
diff --git a/spec/views/layouts/_head.html.haml_spec.rb b/spec/views/layouts/_head.html.haml_spec.rb
index cbb4199954a..70cdc08b4b6 100644
--- a/spec/views/layouts/_head.html.haml_spec.rb
+++ b/spec/views/layouts/_head.html.haml_spec.rb
@@ -70,6 +70,23 @@ describe 'layouts/_head' do
     expect(rendered).to match('<link rel="stylesheet" media="all" href="/stylesheets/highlight/themes/solarised-light.css" />')
   end
 
+  context 'when an asset_host is set and snowplow url is set' do
+    let(:asset_host) { 'http://test.host' }
+
+    before do
+      allow(ActionController::Base).to receive(:asset_host).and_return(asset_host)
+      allow(Gitlab::CurrentSettings).to receive(:snowplow_enabled?).and_return(true)
+      allow(Gitlab::CurrentSettings).to receive(:snowplow_collector_hostname).and_return('www.snow.plow')
+    end
+
+    it 'add a snowplow script tag with asset host' do
+      render
+      expect(rendered).to match('http://test.host/assets/snowplow/')
+      expect(rendered).to match('window.snowplow')
+      expect(rendered).to match('www.snow.plow')
+    end
+  end
+
   def stub_helper_with_safe_string(method)
     allow_any_instance_of(PageLayoutHelper).to receive(method)
       .and_return(%q{foo" http-equiv="refresh}.html_safe)