Use a separate table for storing push eventssplit-events-into-push-events

7 files changed, 322 insertions, 1 deletions

diff --git a/app/models/concerns/sha_attribute.rb b/app/models/concerns/sha_attribute.rb
new file mode 100644
index 00000000000..3935b9fee9f
--- /dev/null
+++ b/app/models/concerns/sha_attribute.rb
@@ -0,0 +1,9 @@
+module ShaAttribute
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def sha_attribute(name)
+      attribute(name, Gitlab::Database::ShaAttribute.new)
+    end
+  end
+end
diff --git a/app/models/push_event.rb b/app/models/push_event.rb
new file mode 100644
index 00000000000..29fbafe934d
--- /dev/null
+++ b/app/models/push_event.rb
@@ -0,0 +1,20 @@
+class PushEvent < ActiveRecord::Base
+  include ShaAttribute
+
+  belongs_to :project
+  belongs_to :user
+
+  sha_attribute :first_commit
+  sha_attribute :last_commit
+
+  enum action: {
+    created: 0,
+    removed: 1,
+    pushed: 2
+  }
+
+  enum ref_type: {
+    branch: 0,
+    tag: 1
+  }
+end
diff --git a/db/migrate/20170608152748_create_push_events_tables.rb b/db/migrate/20170608152748_create_push_events_tables.rb
new file mode 100644
index 00000000000..9439a90b84c
--- /dev/null
+++ b/db/migrate/20170608152748_create_push_events_tables.rb
@@ -0,0 +1,35 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreatePushEventsTables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    create_table :push_events, id: false do |t|
+      t.datetime_with_timezone :created_at, null: false
+
+      t.references :user,
+        null: false, foreign_key: { on_delete: :cascade }
+
+      t.references :project,
+        null: false, foreign_key: { on_delete: :cascade }
+
+      t.bigint :commit_count, null: false
+
+      t.integer :action, null: false, limit: 2
+      t.integer :ref_type, null: false, limit: 2
+
+      t.binary :first_commit
+      t.binary :last_commit
+
+      t.text :ref
+      t.string :commit_message, limit: 70
+
+      t.index [:project_id, :created_at]
+      t.index [:user_id, :created_at]
+    end
+  end
+end
diff --git a/db/post_migrate/20170627101016_schedule_event_migrations.rb b/db/post_migrate/20170627101016_schedule_event_migrations.rb
new file mode 100644
index 00000000000..b63088e5e62
--- /dev/null
+++ b/db/post_migrate/20170627101016_schedule_event_migrations.rb
@@ -0,0 +1,27 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class ScheduleEventMigrations < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class Event < ActiveRecord::Base
+    self.table_name = 'events'
+  end
+
+  def up
+    Event.where(action: 5).in_batches do |relation|
+      jobs = relation.pluck(:id).map do |id|
+        ['MigrateEventsToPushEvents', [id]]
+      end
+
+      BackgroundMigrationWorker.perform_bulk(*jobs)
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 028556bdccf..dd048f7cc17 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170621102400) do
+ActiveRecord::Schema.define(version: 20170627101016) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1150,6 +1150,22 @@ ActiveRecord::Schema.define(version: 20170621102400) do
 
   add_index "protected_tags", ["project_id"], name: "index_protected_tags_on_project_id", using: :btree
 
+  create_table "push_events", id: false, force: :cascade do |t|
+    t.datetime "created_at", null: false
+    t.integer "user_id", null: false
+    t.integer "project_id", null: false
+    t.integer "commit_count", limit: 8, null: false
+    t.integer "action", limit: 2, null: false
+    t.integer "ref_type", limit: 2, null: false
+    t.binary "first_commit"
+    t.binary "last_commit"
+    t.text "ref"
+    t.string "commit_message", limit: 70
+  end
+
+  add_index "push_events", ["project_id", "created_at"], name: "index_push_events_on_project_id_and_created_at", using: :btree
+  add_index "push_events", ["user_id", "created_at"], name: "index_push_events_on_user_id_and_created_at", using: :btree
+
   create_table "redirect_routes", force: :cascade do |t|
     t.integer "source_id", null: false
     t.string "source_type", null: false
@@ -1562,6 +1578,8 @@ ActiveRecord::Schema.define(version: 20170621102400) do
   add_foreign_key "protected_tag_create_access_levels", "namespaces", column: "group_id"
   add_foreign_key "protected_tag_create_access_levels", "protected_tags"
   add_foreign_key "protected_tag_create_access_levels", "users"
+  add_foreign_key "push_events", "projects", on_delete: :cascade
+  add_foreign_key "push_events", "users", on_delete: :cascade
   add_foreign_key "subscriptions", "projects", on_delete: :cascade
   add_foreign_key "system_note_metadata", "notes", name: "fk_d83a918cb1", on_delete: :cascade
   add_foreign_key "timelogs", "issues", name: "fk_timelogs_issues_issue_id", on_delete: :cascade
diff --git a/lib/gitlab/background_migration/migrate_events_to_push_events.rb b/lib/gitlab/background_migration/migrate_events_to_push_events.rb
new file mode 100644
index 00000000000..5744789183b
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_events_to_push_events.rb
@@ -0,0 +1,178 @@
+module Gitlab
+  module BackgroundMigration
+    # Class for migrating push events from the "events" table into the
+    # "push_events" table.
+    class MigrateEventsToPushEvents
+      class Event < ActiveRecord::Base
+        self.table_name = 'events'
+
+        serialize :data # rubocop:disable Cop/ActiverecordSerialize
+
+        BLANK_REF = ('0' * 40).freeze
+        TAG_REF_PREFIX = 'refs/tags/'.freeze
+        MAX_INDEX = 69
+
+        def commit_summary
+          commit = commits.last
+
+          return nil unless commit
+
+          index = commit[:message].index("\n") || MAX_INDEX
+          index = MAX_INDEX if index > MAX_INDEX
+
+          commit[:message][0..index].strip
+        end
+
+        def first_commit_sha
+          if commits?
+            commits.first[:id]
+          elsif create?
+            nil
+          else
+            data[:before]
+          end
+        end
+
+        def last_commit_sha
+          if commits?
+            commits.last[:id]
+          elsif remove?
+            nil
+          else
+            data[:after]
+          end
+        end
+
+        def commits?
+          commits.any?
+        end
+
+        def commits
+          data[:commits]
+        end
+
+        def commit_count
+          data[:total_commits_count]
+        end
+
+        def ref
+          data[:ref]
+        end
+
+        def trimmed_ref_name
+          if ref_type == :tag
+            ref[10..-1]
+          else
+            ref[11..-1]
+          end
+        end
+
+        def create?
+          data[:before] == BLANK_REF
+        end
+
+        def remove?
+          data[:after] == BLANK_REF
+        end
+
+        def push_action
+          if create?
+            :created
+          elsif remove?
+            :removed
+          else
+            :pushed
+          end
+        end
+
+        def ref_type
+          if ref.start_with?(TAG_REF_PREFIX)
+            :tag
+          else
+            :branch
+          end
+        end
+      end
+
+      class PushEvent < ActiveRecord::Base
+        self.table_name = 'push_events'
+
+        enum action: {
+          created: 0,
+          removed: 1,
+          pushed: 2
+        }
+
+        enum ref_type: {
+          branch: 0,
+          tag: 1
+        }
+      end
+
+      # id - The ID of the row in the "events" table to migrate.
+      def perform(id)
+        unless migrate?
+          Rails.logger
+            .info("Not migrating event #{id} as the events or push_events table does not exist")
+
+          return
+        end
+
+        event = find_event(id)
+
+        # If the event was removed in the mean time we just skip it.
+        unless event
+          Rails.logger
+            .info("Not migrating event #{id} as the row no longer exists")
+
+          return
+        end
+
+        create_push_event(event)
+      end
+
+      def create_push_event(event)
+        first_commit = pack(event.first_commit_sha)
+        last_commit = pack(event.last_commit_sha)
+
+        begin
+          PushEvent.create!(
+            created_at: event.created_at,
+            user_id: event.author_id,
+            project_id: event.project_id,
+            commit_count: event.commit_count,
+            ref_type: event.ref_type,
+            action: event.push_action,
+            first_commit: first_commit,
+            last_commit: last_commit,
+            ref: event.trimmed_ref_name,
+            commit_message: event.commit_summary
+          )
+        rescue ActiveRecord::InvalidForeignKey => error
+          # If the user or project no longer exists we'll also skip migrating
+          # the event.
+          Rails.logger
+            .info("Not migrating event #{event.id} due to a foreign key error: #{error.message}")
+
+          return
+        end
+
+        event.destroy!
+      end
+
+      def find_event(id)
+        Event
+          .select([:id, :data, :project_id, :created_at, :author_id])
+          .find_by(id: id)
+      end
+
+      def migrate?
+        Event.table_exists? && PushEvent.table_exists?
+      end
+
+      def pack(value)
+        value ? [value].pack('H*') : nil
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb
new file mode 100644
index 00000000000..d9400e04b83
--- /dev/null
+++ b/lib/gitlab/database/sha_attribute.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module Database
+    BINARY_TYPE = if Gitlab::Database.postgresql?
+                    # PostgreSQL defines its own class with slightly different
+                    # behaviour from the default Binary type.
+                    ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea
+                  else
+                    ActiveRecord::Type::Binary
+                  end
+
+    # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa).
+    #
+    # Using ShaAttribute allows you to store SHA1 values as binary while still
+    # using them as if they were stored as string values. This gives you the
+    # ease of use of string values, but without the storage overhead.
+    class ShaAttribute < BINARY_TYPE
+      PACK_FORMAT = 'H*'.freeze
+
+      # Casts binary data to a SHA1 in hexadecimal.
+      def type_cast_from_database(value)
+        value = super
+
+        value ? value.unpack(PACK_FORMAT)[0] : nil
+      end
+
+      # Casts a SHA1 in hexadecimal to the proper binary format.
+      def type_cast_for_database(value)
+        arg = value ? [value].pack(PACK_FORMAT) : nil
+
+        super(arg)
+      end
+    end
+  end
+end