Add send_git_diff helper

20 files changed, 235 insertions, 63 deletions

diff --git a/lib/api/builds.rb b/lib/api/builds.rb
index 2b104f90aa7..0ff8fa74a84 100644
--- a/lib/api/builds.rb
+++ b/lib/api/builds.rb
@@ -33,7 +33,7 @@ module API
       get ':id/repository/commits/:sha/builds' do
         authorize_read_builds!
 
-        commit = user_project.ci_commits.find_by_sha(params[:sha])
+        commit = user_project.pipelines.find_by_sha(params[:sha])
         return not_found! unless commit
 
         builds = commit.builds.order('id DESC')
diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb
index 9bcd33ff19e..323a7086890 100644
--- a/lib/api/commit_statuses.rb
+++ b/lib/api/commit_statuses.rb
@@ -22,8 +22,8 @@ module API
 
         not_found!('Commit') unless user_project.commit(params[:sha])
 
-        ci_commits = user_project.ci_commits.where(sha: params[:sha])
-        statuses = ::CommitStatus.where(commit: ci_commits)
+        pipelines = user_project.pipelines.where(sha: params[:sha])
+        statuses = ::CommitStatus.where(pipeline: pipelines)
         statuses = statuses.latest unless parse_boolean(params[:all])
         statuses = statuses.where(ref: params[:ref]) if params[:ref].present?
         statuses = statuses.where(stage: params[:stage]) if params[:stage].present?
@@ -50,7 +50,7 @@ module API
         commit = @project.commit(params[:sha])
         not_found! 'Commit' unless commit
 
-        # Since the CommitStatus is attached to Ci::Commit (in the future Pipeline)
+        # Since the CommitStatus is attached to Ci::Pipeline (in the future Pipeline)
         # We need to always have the pipeline object
         # To have a valid pipeline object that can be attached to specific MR
         # Other CI service needs to send `ref`
@@ -64,11 +64,11 @@ module API
           ref = branches.first
         end
 
-        ci_commit = @project.ensure_ci_commit(commit.sha, ref)
+        pipeline = @project.ensure_pipeline(commit.sha, ref)
 
         name = params[:name] || params[:context]
-        status = GenericCommitStatus.running_or_pending.find_by(commit: ci_commit, name: name, ref: params[:ref])
-        status ||= GenericCommitStatus.new(project: @project, commit: ci_commit, user: current_user)
+        status = GenericCommitStatus.running_or_pending.find_by(pipeline: pipeline, name: name, ref: params[:ref])
+        status ||= GenericCommitStatus.new(project: @project, pipeline: pipeline, user: current_user)
         status.update(attrs)
 
         case params[:state].to_s
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index db304abe1c3..2e7836dc8fb 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -243,7 +243,7 @@ module API
             should_remove_source_branch: params[:should_remove_source_branch]
           }
 
-          if parse_boolean(params[:merge_when_build_succeeds]) && merge_request.ci_commit && merge_request.ci_commit.active?
+          if parse_boolean(params[:merge_when_build_succeeds]) && merge_request.pipeline && merge_request.pipeline.active?
             ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params).
               execute(merge_request)
           else
diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb
index e1636636934..5270108ef0f 100644
--- a/lib/ci/charts.rb
+++ b/lib/ci/charts.rb
@@ -60,7 +60,7 @@ module Ci
 
     class BuildTime < Chart
       def collect
-        commits = project.ci_commits.last(30)
+        commits = project.pipelines.last(30)
 
         commits.each do |commit|
           @labels << commit.short_sha
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 026a5ac97ca..130f5b0892e 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -12,18 +12,14 @@ module Ci
     attr_reader :before_script, :after_script, :image, :services, :path, :cache
 
     def initialize(config, path = nil)
-      @config = YAML.safe_load(config, [Symbol], [], true)
+      @config = Gitlab::Ci::Config.new(config).to_hash
       @path = path
 
-      unless @config.is_a? Hash
-        raise ValidationError, "YAML should be a hash"
-      end
-
-      @config = @config.deep_symbolize_keys
-
       initial_parsing
 
       validate!
+    rescue Gitlab::Ci::Config::Loader::FormatError => e
+      raise ValidationError, e.message
     end
 
     def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil)
diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/build_data_builder.rb
index 34e949130da..9f45aefda0f 100644
--- a/lib/gitlab/build_data_builder.rb
+++ b/lib/gitlab/build_data_builder.rb
@@ -3,7 +3,7 @@ module Gitlab
     class << self
       def build(build)
         project = build.project
-        commit = build.commit
+        commit = build.pipeline
         user = build.user
 
         data = {
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
new file mode 100644
index 00000000000..ffe633d4b63
--- /dev/null
+++ b/lib/gitlab/ci/config.rb
@@ -0,0 +1,16 @@
+module Gitlab
+  module Ci
+    class Config
+      class LoaderError < StandardError; end
+
+      def initialize(config)
+        loader = Loader.new(config)
+        @config = loader.load!
+      end
+
+      def to_hash
+        @config
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/ci/config/loader.rb
new file mode 100644
index 00000000000..dbf6eb0edbe
--- /dev/null
+++ b/lib/gitlab/ci/config/loader.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module Ci
+    class Config
+      class Loader
+        class FormatError < StandardError; end
+
+        def initialize(config)
+          @config = YAML.safe_load(config, [Symbol], [], true)
+        end
+
+        def valid?
+          @config.is_a?(Hash)
+        end
+
+        def load!
+          unless valid?
+            raise FormatError, 'Invalid configuration format'
+          end
+
+          @config.deep_symbolize_keys
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 42bec913a45..04fa6a3a5de 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -16,6 +16,20 @@ module Gitlab
       database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
     end
 
+    def self.nulls_last_order(field, direction = 'ASC')
+      order = "#{field} #{direction}"
+
+      if Gitlab::Database.postgresql?
+        order << ' NULLS LAST'
+      else
+        # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL
+        # columns. In the (default) ascending order, `0` comes first.
+        order.prepend("#{field} IS NULL, ") if direction == 'ASC'
+      end
+
+      order
+    end
+
     def true_value
       if Gitlab::Database.postgresql?
         "'t'"
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index fd14234c558..978c3f7896d 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -11,7 +11,7 @@ module Gitlab
       #     add_concurrent_index :users, :some_column
       #
       # See Rails' `add_index` for more info on the available arguments.
-      def add_concurrent_index(*args)
+      def add_concurrent_index(table_name, column_name, options = {})
         if transaction_open?
           raise 'add_concurrent_index can not be run inside a transaction, ' \
             'you can disable transactions by calling disable_ddl_transaction! ' \
@@ -19,10 +19,10 @@ module Gitlab
         end
 
         if Database.postgresql?
-          args << { algorithm: :concurrently }
+          options = options.merge({ algorithm: :concurrently })
         end
 
-        add_index(*args)
+        add_index(table_name, column_name, options)
       end
 
       # Updates the value of a column in batches.
diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb
index 202263c6742..72992baffd4 100644
--- a/lib/gitlab/github_import/base_formatter.rb
+++ b/lib/gitlab/github_import/base_formatter.rb
@@ -9,6 +9,10 @@ module Gitlab
         @formatter = Gitlab::ImportFormatter.new
       end
 
+      def create!
+        self.klass.create!(self.attributes)
+      end
+
       private
 
       def gl_user_id(github_id)
diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb
index 7d679eaec6a..2c1b94ef2cd 100644
--- a/lib/gitlab/github_import/comment_formatter.rb
+++ b/lib/gitlab/github_import/comment_formatter.rb
@@ -8,6 +8,7 @@ module Gitlab
           commit_id: raw_data.commit_id,
           line_code: line_code,
           author_id: author_id,
+          type: type,
           created_at: raw_data.created_at,
           updated_at: raw_data.updated_at
         }
@@ -53,6 +54,10 @@ module Gitlab
       def note
         formatter.author_line(author) + body
       end
+
+      def type
+        'LegacyDiffNote' if on_diff?
+      end
     end
   end
 end
diff --git a/lib/gitlab/github_import/hook_formatter.rb b/lib/gitlab/github_import/hook_formatter.rb
new file mode 100644
index 00000000000..db1fabaa18a
--- /dev/null
+++ b/lib/gitlab/github_import/hook_formatter.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module GithubImport
+    class HookFormatter
+      EVENTS = %w[* create delete pull_request push].freeze
+
+      attr_reader :raw
+
+      delegate :id, :name, :active, to: :raw
+
+      def initialize(raw)
+        @raw = raw
+      end
+
+      def config
+        raw.config.attrs
+      end
+
+      def valid?
+        (EVENTS & raw.events).any? && active
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 9d077e79c39..442b4c389fe 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -3,6 +3,9 @@ module Gitlab
     class Importer
       include Gitlab::ShellAdapter
 
+      GITHUB_SAFE_REMAINING_REQUESTS = 100
+      GITHUB_SAFE_SLEEP_TIME = 500
+
       attr_reader :client, :project, :repo, :repo_url
 
       def initialize(project)
@@ -25,14 +28,53 @@ module Gitlab
 
       private
 
+      def turn_auto_pagination_off!
+        client.auto_paginate = false
+      end
+
+      def turn_auto_pagination_on!
+        client.auto_paginate = true
+      end
+
+      def rate_limit
+        client.rate_limit!
+      end
+
+      def rate_limit_exceed?
+        rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS
+      end
+
+      def rate_limit_sleep_time
+        rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
+      end
+
+      def paginate
+        turn_auto_pagination_off!
+
+        sleep rate_limit_sleep_time if rate_limit_exceed?
+
+        data = yield
+
+        last_response = client.last_response
+
+        while last_response.rels[:next]
+          sleep rate_limit_sleep_time if rate_limit_exceed?
+          last_response = last_response.rels[:next].get
+          data.concat(last_response.data) if last_response.data.is_a?(Array)
+        end
+
+        turn_auto_pagination_on!
+
+        data
+      end
+
       def credentials
         @credentials ||= project.import_data.credentials if project.import_data
       end
 
       def import_labels
-        client.labels(repo).each do |raw_data|
-          Label.create!(LabelFormatter.new(project, raw_data).attributes)
-        end
+        labels = paginate { client.labels(repo, per_page: 100) }
+        labels.each { |raw| LabelFormatter.new(project, raw).create! }
 
         true
       rescue ActiveRecord::RecordInvalid => e
@@ -40,9 +82,8 @@ module Gitlab
       end
 
       def import_milestones
-        client.list_milestones(repo, state: :all).each do |raw_data|
-          Milestone.create!(MilestoneFormatter.new(project, raw_data).attributes)
-        end
+        milestones = paginate { client.milestones(repo, state: :all, per_page: 100) }
+        milestones.each { |raw| MilestoneFormatter.new(project, raw).create! }
 
         true
       rescue ActiveRecord::RecordInvalid => e
@@ -50,16 +91,15 @@ module Gitlab
       end
 
       def import_issues
-        client.list_issues(repo, state: :all, sort: :created, direction: :asc).each do |raw_data|
-          gh_issue = IssueFormatter.new(project, raw_data)
+        data = paginate { client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
 
-          if gh_issue.valid?
-            issue = Issue.create!(gh_issue.attributes)
-            apply_labels(gh_issue.number, issue)
+        data.each do |raw|
+          gh_issue = IssueFormatter.new(project, raw)
 
-            if gh_issue.has_comments?
-              import_comments(gh_issue.number, issue)
-            end
+          if gh_issue.valid?
+            issue = gh_issue.create!
+            apply_labels(issue)
+            import_comments(issue) if gh_issue.has_comments?
           end
         end
 
@@ -69,50 +109,68 @@ module Gitlab
       end
 
       def import_pull_requests
-        pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc)
-                              .map { |raw| PullRequestFormatter.new(project, raw) }
-                              .select(&:valid?)
+        hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?)
+        disable_webhooks(hooks)
+
+        pull_requests = paginate { client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
+        pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
 
         source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] }
         target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] }
         branches_removed = source_branches_removed | target_branches_removed
 
-        create_refs(branches_removed)
+        restore_branches(branches_removed)
 
         pull_requests.each do |pull_request|
-          merge_request = MergeRequest.new(pull_request.attributes)
-
-          if merge_request.save
-            apply_labels(pull_request.number, merge_request)
-            import_comments(pull_request.number, merge_request)
-            import_comments_on_diff(pull_request.number, merge_request)
-          end
+          merge_request = pull_request.create!
+          apply_labels(merge_request)
+          import_comments(merge_request)
+          import_comments_on_diff(merge_request)
         end
 
         true
       rescue ActiveRecord::RecordInvalid => e
         raise Projects::ImportService::Error, e.message
       ensure
-        delete_refs(branches_removed)
+        clean_up_restored_branches(branches_removed)
+        clean_up_disabled_webhooks(hooks)
       end
 
-      def create_refs(branches)
+      def disable_webhooks(hooks)
+        update_webhooks(hooks, active: false)
+      end
+
+      def clean_up_disabled_webhooks(hooks)
+        update_webhooks(hooks, active: true)
+      end
+
+      def update_webhooks(hooks, options)
+        hooks.each do |hook|
+          client.edit_hook(repo, hook.id, hook.name, hook.config, options)
+        end
+      end
+
+      def restore_branches(branches)
         branches.each do |name, sha|
+          sleep rate_limit_sleep_time if rate_limit_exceed?
           client.create_ref(repo, "refs/heads/#{name}", sha)
         end
 
         project.repository.fetch_ref(repo_url, '+refs/heads/*', 'refs/heads/*')
       end
 
-      def delete_refs(branches)
+      def clean_up_restored_branches(branches)
         branches.each do |name, _|
+          sleep rate_limit_sleep_time if rate_limit_exceed?
           client.delete_ref(repo, "heads/#{name}")
           project.repository.rm_branch(project.creator, name)
         end
       end
 
-      def apply_labels(number, issuable)
-        issue = client.issue(repo, number)
+      def apply_labels(issuable)
+        sleep rate_limit_sleep_time if rate_limit_exceed?
+
+        issue = client.issue(repo, issuable.iid)
 
         if issue.labels.count > 0
           label_ids = issue.labels.map do |raw|
@@ -123,20 +181,20 @@ module Gitlab
         end
       end
 
-      def import_comments(issue_number, noteable)
-        comments = client.issue_comments(repo, issue_number)
-        create_comments(comments, noteable)
+      def import_comments(issuable)
+        comments = paginate { client.issue_comments(repo, issuable.iid, per_page: 100) }
+        create_comments(issuable, comments)
       end
 
-      def import_comments_on_diff(pull_request_number, merge_request)
-        comments = client.pull_request_comments(repo, pull_request_number)
-        create_comments(comments, merge_request)
+      def import_comments_on_diff(merge_request)
+        comments = paginate { client.pull_request_comments(repo, merge_request.iid, per_page: 100) }
+        create_comments(merge_request, comments)
       end
 
-      def create_comments(comments, noteable)
-        comments.each do |raw_data|
-          comment = CommentFormatter.new(project, raw_data)
-          noteable.notes.create!(comment.attributes)
+      def create_comments(issuable, comments)
+        comments.each do |raw|
+          comment = CommentFormatter.new(project, raw)
+          issuable.notes.create!(comment.attributes)
         end
       end
 
diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb
index c8173913b4e..835ec858b35 100644
--- a/lib/gitlab/github_import/issue_formatter.rb
+++ b/lib/gitlab/github_import/issue_formatter.rb
@@ -20,6 +20,10 @@ module Gitlab
         raw_data.comments > 0
       end
 
+      def klass
+        Issue
+      end
+
       def number
         raw_data.number
       end
diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb
index c2b9d40b511..9f18244e7d7 100644
--- a/lib/gitlab/github_import/label_formatter.rb
+++ b/lib/gitlab/github_import/label_formatter.rb
@@ -9,6 +9,10 @@ module Gitlab
         }
       end
 
+      def klass
+        Label
+      end
+
       private
 
       def color
diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb
index e91a7e328cf..53d4b3102d1 100644
--- a/lib/gitlab/github_import/milestone_formatter.rb
+++ b/lib/gitlab/github_import/milestone_formatter.rb
@@ -14,6 +14,10 @@ module Gitlab
         }
       end
 
+      def klass
+        Milestone
+      end
+
       private
 
       def number
diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb
index a2947b56ad9..498b00cb658 100644
--- a/lib/gitlab/github_import/pull_request_formatter.rb
+++ b/lib/gitlab/github_import/pull_request_formatter.rb
@@ -24,6 +24,10 @@ module Gitlab
         }
       end
 
+      def klass
+        MergeRequest
+      end
+
       def number
         raw_data.number
       end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 96e99dc0088..8930149b12c 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -14,7 +14,7 @@ module Gitlab
 
         [
           SEND_DATA_HEADER,
-          "git-blob:#{encode(params)}",
+          "git-blob:#{encode(params)}"
         ]
       end
 
@@ -26,7 +26,22 @@ module Gitlab
 
         [
           SEND_DATA_HEADER,
-          "git-archive:#{encode(params)}",
+          "git-archive:#{encode(params)}"
+        ]
+      end
+
+      def send_git_diff(repository, diff_refs)
+        from, to = diff_refs
+
+        params = {
+          'RepoPath'  => repository.path_to_repo,
+          'ShaFrom'   => from.sha,
+          'ShaTo'     => to.sha
+        }
+
+        [
+          SEND_DATA_HEADER,
+          "git-diff:#{encode(params)}"
         ]
       end
 
diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake
index 48baecfd2a2..05fcb8e3da5 100644
--- a/lib/tasks/gitlab/setup.rake
+++ b/lib/tasks/gitlab/setup.rake
@@ -19,7 +19,7 @@ namespace :gitlab do
     Rake::Task["setup_postgresql"].invoke
     Rake::Task["db:seed_fu"].invoke
   rescue Gitlab::TaskAbortedByUserError
-    puts "Quitting...".red
+    puts "Quitting...".color(:red)
     exit 1
   end
 end