From 62ef67acc3a8d260aa3e641b350aaecf8d60f1aa Mon Sep 17 00:00:00 2001
From: Robin Bobbitt <ryehle@us.ibm.com>
Date: Fri, 4 Aug 2017 09:17:20 -0400
Subject: Hide read_registry scope when registry is disabled on instance

---
 lib/gitlab/auth.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 7d3aa532750..0a5afeb5202 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -2,7 +2,7 @@ module Gitlab
   module Auth
     MissingPersonalTokenError = Class.new(StandardError)
 
-    REGISTRY_SCOPES = [:read_registry].freeze
+    REGISTRY_SCOPES = Gitlab.config.registry.enabled ? [:read_registry].freeze : [].freeze
 
     # Scopes used for GitLab API access
     API_SCOPES = [:api, :read_user].freeze
-- 
cgit v1.2.1


From 770bcf71bb85c9eff13f4eb14cbd517986da9056 Mon Sep 17 00:00:00 2001
From: Zeger-Jan van de Weg <git@zjvandeweg.nl>
Date: Wed, 30 Aug 2017 20:39:23 +0200
Subject: Form for setting project auto devops settings

---
 lib/gitlab/import_export/import_export.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 78795dd3d92..d164a9adb2d 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -50,6 +50,7 @@ project_tree:
         - :push_event_payload
     - :stages
     - :statuses
+  - :project_auto_devops
   - :triggers
   - :pipeline_schedules
   - :services
@@ -141,4 +142,4 @@ methods:
   events:
     - :action
   push_event_payload:
-    - :action
\ No newline at end of file
+    - :action
-- 
cgit v1.2.1


From 78dad4cf321eb84aa5decdea34704145adca0c3e Mon Sep 17 00:00:00 2001
From: Zeger-Jan van de Weg <git@zjvandeweg.nl>
Date: Mon, 4 Sep 2017 09:27:09 +0200
Subject: Fix tests

---
 lib/gitlab/import_export/import_export.yml   | 2 +-
 lib/gitlab/import_export/relation_factory.rb | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index d164a9adb2d..0ea1699108b 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -50,7 +50,7 @@ project_tree:
         - :push_event_payload
     - :stages
     - :statuses
-  - :project_auto_devops
+  - :auto_devops
   - :triggers
   - :pipeline_schedules
   - :services
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 20580459046..e6e7e3573b0 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -14,6 +14,7 @@ module Gitlab
                     create_access_levels: 'ProtectedTag::CreateAccessLevel',
                     labels: :project_labels,
                     priorities: :label_priorities,
+                    auto_devops: 'ProjectAutoDevops',
                     label: :project_label }.freeze
 
       USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze
-- 
cgit v1.2.1


From bcd70c4c46ae71366580c7352ddb28075cdf0e60 Mon Sep 17 00:00:00 2001
From: Zeger-Jan van de Weg <git@zjvandeweg.nl>
Date: Mon, 4 Sep 2017 15:44:46 +0200
Subject: Incorporate review

---
 lib/gitlab/import_export/relation_factory.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index e6e7e3573b0..7448b806b72 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -14,7 +14,7 @@ module Gitlab
                     create_access_levels: 'ProtectedTag::CreateAccessLevel',
                     labels: :project_labels,
                     priorities: :label_priorities,
-                    auto_devops: 'ProjectAutoDevops',
+                    auto_devops: :project_auto_devops,
                     label: :project_label }.freeze
 
       USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze
-- 
cgit v1.2.1


From f5bb1789ee170cbdfb7580f6ec5de17ce06b6cbc Mon Sep 17 00:00:00 2001
From: Felipe Artur <felipefac@gmail.com>
Date: Mon, 4 Sep 2017 16:55:29 -0300
Subject: Fix specs BACKPORT

---
 lib/gitlab/path_regex.rb | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb
index 894bd5efae5..7c02c9c5c48 100644
--- a/lib/gitlab/path_regex.rb
+++ b/lib/gitlab/path_regex.rb
@@ -26,6 +26,7 @@ module Gitlab
       apple-touch-icon.png
       assets
       autocomplete
+      boards
       ci
       dashboard
       deploy.html
-- 
cgit v1.2.1


From c53f319f883064e6c5e77f9b48a235b23b0f4363 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Tue, 5 Sep 2017 12:17:56 +0200
Subject: Extract a class that represents artifacts file path

---
 lib/gitlab/ci/build/artifacts/metadata/entry.rb | 244 ++++++++++++------------
 lib/gitlab/ci/build/artifacts/path.rb           |  51 +++++
 2 files changed, 173 insertions(+), 122 deletions(-)
 create mode 100644 lib/gitlab/ci/build/artifacts/path.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
index 2e073334abc..321ddfd8c29 100644
--- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
@@ -1,129 +1,129 @@
 module Gitlab
-  module Ci::Build::Artifacts
-    class Metadata
-      ##
-      # Class that represents an entry (path and metadata) to a file or
-      # directory in GitLab CI Build Artifacts binary file / archive
-      #
-      # This is IO-operations safe class, that does similar job to
-      # Ruby's Pathname but without the risk of accessing filesystem.
-      #
-      # This class is working only with UTF-8 encoded paths.
-      #
-      class Entry
-        attr_reader :path, :entries
-        attr_accessor :name
-
-        def initialize(path, entries)
-          @path = path.dup.force_encoding('UTF-8')
-          @entries = entries
-
-          if path.include?("\0")
-            raise ArgumentError, 'Path contains zero byte character!'
-          end
+  module Ci
+    module Build
+      module Artifacts
+        class Metadata
+          ##
+          # Class that represents an entry (path and metadata) to a file or
+          # directory in GitLab CI Build Artifacts binary file / archive
+          #
+          # This is IO-operations safe class, that does similar job to
+          # Ruby's Pathname but without the risk of accessing filesystem.
+          #
+          # This class is working only with UTF-8 encoded paths.
+          #
+          class Entry
+            attr_reader :entries
+            attr_accessor :name
+
+            def initialize(path, entries)
+              @entries = entries
+              @path = Artifacts::Path.new(path)
+            end
+
+            delegate :empty?, to: :children
+
+            def directory?
+              blank_node? || @path.directory?
+            end
+
+            def file?
+              !directory?
+            end
+
+            def blob
+              return unless file?
+
+              @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil)
+            end
+
+            def has_parent?
+              nodes > 0
+            end
+
+            def parent
+              return nil unless has_parent?
+              self.class.new(@path.to_s.chomp(basename), @entries)
+            end
+
+            def basename
+              (directory? && !blank_node?) ? name + '/' : name
+            end
+
+            def name
+              @name || @path.name
+            end
+
+            def children
+              return [] unless directory?
+              return @children if @children
+
+              child_pattern = %r{^#{Regexp.escape(@path.to_s)}[^/]+/?$}
+              @children = select_entries { |path| path =~ child_pattern }
+            end
+
+            def directories(opts = {})
+              return [] unless directory?
+              dirs = children.select(&:directory?)
+              return dirs unless has_parent? && opts[:parent]
+
+              dotted_parent = parent
+              dotted_parent.name = '..'
+              dirs.prepend(dotted_parent)
+            end
+
+            def files
+              return [] unless directory?
+              children.select(&:file?)
+            end
+
+            def metadata
+              @entries[@path.to_s] || {}
+            end
+
+            def nodes
+              @path.nodes + (file? ? 1 : 0)
+            end
+
+            def blank_node?
+              @path.to_s.empty? # "" is considered to be './'
+            end
+
+            def exists?
+              blank_node? || @entries.include?(@path.to_s)
+            end
+
+            def total_size
+              descendant_pattern = %r{^#{Regexp.escape(@path.to_s)}}
+              entries.sum do |path, entry|
+                (entry[:size] if path =~ descendant_pattern).to_i
+              end
+            end
+
+            def path
+              @path.to_s
+            end
+
+            def to_s
+              @path.to_s
+            end
+
+            def ==(other)
+              path == other.path && @entries == other.entries
+            end
+
+            def inspect
+              "#{self.class.name}: #{self.to_s}"
+            end
 
-          unless path.valid_encoding?
-            raise ArgumentError, 'Path contains non-UTF-8 byte sequence!'
+            private
+
+            def select_entries
+              selected = @entries.select { |path, _metadata| yield path }
+              selected.map { |path, _metadata| self.class.new(path, @entries) }
+            end
           end
         end
-
-        delegate :empty?, to: :children
-
-        def directory?
-          blank_node? || @path.end_with?('/')
-        end
-
-        def file?
-          !directory?
-        end
-
-        def blob
-          return unless file?
-
-          @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil)
-        end
-
-        def has_parent?
-          nodes > 0
-        end
-
-        def parent
-          return nil unless has_parent?
-          self.class.new(@path.chomp(basename), @entries)
-        end
-
-        def basename
-          (directory? && !blank_node?) ? name + '/' : name
-        end
-
-        def name
-          @name || @path.split('/').last.to_s
-        end
-
-        def children
-          return [] unless directory?
-          return @children if @children
-
-          child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$}
-          @children = select_entries { |path| path =~ child_pattern }
-        end
-
-        def directories(opts = {})
-          return [] unless directory?
-          dirs = children.select(&:directory?)
-          return dirs unless has_parent? && opts[:parent]
-
-          dotted_parent = parent
-          dotted_parent.name = '..'
-          dirs.prepend(dotted_parent)
-        end
-
-        def files
-          return [] unless directory?
-          children.select(&:file?)
-        end
-
-        def metadata
-          @entries[@path] || {}
-        end
-
-        def nodes
-          @path.count('/') + (file? ? 1 : 0)
-        end
-
-        def blank_node?
-          @path.empty? # "" is considered to be './'
-        end
-
-        def exists?
-          blank_node? || @entries.include?(@path)
-        end
-
-        def total_size
-          descendant_pattern = %r{^#{Regexp.escape(@path)}}
-          entries.sum do |path, entry|
-            (entry[:size] if path =~ descendant_pattern).to_i
-          end
-        end
-
-        def to_s
-          @path
-        end
-
-        def ==(other)
-          @path == other.path && @entries == other.entries
-        end
-
-        def inspect
-          "#{self.class.name}: #{@path}"
-        end
-
-        private
-
-        def select_entries
-          selected = @entries.select { |path, _metadata| yield path }
-          selected.map { |path, _metadata| self.class.new(path, @entries) }
-        end
       end
     end
   end
diff --git a/lib/gitlab/ci/build/artifacts/path.rb b/lib/gitlab/ci/build/artifacts/path.rb
new file mode 100644
index 00000000000..9cd9b36c5f8
--- /dev/null
+++ b/lib/gitlab/ci/build/artifacts/path.rb
@@ -0,0 +1,51 @@
+module Gitlab
+  module Ci
+    module Build
+      module Artifacts
+        class Path
+          def initialize(path)
+            @path = path.dup.force_encoding('UTF-8')
+          end
+
+          def valid?
+            nonzero? && utf8?
+          end
+
+          def directory?
+            @path.end_with?('/')
+          end
+
+          def name
+            @path.split('/').last.to_s
+          end
+
+          def nodes
+            @path.count('/')
+          end
+
+          def to_s
+            @path.tap do |path|
+              unless nonzero?
+                raise ArgumentError, 'Path contains zero byte character!'
+              end
+
+              unless utf8?
+                raise ArgumentError, 'Path contains non-UTF-8 byte sequence!'
+              end
+            end
+          end
+
+          private
+
+          def nonzero?
+            @path.exclude?("\0")
+          end
+
+          def utf8?
+            @path.valid_encoding?
+          end
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From 52e52f4a172ae5aa54eac4a22d98610ec5aea1b0 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Tue, 5 Sep 2017 12:20:09 +0200
Subject: Make it explicit that workhorse needs artifact path

---
 lib/gitlab/workhorse.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index e5ad9b5a40c..7a94af2f8f1 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -121,10 +121,10 @@ module Gitlab
         ]
       end
 
-      def send_artifacts_entry(build, entry)
+      def send_artifacts_entry(build, path)
         params = {
           'Archive' => build.artifacts_file.path,
-          'Entry' => Base64.encode64(entry.path)
+          'Entry' => Base64.encode64(path.to_s)
         }
 
         [
-- 
cgit v1.2.1


From 0085f5e0aef9e2d6379d2cbc03540c34dd43dc5a Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Thu, 31 Aug 2017 12:24:25 +0200
Subject: start improving memory use on I/E

---
 lib/gitlab/import_export/project_tree_restorer.rb | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index cbc8d170936..6fffed68c7e 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -59,8 +59,11 @@ module Gitlab
 
           next unless relation_hash_list
 
-          relation_hash = create_relation(relation_key, relation_hash_list)
-          saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
+          [relation_hash_list].flatten.each_slice(10) do |relation_hash_batch|
+
+            relation_hash = create_relation(relation_key, relation_hash_batch)
+            saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
+          end
         end
         saved.all?
       end
@@ -77,7 +80,7 @@ module Gitlab
         @project.update_columns(project_params)
         @project
       end
-
+git che
       def project_params
         @tree_hash.reject do |key, value|
           # return params that are not 1 to many or 1 to 1 relations
-- 
cgit v1.2.1


From 690b9865f48c41f16dd0c70b37dbcc66fee4c67c Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Fri, 1 Sep 2017 17:20:09 +0200
Subject: some more performance refactoring

---
 lib/gitlab/import_export/project_tree_restorer.rb | 45 +++++++++++++++++------
 1 file changed, 33 insertions(+), 12 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 6fffed68c7e..a288b7a5b56 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -9,6 +9,7 @@ module Gitlab
         @user = user
         @shared = shared
         @project = project
+        @project_id = project.id
       end
 
       def restore
@@ -48,24 +49,42 @@ module Gitlab
       # the configuration yaml file too.
       # Finally, it updates each attribute in the newly imported project.
       def create_relations
-        saved = []
+        @saved = []
         default_relation_list.each do |relation|
           next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present?
+          if relation.is_a?(Hash)
+            create_sub_relations(relation, @tree_hash)
+          else
+            relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
+            relation_hash_list = @tree_hash[relation_key.to_s]
+            save_relation_hash(relation_hash_list, relation_key, saved)
+          end
 
-          create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash)
 
-          relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
-          relation_hash_list = @tree_hash[relation_key.to_s]
+          # relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
+          # relation_hash_list = @tree_hash[relation_key.to_s]
+          #
+          # next unless relation_hash_list
+          #
+          # if relation_hash_list.is_a?(Array)
+          #   [relation_hash_list].flatten.each_slice(15) do |relation_hash_batch|
+          #     save_relation_hash(relation_hash_batch, relation_key, saved)
+          #   end
+          # else
+          #   save_relation_hash(relation_hash_list, relation_key, saved)
+          # end
 
-          next unless relation_hash_list
+        end
+        @saved.all?
+      end
 
-          [relation_hash_list].flatten.each_slice(10) do |relation_hash_batch|
+      def save_relation_hash(relation_hash_batch, relation_key)
+        relation_hash = create_relation(relation_key, relation_hash_batch)
+        @saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
+        @restored_project = nil
+        @project = nil
 
-            relation_hash = create_relation(relation_key, relation_hash_batch)
-            saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
-          end
-        end
-        saved.all?
+        @project = Project.find_by_id(@project_id)
       end
 
       def default_relation_list
@@ -80,7 +99,7 @@ module Gitlab
         @project.update_columns(project_params)
         @project
       end
-git che
+
       def project_params
         @tree_hash.reject do |key, value|
           # return params that are not 1 to many or 1 to 1 relations
@@ -110,6 +129,8 @@ git che
             relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
             relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
           end
+
+          save_relation_hash(relation_item, relation_key)
         end
       end
 
-- 
cgit v1.2.1


From 025a268881d5441c191474759a84c3a28c6d1965 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Sat, 2 Sep 2017 16:17:41 +0200
Subject: more refactoring

---
 lib/gitlab/import_export/project_tree_restorer.rb | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index a288b7a5b56..b53de2a33e1 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -80,6 +80,7 @@ module Gitlab
 
       def save_relation_hash(relation_hash_batch, relation_key)
         relation_hash = create_relation(relation_key, relation_hash_batch)
+
         @saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
         @restored_project = nil
         @project = nil
@@ -115,7 +116,8 @@ module Gitlab
       # issue, finds any subrelations such as notes, creates them and assign them back to the hash
       #
       # Recursively calls this method if the sub-relation is a hash containing more sub-relations
-      def create_sub_relations(relation, tree_hash)
+      def create_sub_relations(relation, tree_hash, save = true)
+        tree_hash = tree_hash.dup
         relation_key = relation.keys.first.to_s
         return if tree_hash[relation_key].blank?
 
@@ -124,13 +126,13 @@ module Gitlab
             # We just use author to get the user ID, do not attempt to create an instance.
             next if sub_relation == :author
 
-            create_sub_relations(sub_relation, relation_item) if sub_relation.is_a?(Hash)
+            create_sub_relations(sub_relation, relation_item, false) if sub_relation.is_a?(Hash)
 
             relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
             relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
           end
 
-          save_relation_hash(relation_item, relation_key)
+          save_relation_hash([relation_item], relation_key) if save
         end
       end
 
-- 
cgit v1.2.1


From a54c50d1469737372398380f819223062bc27f21 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Sun, 3 Sep 2017 12:00:07 +0200
Subject: remove relation after used from the hash tree

---
 lib/gitlab/import_export/project_tree_restorer.rb | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index b53de2a33e1..3c9598887bf 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -57,7 +57,7 @@ module Gitlab
           else
             relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
             relation_hash_list = @tree_hash[relation_key.to_s]
-            save_relation_hash(relation_hash_list, relation_key, saved)
+            save_relation_hash(relation_hash_list, relation_key, @saved)
           end
 
 
@@ -82,10 +82,7 @@ module Gitlab
         relation_hash = create_relation(relation_key, relation_hash_batch)
 
         @saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
-        @restored_project = nil
-        @project = nil
-
-        @project = Project.find_by_id(@project_id)
+        @restored_project = Project.find_by_id(@project_id)
       end
 
       def default_relation_list
@@ -117,7 +114,6 @@ module Gitlab
       #
       # Recursively calls this method if the sub-relation is a hash containing more sub-relations
       def create_sub_relations(relation, tree_hash, save = true)
-        tree_hash = tree_hash.dup
         relation_key = relation.keys.first.to_s
         return if tree_hash[relation_key].blank?
 
@@ -133,6 +129,7 @@ module Gitlab
           end
 
           save_relation_hash([relation_item], relation_key) if save
+          tree_hash.delete(relation_key) if save
         end
       end
 
-- 
cgit v1.2.1


From 4a9dcfdc721ff6dd5926be5f20a75a691a5e85f0 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Sun, 3 Sep 2017 19:16:18 +0200
Subject: enable AR caching, refactor tree hash loop

---
 lib/gitlab/import_export/project_tree_restorer.rb | 39 ++++++++++++++++-------
 1 file changed, 27 insertions(+), 12 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 3c9598887bf..eeb3024c83a 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -23,8 +23,10 @@ module Gitlab
 
         @project_members = @tree_hash.delete('project_members')
 
-        ActiveRecord::Base.no_touching do
-          create_relations
+          ActiveRecord::Base.uncached do
+            ActiveRecord::Base.no_touching do
+            create_relations
+          end
         end
       rescue => e
         @shared.error(e)
@@ -52,6 +54,7 @@ module Gitlab
         @saved = []
         default_relation_list.each do |relation|
           next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present?
+
           if relation.is_a?(Hash)
             create_sub_relations(relation, @tree_hash)
           else
@@ -82,6 +85,10 @@ module Gitlab
         relation_hash = create_relation(relation_key, relation_hash_batch)
 
         @saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
+        @restored_project = nil
+        @project = nil
+        relation_hash = nil
+        relation_hash_batch = nil
         @restored_project = Project.find_by_id(@project_id)
       end
 
@@ -117,20 +124,28 @@ module Gitlab
         relation_key = relation.keys.first.to_s
         return if tree_hash[relation_key].blank?
 
-        [tree_hash[relation_key]].flatten.each do |relation_item|
-          relation.values.flatten.each do |sub_relation|
-            # We just use author to get the user ID, do not attempt to create an instance.
-            next if sub_relation == :author
+        tree_array = [tree_hash[relation_key]].flatten
 
-            create_sub_relations(sub_relation, relation_item, false) if sub_relation.is_a?(Hash)
+        while relation_item = tree_array.shift
+            relation.values.flatten.each do |sub_relation|
+              # We just use author to get the user ID, do not attempt to create an instance.
+              next if sub_relation == :author
 
-            relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
-            relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
-          end
+              create_sub_relations(sub_relation, relation_item, false) if sub_relation.is_a?(Hash)
 
-          save_relation_hash([relation_item], relation_key) if save
-          tree_hash.delete(relation_key) if save
+              relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
+              relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
+            end
+
+            if save
+              save_relation_hash([relation_item], relation_key)
+            end
+
+          tree_hash[relation_key].delete(relation_item) if save
+          relation_item = nil
         end
+
+        tree_hash.delete(relation_key) if save
       end
 
       def assign_relation_hash(relation_item, sub_relation)
-- 
cgit v1.2.1


From d3348474d4b611ee8b99d860c3a0f99b0d4f9b60 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Sun, 3 Sep 2017 20:01:14 +0200
Subject: some more refactoring

---
 lib/gitlab/import_export/project_tree_restorer.rb | 32 +++++++++++------------
 1 file changed, 15 insertions(+), 17 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index eeb3024c83a..0d26f21d766 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -60,7 +60,7 @@ module Gitlab
           else
             relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
             relation_hash_list = @tree_hash[relation_key.to_s]
-            save_relation_hash(relation_hash_list, relation_key, @saved)
+            save_relation_hash(relation_hash_list, relation_key)
           end
 
 
@@ -85,10 +85,6 @@ module Gitlab
         relation_hash = create_relation(relation_key, relation_hash_batch)
 
         @saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
-        @restored_project = nil
-        @project = nil
-        relation_hash = nil
-        relation_hash_batch = nil
         @restored_project = Project.find_by_id(@project_id)
       end
 
@@ -127,27 +123,29 @@ module Gitlab
         tree_array = [tree_hash[relation_key]].flatten
 
         while relation_item = tree_array.shift
-            relation.values.flatten.each do |sub_relation|
-              # We just use author to get the user ID, do not attempt to create an instance.
-              next if sub_relation == :author
-
-              create_sub_relations(sub_relation, relation_item, false) if sub_relation.is_a?(Hash)
-
-              relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
-              relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
-            end
+            process_sub_relation(relation, relation_item)
 
             if save
               save_relation_hash([relation_item], relation_key)
+              tree_hash[relation_key].delete(relation_item)
             end
-
-          tree_hash[relation_key].delete(relation_item) if save
-          relation_item = nil
         end
 
         tree_hash.delete(relation_key) if save
       end
 
+      def process_sub_relation(relation, relation_item)
+        relation.values.flatten.each do |sub_relation|
+          # We just use author to get the user ID, do not attempt to create an instance.
+          next if sub_relation == :author
+
+          create_sub_relations(sub_relation, relation_item, false) if sub_relation.is_a?(Hash)
+
+          relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
+          relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
+        end
+      end
+
       def assign_relation_hash(relation_item, sub_relation)
         if sub_relation.is_a?(Hash)
           relation_hash = relation_item[sub_relation.keys.first.to_s]
-- 
cgit v1.2.1


From 8ff9fc55cf876a61fcfde421b71d204d9fd33723 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Sun, 3 Sep 2017 20:51:50 +0200
Subject: some more refactoring

---
 lib/gitlab/import_export/project_tree_restorer.rb | 21 ++++-----------------
 lib/gitlab/import_export/shared.rb                |  2 +-
 2 files changed, 5 insertions(+), 18 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 0d26f21d766..82ca24fd2f8 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -23,8 +23,8 @@ module Gitlab
 
         @project_members = @tree_hash.delete('project_members')
 
-          ActiveRecord::Base.uncached do
-            ActiveRecord::Base.no_touching do
+        ActiveRecord::Base.uncached do
+          ActiveRecord::Base.no_touching do
             create_relations
           end
         end
@@ -62,21 +62,6 @@ module Gitlab
             relation_hash_list = @tree_hash[relation_key.to_s]
             save_relation_hash(relation_hash_list, relation_key)
           end
-
-
-          # relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
-          # relation_hash_list = @tree_hash[relation_key.to_s]
-          #
-          # next unless relation_hash_list
-          #
-          # if relation_hash_list.is_a?(Array)
-          #   [relation_hash_list].flatten.each_slice(15) do |relation_hash_batch|
-          #     save_relation_hash(relation_hash_batch, relation_key, saved)
-          #   end
-          # else
-          #   save_relation_hash(relation_hash_list, relation_key, saved)
-          # end
-
         end
         @saved.all?
       end
@@ -123,12 +108,14 @@ module Gitlab
         tree_array = [tree_hash[relation_key]].flatten
 
         while relation_item = tree_array.shift
+          Project.transaction do
             process_sub_relation(relation, relation_item)
 
             if save
               save_relation_hash([relation_item], relation_key)
               tree_hash[relation_key].delete(relation_item)
             end
+          end
         end
 
         tree_hash.delete(relation_key) if save
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
index 5d6de8bc475..9fd0b709ef2 100644
--- a/lib/gitlab/import_export/shared.rb
+++ b/lib/gitlab/import_export/shared.rb
@@ -16,7 +16,7 @@ module Gitlab
         error_out(error.message, caller[0].dup)
         @errors << error.message
         # Debug:
-        Rails.logger.error(error.backtrace)
+        Rails.logger.error(error.backtrace.join("\n"))
       end
 
       private
-- 
cgit v1.2.1


From be99f82478e023e41a543a0b9348e27d56fb3c13 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Tue, 5 Sep 2017 11:02:34 +0200
Subject: started refactoring reader

---
 lib/gitlab/import_export/reader.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb
index eb7f5120592..4f061959047 100644
--- a/lib/gitlab/import_export/reader.rb
+++ b/lib/gitlab/import_export/reader.rb
@@ -18,7 +18,9 @@ module Gitlab
         attributes = @attributes_finder.find(:project)
         project_attributes = attributes.is_a?(Hash) ? attributes[:project] : {}
 
-        project_attributes.merge(include: build_hash(@tree))
+        build_hash(@tree).each do |sub_hash|
+          yield(project_attributes.merge(include: sub_hash))
+        end
       rescue => e
         @shared.error(e)
         false
-- 
cgit v1.2.1


From 4119206f76d802413850dc9fe1fa4715c3bd6fc0 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Tue, 5 Sep 2017 17:10:57 +0200
Subject: fix export performance of CI builds

---
 lib/gitlab/import_export/import_export.yml | 1 +
 lib/gitlab/import_export/reader.rb         | 4 +---
 2 files changed, 2 insertions(+), 3 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 78795dd3d92..ec73846d844 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -116,6 +116,7 @@ excluded_attributes:
   statuses:
     - :trace
     - :token
+    - :when
   push_event_payload:
     - :event_id
 
diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb
index 4f061959047..eb7f5120592 100644
--- a/lib/gitlab/import_export/reader.rb
+++ b/lib/gitlab/import_export/reader.rb
@@ -18,9 +18,7 @@ module Gitlab
         attributes = @attributes_finder.find(:project)
         project_attributes = attributes.is_a?(Hash) ? attributes[:project] : {}
 
-        build_hash(@tree).each do |sub_hash|
-          yield(project_attributes.merge(include: sub_hash))
-        end
+        project_attributes.merge(include: build_hash(@tree))
       rescue => e
         @shared.error(e)
         false
-- 
cgit v1.2.1


From d8d05e5f284a3e3324ee51149ff278ba1dfbd43c Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Tue, 5 Sep 2017 17:24:57 +0200
Subject: refactor code a little

---
 lib/gitlab/import_export/project_tree_restorer.rb | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 82ca24fd2f8..7f24eae7ef8 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -10,6 +10,7 @@ module Gitlab
         @shared = shared
         @project = project
         @project_id = project.id
+        @saved = []
       end
 
       def restore
@@ -51,7 +52,6 @@ module Gitlab
       # the configuration yaml file too.
       # Finally, it updates each attribute in the newly imported project.
       def create_relations
-        @saved = []
         default_relation_list.each do |relation|
           next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present?
 
@@ -59,10 +59,10 @@ module Gitlab
             create_sub_relations(relation, @tree_hash)
           else
             relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
-            relation_hash_list = @tree_hash[relation_key.to_s]
-            save_relation_hash(relation_hash_list, relation_key)
+            save_relation_hash(@tree_hash[relation_key.to_s], relation_key)
           end
         end
+
         @saved.all?
       end
 
@@ -70,6 +70,8 @@ module Gitlab
         relation_hash = create_relation(relation_key, relation_hash_batch)
 
         @saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
+
+        # Restore the project again, extra query but let us skip holding the AR objects in memory
         @restored_project = Project.find_by_id(@project_id)
       end
 
@@ -107,10 +109,17 @@ module Gitlab
 
         tree_array = [tree_hash[relation_key]].flatten
 
+        # Avoid keeping a possible heavy object in memory once we are done with it
         while relation_item = tree_array.shift
+          # The transaction at this level is less speedy than one single transaction
+          # But we can't have it in the upper level or GC won't get rid of the AR objects
+          # after we save the batch.
           Project.transaction do
             process_sub_relation(relation, relation_item)
 
+            # For every subrelation that hangs from Project, save the associated records alltogether
+            # This effectively batches all records per subrelation item, only keeping those in memory
+            # We have to keep in mind that more batch granularity << Memory, but >> Slowness
             if save
               save_relation_hash([relation_item], relation_key)
               tree_hash[relation_key].delete(relation_item)
-- 
cgit v1.2.1


From 104f221b601621cdfdd083f82780188c84190a1c Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Tue, 5 Sep 2017 21:06:27 +0200
Subject: fix specs

---
 lib/gitlab/import_export/project_tree_restorer.rb | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 7f24eae7ef8..d365e9ad40b 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -72,7 +72,7 @@ module Gitlab
         @saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
 
         # Restore the project again, extra query but let us skip holding the AR objects in memory
-        @restored_project = Project.find_by_id(@project_id)
+        @restored_project = Project.find(@project_id)
       end
 
       def default_relation_list
@@ -153,14 +153,12 @@ module Gitlab
       end
 
       def create_relation(relation, relation_hash_list)
-        relation_type = relation.to_sym
-
         relation_array = [relation_hash_list].flatten.map do |relation_hash|
-          Gitlab::ImportExport::RelationFactory.create(relation_sym: relation_type,
-                                                       relation_hash: parsed_relation_hash(relation_hash, relation_type),
+          Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
+                                                       relation_hash: parsed_relation_hash(relation_hash, relation.to_sym),
                                                        members_mapper: members_mapper,
                                                        user: @user,
-                                                       project: restored_project)
+                                                       project: @restored_project)
         end.compact
 
         relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
-- 
cgit v1.2.1


From 7e38cc9cc215682fc20a7c036dee763df4409198 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Wed, 6 Sep 2017 08:56:38 +0200
Subject: small refactor

---
 lib/gitlab/import_export/project_tree_restorer.rb | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index d365e9ad40b..f8775729bef 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -10,7 +10,7 @@ module Gitlab
         @shared = shared
         @project = project
         @project_id = project.id
-        @saved = []
+        @saved = true
       end
 
       def restore
@@ -63,16 +63,16 @@ module Gitlab
           end
         end
 
-        @saved.all?
+        @saved
       end
 
       def save_relation_hash(relation_hash_batch, relation_key)
         relation_hash = create_relation(relation_key, relation_hash_batch)
 
-        @saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
+        @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash)
 
-        # Restore the project again, extra query but let us skip holding the AR objects in memory
-        @restored_project = Project.find(@project_id)
+        # Restore the project again, extra query that skips holding the AR objects in memory
+        @restored_project.reload
       end
 
       def default_relation_list
-- 
cgit v1.2.1


From 89ca01eb1a2eac5472f3957bb425700dcac82a7a Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Wed, 6 Sep 2017 10:09:24 +0200
Subject: fix spec failures

---
 lib/gitlab/import_export/project_tree_restorer.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index f8775729bef..4c17f13b851 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -72,7 +72,7 @@ module Gitlab
         @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash)
 
         # Restore the project again, extra query that skips holding the AR objects in memory
-        @restored_project.reload
+        @restored_project = Project.find(@project_id)
       end
 
       def default_relation_list
-- 
cgit v1.2.1


From e5f91e524628faab81c1d61ebcaa00eaf0954ff6 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Wed, 6 Sep 2017 10:16:11 +0200
Subject: refactor code based on feedback

---
 lib/gitlab/import_export/project_tree_restorer.rb | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 4c17f13b851..dec8a4c8cab 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -53,11 +53,9 @@ module Gitlab
       # Finally, it updates each attribute in the newly imported project.
       def create_relations
         default_relation_list.each do |relation|
-          next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present?
-
           if relation.is_a?(Hash)
             create_sub_relations(relation, @tree_hash)
-          else
+          elsif @tree_hash[relation.to_s].present?
             relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
             save_relation_hash(@tree_hash[relation_key.to_s], relation_key)
           end
-- 
cgit v1.2.1


From 9e1f8ac2a3edb32f672ef2ad5424e99425f67c92 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Wed, 6 Sep 2017 11:11:02 +0200
Subject: refactored code

---
 lib/gitlab/import_export/project_tree_restorer.rb | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index dec8a4c8cab..3bc095a99a9 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -56,8 +56,7 @@ module Gitlab
           if relation.is_a?(Hash)
             create_sub_relations(relation, @tree_hash)
           elsif @tree_hash[relation.to_s].present?
-            relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
-            save_relation_hash(@tree_hash[relation_key.to_s], relation_key)
+            save_relation_hash(@tree_hash[relation.to_s], relation)
           end
         end
 
@@ -101,7 +100,7 @@ module Gitlab
       # issue, finds any subrelations such as notes, creates them and assign them back to the hash
       #
       # Recursively calls this method if the sub-relation is a hash containing more sub-relations
-      def create_sub_relations(relation, tree_hash, save = true)
+      def create_sub_relations(relation, tree_hash, save: true)
         relation_key = relation.keys.first.to_s
         return if tree_hash[relation_key].blank?
 
@@ -133,7 +132,7 @@ module Gitlab
           # We just use author to get the user ID, do not attempt to create an instance.
           next if sub_relation == :author
 
-          create_sub_relations(sub_relation, relation_item, false) if sub_relation.is_a?(Hash)
+          create_sub_relations(sub_relation, relation_item, save: false) if sub_relation.is_a?(Hash)
 
           relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
           relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
-- 
cgit v1.2.1


From b03d9e506d88cbfa13d34e12e0595dbb92f46e7d Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Wed, 6 Sep 2017 11:42:14 +0200
Subject: Remove unneeded string interpolation from entry class

---
 lib/gitlab/ci/build/artifacts/metadata/entry.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
index 321ddfd8c29..22941d48edf 100644
--- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
@@ -113,7 +113,7 @@ module Gitlab
             end
 
             def inspect
-              "#{self.class.name}: #{self.to_s}"
+              "#{self.class.name}: #{self}"
             end
 
             private
-- 
cgit v1.2.1


From 966b1128d884a318dad4277e23368334fe67e836 Mon Sep 17 00:00:00 2001
From: "micael.bergeron" <micael.bergeron@solutionstlm.com>
Date: Sat, 29 Jul 2017 11:04:42 -0400
Subject: WIP: refactor the first-contributor to Issuable

this will remove the need make N queries (per-note) at the
cost of having to mark notes with an attribute

this opens up the possibility for other special roles for notes
---
 lib/gitlab/access.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb
index 4714ab18cc1..b4012ebbb99 100644
--- a/lib/gitlab/access.rb
+++ b/lib/gitlab/access.rb
@@ -67,10 +67,14 @@ module Gitlab
       def protection_values
         protection_options.values
       end
+
+      def human_access(access)
+        options_with_owner.key(access)
+      end
     end
 
     def human_access
-      Gitlab::Access.options_with_owner.key(access_field)
+      Gitlab::Access.human_access(access_field)
     end
 
     def owner?
-- 
cgit v1.2.1


From bca72f5906ed38dc231ef066231238758c1cb42d Mon Sep 17 00:00:00 2001
From: "micael.bergeron" <micael.bergeron@solutionstlm.com>
Date: Sun, 3 Sep 2017 07:45:44 -0400
Subject: wip: fake its a binary diff

---
 lib/gitlab/encoding_helper.rb | 16 ++++++++++++----
 lib/gitlab/git/diff.rb        | 16 +++++++++++++++-
 2 files changed, 27 insertions(+), 5 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
index 8ddc91e341d..c5e173ba55a 100644
--- a/lib/gitlab/encoding_helper.rb
+++ b/lib/gitlab/encoding_helper.rb
@@ -13,6 +13,8 @@ module Gitlab
     # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
     ENCODING_CONFIDENCE_THRESHOLD = 50
 
+    #
+    # 
     def encode!(message)
       return nil unless message.respond_to? :force_encoding
 
@@ -22,20 +24,26 @@ module Gitlab
 
       # return message if message type is binary
       detect = CharlockHolmes::EncodingDetector.detect(message)
-      return message.force_encoding("BINARY") if detect && detect[:type] == :binary
+      return message.force_encoding("BINARY") if binary?(message, detect)
 
-      # force detected encoding if we have sufficient confidence.
       if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
+        # force detected encoding if we have sufficient confidence.
         message.force_encoding(detect[:encoding])
       end
 
       # encode and clean the bad chars
       message.replace clean(message)
-    rescue
+    rescue => e
+      byebug
       encoding = detect ? detect[:encoding] : "unknown"
       "--broken encoding: #{encoding}"
     end
 
+    def binary?(message, detect=nil)
+      detect ||= CharlockHolmes::EncodingDetector.detect(message)
+      detect && detect[:type] == :binary && detect[:confidence] == 100
+    end
+
     def encode_utf8(message)
       detect = CharlockHolmes::EncodingDetector.detect(message)
       if detect && detect[:encoding]
@@ -50,7 +58,7 @@ module Gitlab
         clean(message)
       end
     end
-
+      
     private
 
     def clean(message)
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index ce3d65062e8..e544f255a5d 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -116,6 +116,13 @@ module Gitlab
 
           filtered_opts
         end
+
+        # Return a binary diff message like:
+        # 
+        # "Binary files a/file/path and b/file/path differ\n"
+        def binary_message(old_path, new_path)
+          "Binary files #{old_path} and #{new_path} differ\n"
+        end
       end
 
       def initialize(raw_diff, expanded: true)
@@ -214,7 +221,14 @@ module Gitlab
         # binary we're not going to display anything so we skip the size check.
         return if !patch.delta.binary? && prune_large_patch(patch)
 
-        @diff = encode!(strip_diff_headers(patch.to_s))
+        diff = strip_diff_headers(patch.to_s)
+        @diff = if binary?(diff)
+                  # the diff is binary, let's make a message for it
+                  Diff::binary_message(patch.delta.old_file[:path],
+                                       patch.delta.new_file[:path])
+                else
+                  encode!(diff)
+                end
       end
 
       def init_from_hash(hash)
-- 
cgit v1.2.1


From c9aa7932152bd10f6fc7b87b3263922aa4b911b7 Mon Sep 17 00:00:00 2001
From: "micael.bergeron" <micael.bergeron@solutionstlm.com>
Date: Mon, 4 Sep 2017 13:34:15 -0400
Subject: revert to using a simple representation

---
 lib/gitlab/encoding_helper.rb | 17 ++++++++++++-----
 lib/gitlab/git/blob.rb        | 12 ++++--------
 lib/gitlab/git/diff.rb        | 16 ++++++++--------
 3 files changed, 24 insertions(+), 21 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
index c5e173ba55a..21a14141c87 100644
--- a/lib/gitlab/encoding_helper.rb
+++ b/lib/gitlab/encoding_helper.rb
@@ -24,7 +24,7 @@ module Gitlab
 
       # return message if message type is binary
       detect = CharlockHolmes::EncodingDetector.detect(message)
-      return message.force_encoding("BINARY") if binary?(message, detect)
+      return message.force_encoding("BINARY") if all_binary?(message, detect)
 
       if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
         # force detected encoding if we have sufficient confidence.
@@ -34,14 +34,21 @@ module Gitlab
       # encode and clean the bad chars
       message.replace clean(message)
     rescue => e
-      byebug
       encoding = detect ? detect[:encoding] : "unknown"
       "--broken encoding: #{encoding}"
     end
 
-    def binary?(message, detect=nil)
-      detect ||= CharlockHolmes::EncodingDetector.detect(message)
-      detect && detect[:type] == :binary && detect[:confidence] == 100
+    def all_binary?(data, detect=nil)
+      detect ||= CharlockHolmes::EncodingDetector.detect(data)
+      detect && detect[:type] == :binary
+    end
+
+    def libgit2_binary?(data)
+      # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
+      # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
+      # which is what we use below to keep a consistent behavior.
+      detect = CharlockHolmes::EncodingDetector.new(8000).detect(data)
+      all_binary?(data, detect)
     end
 
     def encode_utf8(message)
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index 7780f4e4d4f..2e6edb8be0d 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -42,14 +42,6 @@ module Gitlab
           end
         end
 
-        def binary?(data)
-          # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
-          # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
-          # which is what we use below to keep a consistent behavior.
-          detect = CharlockHolmes::EncodingDetector.new(8000).detect(data)
-          detect && detect[:type] == :binary
-        end
-
         # Returns an array of Blob instances, specified in blob_references as
         # [[commit_sha, path], [commit_sha, path], ...]. If blob_size_limit < 0 then the
         # full blob contents are returned. If blob_size_limit >= 0 then each blob will
@@ -169,6 +161,10 @@ module Gitlab
             end
           end
         end
+
+        def binary?(data)
+          EncodingHelper.libgit2_binary?(data)
+        end
       end
 
       def initialize(options)
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index e544f255a5d..17defe55af0 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -197,6 +197,13 @@ module Gitlab
         @collapsed = true
       end
 
+      def json_safe_diff
+        return @diff unless all_binary?(@diff)
+
+        # the diff is binary, let's make a message for it
+        Diff::binary_message(@old_path, @new_path)
+      end
+
       private
 
       def init_from_rugged(rugged)
@@ -221,14 +228,7 @@ module Gitlab
         # binary we're not going to display anything so we skip the size check.
         return if !patch.delta.binary? && prune_large_patch(patch)
 
-        diff = strip_diff_headers(patch.to_s)
-        @diff = if binary?(diff)
-                  # the diff is binary, let's make a message for it
-                  Diff::binary_message(patch.delta.old_file[:path],
-                                       patch.delta.new_file[:path])
-                else
-                  encode!(diff)
-                end
+        @diff = encode!(strip_diff_headers(patch.to_s))
       end
 
       def init_from_hash(hash)
-- 
cgit v1.2.1


From dbaed90c8d3dffb2dd970f1621f551dd322db5ee Mon Sep 17 00:00:00 2001
From: "micael.bergeron" <micael.bergeron@solutionstlm.com>
Date: Mon, 4 Sep 2017 15:32:57 -0400
Subject: fix refactoring error with Blob.binary?

remove some lint
---
 lib/gitlab/encoding_helper.rb | 8 ++++----
 lib/gitlab/git/blob.rb        | 8 ++++----
 lib/gitlab/git/diff.rb        | 4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
index 21a14141c87..8ac756a0b6a 100644
--- a/lib/gitlab/encoding_helper.rb
+++ b/lib/gitlab/encoding_helper.rb
@@ -14,7 +14,7 @@ module Gitlab
     ENCODING_CONFIDENCE_THRESHOLD = 50
 
     #
-    # 
+    #
     def encode!(message)
       return nil unless message.respond_to? :force_encoding
 
@@ -33,12 +33,12 @@ module Gitlab
 
       # encode and clean the bad chars
       message.replace clean(message)
-    rescue => e
+    rescue
       encoding = detect ? detect[:encoding] : "unknown"
       "--broken encoding: #{encoding}"
     end
 
-    def all_binary?(data, detect=nil)
+    def all_binary?(data, detect = nil)
       detect ||= CharlockHolmes::EncodingDetector.detect(data)
       detect && detect[:type] == :binary
     end
@@ -65,7 +65,7 @@ module Gitlab
         clean(message)
       end
     end
-      
+
     private
 
     def clean(message)
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index 2e6edb8be0d..e5391b8bf8a 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -57,6 +57,10 @@ module Gitlab
           end
         end
 
+        def binary?(data)
+          EncodingHelper.libgit2_binary?(data)
+        end
+
         private
 
         # Recursive search of blob id by path
@@ -161,10 +165,6 @@ module Gitlab
             end
           end
         end
-
-        def binary?(data)
-          EncodingHelper.libgit2_binary?(data)
-        end
       end
 
       def initialize(options)
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index 17defe55af0..b3237d5496a 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -118,7 +118,7 @@ module Gitlab
         end
 
         # Return a binary diff message like:
-        # 
+        #
         # "Binary files a/file/path and b/file/path differ\n"
         def binary_message(old_path, new_path)
           "Binary files #{old_path} and #{new_path} differ\n"
@@ -201,7 +201,7 @@ module Gitlab
         return @diff unless all_binary?(@diff)
 
         # the diff is binary, let's make a message for it
-        Diff::binary_message(@old_path, @new_path)
+        Diff.binary_message(@old_path, @new_path)
       end
 
       private
-- 
cgit v1.2.1


From a6af5522d7cefaa7d08d28448b5059be07328a2f Mon Sep 17 00:00:00 2001
From: "micael.bergeron" <micael.bergeron@solutionstlm.com>
Date: Tue, 5 Sep 2017 13:16:08 -0400
Subject: renames ambiguous methods and add spec

---
 lib/gitlab/encoding_helper.rb | 10 +++++-----
 lib/gitlab/git/blob.rb        |  2 +-
 lib/gitlab/git/diff.rb        |  4 +++-
 3 files changed, 9 insertions(+), 7 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
index 8ac756a0b6a..c50417f4d4f 100644
--- a/lib/gitlab/encoding_helper.rb
+++ b/lib/gitlab/encoding_helper.rb
@@ -24,7 +24,7 @@ module Gitlab
 
       # return message if message type is binary
       detect = CharlockHolmes::EncodingDetector.detect(message)
-      return message.force_encoding("BINARY") if all_binary?(message, detect)
+      return message.force_encoding("BINARY") if detect_binary?(message, detect)
 
       if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
         # force detected encoding if we have sufficient confidence.
@@ -38,17 +38,17 @@ module Gitlab
       "--broken encoding: #{encoding}"
     end
 
-    def all_binary?(data, detect = nil)
+    def detect_binary?(data, detect = nil)
       detect ||= CharlockHolmes::EncodingDetector.detect(data)
-      detect && detect[:type] == :binary
+      detect && detect[:type] == :binary && detect[:confidence] == 100
     end
 
-    def libgit2_binary?(data)
+    def detect_libgit2_binary?(data)
       # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
       # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
       # which is what we use below to keep a consistent behavior.
       detect = CharlockHolmes::EncodingDetector.new(8000).detect(data)
-      all_binary?(data, detect)
+      detect && detect[:type] == :binary
     end
 
     def encode_utf8(message)
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index e5391b8bf8a..8d96826f6ee 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -58,7 +58,7 @@ module Gitlab
         end
 
         def binary?(data)
-          EncodingHelper.libgit2_binary?(data)
+          EncodingHelper.detect_libgit2_binary?(data)
         end
 
         private
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index b3237d5496a..a23c8cf0dd1 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -120,6 +120,8 @@ module Gitlab
         # Return a binary diff message like:
         #
         # "Binary files a/file/path and b/file/path differ\n"
+        # This is used when we detect that a diff is binary
+        # using CharlockHolmes when Rugged treats it as text.
         def binary_message(old_path, new_path)
           "Binary files #{old_path} and #{new_path} differ\n"
         end
@@ -198,7 +200,7 @@ module Gitlab
       end
 
       def json_safe_diff
-        return @diff unless all_binary?(@diff)
+        return @diff unless detect_binary?(@diff)
 
         # the diff is binary, let's make a message for it
         Diff.binary_message(@old_path, @new_path)
-- 
cgit v1.2.1


From c1f66cc7f0658621f2bfcb09f0269da675f57e84 Mon Sep 17 00:00:00 2001
From: "micael.bergeron" <micael.bergeron@solutionstlm.com>
Date: Wed, 6 Sep 2017 09:15:27 -0400
Subject: remove useless comment

---
 lib/gitlab/encoding_helper.rb | 2 --
 1 file changed, 2 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
index c50417f4d4f..7b3483a7f96 100644
--- a/lib/gitlab/encoding_helper.rb
+++ b/lib/gitlab/encoding_helper.rb
@@ -13,8 +13,6 @@ module Gitlab
     # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
     ENCODING_CONFIDENCE_THRESHOLD = 50
 
-    #
-    #
     def encode!(message)
       return nil unless message.respond_to? :force_encoding
 
-- 
cgit v1.2.1


From 4df54f260751a832ebf0b8c18524020d6604994b Mon Sep 17 00:00:00 2001
From: Alexander Keramidas <dev.alexkeramidas@gmail.com>
Date: Tue, 29 Aug 2017 11:57:41 +0300
Subject: Profile updates from providers

---
 lib/gitlab/ldap/user.rb        |  4 ++--
 lib/gitlab/o_auth/auth_hash.rb | 17 +++++++++++++++--
 lib/gitlab/o_auth/user.rb      | 32 +++++++++++++++++++++-----------
 lib/gitlab/saml/user.rb        |  2 +-
 4 files changed, 39 insertions(+), 16 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb
index 39180dc17d9..3bf27b37ae6 100644
--- a/lib/gitlab/ldap/user.rb
+++ b/lib/gitlab/ldap/user.rb
@@ -36,7 +36,7 @@ module Gitlab
       end
 
       def find_by_email
-        ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_email?
+        ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_attribute?(:email)
       end
 
       def update_user_attributes
@@ -60,7 +60,7 @@ module Gitlab
         ldap_config.block_auto_created_users
       end
 
-      def sync_email_from_provider?
+      def sync_profile_from_provider?
         true
       end
 
diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb
index 7d6911a1ab3..1f331b1e91d 100644
--- a/lib/gitlab/o_auth/auth_hash.rb
+++ b/lib/gitlab/o_auth/auth_hash.rb
@@ -32,8 +32,21 @@ module Gitlab
         @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase)
       end
 
-      def has_email?
-        get_info(:email).present?
+      def location
+        location = get_info(:address)
+        if location.is_a?(Hash)
+          [location.locality.presence, location.country.presence].compact.join(', ')
+        else
+          location
+        end
+      end
+
+      def has_attribute?(attribute)
+        if attribute == :location
+          get_info(:address).present?
+        else
+          get_info(attribute).present?
+        end
       end
 
       private
diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb
index e8330917e91..7704bf715e4 100644
--- a/lib/gitlab/o_auth/user.rb
+++ b/lib/gitlab/o_auth/user.rb
@@ -12,7 +12,7 @@ module Gitlab
 
       def initialize(auth_hash)
         self.auth_hash = auth_hash
-        update_email
+        update_profile if sync_profile_from_provider?
       end
 
       def persisted?
@@ -184,20 +184,30 @@ module Gitlab
         }
       end
 
-      def sync_email_from_provider?
-        auth_hash.provider.to_s == Gitlab.config.omniauth.sync_email_from_provider.to_s
+      def sync_profile_from_provider?
+        providers = Gitlab.config.omniauth.sync_profile_from_provider
+
+        if providers.is_a?(Array)
+          providers.include?(auth_hash.provider)
+        else
+          providers
+        end
       end
 
-      def update_email
-        if auth_hash.has_email? && sync_email_from_provider?
-          if persisted?
-            gl_user.skip_reconfirmation!
-            gl_user.email = auth_hash.email
-          end
+      def update_profile
+        user_synced_attributes_metadata = gl_user.user_synced_attributes_metadata || gl_user.build_user_synced_attributes_metadata
 
-          gl_user.external_email = true
-          gl_user.email_provider = auth_hash.provider
+        UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key|
+          if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key)
+            gl_user[key] = auth_hash.public_send(key) # rubocop:disable GitlabSecurity/PublicSend
+            user_synced_attributes_metadata.set_attribute_synced(key, true)
+          else
+            user_synced_attributes_metadata.set_attribute_synced(key, false)
+          end
         end
+
+        user_synced_attributes_metadata.provider = auth_hash.provider
+        gl_user.user_synced_attributes_metadata = user_synced_attributes_metadata
       end
 
       def log
diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb
index 8a7cc690046..0f323a9e8b2 100644
--- a/lib/gitlab/saml/user.rb
+++ b/lib/gitlab/saml/user.rb
@@ -40,7 +40,7 @@ module Gitlab
       end
 
       def find_by_email
-        if auth_hash.has_email?
+        if auth_hash.has_attribute?(:email)
           user = ::User.find_by(email: auth_hash.email.downcase)
           user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) if user
           user
-- 
cgit v1.2.1


From 759f34bd0a250cb2cdf1b718837b56bb28fa1939 Mon Sep 17 00:00:00 2001
From: Nick Thomas <nick@gitlab.com>
Date: Wed, 6 Sep 2017 12:29:14 +0100
Subject: Backport EE fixes to Gitlab::UrlSanitizer to CE

---
 lib/gitlab/url_sanitizer.rb | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb
index c81dc7e30d0..9c26490f40f 100644
--- a/lib/gitlab/url_sanitizer.rb
+++ b/lib/gitlab/url_sanitizer.rb
@@ -9,7 +9,7 @@ module Gitlab
     end
 
     def self.valid?(url)
-      return false unless url
+      return false unless url.present?
 
       Addressable::URI.parse(url.strip)
 
@@ -29,13 +29,13 @@ module Gitlab
 
     def masked_url
       url = @url.dup
-      url.password = "*****" unless url.password.nil?
-      url.user = "*****" unless url.user.nil?
+      url.password = "*****" if url.password.present?
+      url.user = "*****" if url.user.present?
       url.to_s
     end
 
     def credentials
-      @credentials ||= { user: @url.user, password: @url.password }
+      @credentials ||= { user: @url.user.presence, password: @url.password.presence }
     end
 
     def full_url
@@ -47,8 +47,8 @@ module Gitlab
     def generate_full_url
       return @url unless valid_credentials?
       @full_url = @url.dup
-      @full_url.user = credentials[:user]
-      @full_url.password = credentials[:password]
+      @full_url.user = credentials[:user].presence
+      @full_url.password = credentials[:password].presence
       @full_url
     end
 
-- 
cgit v1.2.1


From c8bdb20228b34130c7f0525ad92140702dce1e20 Mon Sep 17 00:00:00 2001
From: Nick Thomas <nick@gitlab.com>
Date: Wed, 6 Sep 2017 15:20:25 +0100
Subject: Remove blank passwords from sanitized URLs

---
 lib/gitlab/url_sanitizer.rb | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb
index 9c26490f40f..703adae12cb 100644
--- a/lib/gitlab/url_sanitizer.rb
+++ b/lib/gitlab/url_sanitizer.rb
@@ -19,7 +19,12 @@ module Gitlab
     end
 
     def initialize(url, credentials: nil)
-      @url = Addressable::URI.parse(url.strip)
+      @url = Addressable::URI.parse(url.to_s.strip)
+
+      %i[user password].each do |symbol|
+        credentials[symbol] = credentials[symbol].presence if credentials&.key?(symbol)
+      end
+
       @credentials = credentials
     end
 
@@ -47,8 +52,10 @@ module Gitlab
     def generate_full_url
       return @url unless valid_credentials?
       @full_url = @url.dup
-      @full_url.user = credentials[:user].presence
-      @full_url.password = credentials[:password].presence
+
+      @full_url.password = credentials[:password]
+      @full_url.user = credentials[:user]
+
       @full_url
     end
 
-- 
cgit v1.2.1


From 235b105c917c16ab14a79ba13280aff4fd9f1cf9 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Wed, 30 Aug 2017 15:38:16 +0200
Subject: Finish migration to the new events setup

This finishes the procedure for migrating events from the old format
into the new format. Code no longer uses the old setup and the database
tables used during the migration process are swapped, with the old table
being dropped.

While the database migration can be reversed this will 1) take a lot of
time as data has to be coped around 2) won't restore data in the
"events.data" column as we have no way of restoring this.

Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/37241
---
 lib/gitlab/import_export.rb                  | 2 +-
 lib/gitlab/import_export/relation_factory.rb | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index 3470a09eaf0..50ee879129c 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -3,7 +3,7 @@ module Gitlab
     extend self
 
     # For every version update, the version history in import_export.md has to be kept up to date.
-    VERSION = '0.1.8'.freeze
+    VERSION = '0.2.0'.freeze
     FILENAME_LIMIT = 50
 
     def export_path(relative_path:)
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 20580459046..d563a87dcfd 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -69,7 +69,6 @@ module Gitlab
         reset_tokens!
         remove_encrypted_attributes!
 
-        @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data']
         set_st_diff_commits if @relation_name == :merge_request_diff
         set_diff if @relation_name == :merge_request_diff_files
       end
-- 
cgit v1.2.1


From 41ef94e777d9c9d10a8b64b1498f57a8e5847e23 Mon Sep 17 00:00:00 2001
From: Ahmad Sherif <me@ahmadsherif.com>
Date: Mon, 28 Aug 2017 09:31:41 +0200
Subject: Migrate creating/deleting a branch to Gitaly

---
 lib/gitlab/git/repository.rb            | 37 +++++++++++++++++++++---------
 lib/gitlab/gitaly_client/ref_service.rb | 40 ++++++++++++++++++++++++++++++++-
 lib/gitlab/github_import/importer.rb    |  2 +-
 3 files changed, 66 insertions(+), 13 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 75d4efc0bc5..efa13590a2c 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -18,6 +18,7 @@ module Gitlab
       InvalidBlobName = Class.new(StandardError)
       InvalidRef = Class.new(StandardError)
       GitError = Class.new(StandardError)
+      DeleteBranchError = Class.new(StandardError)
 
       class << self
         # Unlike `new`, `create` takes the storage path, not the storage name
@@ -653,10 +654,16 @@ module Gitlab
       end
 
       # Delete the specified branch from the repository
-      #
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476
       def delete_branch(branch_name)
-        rugged.branches.delete(branch_name)
+        gitaly_migrate(:delete_branch) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.delete_branch(branch_name)
+          else
+            rugged.branches.delete(branch_name)
+          end
+        end
+      rescue Rugged::ReferenceError, CommandError => e
+        raise DeleteBranchError, e
       end
 
       def delete_refs(*ref_names)
@@ -681,15 +688,14 @@ module Gitlab
       # Examples:
       #   create_branch("feature")
       #   create_branch("other-feature", "master")
-      #
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476
       def create_branch(ref, start_point = "HEAD")
-        rugged_ref = rugged.branches.create(ref, start_point)
-        target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
-        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
-      rescue Rugged::ReferenceError => e
-        raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/
-        raise InvalidRef.new("Invalid reference #{start_point}")
+        gitaly_migrate(:create_branch) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.create_branch(ref, start_point)
+          else
+            rugged_create_branch(ref, start_point)
+          end
+        end
       end
 
       # Delete the specified remote from this repository.
@@ -1226,6 +1232,15 @@ module Gitlab
         false
       end
 
+      def rugged_create_branch(ref, start_point)
+        rugged_ref = rugged.branches.create(ref, start_point)
+        target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
+      rescue Rugged::ReferenceError => e
+        raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/
+        raise InvalidRef.new("Invalid reference #{start_point}")
+      end
+
       def gitaly_copy_gitattributes(revision)
         gitaly_repository_client.apply_gitattributes(revision)
       end
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index a1a25cf2079..8ef873d5848 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -79,7 +79,7 @@ module Gitlab
       end
 
       def find_branch(branch_name)
-        request = Gitaly::DeleteBranchRequest.new(
+        request = Gitaly::FindBranchRequest.new(
           repository: @gitaly_repo,
           name: GitalyClient.encode(branch_name)
         )
@@ -92,6 +92,40 @@ module Gitlab
         Gitlab::Git::Branch.new(@repository, encode!(branch.name.dup), branch.target_commit.id, target_commit)
       end
 
+      def create_branch(ref, start_point)
+        request = Gitaly::CreateBranchRequest.new(
+          repository: @gitaly_repo,
+          name: GitalyClient.encode(ref),
+          start_point: GitalyClient.encode(start_point)
+        )
+
+        response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request)
+
+        case response.status
+        when :OK
+          branch = response.branch
+          target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit)
+          Gitlab::Git::Branch.new(@repository, branch.name, branch.target_commit.id, target_commit)
+        when :ERR_INVALID
+          invalid_ref!("Invalid ref name")
+        when :ERR_EXISTS
+          invalid_ref!("Branch #{ref} already exists")
+        when :ERR_INVALID_START_POINT
+          invalid_ref!("Invalid reference #{start_point}")
+        else
+          raise "Unknown response status: #{response.status}"
+        end
+      end
+
+      def delete_branch(branch_name)
+        request = Gitaly::DeleteBranchRequest.new(
+          repository: @gitaly_repo,
+          name: GitalyClient.encode(branch_name)
+        )
+
+        GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request)
+      end
+
       private
 
       def consume_refs_response(response)
@@ -163,6 +197,10 @@ module Gitlab
 
         Gitlab::Git::Commit.decorate(@repository, hash)
       end
+
+      def invalid_ref!(message)
+        raise Gitlab::Git::Repository::InvalidRef.new(message)
+      end
     end
   end
 end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 373062b354b..b8c07460ebb 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -166,7 +166,7 @@ module Gitlab
 
       def remove_branch(name)
         project.repository.delete_branch(name)
-      rescue Rugged::ReferenceError
+      rescue Gitlab::Git::Repository::DeleteBranchFailed
         errors << { type: :remove_branch, name: name }
       end
 
-- 
cgit v1.2.1


From 6c49a628000605d1beb120431003abb329b9fd16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= <ruben@gitlab.com>
Date: Thu, 17 Aug 2017 10:37:36 -0500
Subject: Restore some changes from !9199

---
 lib/gitlab/themes.rb | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 lib/gitlab/themes.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
new file mode 100644
index 00000000000..19ab76ae80f
--- /dev/null
+++ b/lib/gitlab/themes.rb
@@ -0,0 +1,87 @@
+module Gitlab
+  # Module containing GitLab's application theme definitions and helper methods
+  # for accessing them.
+  module Themes
+    extend self
+
+    # Theme ID used when no `default_theme` configuration setting is provided.
+    APPLICATION_DEFAULT = 2
+
+    # Struct class representing a single Theme
+    Theme = Struct.new(:id, :name, :css_class)
+
+    # All available Themes
+    THEMES = [
+      Theme.new(1, 'Graphite', 'ui_graphite'),
+      Theme.new(2, 'Charcoal', 'ui_charcoal'),
+      Theme.new(3, 'Green',    'ui_green'),
+      Theme.new(4, 'Black',    'ui_black'),
+      Theme.new(5, 'Violet',   'ui_violet'),
+      Theme.new(6, 'Blue',     'ui_blue')
+    ].freeze
+
+    # Convenience method to get a space-separated String of all the theme
+    # classes that might be applied to the `body` element
+    #
+    # Returns a String
+    def body_classes
+      THEMES.collect(&:css_class).uniq.join(' ')
+    end
+
+    # Get a Theme by its ID
+    #
+    # If the ID is invalid, returns the default Theme.
+    #
+    # id - Integer ID
+    #
+    # Returns a Theme
+    def by_id(id)
+      THEMES.detect { |t| t.id == id } || default
+    end
+
+    # Returns the number of defined Themes
+    def count
+      THEMES.size
+    end
+
+    # Get the default Theme
+    #
+    # Returns a Theme
+    def default
+      by_id(default_id)
+    end
+
+    # Iterate through each Theme
+    #
+    # Yields the Theme object
+    def each(&block)
+      THEMES.each(&block)
+    end
+
+    # Get the Theme for the specified user, or the default
+    #
+    # user - User record
+    #
+    # Returns a Theme
+    def for_user(user)
+      if user
+        by_id(user.theme_id)
+      else
+        default
+      end
+    end
+
+    private
+
+    def default_id
+      id = Gitlab.config.gitlab.default_theme.to_i
+
+      # Prevent an invalid configuration setting from causing an infinite loop
+      if id < THEMES.first.id || id > THEMES.last.id
+        APPLICATION_DEFAULT
+      else
+        id
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From 3c815f97d32d78abc0eea4f0dd2397e646cd9fd6 Mon Sep 17 00:00:00 2001
From: Annabel Dunstone Gray <annabel.dunstone@gmail.com>
Date: Thu, 31 Aug 2017 15:08:30 -0500
Subject: Add functionality for two themes

---
 lib/gitlab/themes.rb | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
index 19ab76ae80f..1714b6d2c99 100644
--- a/lib/gitlab/themes.rb
+++ b/lib/gitlab/themes.rb
@@ -12,12 +12,8 @@ module Gitlab
 
     # All available Themes
     THEMES = [
-      Theme.new(1, 'Graphite', 'ui_graphite'),
-      Theme.new(2, 'Charcoal', 'ui_charcoal'),
-      Theme.new(3, 'Green',    'ui_green'),
-      Theme.new(4, 'Black',    'ui_black'),
-      Theme.new(5, 'Violet',   'ui_violet'),
-      Theme.new(6, 'Blue',     'ui_blue')
+      Theme.new(1, 'Indigo', 'ui_indigo'),
+      Theme.new(2, 'Dark', 'ui_dark')
     ].freeze
 
     # Convenience method to get a space-separated String of all the theme
-- 
cgit v1.2.1


From c3ac7311525bc23bd8b75addec34654aa87ebbae Mon Sep 17 00:00:00 2001
From: Annabel Dunstone Gray <annabel.dunstone@gmail.com>
Date: Tue, 5 Sep 2017 16:57:31 -0700
Subject: Add blue theme

---
 lib/gitlab/themes.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
index 1714b6d2c99..e0be799b551 100644
--- a/lib/gitlab/themes.rb
+++ b/lib/gitlab/themes.rb
@@ -13,7 +13,8 @@ module Gitlab
     # All available Themes
     THEMES = [
       Theme.new(1, 'Indigo', 'ui_indigo'),
-      Theme.new(2, 'Dark', 'ui_dark')
+      Theme.new(2, 'Dark', 'ui_dark'),
+      Theme.new(3, 'Blue', 'ui_blue')
     ].freeze
 
     # Convenience method to get a space-separated String of all the theme
-- 
cgit v1.2.1


From 7b567597eef25ca3af8af63a71bcc4dfefc2a694 Mon Sep 17 00:00:00 2001
From: Annabel Dunstone Gray <annabel.dunstone@gmail.com>
Date: Wed, 6 Sep 2017 08:09:54 -0700
Subject: Fix specs; start on light theme

---
 lib/gitlab/themes.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
index e0be799b551..5e1e65854a3 100644
--- a/lib/gitlab/themes.rb
+++ b/lib/gitlab/themes.rb
@@ -14,7 +14,9 @@ module Gitlab
     THEMES = [
       Theme.new(1, 'Indigo', 'ui_indigo'),
       Theme.new(2, 'Dark', 'ui_dark'),
-      Theme.new(3, 'Blue', 'ui_blue')
+      Theme.new(3, 'Light', 'ui_light'),
+      Theme.new(4, 'Blue', 'ui_blue'),
+      Theme.new(5, 'Green', 'ui_green'),
     ].freeze
 
     # Convenience method to get a space-separated String of all the theme
-- 
cgit v1.2.1


From 6ee158197dc97924ab6f53ce6075cb3a8b9786ce Mon Sep 17 00:00:00 2001
From: Annabel Dunstone Gray <annabel.dunstone@gmail.com>
Date: Wed, 6 Sep 2017 11:37:40 -0700
Subject: Add previews in preferences

---
 lib/gitlab/themes.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
index 5e1e65854a3..e871557e243 100644
--- a/lib/gitlab/themes.rb
+++ b/lib/gitlab/themes.rb
@@ -16,7 +16,7 @@ module Gitlab
       Theme.new(2, 'Dark', 'ui_dark'),
       Theme.new(3, 'Light', 'ui_light'),
       Theme.new(4, 'Blue', 'ui_blue'),
-      Theme.new(5, 'Green', 'ui_green'),
+      Theme.new(5, 'Green', 'ui_green')
     ].freeze
 
     # Convenience method to get a space-separated String of all the theme
-- 
cgit v1.2.1


From c2e99b40f71ca7036cc07596aae164e92378263b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= <alejorro70@gmail.com>
Date: Wed, 6 Sep 2017 17:47:25 -0300
Subject: Implement fix for n+1 issue on `flatten_tree` helper

---
 lib/gitlab/git/tree.rb                     | 11 +++++++----
 lib/gitlab/gitaly_client/commit_service.rb |  6 +++---
 2 files changed, 10 insertions(+), 7 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index b54962a4456..5cf336af3c6 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -5,7 +5,7 @@ module Gitlab
     class Tree
       include Gitlab::EncodingHelper
 
-      attr_accessor :id, :root_id, :name, :path, :type,
+      attr_accessor :id, :root_id, :name, :path, :flat_path, :type,
         :mode, :commit_id, :submodule_url
 
       class << self
@@ -19,8 +19,7 @@ module Gitlab
 
           Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled|
             if is_enabled
-              client = Gitlab::GitalyClient::CommitService.new(repository)
-              client.tree_entries(repository, sha, path)
+              repository.gitaly_commit_client.tree_entries(repository, sha, path)
             else
               tree_entries_from_rugged(repository, sha, path)
             end
@@ -88,7 +87,7 @@ module Gitlab
       end
 
       def initialize(options)
-        %w(id root_id name path type mode commit_id).each do |key|
+        %w(id root_id name path flat_path type mode commit_id).each do |key|
           self.send("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend
         end
       end
@@ -101,6 +100,10 @@ module Gitlab
         encode! @path
       end
 
+      def flat_path
+        encode! @flat_path
+      end
+
       def dir?
         type == :tree
       end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 21a32a7e0db..0825a3a7694 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -88,14 +88,14 @@ module Gitlab
 
         response.flat_map do |message|
           message.entries.map do |gitaly_tree_entry|
-            entry_path = gitaly_tree_entry.path.dup
             Gitlab::Git::Tree.new(
               id: gitaly_tree_entry.oid,
               root_id: gitaly_tree_entry.root_oid,
               type: gitaly_tree_entry.type.downcase,
               mode: gitaly_tree_entry.mode.to_s(8),
-              name: File.basename(entry_path),
-              path: entry_path,
+              name: File.basename(gitaly_tree_entry.path),
+              path: GitalyClient.encode(gitaly_tree_entry.path),
+              flat_path: GitalyClient.encode(gitaly_tree_entry.flat_path),
               commit_id: gitaly_tree_entry.commit_oid
             )
           end
-- 
cgit v1.2.1


From f7c8032e0993a6dc6bb808b0f2234324d3fe9707 Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Wed, 6 Sep 2017 22:41:15 -0700
Subject: Add JSON logger in `log/api_json.log` for Grape API endpoints

Closes #36189
---
 lib/gitlab/api_logger.rb | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 lib/gitlab/api_logger.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/api_logger.rb b/lib/gitlab/api_logger.rb
new file mode 100644
index 00000000000..09122b233ea
--- /dev/null
+++ b/lib/gitlab/api_logger.rb
@@ -0,0 +1,8 @@
+module Gitlab
+  class ApiLogger < ::Logger
+
+    def format_message(severity, timestamp, progname, message)
+      super + "\n"
+    end
+  end
+end
-- 
cgit v1.2.1


From 99dcf870dd46299ffec2e5067c9b3223de3315ce Mon Sep 17 00:00:00 2001
From: Annabel Dunstone Gray <annabel.dunstone@gmail.com>
Date: Thu, 7 Sep 2017 06:43:12 -0700
Subject: Use indigo as default theme

---
 lib/gitlab/themes.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
index e871557e243..37054d98102 100644
--- a/lib/gitlab/themes.rb
+++ b/lib/gitlab/themes.rb
@@ -5,7 +5,7 @@ module Gitlab
     extend self
 
     # Theme ID used when no `default_theme` configuration setting is provided.
-    APPLICATION_DEFAULT = 2
+    APPLICATION_DEFAULT = 1
 
     # Struct class representing a single Theme
     Theme = Struct.new(:id, :name, :css_class)
-- 
cgit v1.2.1


From c304dfd4d6ca0f52537044742bb6dd6c219bdbbf Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Thu, 7 Sep 2017 07:02:46 -0700
Subject: Fix Rubocop failures in API logger

---
 lib/gitlab/api_logger.rb | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/api_logger.rb b/lib/gitlab/api_logger.rb
index 09122b233ea..dcb194a1b89 100644
--- a/lib/gitlab/api_logger.rb
+++ b/lib/gitlab/api_logger.rb
@@ -1,6 +1,5 @@
 module Gitlab
   class ApiLogger < ::Logger
-
     def format_message(severity, timestamp, progname, message)
       super + "\n"
     end
-- 
cgit v1.2.1


From 26264625cbc2692bc113c1e701cb15eecbd010d2 Mon Sep 17 00:00:00 2001
From: Ruben Davila <rdavila84@gmail.com>
Date: Thu, 7 Sep 2017 09:10:39 -0500
Subject: Small refactor after last code review.

---
 lib/gitlab/themes.rb | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
index 37054d98102..d43eff5ba4a 100644
--- a/lib/gitlab/themes.rb
+++ b/lib/gitlab/themes.rb
@@ -73,13 +73,11 @@ module Gitlab
     private
 
     def default_id
-      id = Gitlab.config.gitlab.default_theme.to_i
+      @default_id ||= begin
+        id = Gitlab.config.gitlab.default_theme.to_i
+        theme_ids = THEMES.map(&:id)
 
-      # Prevent an invalid configuration setting from causing an infinite loop
-      if id < THEMES.first.id || id > THEMES.last.id
-        APPLICATION_DEFAULT
-      else
-        id
+        theme_ids.include?(id) ? id : APPLICATION_DEFAULT
       end
     end
   end
-- 
cgit v1.2.1


From 35dec2c3e87f2f44c3ab0269e7f737afdc28801a Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Thu, 7 Sep 2017 07:48:13 -0700
Subject: Use a custom GrapeLogging formatter to get the timestamp

---
 lib/gitlab/api_logger.rb                              |  7 -------
 .../formatters/lograge_with_timestamp.rb              | 19 +++++++++++++++++++
 2 files changed, 19 insertions(+), 7 deletions(-)
 delete mode 100644 lib/gitlab/api_logger.rb
 create mode 100644 lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/api_logger.rb b/lib/gitlab/api_logger.rb
deleted file mode 100644
index dcb194a1b89..00000000000
--- a/lib/gitlab/api_logger.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-module Gitlab
-  class ApiLogger < ::Logger
-    def format_message(severity, timestamp, progname, message)
-      super + "\n"
-    end
-  end
-end
diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb
new file mode 100644
index 00000000000..1e1fdabca93
--- /dev/null
+++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb
@@ -0,0 +1,19 @@
+module Gitlab
+  module GrapeLogging
+    module Formatters
+      class LogrageWithTimestamp
+        def call(severity, datetime, _, data)
+          time = data.delete :time
+          attributes = {
+            time: datetime.utc.iso8601(3),
+            severity: severity,
+            duration: time[:total],
+            db: time[:db],
+            view: time[:view]
+          }.merge(data)
+          ::Lograge.formatter.call(attributes) + "\n"
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From 94680e1448f9e3af3dfc43e33a74609cdc0ecb69 Mon Sep 17 00:00:00 2001
From: Andrew Newdigate <andrew@gitlab.com>
Date: Thu, 7 Sep 2017 17:39:00 +0000
Subject: Gitaly feature toggles are on by default in development environments

---
 lib/gitlab/gitaly_client.rb | 36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 9a5f4f598b2..a3dc2cd0b60 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -70,21 +70,41 @@ module Gitlab
       params['gitaly_token'].presence || Gitlab.config.gitaly['token']
     end
 
-    def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN)
+    # Evaluates whether a feature toggle is on or off
+    def self.feature_enabled?(feature_name, status: MigrationStatus::OPT_IN)
+      # Disabled features are always off!
       return false if status == MigrationStatus::DISABLED
 
-      feature = Feature.get("gitaly_#{feature}")
+      feature = Feature.get("gitaly_#{feature_name}")
 
-      # If the feature hasn't been set, turn it on if it's opt-out
-      return status == MigrationStatus::OPT_OUT unless Feature.persisted?(feature)
+      # If the feature has been set, always evaluate
+      if Feature.persisted?(feature)
+        if feature.percentage_of_time_value > 0
+          # Probabilistically enable this feature
+          return Random.rand() * 100 < feature.percentage_of_time_value
+        end
+
+        return feature.enabled?
+      end
 
-      if feature.percentage_of_time_value > 0
-        # Probabilistically enable this feature
-        return Random.rand() * 100 < feature.percentage_of_time_value
+      # If the feature has not been set, the default depends
+      # on it's status
+      case status
+      when MigrationStatus::OPT_OUT
+        true
+      when MigrationStatus::OPT_IN
+        opt_into_all_features?
+      else
+        false
       end
+    end
 
-      feature.enabled?
+    # opt_into_all_features? returns true when the current environment
+    # is one in which we opt into features automatically
+    def self.opt_into_all_features?
+      Rails.env.development? || ENV["GITALY_FEATURE_DEFAULT_ON"] == "1"
     end
+    private_class_method :opt_into_all_features?
 
     def self.migrate(feature, status: MigrationStatus::OPT_IN)
       is_enabled  = feature_enabled?(feature, status: status)
-- 
cgit v1.2.1


From 39298575a819ade6ad4f9e37a7f22592a05d21f8 Mon Sep 17 00:00:00 2001
From: Tiago Botelho <tiagonbotelho@hotmail.com>
Date: Mon, 4 Sep 2017 18:55:04 +0100
Subject: Adds exclusive lease to Git garbage collect worker.

---
 lib/gitlab/exclusive_lease.rb | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb
index 3784f6c4947..3f7b42456af 100644
--- a/lib/gitlab/exclusive_lease.rb
+++ b/lib/gitlab/exclusive_lease.rb
@@ -25,6 +25,12 @@ module Gitlab
       end
     EOS
 
+    def self.get_uuid(key)
+      Gitlab::Redis::SharedState.with do |redis|
+        redis.get(redis_shared_state_key(key)) || false
+      end
+    end
+
     def self.cancel(key, uuid)
       Gitlab::Redis::SharedState.with do |redis|
         redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid])
@@ -35,10 +41,10 @@ module Gitlab
       "gitlab:exclusive_lease:#{key}"
     end
 
-    def initialize(key, timeout:)
+    def initialize(key, uuid: nil, timeout:)
       @redis_shared_state_key = self.class.redis_shared_state_key(key)
       @timeout = timeout
-      @uuid = SecureRandom.uuid
+      @uuid = uuid || SecureRandom.uuid
     end
 
     # Try to obtain the lease. Return lease UUID on success,
-- 
cgit v1.2.1


From bc10afb600a8079fe250e7c82bf16763a8fed28f Mon Sep 17 00:00:00 2001
From: Robert Speicher <robert@gitlab.com>
Date: Thu, 29 Jun 2017 01:22:22 +0000
Subject: Merge branch 'dm-go-get-xss' into 'security-9-3'

Fix XSS issue in go-get handling

See merge request !2128
---
 lib/gitlab/middleware/go.rb | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb
index 6023fa1820f..f42168c720e 100644
--- a/lib/gitlab/middleware/go.rb
+++ b/lib/gitlab/middleware/go.rb
@@ -3,6 +3,10 @@
 module Gitlab
   module Middleware
     class Go
+      include ActionView::Helpers::TagHelper
+
+      PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze
+
       def initialize(app)
         @app = app
       end
@@ -10,17 +14,20 @@ module Gitlab
       def call(env)
         request = Rack::Request.new(env)
 
-        if go_request?(request)
-          render_go_doc(request)
-        else
-          @app.call(env)
-        end
+        render_go_doc(request) || @app.call(env)
       end
 
       private
 
       def render_go_doc(request)
-        body = go_body(request)
+        return unless go_request?(request)
+
+        path = project_path(request)
+        return unless path
+
+        body = go_body(path)
+        return unless body
+
         response = Rack::Response.new(body, 200, { 'Content-Type' => 'text/html' })
         response.finish
       end
@@ -29,11 +36,13 @@ module Gitlab
         request["go-get"].to_i == 1 && request.env["PATH_INFO"].present?
       end
 
-      def go_body(request)
-        project_url = URI.join(Gitlab.config.gitlab.url, project_path(request))
+      def go_body(path)
+        project_url = URI.join(Gitlab.config.gitlab.url, path)
         import_prefix = strip_url(project_url.to_s)
 
-        "<!DOCTYPE html><html><head><meta content='#{import_prefix} git #{project_url}.git' name='go-import'></head></html>\n"
+        meta_tag = tag :meta, name: 'go-import', content: "#{import_prefix} git #{project_url}.git"
+        head_tag = content_tag :head, meta_tag
+        content_tag :html, head_tag
       end
 
       def strip_url(url)
@@ -44,6 +53,10 @@ module Gitlab
         path_info = request.env["PATH_INFO"]
         path_info.sub!(/^\//, '')
 
+        project_path_match = "#{path_info}/".match(PROJECT_PATH_REGEX)
+        return unless project_path_match
+        path = project_path_match[1]
+
         # Go subpackages may be in the form of `namespace/project/path1/path2/../pathN`.
         # In a traditional project with a single namespace, this would denote repo
         # `namespace/project` with subpath `path1/path2/../pathN`, but with nested
@@ -51,7 +64,7 @@ module Gitlab
         # `path2/../pathN`, for example.
 
         # We find all potential project paths out of the path segments
-        path_segments = path_info.split('/')
+        path_segments = path.split('/')
         simple_project_path = path_segments.first(2).join('/')
 
         # If the path is at most 2 segments long, it is a simple `namespace/project` path and we're done
-- 
cgit v1.2.1


From 88568317a8829a50a22c1151b9580c0d8fe60c1f Mon Sep 17 00:00:00 2001
From: Travis Miller <travis@travismiller.com>
Date: Mon, 4 Sep 2017 15:52:00 -0500
Subject: Add GitLab-Pages version to Admin Dashboard

---
 lib/gitlab/pages.rb | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 lib/gitlab/pages.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb
new file mode 100644
index 00000000000..981ef8faa9a
--- /dev/null
+++ b/lib/gitlab/pages.rb
@@ -0,0 +1,5 @@
+module Gitlab
+  module Pages
+    VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze
+  end
+end
-- 
cgit v1.2.1


From 572e90125f56e70cc6220f5ad45c6fc1cc0c339e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20Coutable?= <remy@rymai.me>
Date: Fri, 8 Sep 2017 14:04:44 +0200
Subject: Backport the EE signature of Gitlab::Workhorse#send_artifacts_entry
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Rémy Coutable <remy@rymai.me>
---
 lib/gitlab/workhorse.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 7a94af2f8f1..17550cf9074 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -121,10 +121,10 @@ module Gitlab
         ]
       end
 
-      def send_artifacts_entry(build, path)
+      def send_artifacts_entry(build, entry)
         params = {
           'Archive' => build.artifacts_file.path,
-          'Entry' => Base64.encode64(path.to_s)
+          'Entry' => Base64.encode64(entry.to_s)
         }
 
         [
-- 
cgit v1.2.1


From 52a2423e373e6552023faaff43ae07dbb7423c00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= <ruben@gitlab.com>
Date: Fri, 8 Sep 2017 14:48:44 +0000
Subject: Revert "Merge branch
 '35012-navigation-add-option-to-change-navigation-color-palette' into
 'master'"

This reverts merge request !13619
---
 lib/gitlab/themes.rb | 84 ----------------------------------------------------
 1 file changed, 84 deletions(-)
 delete mode 100644 lib/gitlab/themes.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
deleted file mode 100644
index d43eff5ba4a..00000000000
--- a/lib/gitlab/themes.rb
+++ /dev/null
@@ -1,84 +0,0 @@
-module Gitlab
-  # Module containing GitLab's application theme definitions and helper methods
-  # for accessing them.
-  module Themes
-    extend self
-
-    # Theme ID used when no `default_theme` configuration setting is provided.
-    APPLICATION_DEFAULT = 1
-
-    # Struct class representing a single Theme
-    Theme = Struct.new(:id, :name, :css_class)
-
-    # All available Themes
-    THEMES = [
-      Theme.new(1, 'Indigo', 'ui_indigo'),
-      Theme.new(2, 'Dark', 'ui_dark'),
-      Theme.new(3, 'Light', 'ui_light'),
-      Theme.new(4, 'Blue', 'ui_blue'),
-      Theme.new(5, 'Green', 'ui_green')
-    ].freeze
-
-    # Convenience method to get a space-separated String of all the theme
-    # classes that might be applied to the `body` element
-    #
-    # Returns a String
-    def body_classes
-      THEMES.collect(&:css_class).uniq.join(' ')
-    end
-
-    # Get a Theme by its ID
-    #
-    # If the ID is invalid, returns the default Theme.
-    #
-    # id - Integer ID
-    #
-    # Returns a Theme
-    def by_id(id)
-      THEMES.detect { |t| t.id == id } || default
-    end
-
-    # Returns the number of defined Themes
-    def count
-      THEMES.size
-    end
-
-    # Get the default Theme
-    #
-    # Returns a Theme
-    def default
-      by_id(default_id)
-    end
-
-    # Iterate through each Theme
-    #
-    # Yields the Theme object
-    def each(&block)
-      THEMES.each(&block)
-    end
-
-    # Get the Theme for the specified user, or the default
-    #
-    # user - User record
-    #
-    # Returns a Theme
-    def for_user(user)
-      if user
-        by_id(user.theme_id)
-      else
-        default
-      end
-    end
-
-    private
-
-    def default_id
-      @default_id ||= begin
-        id = Gitlab.config.gitlab.default_theme.to_i
-        theme_ids = THEMES.map(&:id)
-
-        theme_ids.include?(id) ? id : APPLICATION_DEFAULT
-      end
-    end
-  end
-end
-- 
cgit v1.2.1


From 5c6f40ab6e825e8d31e3bfc362e9bcce80a14dba Mon Sep 17 00:00:00 2001
From: Zeger-Jan van de Weg <git@zjvandeweg.nl>
Date: Fri, 8 Sep 2017 19:38:02 +0200
Subject: Add usage ping for Auto DevOps

Fixes gitlab-org/gitlab-ce#37648
---
 lib/gitlab/usage_data.rb | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 3cf26625108..47b0be5835f 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -22,6 +22,8 @@ module Gitlab
             ci_builds: ::Ci::Build.count,
             ci_internal_pipelines: ::Ci::Pipeline.internal.count,
             ci_external_pipelines: ::Ci::Pipeline.external.count,
+            ci_pipeline_config_auto_devops: ::Ci::Pipeline.auto_devops_source.count,
+            ci_pipeline_config_repository: ::Ci::Pipeline.repository_source.count,
             ci_runners: ::Ci::Runner.count,
             ci_triggers: ::Ci::Trigger.count,
             ci_pipeline_schedules: ::Ci::PipelineSchedule.count,
-- 
cgit v1.2.1


From 9b177bb7c94df6c7d3868235f75939a41acf8718 Mon Sep 17 00:00:00 2001
From: Annabel Dunstone Gray <annabel.dunstone@gmail.com>
Date: Mon, 11 Sep 2017 15:44:42 +0000
Subject: Revert "Merge branch 'revert-f2421b2b' into 'master'"

This reverts merge request !14148
---
 lib/gitlab/themes.rb | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 lib/gitlab/themes.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
new file mode 100644
index 00000000000..d43eff5ba4a
--- /dev/null
+++ b/lib/gitlab/themes.rb
@@ -0,0 +1,84 @@
+module Gitlab
+  # Module containing GitLab's application theme definitions and helper methods
+  # for accessing them.
+  module Themes
+    extend self
+
+    # Theme ID used when no `default_theme` configuration setting is provided.
+    APPLICATION_DEFAULT = 1
+
+    # Struct class representing a single Theme
+    Theme = Struct.new(:id, :name, :css_class)
+
+    # All available Themes
+    THEMES = [
+      Theme.new(1, 'Indigo', 'ui_indigo'),
+      Theme.new(2, 'Dark', 'ui_dark'),
+      Theme.new(3, 'Light', 'ui_light'),
+      Theme.new(4, 'Blue', 'ui_blue'),
+      Theme.new(5, 'Green', 'ui_green')
+    ].freeze
+
+    # Convenience method to get a space-separated String of all the theme
+    # classes that might be applied to the `body` element
+    #
+    # Returns a String
+    def body_classes
+      THEMES.collect(&:css_class).uniq.join(' ')
+    end
+
+    # Get a Theme by its ID
+    #
+    # If the ID is invalid, returns the default Theme.
+    #
+    # id - Integer ID
+    #
+    # Returns a Theme
+    def by_id(id)
+      THEMES.detect { |t| t.id == id } || default
+    end
+
+    # Returns the number of defined Themes
+    def count
+      THEMES.size
+    end
+
+    # Get the default Theme
+    #
+    # Returns a Theme
+    def default
+      by_id(default_id)
+    end
+
+    # Iterate through each Theme
+    #
+    # Yields the Theme object
+    def each(&block)
+      THEMES.each(&block)
+    end
+
+    # Get the Theme for the specified user, or the default
+    #
+    # user - User record
+    #
+    # Returns a Theme
+    def for_user(user)
+      if user
+        by_id(user.theme_id)
+      else
+        default
+      end
+    end
+
+    private
+
+    def default_id
+      @default_id ||= begin
+        id = Gitlab.config.gitlab.default_theme.to_i
+        theme_ids = THEMES.map(&:id)
+
+        theme_ids.include?(id) ? id : APPLICATION_DEFAULT
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From 25c34608b9ecd73391aaf7fdc66740e43fee5acb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kim=20=22BKC=22=20Carlb=C3=A4cker?=
 <kim.carlbacker@gmail.com>
Date: Wed, 6 Sep 2017 12:55:16 +0200
Subject: Migrate Git::CommitStats to Gitaly

---
 lib/gitlab/git/commit.rb                   |  2 +-
 lib/gitlab/git/commit_stats.rb             | 19 ++++++++++++++++++-
 lib/gitlab/gitaly_client/commit_service.rb |  8 ++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 5ee6669050c..1f370686186 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -352,7 +352,7 @@ module Gitlab
       end
 
       def stats
-        Gitlab::Git::CommitStats.new(self)
+        Gitlab::Git::CommitStats.new(@repository, self)
       end
 
       def to_patch(options = {})
diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb
index 00acb4763e9..6bf49a0af18 100644
--- a/lib/gitlab/git/commit_stats.rb
+++ b/lib/gitlab/git/commit_stats.rb
@@ -10,12 +10,29 @@ module Gitlab
       # Instantiate a CommitStats object
       #
       # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323
-      def initialize(commit)
+      def initialize(repo, commit)
         @id = commit.id
         @additions = 0
         @deletions = 0
         @total = 0
 
+        repo.gitaly_migrate(:commit_stats) do |is_enabled|
+          if is_enabled
+            gitaly_stats(repo, commit)
+          else
+            rugged_stats(commit)
+          end
+        end
+      end
+
+      def gitaly_stats(repo, commit)
+        stats = repo.gitaly_commit_client.commit_stats(@id)
+        @additions = stats.additions
+        @deletions = stats.deletions
+        @total = @additions + @deletions
+      end
+
+      def rugged_stats(commit)
         diff = commit.rugged_diff_from_parent
 
         diff.each_patch do |p|
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 0825a3a7694..1ba1a7830a4 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -204,6 +204,14 @@ module Gitlab
         response.sum(&:data)
       end
 
+      def commit_stats(revision)
+        request = Gitaly::CommitStatsRequest.new(
+          repository: @gitaly_repo,
+          revision: GitalyClient.encode(revision)
+        )
+        GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request)
+      end
+
       private
 
       def commit_diff_request_params(commit, options = {})
-- 
cgit v1.2.1


From 74bf291c78538641bb871adf8d998bdf0f8e2508 Mon Sep 17 00:00:00 2001
From: Zeger-Jan van de Weg <git@zjvandeweg.nl>
Date: Tue, 12 Sep 2017 12:33:48 +0200
Subject: Add auto devops enabled/disabled to usage ping

---
 lib/gitlab/usage_data.rb | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 47b0be5835f..36708078136 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -27,6 +27,8 @@ module Gitlab
             ci_runners: ::Ci::Runner.count,
             ci_triggers: ::Ci::Trigger.count,
             ci_pipeline_schedules: ::Ci::PipelineSchedule.count,
+            auto_devops_enabled: ::ProjectAutoDevops.enabled.count,
+            auto_devops_disabled: ::ProjectAutoDevops.disabled.count,
             deploy_keys: DeployKey.count,
             deployments: Deployment.count,
             environments: ::Environment.count,
-- 
cgit v1.2.1


From a31e0aff224d4047e10fae98b6f9dc9f84b7457a Mon Sep 17 00:00:00 2001
From: Micael Bergeron <mbergeron@gitlab.com>
Date: Tue, 12 Sep 2017 14:07:31 +0000
Subject: Resolve "Error 500 in non-UTF8 branch names"

---
 lib/gitlab/git.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index 8c9acbc9fbe..b4b6326cfdd 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -11,7 +11,7 @@ module Gitlab
       include Gitlab::EncodingHelper
 
       def ref_name(ref)
-        encode! ref.sub(/\Arefs\/(tags|heads|remotes)\//, '')
+        encode_utf8(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '')
       end
 
       def branch_name(ref)
-- 
cgit v1.2.1


From 4db6b8f0bb35fabf48c2d795ef3151810bb8251c Mon Sep 17 00:00:00 2001
From: Maxim Rydkin <maks.rydkin@gmail.com>
Date: Tue, 12 Sep 2017 17:02:11 +0000
Subject: Decrease Cyclomatic Complexity threshold to 13

---
 lib/gitlab/conflict/parser.rb | 28 +++++++++++++++++++---------
 lib/gitlab/mail_room.rb       | 27 +++++++++++++++++----------
 2 files changed, 36 insertions(+), 19 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb
index 84f9ecd3d23..e3678c914db 100644
--- a/lib/gitlab/conflict/parser.rb
+++ b/lib/gitlab/conflict/parser.rb
@@ -12,12 +12,7 @@ module Gitlab
       MissingEndDelimiter = Class.new(ParserError)
 
       def parse(text, our_path:, their_path:, parent_file: nil)
-        raise UnmergeableFile if text.blank? # Typically a binary file
-        raise UnmergeableFile if text.length > 200.kilobytes
-
-        text.force_encoding('UTF-8')
-
-        raise UnsupportedEncoding unless text.valid_encoding?
+        validate_text!(text)
 
         line_obj_index = 0
         line_old = 1
@@ -32,15 +27,15 @@ module Gitlab
           full_line = line.delete("\n")
 
           if full_line == conflict_start
-            raise UnexpectedDelimiter unless type.nil?
+            validate_delimiter!(type.nil?)
 
             type = 'new'
           elsif full_line == conflict_middle
-            raise UnexpectedDelimiter unless type == 'new'
+            validate_delimiter!(type == 'new')
 
             type = 'old'
           elsif full_line == conflict_end
-            raise UnexpectedDelimiter unless type == 'old'
+            validate_delimiter!(type == 'old')
 
             type = nil
           elsif line[0] == '\\'
@@ -59,6 +54,21 @@ module Gitlab
 
         lines
       end
+
+      private
+
+      def validate_text!(text)
+        raise UnmergeableFile if text.blank? # Typically a binary file
+        raise UnmergeableFile if text.length > 200.kilobytes
+
+        text.force_encoding('UTF-8')
+
+        raise UnsupportedEncoding unless text.valid_encoding?
+      end
+
+      def validate_delimiter!(condition)
+        raise UnexpectedDelimiter unless condition
+      end
     end
   end
 end
diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb
index 9f432673a6e..344784c866f 100644
--- a/lib/gitlab/mail_room.rb
+++ b/lib/gitlab/mail_room.rb
@@ -4,6 +4,15 @@ require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues)
 
 module Gitlab
   module MailRoom
+    DEFAULT_CONFIG = {
+      enabled: false,
+      port: 143,
+      ssl: false,
+      start_tls: false,
+      mailbox: 'inbox',
+      idle_timeout: 60
+    }.freeze
+
     class << self
       def enabled?
         config[:enabled] && config[:address]
@@ -22,16 +31,10 @@ module Gitlab
       def fetch_config
         return {} unless File.exist?(config_file)
 
-        rails_env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
-        all_config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys
-
-        config = all_config[:incoming_email] || {}
-        config[:enabled] = false if config[:enabled].nil?
-        config[:port] = 143 if config[:port].nil?
-        config[:ssl] = false if config[:ssl].nil?
-        config[:start_tls] = false if config[:start_tls].nil?
-        config[:mailbox] = 'inbox' if config[:mailbox].nil?
-        config[:idle_timeout] = 60 if config[:idle_timeout].nil?
+        config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys[:incoming_email] || {}
+        config = DEFAULT_CONFIG.merge(config) do |_key, oldval, newval|
+          newval.nil? ? oldval : newval
+        end
 
         if config[:enabled] && config[:address]
           gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env)
@@ -45,6 +48,10 @@ module Gitlab
         config
       end
 
+      def rails_env
+        @rails_env ||= ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+      end
+
       def config_file
         ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../../config/gitlab.yml', __FILE__)
       end
-- 
cgit v1.2.1


From 060fc3905de795ff206260b97afd18177f876316 Mon Sep 17 00:00:00 2001
From: Maxim Rydkin <maks.rydkin@gmail.com>
Date: Wed, 6 Sep 2017 14:08:06 +0300
Subject: move `lib/ci/ansi2html.rb` into `lib/gitlab/ci/ansi2html.rb`

---
 lib/gitlab/ci/ansi2html.rb    | 333 ++++++++++++++++++++++++++++++++++++++++++
 lib/gitlab/ci/trace/stream.rb |   4 +-
 2 files changed, 335 insertions(+), 2 deletions(-)
 create mode 100644 lib/gitlab/ci/ansi2html.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb
new file mode 100644
index 00000000000..ad78ae244b2
--- /dev/null
+++ b/lib/gitlab/ci/ansi2html.rb
@@ -0,0 +1,333 @@
+# ANSI color library
+#
+# Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code
+module Gitlab
+  module Ci
+    module Ansi2html
+      # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107)
+      COLOR = {
+        0 => 'black', # not that this is gray in the intense color table
+        1 => 'red',
+        2 => 'green',
+        3 => 'yellow',
+        4 => 'blue',
+        5 => 'magenta',
+        6 => 'cyan',
+        7 => 'white', # not that this is gray in the dark (aka default) color table
+      }.freeze
+
+      STYLE_SWITCHES = {
+        bold:       0x01,
+        italic:     0x02,
+        underline:  0x04,
+        conceal:    0x08,
+        cross:      0x10
+      }.freeze
+
+      def self.convert(ansi, state = nil)
+        Converter.new.convert(ansi, state)
+      end
+
+      class Converter
+        def on_0(s) reset()                            end
+
+        def on_1(s) enable(STYLE_SWITCHES[:bold])      end
+
+        def on_3(s) enable(STYLE_SWITCHES[:italic])    end
+
+        def on_4(s) enable(STYLE_SWITCHES[:underline]) end
+
+        def on_8(s) enable(STYLE_SWITCHES[:conceal])   end
+
+        def on_9(s) enable(STYLE_SWITCHES[:cross])     end
+
+        def on_21(s) disable(STYLE_SWITCHES[:bold])      end
+
+        def on_22(s) disable(STYLE_SWITCHES[:bold])      end
+
+        def on_23(s) disable(STYLE_SWITCHES[:italic])    end
+
+        def on_24(s) disable(STYLE_SWITCHES[:underline]) end
+
+        def on_28(s) disable(STYLE_SWITCHES[:conceal])   end
+
+        def on_29(s) disable(STYLE_SWITCHES[:cross])     end
+
+        def on_30(s) set_fg_color(0) end
+
+        def on_31(s) set_fg_color(1) end
+
+        def on_32(s) set_fg_color(2) end
+
+        def on_33(s) set_fg_color(3) end
+
+        def on_34(s) set_fg_color(4) end
+
+        def on_35(s) set_fg_color(5) end
+
+        def on_36(s) set_fg_color(6) end
+
+        def on_37(s) set_fg_color(7) end
+
+        def on_38(s) set_fg_color_256(s) end
+
+        def on_39(s) set_fg_color(9) end
+
+        def on_40(s) set_bg_color(0) end
+
+        def on_41(s) set_bg_color(1) end
+
+        def on_42(s) set_bg_color(2) end
+
+        def on_43(s) set_bg_color(3) end
+
+        def on_44(s) set_bg_color(4) end
+
+        def on_45(s) set_bg_color(5) end
+
+        def on_46(s) set_bg_color(6) end
+
+        def on_47(s) set_bg_color(7) end
+
+        def on_48(s) set_bg_color_256(s) end
+
+        def on_49(s) set_bg_color(9) end
+
+        def on_90(s) set_fg_color(0, 'l') end
+
+        def on_91(s) set_fg_color(1, 'l') end
+
+        def on_92(s) set_fg_color(2, 'l') end
+
+        def on_93(s) set_fg_color(3, 'l') end
+
+        def on_94(s) set_fg_color(4, 'l') end
+
+        def on_95(s) set_fg_color(5, 'l') end
+
+        def on_96(s) set_fg_color(6, 'l') end
+
+        def on_97(s) set_fg_color(7, 'l') end
+
+        def on_99(s) set_fg_color(9, 'l') end
+
+        def on_100(s) set_bg_color(0, 'l') end
+
+        def on_101(s) set_bg_color(1, 'l') end
+
+        def on_102(s) set_bg_color(2, 'l') end
+
+        def on_103(s) set_bg_color(3, 'l') end
+
+        def on_104(s) set_bg_color(4, 'l') end
+
+        def on_105(s) set_bg_color(5, 'l') end
+
+        def on_106(s) set_bg_color(6, 'l') end
+
+        def on_107(s) set_bg_color(7, 'l') end
+
+        def on_109(s) set_bg_color(9, 'l') end
+
+        attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask
+
+        STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze
+
+        def convert(stream, new_state)
+          reset_state
+          restore_state(new_state, stream) if new_state.present?
+
+          append = false
+          truncated = false
+
+          cur_offset = stream.tell
+          if cur_offset > @offset
+            @offset = cur_offset
+            truncated = true
+          else
+            stream.seek(@offset)
+            append = @offset > 0
+          end
+          start_offset = @offset
+
+          open_new_tag
+
+          stream.each_line do |line|
+            s = StringScanner.new(line)
+            until s.eos?
+              if s.scan(/\e([@-_])(.*?)([@-~])/)
+                handle_sequence(s)
+              elsif s.scan(/\e(([@-_])(.*?)?)?$/)
+                break
+              elsif s.scan(/</)
+                @out << '&lt;'
+              elsif s.scan(/\r?\n/)
+                @out << '<br>'
+              else
+                @out << s.scan(/./m)
+              end
+              @offset += s.matched_size
+            end
+          end
+
+          close_open_tags()
+
+          OpenStruct.new(
+            html: @out.force_encoding(Encoding.default_external),
+            state: state,
+            append: append,
+            truncated: truncated,
+            offset: start_offset,
+            size: stream.tell - start_offset,
+            total: stream.size
+          )
+        end
+
+        def handle_sequence(s)
+          indicator = s[1]
+          commands = s[2].split ';'
+          terminator = s[3]
+
+          # We are only interested in color and text style changes - triggered by
+          # sequences starting with '\e[' and ending with 'm'. Any other control
+          # sequence gets stripped (including stuff like "delete last line")
+          return unless indicator == '[' && terminator == 'm'
+
+          close_open_tags()
+
+          if commands.empty?()
+            reset()
+            return
+          end
+
+          evaluate_command_stack(commands)
+
+          open_new_tag
+        end
+
+        def evaluate_command_stack(stack)
+          return unless command = stack.shift()
+
+          if self.respond_to?("on_#{command}", true)
+            self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend
+          end
+
+          evaluate_command_stack(stack)
+        end
+
+        def open_new_tag
+          css_classes = []
+
+          unless @fg_color.nil?
+            fg_color = @fg_color
+            # Most terminals show bold colored text in the light color variant
+            # Let's mimic that here
+            if @style_mask & STYLE_SWITCHES[:bold] != 0
+              fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1')
+            end
+            css_classes << fg_color
+          end
+          css_classes << @bg_color unless @bg_color.nil?
+
+          STYLE_SWITCHES.each do |css_class, flag|
+            css_classes << "term-#{css_class}" if @style_mask & flag != 0
+          end
+
+          return if css_classes.empty?
+
+          @out << %{<span class="#{css_classes.join(' ')}">}
+          @n_open_tags += 1
+        end
+
+        def close_open_tags
+          while @n_open_tags > 0
+            @out << %{</span>}
+            @n_open_tags -= 1
+          end
+        end
+
+        def reset_state
+          @offset = 0
+          @n_open_tags = 0
+          @out = ''
+          reset
+        end
+
+        def state
+          state = STATE_PARAMS.inject({}) do |h, param|
+            h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend
+            h
+          end
+          Base64.urlsafe_encode64(state.to_json)
+        end
+
+        def restore_state(new_state, stream)
+          state = Base64.urlsafe_decode64(new_state)
+          state = JSON.parse(state, symbolize_names: true)
+          return if state[:offset].to_i > stream.size
+
+          STATE_PARAMS.each do |param|
+            send("#{param}=".to_sym, state[param]) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+
+        def reset
+          @fg_color = nil
+          @bg_color = nil
+          @style_mask = 0
+        end
+
+        def enable(flag)
+          @style_mask |= flag
+        end
+
+        def disable(flag)
+          @style_mask &= ~flag
+        end
+
+        def set_fg_color(color_index, prefix = nil)
+          @fg_color = get_term_color_class(color_index, ["fg", prefix])
+        end
+
+        def set_bg_color(color_index, prefix = nil)
+          @bg_color = get_term_color_class(color_index, ["bg", prefix])
+        end
+
+        def get_term_color_class(color_index, prefix)
+          color_name = COLOR[color_index]
+          return nil if color_name.nil?
+
+          get_color_class(["term", prefix, color_name])
+        end
+
+        def set_fg_color_256(command_stack)
+          css_class = get_xterm_color_class(command_stack, "fg")
+          @fg_color = css_class unless css_class.nil?
+        end
+
+        def set_bg_color_256(command_stack)
+          css_class = get_xterm_color_class(command_stack, "bg")
+          @bg_color = css_class unless css_class.nil?
+        end
+
+        def get_xterm_color_class(command_stack, prefix)
+          # the 38 and 48 commands have to be followed by "5" and the color index
+          return unless command_stack.length >= 2
+          return unless command_stack[0] == "5"
+
+          command_stack.shift() # ignore the "5" command
+          color_index = command_stack.shift().to_i
+
+          return unless color_index >= 0
+          return unless color_index <= 255
+
+          get_color_class(["xterm", prefix, color_index])
+        end
+
+        def get_color_class(segments)
+          [segments].flatten.compact.join('-')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index 8503ecf8700..ab3408f48d6 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -56,13 +56,13 @@ module Gitlab
         end
 
         def html_with_state(state = nil)
-          ::Ci::Ansi2html.convert(stream, state)
+          ::Gitlab::Ci::Ansi2html.convert(stream, state)
         end
 
         def html(last_lines: nil)
           text = raw(last_lines: last_lines)
           buffer = StringIO.new(text)
-          ::Ci::Ansi2html.convert(buffer).html
+          ::Gitlab::Ci::Ansi2html.convert(buffer).html
         end
 
         def extract_coverage(regex)
-- 
cgit v1.2.1


From f364cc34ea7562e782964b2c65428055d525e440 Mon Sep 17 00:00:00 2001
From: Maxim Rydkin <maks.rydkin@gmail.com>
Date: Wed, 6 Sep 2017 14:11:00 +0300
Subject: move `lib/ci/charts.rb` into `lib/gitlab/ci/charts.rb`

---
 lib/gitlab/ci/charts.rb | 118 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 118 insertions(+)
 create mode 100644 lib/gitlab/ci/charts.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb
new file mode 100644
index 00000000000..ac4bd5d6684
--- /dev/null
+++ b/lib/gitlab/ci/charts.rb
@@ -0,0 +1,118 @@
+module Gitlab
+  module Ci
+    module Charts
+      module DailyInterval
+        def grouped_count(query)
+          query
+            .group("DATE(#{Ci::Pipeline.table_name}.created_at)")
+            .count(:created_at)
+            .transform_keys { |date| date.strftime(@format) }
+        end
+
+        def interval_step
+          @interval_step ||= 1.day
+        end
+      end
+
+      module MonthlyInterval
+        def grouped_count(query)
+          if Gitlab::Database.postgresql?
+            query
+              .group("to_char(#{Ci::Pipeline.table_name}.created_at, '01 Month YYYY')")
+              .count(:created_at)
+              .transform_keys(&:squish)
+          else
+            query
+              .group("DATE_FORMAT(#{Ci::Pipeline.table_name}.created_at, '01 %M %Y')")
+              .count(:created_at)
+          end
+        end
+
+        def interval_step
+          @interval_step ||= 1.month
+        end
+      end
+
+      class Chart
+        attr_reader :labels, :total, :success, :project, :pipeline_times
+
+        def initialize(project)
+          @labels = []
+          @total = []
+          @success = []
+          @pipeline_times = []
+          @project = project
+
+          collect
+        end
+
+        def collect
+          query = project.pipelines
+            .where("? > #{Ci::Pipeline.table_name}.created_at AND #{Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection
+
+          totals_count  = grouped_count(query)
+          success_count = grouped_count(query.success)
+
+          current = @from
+          while current < @to
+            label = current.strftime(@format)
+
+            @labels  << label
+            @total   << (totals_count[label] || 0)
+            @success << (success_count[label] || 0)
+
+            current += interval_step
+          end
+        end
+      end
+
+      class YearChart < Chart
+        include MonthlyInterval
+
+        def initialize(*)
+          @to     = Date.today.end_of_month
+          @from   = @to.years_ago(1).beginning_of_month
+          @format = '%d %B %Y'
+
+          super
+        end
+      end
+
+      class MonthChart < Chart
+        include DailyInterval
+
+        def initialize(*)
+          @to     = Date.today
+          @from   = @to - 30.days
+          @format = '%d %B'
+
+          super
+        end
+      end
+
+      class WeekChart < Chart
+        include DailyInterval
+
+        def initialize(*)
+          @to     = Date.today
+          @from   = @to - 7.days
+          @format = '%d %B'
+
+          super
+        end
+      end
+
+      class PipelineTime < Chart
+        def collect
+          commits = project.pipelines.last(30)
+
+          commits.each do |commit|
+            @labels << commit.short_sha
+            duration = commit.duration || 0
+            @pipeline_times << (duration / 60)
+          end
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From c295d3362b610945de5c722616b39323a48c377e Mon Sep 17 00:00:00 2001
From: Maxim Rydkin <maks.rydkin@gmail.com>
Date: Wed, 6 Sep 2017 14:13:52 +0300
Subject: move `lib/ci/model.rb` into `lib/gitlab/ci/model.rb`

---
 lib/gitlab/ci/model.rb | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 lib/gitlab/ci/model.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb
new file mode 100644
index 00000000000..3994a50772b
--- /dev/null
+++ b/lib/gitlab/ci/model.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  module Ci
+    module Model
+      def table_name_prefix
+        "ci_"
+      end
+
+      def model_name
+        @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last)
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From c45ace8972f18af1f232f9074d6e4104fc4f0c14 Mon Sep 17 00:00:00 2001
From: Maxim Rydkin <maks.rydkin@gmail.com>
Date: Wed, 6 Sep 2017 14:23:24 +0300
Subject: move `lib/ci/gitlab_ci_yaml_processor.rb` into
 `lib/gitlab/ci/yaml_processor.rb`

---
 lib/gitlab/ci/yaml_processor.rb | 253 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 253 insertions(+)
 create mode 100644 lib/gitlab/ci/yaml_processor.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
new file mode 100644
index 00000000000..7582964b24e
--- /dev/null
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -0,0 +1,253 @@
+module Gitlab
+  module Ci
+    class YamlProcessor
+      ValidationError = Class.new(StandardError)
+
+      include Gitlab::Ci::Config::Entry::LegacyValidationHelpers
+
+      attr_reader :path, :cache, :stages, :jobs
+
+      def initialize(config, path = nil)
+        @ci_config = Gitlab::Ci::Config.new(config)
+        @config = @ci_config.to_hash
+        @path = path
+
+        unless @ci_config.valid?
+          raise ValidationError, @ci_config.errors.first
+        end
+
+        initial_parsing
+      rescue Gitlab::Ci::Config::Loader::FormatError => e
+        raise ValidationError, e.message
+      end
+
+      def builds_for_stage_and_ref(stage, ref, tag = false, source = nil)
+        jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _|
+          build_attributes(name)
+        end
+      end
+
+      def builds
+        @jobs.map do |name, _|
+          build_attributes(name)
+        end
+      end
+
+      def stage_seeds(pipeline)
+        seeds = @stages.uniq.map do |stage|
+          builds = pipeline_stage_builds(stage, pipeline)
+
+          Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
+        end
+
+        seeds.compact
+      end
+
+      def build_attributes(name)
+        job = @jobs[name.to_sym] || {}
+
+        { stage_idx: @stages.index(job[:stage]),
+          stage: job[:stage],
+          commands: job[:commands],
+          tag_list: job[:tags] || [],
+          name: job[:name].to_s,
+          allow_failure: job[:ignore],
+          when: job[:when] || 'on_success',
+          environment: job[:environment_name],
+          coverage_regex: job[:coverage],
+          yaml_variables: yaml_variables(name),
+          options: {
+            image: job[:image],
+            services: job[:services],
+            artifacts: job[:artifacts],
+            cache: job[:cache],
+            dependencies: job[:dependencies],
+            before_script: job[:before_script],
+            script: job[:script],
+            after_script: job[:after_script],
+            environment: job[:environment],
+            retry: job[:retry]
+          }.compact }
+      end
+
+      def self.validation_message(content)
+        return 'Please provide content of .gitlab-ci.yml' if content.blank?
+
+        begin
+          Gitlab::Ci::YamlProcessor.new(content)
+          nil
+        rescue ValidationError, Psych::SyntaxError => e
+          e.message
+        end
+      end
+
+      private
+
+      def pipeline_stage_builds(stage, pipeline)
+        builds = builds_for_stage_and_ref(
+          stage, pipeline.ref, pipeline.tag?, pipeline.source)
+
+        builds.select do |build|
+          job = @jobs[build.fetch(:name).to_sym]
+          has_kubernetes = pipeline.has_kubernetes_active?
+          only_kubernetes = job.dig(:only, :kubernetes)
+          except_kubernetes = job.dig(:except, :kubernetes)
+
+          [!only_kubernetes && !except_kubernetes,
+           only_kubernetes && has_kubernetes,
+           except_kubernetes && !has_kubernetes].any?
+        end
+      end
+
+      def jobs_for_ref(ref, tag = false, source = nil)
+        @jobs.select do |_, job|
+          process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source)
+        end
+      end
+
+      def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil)
+        jobs_for_ref(ref, tag, source).select do |_, job|
+          job[:stage] == stage
+        end
+      end
+
+      def initial_parsing
+        ##
+        # Global config
+        #
+        @before_script = @ci_config.before_script
+        @image = @ci_config.image
+        @after_script = @ci_config.after_script
+        @services = @ci_config.services
+        @variables = @ci_config.variables
+        @stages = @ci_config.stages
+        @cache = @ci_config.cache
+
+        ##
+        # Jobs
+        #
+        @jobs = @ci_config.jobs
+
+        @jobs.each do |name, job|
+          # logical validation for job
+
+          validate_job_stage!(name, job)
+          validate_job_dependencies!(name, job)
+          validate_job_environment!(name, job)
+        end
+      end
+
+      def yaml_variables(name)
+        variables = (@variables || {})
+          .merge(job_variables(name))
+
+        variables.map do |key, value|
+          { key: key.to_s, value: value, public: true }
+        end
+      end
+
+      def job_variables(name)
+        job = @jobs[name.to_sym]
+        return {} unless job
+
+        job[:variables] || {}
+      end
+
+      def validate_job_stage!(name, job)
+        return unless job[:stage]
+
+        unless job[:stage].is_a?(String) && job[:stage].in?(@stages)
+          raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}"
+        end
+      end
+
+      def validate_job_dependencies!(name, job)
+        return unless job[:dependencies]
+
+        stage_index = @stages.index(job[:stage])
+
+        job[:dependencies].each do |dependency|
+          raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym]
+
+          unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index
+            raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages"
+          end
+        end
+      end
+
+      def validate_job_environment!(name, job)
+        return unless job[:environment]
+        return unless job[:environment].is_a?(Hash)
+
+        environment = job[:environment]
+        validate_on_stop_job!(name, environment, environment[:on_stop])
+      end
+
+      def validate_on_stop_job!(name, environment, on_stop)
+        return unless on_stop
+
+        on_stop_job = @jobs[on_stop.to_sym]
+        unless on_stop_job
+          raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined"
+        end
+
+        unless on_stop_job[:environment]
+          raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined"
+        end
+
+        unless on_stop_job[:environment][:name] == environment[:name]
+          raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name"
+        end
+
+        unless on_stop_job[:environment][:action] == 'stop'
+          raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined"
+        end
+      end
+
+      def process?(only_params, except_params, ref, tag, source)
+        if only_params.present?
+          return false unless matching?(only_params, ref, tag, source)
+        end
+
+        if except_params.present?
+          return false if matching?(except_params, ref, tag, source)
+        end
+
+        true
+      end
+
+      def matching?(patterns, ref, tag, source)
+        patterns.any? do |pattern|
+          pattern, path = pattern.split('@', 2)
+          matches_path?(path) && matches_pattern?(pattern, ref, tag, source)
+        end
+      end
+
+      def matches_path?(path)
+        return true unless path
+
+        path == self.path
+      end
+
+      def matches_pattern?(pattern, ref, tag, source)
+        return true if tag && pattern == 'tags'
+        return true if !tag && pattern == 'branches'
+        return true if source_to_pattern(source) == pattern
+
+        if pattern.first == "/" && pattern.last == "/"
+          Regexp.new(pattern[1...-1]) =~ ref
+        else
+          pattern == ref
+        end
+      end
+
+      def source_to_pattern(source)
+        if %w[api external web].include?(source)
+          source
+        else
+          source&.pluralize
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From 5b296f81e2f49fc7dc2d3b091340a9a8b86eae97 Mon Sep 17 00:00:00 2001
From: Maxim Rydkin <maks.rydkin@gmail.com>
Date: Wed, 6 Sep 2017 14:26:15 +0300
Subject: move `lib/ci/mask_secret.rb` into `lib/gitlab/ci/mask_secret.rb`

---
 lib/gitlab/ci/mask_secret.rb | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 lib/gitlab/ci/mask_secret.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/ci/mask_secret.rb b/lib/gitlab/ci/mask_secret.rb
new file mode 100644
index 00000000000..0daddaa638c
--- /dev/null
+++ b/lib/gitlab/ci/mask_secret.rb
@@ -0,0 +1,12 @@
+module Gitlab
+  module Ci::MaskSecret
+    class << self
+      def mask!(value, token)
+        return value unless value.present? && token.present?
+
+        value.gsub!(token, 'x' * token.length)
+        value
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From e83a8187510b3c44cfd699109ac0bcf02f693fbd Mon Sep 17 00:00:00 2001
From: Maxim Rydkin <maks.rydkin@gmail.com>
Date: Wed, 6 Sep 2017 14:31:37 +0300
Subject: adds changelog

fix CI

fix CI
---
 lib/gitlab/ci/charts.rb | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb
index ac4bd5d6684..7df7b542d91 100644
--- a/lib/gitlab/ci/charts.rb
+++ b/lib/gitlab/ci/charts.rb
@@ -4,7 +4,7 @@ module Gitlab
       module DailyInterval
         def grouped_count(query)
           query
-            .group("DATE(#{Ci::Pipeline.table_name}.created_at)")
+            .group("DATE(#{::Ci::Pipeline.table_name}.created_at)")
             .count(:created_at)
             .transform_keys { |date| date.strftime(@format) }
         end
@@ -18,12 +18,12 @@ module Gitlab
         def grouped_count(query)
           if Gitlab::Database.postgresql?
             query
-              .group("to_char(#{Ci::Pipeline.table_name}.created_at, '01 Month YYYY')")
+              .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')")
               .count(:created_at)
               .transform_keys(&:squish)
           else
             query
-              .group("DATE_FORMAT(#{Ci::Pipeline.table_name}.created_at, '01 %M %Y')")
+              .group("DATE_FORMAT(#{::Ci::Pipeline.table_name}.created_at, '01 %M %Y')")
               .count(:created_at)
           end
         end
@@ -48,7 +48,7 @@ module Gitlab
 
         def collect
           query = project.pipelines
-            .where("? > #{Ci::Pipeline.table_name}.created_at AND #{Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection
+            .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection
 
           totals_count  = grouped_count(query)
           success_count = grouped_count(query.success)
-- 
cgit v1.2.1


From bb1cf2aaf9d392ce9dfbe235af80d65719dbb343 Mon Sep 17 00:00:00 2001
From: Alexis Reigel <mail@koffeinfrei.org>
Date: Wed, 13 Sep 2017 11:31:37 +0000
Subject: Fix: GPG tmp dir removal race condition

---
 lib/gitlab/gpg.rb | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb
index 025f826e65f..0d5039ddf5f 100644
--- a/lib/gitlab/gpg.rb
+++ b/lib/gitlab/gpg.rb
@@ -69,11 +69,17 @@ module Gitlab
 
     def optimistic_using_tmp_keychain
       previous_dir = current_home_dir
-      Dir.mktmpdir do |dir|
-        GPGME::Engine.home_dir = dir
-        yield
-      end
+      tmp_dir = Dir.mktmpdir
+      GPGME::Engine.home_dir = tmp_dir
+      yield
     ensure
+      # Ignore any errors when removing the tmp directory, as we may run into a
+      # race condition:
+      # The `gpg-agent` agent process may clean up some files as well while
+      # `FileUtils.remove_entry` is iterating the directory and removing all
+      # its contained files and directories recursively, which could raise an
+      # error.
+      FileUtils.remove_entry(tmp_dir, true)
       GPGME::Engine.home_dir = previous_dir
     end
   end
-- 
cgit v1.2.1