Add latest changes from gitlab-org/gitlab@master

10 files changed, 374 insertions, 21 deletions

diff --git a/lib/api/entities/label_basic.rb b/lib/api/entities/label_basic.rb
index ed52688638e..7c846180558 100644
--- a/lib/api/entities/label_basic.rb
+++ b/lib/api/entities/label_basic.rb
@@ -3,7 +3,11 @@
 module API
   module Entities
     class LabelBasic < Grape::Entity
-      expose :id, :name, :color, :description, :description_html, :text_color
+      expose :id, :name, :description, :description_html, :text_color
+
+      expose :color do |label, options|
+        label.color.to_s
+      end
     end
   end
 end
diff --git a/lib/api/validations/validators/file_path.rb b/lib/api/validations/validators/file_path.rb
index 246c445658f..268ddc29d4e 100644
--- a/lib/api/validations/validators/file_path.rb
+++ b/lib/api/validations/validators/file_path.rb
@@ -8,8 +8,7 @@ module API
           options = @option.is_a?(Hash) ? @option : {}
           path_allowlist = options.fetch(:allowlist, [])
           path = params[attr_name]
-          path = Gitlab::Utils.check_path_traversal!(path)
-          Gitlab::Utils.check_allowed_absolute_path!(path, path_allowlist)
+          Gitlab::Utils.check_allowed_absolute_path_and_path_traversal!(path, path_allowlist)
         rescue StandardError
           raise Grape::Exceptions::Validation.new(
             params: [@scope.full_name(attr_name)],
diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb
new file mode 100644
index 00000000000..3843356af69
--- /dev/null
+++ b/lib/gitlab/background_migration/encrypt_integration_properties.rb
@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Migrates the integration.properties column from plaintext to encrypted text.
+    class EncryptIntegrationProperties
+      # The Integration model, with just the relevant bits.
+      class Integration < ActiveRecord::Base
+        include EachBatch
+
+        ALGORITHM = 'aes-256-gcm'
+
+        self.table_name = 'integrations'
+        self.inheritance_column = :_type_disabled
+
+        scope :with_properties, -> { where.not(properties: nil) }
+        scope :not_already_encrypted, -> { where(encrypted_properties: nil) }
+        scope :for_batch, ->(range) { where(id: range) }
+
+        attr_encrypted :encrypted_properties_tmp,
+                       attribute: :encrypted_properties,
+                       mode: :per_attribute_iv,
+                       key: ::Settings.attr_encrypted_db_key_base_32,
+                       algorithm: ALGORITHM,
+                       marshal: true,
+                       marshaler: ::Gitlab::Json,
+                       encode: false,
+                       encode_iv: false
+
+        # See 'Integration#reencrypt_properties'
+        def encrypt_properties
+          data = ::Gitlab::Json.parse(properties)
+          iv = generate_iv(ALGORITHM)
+          ep = self.class.encrypt(:encrypted_properties_tmp, data, { iv: iv })
+
+          [ep, iv]
+        end
+      end
+
+      def perform(start_id, stop_id)
+        batch_query = Integration.with_properties.not_already_encrypted.for_batch(start_id..stop_id)
+        encrypt_batch(batch_query)
+        mark_job_as_succeeded(start_id, stop_id)
+      end
+
+      private
+
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+          self.class.name.demodulize,
+          arguments
+        )
+      end
+
+      # represent binary string as a PSQL binary literal:
+      # https://www.postgresql.org/docs/9.4/datatype-binary.html
+      def bytea(value)
+        "'\\x#{value.unpack1('H*')}'::bytea"
+      end
+
+      def encrypt_batch(batch_query)
+        values = batch_query.select(:id, :properties).map do |record|
+          encrypted_properties, encrypted_properties_iv = record.encrypt_properties
+          "(#{record.id}, #{bytea(encrypted_properties)}, #{bytea(encrypted_properties_iv)})"
+        end
+
+        return if values.empty?
+
+        Integration.connection.execute(<<~SQL.squish)
+          WITH cte(cte_id, cte_encrypted_properties, cte_encrypted_properties_iv)
+            AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
+            SELECT *
+            FROM (VALUES #{values.join(',')}) AS t (id, encrypted_properties, encrypted_properties_iv)
+          )
+          UPDATE #{Integration.table_name}
+          SET encrypted_properties = cte_encrypted_properties
+            , encrypted_properties_iv = cte_encrypted_properties_iv
+          FROM cte
+          WHERE cte_id = id
+        SQL
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb
index f57959f9547..ff8641c014d 100644
--- a/lib/gitlab/ci/parsers/security/common.rb
+++ b/lib/gitlab/ci/parsers/security/common.rb
@@ -42,14 +42,19 @@ module Gitlab
           attr_reader :json_data, :report, :validate
 
           def valid?
-            if Feature.enabled?(:enforce_security_report_validation)
-              if !validate || schema_validator.valid?
-                report.schema_validation_status = :valid_schema
-                true
+            if Feature.enabled?(:show_report_validation_warnings)
+              # We want validation to happen regardless of VALIDATE_SCHEMA CI variable
+              schema_validation_passed = schema_validator.valid?
+
+              if validate
+                schema_validator.errors.each { |error| report.add_error('Schema', error) } unless schema_validation_passed
+
+                schema_validation_passed
               else
-                report.schema_validation_status = :invalid_schema
-                schema_validator.errors.each { |error| report.add_error('Schema', error) }
-                false
+                # We treat all schema validation errors as warnings
+                schema_validator.errors.each { |error| report.add_warning('Schema', error) }
+
+                true
               end
             else
               return true if !validate || schema_validator.valid?
diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb
index fbf8c81ac36..8c528056d0c 100644
--- a/lib/gitlab/ci/reports/security/report.rb
+++ b/lib/gitlab/ci/reports/security/report.rb
@@ -6,7 +6,7 @@ module Gitlab
       module Security
         class Report
           attr_reader :created_at, :type, :pipeline, :findings, :scanners, :identifiers
-          attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version, :schema_validation_status
+          attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version, :schema_validation_status, :warnings
 
           delegate :project_id, to: :pipeline
 
@@ -19,6 +19,7 @@ module Gitlab
             @identifiers = {}
             @scanned_resources = []
             @errors = []
+            @warnings = []
           end
 
           def commit_sha
@@ -29,6 +30,10 @@ module Gitlab
             errors << { type: type, message: message }
           end
 
+          def add_warning(type, message)
+            warnings << { type: type, message: message }
+          end
+
           def errored?
             errors.present?
           end
diff --git a/lib/gitlab/color.rb b/lib/gitlab/color.rb
new file mode 100644
index 00000000000..e0caabb0ec6
--- /dev/null
+++ b/lib/gitlab/color.rb
@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class Color
+    PATTERN = /\A\#(?:[0-9A-Fa-f]{3}){1,2}\Z/.freeze
+
+    def initialize(value)
+      @value = value&.strip&.freeze
+    end
+
+    module Constants
+      DARK = Color.new('#333333')
+      LIGHT = Color.new('#FFFFFF')
+
+      COLOR_NAME_TO_HEX = {
+        black: '#000000',
+        silver: '#C0C0C0',
+        gray: '#808080',
+        white: '#FFFFFF',
+        maroon: '#800000',
+        red: '#FF0000',
+        purple: '#800080',
+        fuchsia: '#FF00FF',
+        green: '#008000',
+        lime: '#00FF00',
+        olive: '#808000',
+        yellow: '#FFFF00',
+        navy: '#000080',
+        blue: '#0000FF',
+        teal: '#008080',
+        aqua: '#00FFFF',
+        orange: '#FFA500',
+        aliceblue: '#F0F8FF',
+        antiquewhite: '#FAEBD7',
+        aquamarine: '#7FFFD4',
+        azure: '#F0FFFF',
+        beige: '#F5F5DC',
+        bisque: '#FFE4C4',
+        blanchedalmond: '#FFEBCD',
+        blueviolet: '#8A2BE2',
+        brown: '#A52A2A',
+        burlywood: '#DEB887',
+        cadetblue: '#5F9EA0',
+        chartreuse: '#7FFF00',
+        chocolate: '#D2691E',
+        coral: '#FF7F50',
+        cornflowerblue: '#6495ED',
+        cornsilk: '#FFF8DC',
+        crimson: '#DC143C',
+        darkblue: '#00008B',
+        darkcyan: '#008B8B',
+        darkgoldenrod: '#B8860B',
+        darkgray: '#A9A9A9',
+        darkgreen: '#006400',
+        darkgrey: '#A9A9A9',
+        darkkhaki: '#BDB76B',
+        darkmagenta: '#8B008B',
+        darkolivegreen: '#556B2F',
+        darkorange: '#FF8C00',
+        darkorchid: '#9932CC',
+        darkred: '#8B0000',
+        darksalmon: '#E9967A',
+        darkseagreen: '#8FBC8F',
+        darkslateblue: '#483D8B',
+        darkslategray: '#2F4F4F',
+        darkslategrey: '#2F4F4F',
+        darkturquoise: '#00CED1',
+        darkviolet: '#9400D3',
+        deeppink: '#FF1493',
+        deepskyblue: '#00BFFF',
+        dimgray: '#696969',
+        dimgrey: '#696969',
+        dodgerblue: '#1E90FF',
+        firebrick: '#B22222',
+        floralwhite: '#FFFAF0',
+        forestgreen: '#228B22',
+        gainsboro: '#DCDCDC',
+        ghostwhite: '#F8F8FF',
+        gold: '#FFD700',
+        goldenrod: '#DAA520',
+        greenyellow: '#ADFF2F',
+        grey: '#808080',
+        honeydew: '#F0FFF0',
+        hotpink: '#FF69B4',
+        indianred: '#CD5C5C',
+        indigo: '#4B0082',
+        ivory: '#FFFFF0',
+        khaki: '#F0E68C',
+        lavender: '#E6E6FA',
+        lavenderblush: '#FFF0F5',
+        lawngreen: '#7CFC00',
+        lemonchiffon: '#FFFACD',
+        lightblue: '#ADD8E6',
+        lightcoral: '#F08080',
+        lightcyan: '#E0FFFF',
+        lightgoldenrodyellow: '#FAFAD2',
+        lightgray: '#D3D3D3',
+        lightgreen: '#90EE90',
+        lightgrey: '#D3D3D3',
+        lightpink: '#FFB6C1',
+        lightsalmon: '#FFA07A',
+        lightseagreen: '#20B2AA',
+        lightskyblue: '#87CEFA',
+        lightslategray: '#778899',
+        lightslategrey: '#778899',
+        lightsteelblue: '#B0C4DE',
+        lightyellow: '#FFFFE0',
+        limegreen: '#32CD32',
+        linen: '#FAF0E6',
+        mediumaquamarine: '#66CDAA',
+        mediumblue: '#0000CD',
+        mediumorchid: '#BA55D3',
+        mediumpurple: '#9370DB',
+        mediumseagreen: '#3CB371',
+        mediumslateblue: '#7B68EE',
+        mediumspringgreen: '#00FA9A',
+        mediumturquoise: '#48D1CC',
+        mediumvioletred: '#C71585',
+        midnightblue: '#191970',
+        mintcream: '#F5FFFA',
+        mistyrose: '#FFE4E1',
+        moccasin: '#FFE4B5',
+        navajowhite: '#FFDEAD',
+        oldlace: '#FDF5E6',
+        olivedrab: '#6B8E23',
+        orangered: '#FF4500',
+        orchid: '#DA70D6',
+        palegoldenrod: '#EEE8AA',
+        palegreen: '#98FB98',
+        paleturquoise: '#AFEEEE',
+        palevioletred: '#DB7093',
+        papayawhip: '#FFEFD5',
+        peachpuff: '#FFDAB9',
+        peru: '#CD853F',
+        pink: '#FFC0CB',
+        plum: '#DDA0DD',
+        powderblue: '#B0E0E6',
+        rosybrown: '#BC8F8F',
+        royalblue: '#4169E1',
+        saddlebrown: '#8B4513',
+        salmon: '#FA8072',
+        sandybrown: '#F4A460',
+        seagreen: '#2E8B57',
+        seashell: '#FFF5EE',
+        sienna: '#A0522D',
+        skyblue: '#87CEEB',
+        slateblue: '#6A5ACD',
+        slategray: '#708090',
+        slategrey: '#708090',
+        snow: '#FFFAFA',
+        springgreen: '#00FF7F',
+        steelblue: '#4682B4',
+        tan: '#D2B48C',
+        thistle: '#D8BFD8',
+        tomato: '#FF6347',
+        turquoise: '#40E0D0',
+        violet: '#EE82EE',
+        wheat: '#F5DEB3',
+        whitesmoke: '#F5F5F5',
+        yellowgreen: '#9ACD32',
+        rebeccapurple: '#663399'
+      }.stringify_keys.transform_values { Color.new(_1) }.freeze
+    end
+
+    def self.of(color)
+      raise ArgumentError, 'No color spec' unless color
+      return color if color.is_a?(self)
+
+      color = color.to_s.strip
+      Constants::COLOR_NAME_TO_HEX[color.downcase] || new(color)
+    end
+
+    def to_s
+      @value.to_s
+    end
+
+    def as_json(_options = nil)
+      to_s
+    end
+
+    def eql(other)
+      return false unless other.is_a?(self.class)
+
+      to_s == other.to_s
+    end
+    alias_method :==, :eql
+
+    def valid?
+      PATTERN.match?(@value)
+    end
+
+    def light?
+      valid? && rgb.sum > 500
+    end
+
+    def luminosity
+      return :light if light?
+
+      :dark
+    end
+
+    def contrast
+      return Constants::DARK if light?
+
+      Constants::LIGHT
+    end
+
+    private
+
+    def rgb
+      return [] unless valid?
+
+      @rgb ||= begin
+        if @value.length == 4
+          @value[1, 4].scan(/./).map { |v| (v * 2).hex }
+        else
+          @value[1, 7].scan(/.{2}/).map(&:hex)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/type/color.rb b/lib/gitlab/database/type/color.rb
new file mode 100644
index 00000000000..32ea33a42f3
--- /dev/null
+++ b/lib/gitlab/database/type/color.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Type
+      class Color < ActiveModel::Type::Value
+        def serialize(value)
+          value.to_s if value
+        end
+
+        def serializable?(value)
+          value.nil? || value.is_a?(::String) || value.is_a?(::Gitlab::Color)
+        end
+
+        def cast_value(value)
+          ::Gitlab::Color.new(value.to_s)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb
index 368b621bdfb..9824b46554f 100644
--- a/lib/gitlab/json.rb
+++ b/lib/gitlab/json.rb
@@ -16,6 +16,9 @@ module Gitlab
       # @return [Boolean, String, Array, Hash]
       # @raise [JSON::ParserError] raised if parsing fails
       def parse(string, opts = {})
+        # Parse nil as nil
+        return if string.nil?
+
         # First we should ensure this really is a string, not some other
         # type which purports to be a string. This handles some legacy
         # usage of the JSON class.
@@ -30,6 +33,7 @@ module Gitlab
       end
 
       alias_method :parse!, :parse
+      alias_method :load, :parse
 
       # Restricted method for converting a Ruby object to JSON. If you
       # need to pass options to this, you should use `.generate` instead,
@@ -67,6 +71,14 @@ module Gitlab
         ::JSON.pretty_generate(object, opts)
       end
 
+      # The standard parser error we should be returning. Defined in a method
+      # so we can potentially override it later.
+      #
+      # @return [JSON::ParserError]
+      def parser_error
+        ::JSON::ParserError
+      end
+
       private
 
       # Convert JSON string into Ruby through toggleable adapters.
@@ -134,14 +146,6 @@ module Gitlab
         opts
       end
 
-      # The standard parser error we should be returning. Defined in a method
-      # so we can potentially override it later.
-      #
-      # @return [JSON::ParserError]
-      def parser_error
-        ::JSON::ParserError
-      end
-
       # @param [Nil, Boolean] an extracted :legacy_mode key from the opts hash
       # @return [Boolean]
       def legacy_mode_enabled?(arg_value)
diff --git a/lib/gitlab/performance_bar/stats.rb b/lib/gitlab/performance_bar/stats.rb
index cf524e69454..8743772eef6 100644
--- a/lib/gitlab/performance_bar/stats.rb
+++ b/lib/gitlab/performance_bar/stats.rb
@@ -25,8 +25,8 @@ module Gitlab
         log_queries(id, data, 'active-record')
         log_queries(id, data, 'gitaly')
         log_queries(id, data, 'redis')
-      rescue StandardError => err
-        logger.error(message: "failed to process request id #{id}: #{err.message}")
+      rescue StandardError => e
+        logger.error(message: "failed to process request id #{id}: #{e.message}")
       end
 
       private
@@ -34,6 +34,8 @@ module Gitlab
       def request(id)
         # Peek gem stores request data under peek:requests:request_id key
         json_data = @redis.get("peek:requests:#{id}")
+        raise "No data for #{id}" if json_data.nil?
+
         Gitlab::Json.parse(json_data)
       end
 
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index 608545baf74..a337fcc43c4 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -37,6 +37,13 @@ module Gitlab
       raise StandardError, "path #{path} is not allowed"
     end
 
+    def check_allowed_absolute_path_and_path_traversal!(path, path_allowlist)
+      traversal_path = check_path_traversal!(path)
+      raise StandardError, "path is not a string!" unless traversal_path.is_a?(String)
+
+      check_allowed_absolute_path!(traversal_path, path_allowlist)
+    end
+
     def decode_path(encoded_path)
       decoded = CGI.unescape(encoded_path)
       if decoded != CGI.unescape(decoded)