Merge #7554

7554: Fix ruby version parsing in compact index r=deivid-rodriguez a=kou

### What was the end-user problem that led to this PR?

The end-user can't use multiple Ruby version requirements (e.g. ">= 2.2, < 2.7.dev") provided by each gem to resolve suitable version. 

See also: https://github.com/bundler/bundler/pull/7522#issuecomment-571090513

### What was your diagnosis of the problem?

https://rubygems.org/info/ffi returns the following information:

```text
1.11.3-x86-mingw32 |checksum:99e4845796c8dec1c3fc80dc772860a01633b33291bd7534007f5c7724f0b876,ruby:>= 2.2, < 2.7.dev
```

The current parser doesn't parse the line correctly. It drops "< 2.7.dev" requirement for Ruby version.

It means that the parsed requirement is "ruby >= 2.2" instead of "ruby >=2.2 && ruby < 2.7.dev".

### What is your fix for the problem, implemented in this PR?

I implemented a new parsing logic for "requirements". It assumes that "requirements" has at most three fields: "checksum", "ruby" and "rubygems". Because https://github.com/rubygems/compact_index/blob/master/lib/compact_index/gem_version.rb#L26-L28 implements so.

### Why did you choose this fix out of the possible options?

I chose this fix because it's difficult to implement generic parsing logic without "," escape in version. For example, "," leading to "2.2" and "," leading to "2.7.dev" in "ruby:>= 2.2, < 2.7.dev,rubygems:> 1.3.1" are conflict. We need to know field name such as "ruby" and "rubygems" to detect "," as field delimiter.

"," in "dependencies" are escaped as "&": https://github.com/rubygems/compact_index/blob/master/lib/compact_index/gem_version.rb#L37


Co-authored-by: Sutou Kouhei <kou@clear-code.com>

3 files changed, 245 insertions, 13 deletions

diff --git a/lib/bundler/compact_index_client/cache.rb b/lib/bundler/compact_index_client/cache.rb
index f6105d3bb3..ef8517e8f6 100644
--- a/lib/bundler/compact_index_client/cache.rb
+++ b/lib/bundler/compact_index_client/cache.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "gem_parser"
+
 module Bundler
   class CompactIndexClient
     class Cache
@@ -92,19 +94,9 @@ module Bundler
         header ? lines[header + 1..-1] : lines
       end
 
-      def parse_gem(string)
-        version_and_platform, rest = string.split(" ", 2)
-        version, platform = version_and_platform.split("-", 2)
-        dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest
-        dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : []
-        requirements = requirements ? requirements.map {|r| parse_dependency(r) } : []
-        [version, platform, dependencies, requirements]
-      end
-
-      def parse_dependency(string)
-        dependency = string.split(":")
-        dependency[-1] = dependency[-1].split("&") if dependency.size > 1
-        dependency
+      def parse_gem(line)
+        @gem_parser ||= GemParser.new
+        @gem_parser.parse(line)
       end
 
       def info_roots
diff --git a/lib/bundler/compact_index_client/gem_parser.rb b/lib/bundler/compact_index_client/gem_parser.rb
new file mode 100644
index 0000000000..117da64ce6
--- /dev/null
+++ b/lib/bundler/compact_index_client/gem_parser.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Bundler
+  class CompactIndexClient
+    class GemParser
+      def parse(line)
+        version_and_platform, rest = line.split(" ", 2)
+        version, platform = version_and_platform.split("-", 2)
+        dependencies, requirements = rest.split("|", 2) if rest
+        dependencies = dependencies ? parse_dependencies(dependencies) : []
+        requirements = requirements ? parse_requirements(requirements) : []
+        [version, platform, dependencies, requirements]
+      end
+
+    private
+
+      def parse_dependencies(raw_dependencies)
+        raw_dependencies.split(",").map {|d| parse_dependency(d) }
+      end
+
+      def parse_dependency(raw_dependency)
+        dependency = raw_dependency.split(":")
+        dependency[-1] = dependency[-1].split("&") if dependency.size > 1
+        dependency
+      end
+
+      # Parse the following format:
+      #
+      #   line = "checksum:#{checksum}"
+      #   line << ",ruby:#{ruby_version}" if ruby_version && ruby_version != ">= 0"
+      #   line << ",rubygems:#{rubygems_version}" if rubygems_version && rubygems_version != ">= 0"
+      #
+      # See compact_index/gem_version.rb for details.
+      #
+      # We can't use parse_dependencies for requirements because "," in
+      # ruby_version and rubygems_version isn't escaped as "&". For example,
+      # "checksum:XXX,ruby:>=2.2, < 2.7.dev" can't be parsed as expected.
+      def parse_requirements(raw_requirements)
+        requirements = []
+        checksum = raw_requirements.match(/\A(checksum):([^,]+)/)
+        if checksum
+          requirements << [checksum[1], [checksum[2]]]
+          raw_requirements = checksum.post_match
+          if raw_requirements.start_with?(",")
+            raw_requirements = raw_requirements[1..-1]
+          end
+        end
+        rubygems = raw_requirements.match(/(rubygems):(.+)\z/)
+        if rubygems
+          raw_requirements = rubygems.pre_match
+          if raw_requirements.start_with?(",")
+            raw_requirements = raw_requirements[1..-1]
+          end
+        end
+        ruby = raw_requirements.match(/\A(ruby):(.+)\z/)
+        if ruby
+          requirements << [ruby[1], ruby[2].split(/\s*,\s*/)]
+        end
+        if rubygems
+          requirements << [rubygems[1], rubygems[2].split(/\s*,\s*/)]
+        end
+        requirements
+      end
+    end
+  end
+end
diff --git a/spec/bundler/compact_index_client/gem_parser_spec.rb b/spec/bundler/compact_index_client/gem_parser_spec.rb
new file mode 100644
index 0000000000..5f5305426f
--- /dev/null
+++ b/spec/bundler/compact_index_client/gem_parser_spec.rb
@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require "bundler/compact_index_client/gem_parser"
+
+RSpec.describe Bundler::CompactIndexClient::GemParser do
+  def parse(line)
+    parser = Bundler::CompactIndexClient::GemParser.new
+    parser.parse(line)
+  end
+
+  context "platform" do
+    it "existent" do
+      checksum = "d5956d2bcb509af2cd07c90d9e5fdb331be8845a75bfd823a31c147b52cff471"
+      line = "1.11.3-java |checksum:#{checksum}"
+      expected = [
+        "1.11.3",
+        "java",
+        [],
+        [
+          ["checksum", [checksum]],
+        ],
+      ]
+      expect(parse(line)).to eq expected
+    end
+
+    it "nonexistent" do
+      checksum = "6da2eb3c4867e64df28d3e0b1008422dfacda7c046f9a8f3c56c52505b195e81"
+      line = "1.11.3 |checksum:#{checksum}"
+      expected = [
+        "1.11.3",
+        nil,
+        [],
+        [
+          ["checksum", [checksum]],
+        ],
+      ]
+      expect(parse(line)).to eq expected
+    end
+  end
+
+  context "dependencies" do
+    it "nothing" do
+      checksum = "6da2eb3c4867e64df28d3e0b1008422dfacda7c046f9a8f3c56c52505b195e81"
+      line = "1.11.3 |checksum:#{checksum}"
+      expected = [
+        "1.11.3",
+        nil,
+        [],
+        [
+          ["checksum", [checksum]],
+        ],
+      ]
+      expect(parse(line)).to eq expected
+    end
+
+    it "one" do
+      checksum = "5f0b378d12ab5665e2b6a1525274de97350238963002583cf088dae988527647"
+      line = "0.3.2 bones:>= 2.4.2|checksum:#{checksum}"
+      expected = [
+        "0.3.2",
+        nil,
+        [
+          ["bones", [">= 2.4.2"]],
+        ],
+        [
+          ["checksum", [checksum]],
+        ],
+      ]
+      expect(parse(line)).to eq expected
+    end
+
+    it "multiple" do
+      checksum = "199e892ada86c44d1f2e110b822d5da46b52fa2cbd2f00d89695b4cf610f9927"
+      line = "3.1.2 native-package-installer:>= 0,pkg-config:>= 0|checksum:#{checksum}"
+      expected = [
+        "3.1.2",
+        nil,
+        [
+          ["native-package-installer", [">= 0"]],
+          ["pkg-config", [">= 0"]],
+        ],
+        [
+          ["checksum", [checksum]],
+        ],
+      ]
+      expect(parse(line)).to eq expected
+    end
+
+    context "version" do
+      it "multiple" do
+        checksum = "1ec894b8090cb2c9393153552be2f3b6b1975265cbc1e0a3c6b28ebfea7e76a1"
+        line = "3.1.5 multi_json:< 1.3&>= 1.0|checksum:#{checksum}"
+        expected = [
+          "3.1.5",
+          nil,
+          [
+            ["multi_json", ["< 1.3", ">= 1.0"]],
+          ],
+          [
+            ["checksum", [checksum]],
+          ],
+        ]
+        expect(parse(line)).to eq expected
+      end
+    end
+  end
+
+  context "requirements" do
+    context "ruby" do
+      it "one version" do
+        checksum ="6da2eb3c4867e64df28d3e0b1008422dfacda7c046f9a8f3c56c52505b195e81"
+        line = "1.11.3 |checksum:#{checksum},ruby:>= 2.0"
+        expected = [
+          "1.11.3",
+          nil,
+          [],
+          [
+            ["checksum", [checksum]],
+            ["ruby", [">= 2.0"]],
+          ],
+        ]
+        expect(parse(line)).to eq expected
+      end
+
+      it "multiple versions" do
+        checksum = "99e4845796c8dec1c3fc80dc772860a01633b33291bd7534007f5c7724f0b876"
+        line = "1.11.3-x86-mingw32 |checksum:#{checksum},ruby:>= 2.2, < 2.7.dev"
+        expected = [
+          "1.11.3",
+          "x86-mingw32",
+          [],
+          [
+            ["checksum", [checksum]],
+            ["ruby", [">= 2.2", "< 2.7.dev"]],
+          ],
+        ]
+        expect(parse(line)).to eq expected
+      end
+
+      it "with rubygems" do
+        checksum = "7a82b358f00da749b01f8c84df8e8eb21c1bc389740aab9a2bf4ce59894564ac"
+        line = "1.9.23.pre1 |checksum:#{checksum},ruby:>= 1.9, < 2.7.dev,rubygems:> 1.3.1"
+        expected = [
+          "1.9.23.pre1",
+          nil,
+          [],
+          [
+            ["checksum", [checksum]],
+            ["ruby", [">= 1.9", "< 2.7.dev"]],
+            ["rubygems", ["> 1.3.1"]],
+          ],
+        ]
+        expect(parse(line)).to eq expected
+      end
+    end
+
+    context "rubygems" do
+      it "existent" do
+        checksum = "91ddb4c1b5482a4aff957f6733e282ce2767b2d3051138e0203e39d6df4eba10"
+        line = "1.0.12.pre |checksum:#{checksum},rubygems:> 1.3.1"
+        expected = [
+          "1.0.12.pre",
+          nil,
+          [],
+          [
+            ["checksum", [checksum]],
+            ["rubygems", ["> 1.3.1"]],
+          ],
+        ]
+        expect(parse(line)).to eq expected
+      end
+    end
+  end
+end