Make String#rstrip{,!} raise Encoding::CompatibilityError for broken coderange

It's questionable whether we want to allow rstrip to work for strings
where the broken coderange occurs before the trailing whitespace and
not after, but this approach is probably simpler, and I don't think
users should expect string operations like rstrip to work on broken
strings.

In some cases, this changes rstrip to raise
Encoding::CompatibilityError instead of ArgumentError.  However, as
the problem is related to an encoding issue in the receiver, and due
not due to an issue with an argument, I think
Encoding::CompatibilityError is the more appropriate error.

Fixes [Bug #18931]

4 files changed, 30 insertions, 8 deletions

diff --git a/spec/ruby/core/string/rstrip_spec.rb b/spec/ruby/core/string/rstrip_spec.rb
index 4332b33b20..e96ce4120f 100644
--- a/spec/ruby/core/string/rstrip_spec.rb
+++ b/spec/ruby/core/string/rstrip_spec.rb
@@ -69,13 +69,27 @@ describe "String#rstrip!" do
     -> { "".freeze.rstrip!      }.should raise_error(FrozenError)
   end
 
-  it "raises an ArgumentError if the last non-space codepoint is invalid" do
-    s = "abc\xDF".force_encoding(Encoding::UTF_8)
-    s.valid_encoding?.should be_false
-    -> { s.rstrip! }.should raise_error(ArgumentError)
+  ruby_version_is "3.2" do
+    it "raises an Encoding::CompatibilityError if the last non-space codepoint is invalid" do
+      s = "abc\xDF".force_encoding(Encoding::UTF_8)
+      s.valid_encoding?.should be_false
+      -> { s.rstrip! }.should raise_error(Encoding::CompatibilityError)
 
-    s = "abc\xDF   ".force_encoding(Encoding::UTF_8)
-    s.valid_encoding?.should be_false
-    -> { s.rstrip! }.should raise_error(ArgumentError)
+      s = "abc\xDF   ".force_encoding(Encoding::UTF_8)
+      s.valid_encoding?.should be_false
+      -> { s.rstrip! }.should raise_error(Encoding::CompatibilityError)
+    end
+  end
+
+  ruby_version_is ""..."3.2" do
+    it "raises an ArgumentError if the last non-space codepoint is invalid" do
+      s = "abc\xDF".force_encoding(Encoding::UTF_8)
+      s.valid_encoding?.should be_false
+      -> { s.rstrip! }.should raise_error(ArgumentError)
+
+      s = "abc\xDF   ".force_encoding(Encoding::UTF_8)
+      s.valid_encoding?.should be_false
+      -> { s.rstrip! }.should raise_error(ArgumentError)
+    end
   end
 end
diff --git a/string.c b/string.c
index 0ae36e5c72..20d92896c0 100644
--- a/string.c
+++ b/string.c
@@ -9720,6 +9720,9 @@ rstrip_offset(VALUE str, const char *s, const char *e, rb_encoding *enc)
     const char *t;
 
     rb_str_check_dummy_enc(enc);
+    if (rb_enc_str_coderange(str) == ENC_CODERANGE_BROKEN) {
+        rb_raise(rb_eEncCompatError, "invalid byte sequence in %s", rb_enc_name(enc));
+    }
     if (!s || s >= e) return 0;
     t = e;
 
diff --git a/test/net/http/test_httpheader.rb b/test/net/http/test_httpheader.rb
index b1ca9e8225..fe43ff0405 100644
--- a/test/net/http/test_httpheader.rb
+++ b/test/net/http/test_httpheader.rb
@@ -28,7 +28,7 @@ class HTTPHeaderTest < Test::Unit::TestCase
     assert_raise(NoMethodError){ @c.initialize_http_header("foo"=>[]) }
     assert_raise(ArgumentError){ @c.initialize_http_header("foo"=>"a\nb") }
     assert_raise(ArgumentError){ @c.initialize_http_header("foo"=>"a\rb") }
-    assert_raise(ArgumentError){ @c.initialize_http_header("foo"=>"a\xff") }
+    assert_raise(Encoding::CompatibilityError){ @c.initialize_http_header("foo"=>"a\xff") }
   end
 
   def test_initialize_with_symbol
diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb
index 4add7dfe2f..66db5b565d 100644
--- a/test/ruby/test_string.rb
+++ b/test/ruby/test_string.rb
@@ -2817,6 +2817,11 @@ CODE
     assert_equal("\u3042", s5)
 
     assert_raise(Encoding::CompatibilityError) { S("\u3042".encode("ISO-2022-JP")).rstrip! }
+    assert_raise(Encoding::CompatibilityError) { S("abc \x80 ".force_encoding('UTF-8')).rstrip! }
+    assert_raise(Encoding::CompatibilityError) { S("abc\x80 ".force_encoding('UTF-8')).rstrip! }
+    assert_raise(Encoding::CompatibilityError) { S("abc \x80".force_encoding('UTF-8')).rstrip! }
+    assert_raise(Encoding::CompatibilityError) { S("\x80".force_encoding('UTF-8')).rstrip! }
+    assert_raise(Encoding::CompatibilityError) { S(" \x80 ".force_encoding('UTF-8')).rstrip! }
   end
 
   def test_lstrip