Fix for set max entries edge case in setTypeCreate / setTypeMaybeConvert (#12183)

In the judgment in setTypeCreate, we should judge size_hint <= max_entries.

This results in the following inconsistencies:
```
127.0.0.1:6379> config set set-max-intset-entries 5 set-max-listpack-entries 5
OK

127.0.0.1:6379> sadd intset_set1 1 2 3 4 5
(integer) 5
127.0.0.1:6379> object encoding intset_set1
"hashtable"
127.0.0.1:6379> sadd intset_set2 1 2 3 4
(integer) 4
127.0.0.1:6379> sadd intset_set2 5
(integer) 1
127.0.0.1:6379> object encoding intset_set2
"intset"

127.0.0.1:6379> sadd listpack_set1 a 1 2 3 4
(integer) 5
127.0.0.1:6379> object encoding listpack_set1
"hashtable"
127.0.0.1:6379> sadd listpack_set2 a 1 2 3
(integer) 4
127.0.0.1:6379> sadd listpack_set2 4
(integer) 1
127.0.0.1:6379> object encoding listpack_set2
"listpack"
```

This was introduced in #12019, added corresponding tests.

1 files changed, 49 insertions, 2 deletions

diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl
index 2d005c675..768e5cfbf 100644
--- a/tests/unit/type/set.tcl
+++ b/tests/unit/type/set.tcl
@@ -113,14 +113,61 @@ start_server {
         assert_equal {1} [r smismember myset 213244124402402314402033402]
     }
 
-    test "SADD overflows the maximum allowed integers in an intset" {
+foreach type {single multiple single_multiple} {
+    test "SADD overflows the maximum allowed integers in an intset - $type" {
         r del myset
-        for {set i 0} {$i < 512} {incr i} { r sadd myset $i }
+
+        if {$type == "single"} {
+            # All are single sadd commands.
+            for {set i 0} {$i < 512} {incr i} { r sadd myset $i }
+        } elseif {$type == "multiple"} {
+            # One sadd command to add all elements.
+            set args {}
+            for {set i 0} {$i < 512} {incr i} { lappend args $i }
+            r sadd myset {*}$args
+        } elseif {$type == "single_multiple"} {
+            # First one sadd adds an element (creates a key) and then one sadd adds all elements.
+            r sadd myset 1
+            set args {}
+            for {set i 0} {$i < 512} {incr i} { lappend args $i }
+            r sadd myset {*}$args
+        }
+
         assert_encoding intset myset
+        assert_equal 512 [r scard myset]
         assert_equal 1 [r sadd myset 512]
         assert_encoding hashtable myset
     }
 
+    test "SADD overflows the maximum allowed elements in a listpack - $type" {
+        r del myset
+
+        if {$type == "single"} {
+            # All are single sadd commands.
+            r sadd myset a
+            for {set i 0} {$i < 127} {incr i} { r sadd myset $i }
+        } elseif {$type == "multiple"} {
+            # One sadd command to add all elements.
+            set args {}
+            lappend args a
+            for {set i 0} {$i < 127} {incr i} { lappend args $i }
+            r sadd myset {*}$args
+        } elseif {$type == "single_multiple"} {
+            # First one sadd adds an element (creates a key) and then one sadd adds all elements.
+            r sadd myset a
+            set args {}
+            lappend args a
+            for {set i 0} {$i < 127} {incr i} { lappend args $i }
+            r sadd myset {*}$args
+        }
+
+        assert_encoding listpack myset
+        assert_equal 128 [r scard myset]
+        assert_equal 1 [r sadd myset b]
+        assert_encoding hashtable myset
+    }
+}
+
     test {Variadic SADD} {
         r del myset
         assert_equal 3 [r sadd myset a b c]