Attempt to solve MacOS CI issues in GH Actions (#12013)

The MacOS CI in github actions often hangs without any logs. GH argues that
it's due to resource utilization, either running out of disk space, memory, or CPU
starvation, and thus the runner is terminated.

This PR contains multiple attempts to resolve this:
1. introducing pause_process instead of SIGSTOP, which waits for the process
  to stop before resuming the test, possibly resolving race conditions in some tests,
  this was a suspect since there was one test that could result in an infinite loop in that
 case, in practice this didn't help, but still a good idea to keep.
2. disable the `save` config in many tests that don't need it, specifically ones that use
  heavy writes and could create large files.
3. change the `populate` proc to use short pipeline rather than an infinite one.
4. use `--clients 1` in the macos CI so that we don't risk running multiple resource
  demanding tests in parallel.
5. enable `--verbose` to be repeated to elevate verbosity and print more info to stdout
  when a test or a server starts.

9 files changed, 68 insertions, 72 deletions

diff --git a/tests/integration/aof-multi-part.tcl b/tests/integration/aof-multi-part.tcl
index 74f6b4949..1d41a8a83 100644
--- a/tests/integration/aof-multi-part.tcl
+++ b/tests/integration/aof-multi-part.tcl
@@ -755,7 +755,7 @@ tags {"external:skip"} {
     # writing pressure, etc.
 
 
-    start_server {tags {"Multi Part AOF"} overrides {aof-use-rdb-preamble {yes} appendonly {no}}} {
+    start_server {tags {"Multi Part AOF"} overrides {aof-use-rdb-preamble {yes} appendonly {no} save {}}} {
         set dir [get_redis_dir]
         set aof_basename "appendonly.aof"
         set aof_dirname "appendonlydir"
@@ -1173,7 +1173,7 @@ tags {"external:skip"} {
             assert {$d1 eq $d2}
         }
 
-        start_server {overrides {aof-use-rdb-preamble {yes} appendonly {no}}} {
+        start_server {overrides {aof-use-rdb-preamble {yes} appendonly {no} save {}}} {
             set dir [get_redis_dir]
             set aof_basename "appendonly.aof"
             set aof_dirname "appendonlydir"
diff --git a/tests/integration/block-repl.tcl b/tests/integration/block-repl.tcl
index 3f3a86ed8..52b4a53ea 100644
--- a/tests/integration/block-repl.tcl
+++ b/tests/integration/block-repl.tcl
@@ -12,7 +12,7 @@ proc stop_bg_block_op {handle} {
 }
 
 start_server {tags {"repl" "external:skip"}} {
-    start_server {} {
+    start_server {overrides {save {}}} {
         set master [srv -1 client]
         set master_host [srv -1 host]
         set master_port [srv -1 port]
diff --git a/tests/integration/failover.tcl b/tests/integration/failover.tcl
index 2cd944851..21fa3d281 100644
--- a/tests/integration/failover.tcl
+++ b/tests/integration/failover.tcl
@@ -1,6 +1,6 @@
-start_server {tags {"failover external:skip"}} {
-start_server {} {
-start_server {} {
+start_server {tags {"failover external:skip"} overrides {save {}}} {
+start_server {overrides {save {}}} {
+start_server {overrides {save {}}} {
     set node_0 [srv 0 client]
     set node_0_host [srv 0 host]
     set node_0_port [srv 0 port]
@@ -66,13 +66,13 @@ start_server {} {
 
         # Generate a delta between primary and replica
         set load_handler [start_write_load $node_0_host $node_0_port 5]
-        exec kill -SIGSTOP [srv -1 pid]
+        pause_process [srv -1 pid]
         wait_for_condition 50 100 {
             [s 0 total_commands_processed] > 100
         } else {
             fail "Node 0 did not accept writes"
         }
-        exec kill -SIGCONT [srv -1 pid]
+        resume_process [srv -1 pid]
 
         # Execute the failover
         $node_0 failover to $node_1_host $node_1_port
@@ -108,7 +108,7 @@ start_server {} {
 
         wait_for_ofs_sync $node_1 $node_2
         # We stop node 0 to and make sure node 2 is selected
-        exec kill -SIGSTOP $node_0_pid
+        pause_process $node_0_pid
         $node_1 set CASE 1
         $node_1 FAILOVER
 
@@ -118,7 +118,7 @@ start_server {} {
         } else {
             fail "Failover from node 1 to node 2 did not finish"
         }
-        exec kill -SIGCONT $node_0_pid
+        resume_process $node_0_pid
         $node_0 replicaof $node_2_host $node_2_port
 
         wait_for_sync $node_0
@@ -138,7 +138,7 @@ start_server {} {
         set initial_psyncs [s 0 sync_partial_ok]
         set initial_syncs [s 0 sync_full]
 
-        exec kill -SIGSTOP $node_0_pid
+        pause_process $node_0_pid
         # node 0 will never acknowledge this write
         $node_2 set case 2
         $node_2 failover to $node_0_host $node_0_port TIMEOUT 100 FORCE
@@ -155,7 +155,7 @@ start_server {} {
         assert_match *slave* [$node_1 role]
         assert_match *slave* [$node_2 role]
 
-        exec kill -SIGCONT $node_0_pid
+        resume_process $node_0_pid
 
         # Wait for failover to end
         wait_for_condition 50 100 {
@@ -186,7 +186,7 @@ start_server {} {
         set initial_syncs [s 0 sync_full]
 
         # Stop replica so it never catches up
-        exec kill -SIGSTOP [srv -1 pid]
+        pause_process [srv -1 pid]
         $node_0 SET CASE 1
         
         $node_0 failover to [srv -1 host] [srv -1 port] TIMEOUT 500
@@ -197,7 +197,7 @@ start_server {} {
             fail "Failover from node_0 to replica did not finish"
         }
 
-        exec kill -SIGCONT [srv -1 pid]
+        resume_process [srv -1 pid]
 
         # We need to make sure the nodes actually sync back up
         wait_for_ofs_sync $node_0 $node_1
@@ -218,7 +218,7 @@ start_server {} {
         set initial_syncs [s 0 sync_full]
     
         # Stop replica so it never catches up
-        exec kill -SIGSTOP [srv -1 pid]
+        pause_process [srv -1 pid]
         $node_0 SET CASE 2
         
         $node_0 failover to [srv -1 host] [srv -1 port] TIMEOUT 60000
@@ -230,7 +230,7 @@ start_server {} {
         $node_0 failover abort
         assert_match [s 0 master_failover_state] "no-failover"
 
-        exec kill -SIGCONT [srv -1 pid]
+        resume_process [srv -1 pid]
 
         # Just make sure everything is still synced
         wait_for_ofs_sync $node_0 $node_1
@@ -255,11 +255,11 @@ start_server {} {
 
         # We pause the target long enough to send a write command
         # during the pause. This write will not be interrupted.
-        exec kill -SIGSTOP [srv -1 pid]
+        pause_process [srv -1 pid]
         set rd [redis_deferring_client]
         $rd SET FOO BAR
         $node_0 failover to $node_1_host $node_1_port
-        exec kill -SIGCONT [srv -1 pid]
+        resume_process [srv -1 pid]
 
         # Wait for failover to end
         wait_for_condition 50 100 {
diff --git a/tests/integration/replication-2.tcl b/tests/integration/replication-2.tcl
index f9f259211..c18ff24fc 100644
--- a/tests/integration/replication-2.tcl
+++ b/tests/integration/replication-2.tcl
@@ -42,7 +42,7 @@ start_server {tags {"repl external:skip"}} {
         test {No write if min-slaves-max-lag is > of the slave lag} {
             r config set min-slaves-to-write 1
             r config set min-slaves-max-lag 2
-            exec kill -SIGSTOP [srv -1 pid]
+            pause_process [srv -1 pid]
             assert {[r set foo 12345] eq {OK}}
             wait_for_condition 100 100 {
                 [catch {r set foo 12345}] != 0
@@ -52,7 +52,7 @@ start_server {tags {"repl external:skip"}} {
             catch {r set foo 12345} err
             assert_match {NOREPLICAS*} $err
         }
-        exec kill -SIGCONT [srv -1 pid]
+        resume_process [srv -1 pid]
 
         test {min-slaves-to-write is ignored by slaves} {
             r config set min-slaves-to-write 1
diff --git a/tests/integration/replication-4.tcl b/tests/integration/replication-4.tcl
index f772eccb2..867ef364e 100644
--- a/tests/integration/replication-4.tcl
+++ b/tests/integration/replication-4.tcl
@@ -1,5 +1,5 @@
-start_server {tags {"repl network external:skip singledb:skip"}} {
-    start_server {} {
+start_server {tags {"repl network external:skip singledb:skip"} overrides {save {}}} {
+    start_server { overrides {save {}}} {
 
         set master [srv -1 client]
         set master_host [srv -1 host]
@@ -104,7 +104,7 @@ start_server {tags {"repl external:skip"}} {
             assert_equal OK [$master set foo 123]
             assert_equal OK [$master eval "return redis.call('set','foo',12345)" 0]
             # Killing a slave to make it become a lagged slave.
-            exec kill -SIGSTOP [srv 0 pid]
+            pause_process [srv 0 pid]
             # Waiting for slave kill.
             wait_for_condition 100 100 {
                 [catch {$master set foo 123}] != 0
@@ -113,7 +113,7 @@ start_server {tags {"repl external:skip"}} {
             }
             assert_error "*NOREPLICAS*" {$master set foo 123}
             assert_error "*NOREPLICAS*" {$master eval "return redis.call('set','foo',12345)" 0}
-            exec kill -SIGCONT [srv 0 pid]
+            resume_process [srv 0 pid]
         }
     }
 }
@@ -146,12 +146,12 @@ start_server {tags {"repl external:skip"}} {
             $master debug set-active-expire 0
             $master set k 1 px $px_ms
             wait_for_ofs_sync $master $slave
-            exec kill -SIGSTOP [srv 0 pid]
+            pause_process [srv 0 pid]
             $master incr k
             after [expr $px_ms + 1]
             # Stopping the replica for one second to makes sure the INCR arrives
             # to the replica after the key is logically expired.
-            exec kill -SIGCONT [srv 0 pid]
+            resume_process [srv 0 pid]
             wait_for_ofs_sync $master $slave
             # Check that k is logically expired but is present in the replica.
             set res [$slave exists k]
diff --git a/tests/integration/replication-buffer.tcl b/tests/integration/replication-buffer.tcl
index 2e402480d..143dc74aa 100644
--- a/tests/integration/replication-buffer.tcl
+++ b/tests/integration/replication-buffer.tcl
@@ -159,7 +159,7 @@ start_server {} {
         assert {[s repl_backlog_histlen] > [expr 2*10000*10000]}
         assert_equal [s connected_slaves] {2}
 
-        exec kill -SIGSTOP $replica2_pid
+        pause_process $replica2_pid
         r config set client-output-buffer-limit "replica 128k 0 0"
         # trigger output buffer limit check
         r set key [string repeat A [expr 64*1024]]
@@ -178,7 +178,7 @@ start_server {} {
         } else {
             fail "Replication backlog memory is not smaller"
         }
-        exec kill -SIGCONT $replica2_pid
+        resume_process $replica2_pid
     }
     # speed up termination
     $master config set shutdown-timeout 0
diff --git a/tests/integration/replication-psync.tcl b/tests/integration/replication-psync.tcl
index 16f3b8889..dc1df0fa6 100644
--- a/tests/integration/replication-psync.tcl
+++ b/tests/integration/replication-psync.tcl
@@ -9,8 +9,8 @@
 # reconnect with the master, otherwise just the initial synchronization is
 # checked for consistency.
 proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reconnect} {
-    start_server {tags {"repl"}} {
-        start_server {} {
+    start_server {tags {"repl"} overrides {save {}}} {
+        start_server {overrides {save {}}} {
 
             set master [srv -1 client]
             set master_host [srv -1 host]
diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index b4e9ee673..de4d527f4 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -302,7 +302,7 @@ start_server {tags {"repl external:skip"}} {
 
 foreach mdl {no yes} {
     foreach sdl {disabled swapdb} {
-        start_server {tags {"repl external:skip"}} {
+        start_server {tags {"repl external:skip"} overrides {save {}}} {
             set master [srv 0 client]
             $master config set repl-diskless-sync $mdl
             $master config set repl-diskless-sync-delay 5
@@ -310,11 +310,11 @@ foreach mdl {no yes} {
             set master_host [srv 0 host]
             set master_port [srv 0 port]
             set slaves {}
-            start_server {} {
+            start_server {overrides {save {}}} {
                 lappend slaves [srv 0 client]
-                start_server {} {
+                start_server {overrides {save {}}} {
                     lappend slaves [srv 0 client]
-                    start_server {} {
+                    start_server {overrides {save {}}} {
                         lappend slaves [srv 0 client]
                         test "Connect multiple replicas at the same time (issue #141), master diskless=$mdl, replica diskless=$sdl" {
                             # start load handles only inside the test, so that the test can be skipped
@@ -391,11 +391,11 @@ foreach mdl {no yes} {
     }
 }
 
-start_server {tags {"repl external:skip"}} {
+start_server {tags {"repl external:skip"} overrides {save {}}} {
     set master [srv 0 client]
     set master_host [srv 0 host]
     set master_port [srv 0 port]
-    start_server {} {
+    start_server {overrides {save {}}} {
         test "Master stream is correctly processed while the replica has a script in -BUSY state" {
             set load_handle0 [start_write_load $master_host $master_port 3]
             set slave [srv 0 client]
@@ -705,11 +705,11 @@ foreach testType {Successful Aborted} {
 }
 
 test {diskless loading short read} {
-    start_server {tags {"repl"}} {
+    start_server {tags {"repl"} overrides {save ""}} {
         set replica [srv 0 client]
         set replica_host [srv 0 host]
         set replica_port [srv 0 port]
-        start_server {} {
+        start_server {overrides {save ""}} {
             set master [srv 0 client]
             set master_host [srv 0 host]
             set master_port [srv 0 port]
@@ -847,7 +847,7 @@ proc compute_cpu_usage {start end} {
 
 
 # test diskless rdb pipe with multiple replicas, which may drop half way
-start_server {tags {"repl external:skip"}} {
+start_server {tags {"repl external:skip"} overrides {save ""}} {
     set master [srv 0 client]
     $master config set repl-diskless-sync yes
     $master config set repl-diskless-sync-delay 5
@@ -868,10 +868,10 @@ start_server {tags {"repl external:skip"}} {
             set replicas {}
             set replicas_alive {}
             # start one replica that will read the rdb fast, and one that will be slow
-            start_server {} {
+            start_server {overrides {save ""}} {
                 lappend replicas [srv 0 client]
                 lappend replicas_alive [srv 0 client]
-                start_server {} {
+                start_server {overrides {save ""}} {
                     lappend replicas [srv 0 client]
                     lappend replicas_alive [srv 0 client]
 
@@ -913,7 +913,7 @@ start_server {tags {"repl external:skip"}} {
                     if {$all_drop == "timeout"} {
                         $master config set repl-timeout 2
                         # we want the slow replica to hang on a key for very long so it'll reach repl-timeout
-                        exec kill -SIGSTOP [srv -1 pid]
+                        pause_process [srv -1 pid]
                         after 2000
                     }
 
@@ -940,7 +940,7 @@ start_server {tags {"repl external:skip"}} {
                         # master disconnected the slow replica, remove from array
                         set replicas_alive [lreplace $replicas_alive 0 0]
                         # release it
-                        exec kill -SIGCONT [srv -1 pid]
+                        resume_process [srv -1 pid]
                     }
 
                     # make sure we don't have a busy loop going thought epoll_wait
@@ -1000,7 +1000,7 @@ test "diskless replication child being killed is collected" {
     # when diskless master is waiting for the replica to become writable
     # it removes the read event from the rdb pipe so if the child gets killed
     # the replica will hung. and the master may not collect the pid with waitpid
-    start_server {tags {"repl"}} {
+    start_server {tags {"repl"} overrides {save ""}} {
         set master [srv 0 client]
         set master_host [srv 0 host]
         set master_port [srv 0 port]
@@ -1010,7 +1010,7 @@ test "diskless replication child being killed is collected" {
         # put enough data in the db that the rdb file will be bigger than the socket buffers
         $master debug populate 20000 test 10000
         $master config set rdbcompression no
-        start_server {} {
+        start_server {overrides {save ""}} {
             set replica [srv 0 client]
             set loglines [count_log_lines 0]
             $replica config set repl-diskless-load swapdb
@@ -1044,7 +1044,7 @@ test "diskless replication child being killed is collected" {
 foreach mdl {yes no} {
     test "replication child dies when parent is killed - diskless: $mdl" {
         # when master is killed, make sure the fork child can detect that and exit
-        start_server {tags {"repl"}} {
+        start_server {tags {"repl"} overrides {save ""}} {
             set master [srv 0 client]
             set master_host [srv 0 host]
             set master_port [srv 0 port]
@@ -1054,7 +1054,7 @@ foreach mdl {yes no} {
             # create keys that will take 10 seconds to save
             $master config set rdb-key-save-delay 1000
             $master debug populate 10000
-            start_server {} {
+            start_server {overrides {save ""}} {
                 set replica [srv 0 client]
                 $replica replicaof $master_host $master_port
 
@@ -1085,7 +1085,7 @@ test "diskless replication read pipe cleanup" {
     # When we close this pipe (fd), the read handler also needs to be removed from the event loop (if it still registered).
     # Otherwise, next time we will use the same fd, the registration will be fail (panic), because
     # we will use EPOLL_CTL_MOD (the fd still register in the event loop), on fd that already removed from epoll_ctl
-    start_server {tags {"repl"}} {
+    start_server {tags {"repl"} overrides {save ""}} {
         set master [srv 0 client]
         set master_host [srv 0 host]
         set master_port [srv 0 port]
@@ -1097,7 +1097,7 @@ test "diskless replication read pipe cleanup" {
         $master config set rdb-key-save-delay 100000
         $master debug populate 20000 test 10000
         $master config set rdbcompression no
-        start_server {} {
+        start_server {overrides {save ""}} {
             set replica [srv 0 client]
             set loglines [count_log_lines 0]
             $replica config set repl-diskless-load swapdb
@@ -1122,17 +1122,17 @@ test "diskless replication read pipe cleanup" {
 test {replicaof right after disconnection} {
     # this is a rare race condition that was reproduced sporadically by the psync2 unit.
     # see details in #7205
-    start_server {tags {"repl"}} {
+    start_server {tags {"repl"} overrides {save ""}} {
         set replica1 [srv 0 client]
         set replica1_host [srv 0 host]
         set replica1_port [srv 0 port]
         set replica1_log [srv 0 stdout]
-        start_server {} {
+        start_server {overrides {save ""}} {
             set replica2 [srv 0 client]
             set replica2_host [srv 0 host]
             set replica2_port [srv 0 port]
             set replica2_log [srv 0 stdout]
-            start_server {} {
+            start_server {overrides {save ""}} {
                 set master [srv 0 client]
                 set master_host [srv 0 host]
                 set master_port [srv 0 port]
diff --git a/tests/integration/shutdown.tcl b/tests/integration/shutdown.tcl
index 60afc5c7f..b2ec32cbd 100644
--- a/tests/integration/shutdown.tcl
+++ b/tests/integration/shutdown.tcl
@@ -19,8 +19,8 @@ proc fill_up_os_socket_send_buffer_for_repl {idx} {
 
 foreach how {sigterm shutdown} {
     test "Shutting down master waits for replica to catch up ($how)" {
-        start_server {} {
-            start_server {} {
+        start_server {overrides {save ""}} {
+            start_server {overrides {save ""}} {
                 set master [srv -1 client]
                 set master_host [srv -1 host]
                 set master_port [srv -1 port]
@@ -42,8 +42,7 @@ foreach how {sigterm shutdown} {
                 wait_for_ofs_sync $master $replica
 
                 # Pause the replica.
-                exec kill -SIGSTOP $replica_pid
-                after 10
+                pause_process $replica_pid
 
                 # Fill up the OS socket send buffer for the replica connection
                 # to prevent the following INCR from reaching the replica via
@@ -69,7 +68,7 @@ foreach how {sigterm shutdown} {
 
                 # Wake up replica and check if master has waited for it.
                 after 20; # 2 cron intervals
-                exec kill -SIGCONT $replica_pid
+                resume_process $replica_pid
                 wait_for_condition 300 1000 {
                     [$replica get k] eq 2
                 } else {
@@ -86,8 +85,8 @@ foreach how {sigterm shutdown} {
 }
 
 test {Shutting down master waits for replica timeout} {
-    start_server {} {
-        start_server {} {
+    start_server {overrides {save ""}} {
+        start_server {overrides {save ""}} {
             set master [srv -1 client]
             set master_host [srv -1 host]
             set master_port [srv -1 port]
@@ -107,8 +106,7 @@ test {Shutting down master waits for replica timeout} {
             wait_for_ofs_sync $master $replica
 
             # Pause the replica.
-            exec kill -SIGSTOP $replica_pid
-            after 10
+            pause_process $replica_pid
 
             # Fill up the OS socket send buffer for the replica connection to
             # prevent the following INCR k from reaching the replica via the OS.
@@ -129,15 +127,15 @@ test {Shutting down master waits for replica timeout} {
             verify_log_message -1 "*0 of 1 replicas are in sync*" 0
 
             # Wake up replica.
-            exec kill -SIGCONT $replica_pid
+            resume_process $replica_pid
             assert_equal 1 [$replica get k]
         }
     }
 } {} {repl external:skip}
 
 test "Shutting down master waits for replica then fails" {
-    start_server {} {
-        start_server {} {
+    start_server {overrides {save ""}} {
+        start_server {overrides {save ""}} {
             set master [srv -1 client]
             set master_host [srv -1 host]
             set master_port [srv -1 port]
@@ -150,8 +148,7 @@ test "Shutting down master waits for replica then fails" {
             wait_for_sync $replica
 
             # Pause the replica and write a key on master.
-            exec kill -SIGSTOP $replica_pid
-            after 10
+            pause_process $replica_pid
             $master incr k
 
             # Two clients call blocking SHUTDOWN in parallel.
@@ -168,7 +165,7 @@ test "Shutting down master waits for replica then fails" {
             $master config set appendonly yes
 
             # Wake up replica, causing master to continue shutting down.
-            exec kill -SIGCONT $replica_pid
+            resume_process $replica_pid
 
             # SHUTDOWN returns an error to both clients blocking on SHUTDOWN.
             catch { $rd1 read } e1
@@ -190,8 +187,8 @@ test "Shutting down master waits for replica then fails" {
 } {} {repl external:skip}
 
 test "Shutting down master waits for replica then aborted" {
-    start_server {} {
-        start_server {} {
+    start_server {overrides {save ""}} {
+        start_server {overrides {save ""}} {
             set master [srv -1 client]
             set master_host [srv -1 host]
             set master_port [srv -1 port]
@@ -204,8 +201,7 @@ test "Shutting down master waits for replica then aborted" {
             wait_for_sync $replica
 
             # Pause the replica and write a key on master.
-            exec kill -SIGSTOP $replica_pid
-            after 10
+            pause_process $replica_pid
             $master incr k
 
             # Two clients call blocking SHUTDOWN in parallel.
@@ -221,7 +217,7 @@ test "Shutting down master waits for replica then aborted" {
             $master shutdown abort
 
             # Wake up replica, causing master to continue shutting down.
-            exec kill -SIGCONT $replica_pid
+            resume_process $replica_pid
 
             # SHUTDOWN returns an error to both clients blocking on SHUTDOWN.
             catch { $rd1 read } e1