Implement Multi Part AOF mechanism to avoid AOFRW overheads. (#9788)

Implement Multi-Part AOF mechanism to avoid overheads during AOFRW.
Introducing a folder with multiple AOF files tracked by a manifest file.

The main issues with the the original AOFRW mechanism are:
* buffering of commands that are processed during rewrite (consuming a lot of RAM)
* freezes of the main process when the AOFRW completes to drain the remaining part of the buffer and fsync it.
* double disk IO for the data that arrives during AOFRW (had to be written to both the old and new AOF files)

The main modifications of this PR:
1. Remove the AOF rewrite buffer and related code.
2. Divide the AOF into multiple files, they are classified as two types, one is the the `BASE` type,
  it represents the full amount of data (Maybe AOF or RDB format) after each AOFRW, there is only
  one `BASE` file at most. The second is `INCR` type, may have more than one. They represent the
  incremental commands since the last AOFRW.
3. Use a AOF manifest file to record and manage these AOF files mentioned above.
4. The original configuration of `appendfilename` will be the base part of the new file name, for example:
  `appendonly.aof.1.base.rdb` and `appendonly.aof.2.incr.aof`
5. Add manifest-related TCL tests, and modified some existing tests that depend on the `appendfilename`
6. Remove the `aof_rewrite_buffer_length` field in info.
7. Add `aof-disable-auto-gc` configuration. By default we're automatically deleting HISTORY type AOFs.
  It also gives users the opportunity to preserve the history AOFs. just for testing use now.
8. Add AOFRW limiting measure. When the AOFRW failures reaches the threshold (3 times now),
  we will delay the execution of the next AOFRW by 1 minute. If the next AOFRW also fails, it will be
  delayed by 2 minutes. The next is 4, 8, 16, the maximum delay is 60 minutes (1 hour). During the limit
  period, we can still use the 'bgrewriteaof' command to execute AOFRW immediately.
9. Support upgrade (load) data from old version redis.
10. Add `appenddirname` configuration, as the directory name of the append only files. All AOF files and
  manifest file will be placed in this directory.
11. Only the last AOF file (BASE or INCR) can be truncated. Otherwise redis will exit even if
  `aof-load-truncated` is enabled.

Co-authored-by: Oran Agra <oran@redislabs.com>

1 files changed, 47 insertions, 53 deletions

diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl
index aa08b0cd4..bc88f2487 100644
--- a/tests/integration/aof.tcl
+++ b/tests/integration/aof.tcl
@@ -1,36 +1,25 @@
-set defaults { appendonly {yes} appendfilename {appendonly.aof} }
+source tests/support/aofmanifest.tcl
+set defaults { appendonly {yes} appendfilename {appendonly.aof} appenddirname {appendonlydir} auto-aof-rewrite-percentage {0}}
 set server_path [tmpdir server.aof]
-set aof_path "$server_path/appendonly.aof"
-
-proc append_to_aof {str} {
-    upvar fp fp
-    puts -nonewline $fp $str
-}
-
-proc create_aof {code} {
-    upvar fp fp aof_path aof_path
-    set fp [open $aof_path w+]
-    uplevel 1 $code
-    close $fp
-}
-
-proc start_server_aof {overrides code} {
-    upvar defaults defaults srv srv server_path server_path
-    set config [concat $defaults $overrides]
-    set srv [start_server [list overrides $config]]
-    uplevel 1 $code
-    kill_server $srv
-}
+set aof_dirname "appendonlydir"
+set aof_basename "appendonly.aof"
+set aof_dirpath "$server_path/$aof_dirname"
+set aof_file "$server_path/$aof_dirname/${aof_basename}.1$::incr_aof_sufix$::aof_format_suffix"
+set aof_manifest_file "$server_path/$aof_dirname/$aof_basename$::manifest_suffix"
 
 tags {"aof external:skip"} {
-    ## Server can start when aof-load-truncated is set to yes and AOF
-    ## is truncated, with an incomplete MULTI block.
-    create_aof {
+    # Server can start when aof-load-truncated is set to yes and AOF
+    # is truncated, with an incomplete MULTI block.
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand set foo hello]
         append_to_aof [formatCommand multi]
         append_to_aof [formatCommand set bar world]
     }
 
+    create_aof_manifest $aof_dirpath $aof_manifest_file {
+        append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+    }
+
     start_server_aof [list dir $server_path aof-load-truncated yes] {
         test "Unfinished MULTI: Server should start if load-truncated is yes" {
             assert_equal 1 [is_alive $srv]
@@ -38,7 +27,7 @@ tags {"aof external:skip"} {
     }
 
     ## Should also start with truncated AOF without incomplete MULTI block.
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand incr foo]
         append_to_aof [formatCommand incr foo]
         append_to_aof [formatCommand incr foo]
@@ -77,7 +66,7 @@ tags {"aof external:skip"} {
     }
 
     ## Test that the server exits when the AOF contains a format error
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand set foo hello]
         append_to_aof "!!!"
         append_to_aof [formatCommand set foo hello]
@@ -102,7 +91,7 @@ tags {"aof external:skip"} {
     }
 
     ## Test the server doesn't start when the AOF contains an unfinished MULTI
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand set foo hello]
         append_to_aof [formatCommand multi]
         append_to_aof [formatCommand set bar world]
@@ -127,7 +116,7 @@ tags {"aof external:skip"} {
     }
 
     ## Test that the server exits when the AOF contains a short read
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand set foo hello]
         append_to_aof [string range [formatCommand set bar world] 0 end-1]
     }
@@ -153,25 +142,25 @@ tags {"aof external:skip"} {
     ## Test that redis-check-aof indeed sees this AOF is not valid
     test "Short read: Utility should confirm the AOF is not valid" {
         catch {
-            exec src/redis-check-aof $aof_path
+            exec src/redis-check-aof $aof_file
         } result
         assert_match "*not valid*" $result
     }
 
     test "Short read: Utility should show the abnormal line num in AOF" {
-        create_aof {
+        create_aof $aof_dirpath $aof_file {
             append_to_aof [formatCommand set foo hello]
             append_to_aof "!!!"
         }
 
         catch {
-            exec src/redis-check-aof $aof_path
+            exec src/redis-check-aof $aof_file
         } result
         assert_match "*ok_up_to_line=8*" $result
     }
 
     test "Short read: Utility should be able to fix the AOF" {
-        set result [exec src/redis-check-aof --fix $aof_path << "y\n"]
+        set result [exec src/redis-check-aof --fix $aof_file << "y\n"]
         assert_match "*Successfully truncated AOF*" $result
     }
 
@@ -190,7 +179,7 @@ tags {"aof external:skip"} {
     }
 
     ## Test that SPOP (that modifies the client's argc/argv) is correctly free'd
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand sadd set foo]
         append_to_aof [formatCommand sadd set bar]
         append_to_aof [formatCommand spop set]
@@ -209,7 +198,7 @@ tags {"aof external:skip"} {
     }
 
     ## Uses the alsoPropagate() API.
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand sadd set foo]
         append_to_aof [formatCommand sadd set bar]
         append_to_aof [formatCommand sadd set gah]
@@ -229,7 +218,7 @@ tags {"aof external:skip"} {
     }
 
     ## Test that PEXPIREAT is loaded correctly
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand rpush list foo]
         append_to_aof [formatCommand pexpireat list 1000]
         append_to_aof [formatCommand rpush list bar]
@@ -247,14 +236,14 @@ tags {"aof external:skip"} {
         }
     }
 
-    start_server {overrides {appendonly {yes} appendfilename {appendonly.aof}}} {
+    start_server {overrides {appendonly {yes}}} {
         test {Redis should not try to convert DEL into EXPIREAT for EXPIRE -1} {
             r set x 10
             r expire x -1
         }
     }
 
-    start_server {overrides {appendonly {yes} appendfilename {appendonly.aof} appendfsync always}} {
+    start_server {overrides {appendonly {yes} appendfsync always}} {
         test {AOF fsync always barrier issue} {
             set rd [redis_deferring_client]
             # Set a sleep when aof is flushed, so that we have a chance to look
@@ -272,7 +261,7 @@ tags {"aof external:skip"} {
                 r del x
                 r setrange x [expr {int(rand()*5000000)+10000000}] x
                 r debug aof-flush-sleep 500000
-                set aof [file join [lindex [r config get dir] 1] appendonly.aof]
+                set aof [get_last_incr_aof_path r]
                 set size1 [file size $aof]
                 $rd get x
                 after [expr {int(rand()*30)}]
@@ -285,9 +274,9 @@ tags {"aof external:skip"} {
         }
     }
 
-    start_server {overrides {appendonly {yes} appendfilename {appendonly.aof}}} {
+    start_server {overrides {appendonly {yes}}} {
         test {GETEX should not append to AOF} {
-            set aof [file join [lindex [r config get dir] 1] appendonly.aof]
+            set aof [get_last_incr_aof_path r]
             r set foo bar
             set before [file size $aof]
             r getex foo
@@ -297,7 +286,7 @@ tags {"aof external:skip"} {
     }
 
     ## Test that the server exits when the AOF contains a unknown command
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand set foo hello]
         append_to_aof [formatCommand bla foo hello]
         append_to_aof [formatCommand set foo hello]
@@ -322,7 +311,7 @@ tags {"aof external:skip"} {
     }
 
     # Test that LMPOP/BLMPOP work fine with AOF.
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand lpush mylist a b c]
         append_to_aof [formatCommand rpush mylist2 1 2 3]
         append_to_aof [formatCommand lpush mylist3 a b c d e]
@@ -369,7 +358,7 @@ tags {"aof external:skip"} {
     }
 
     # Test that ZMPOP/BZMPOP work fine with AOF.
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand zadd myzset 1 one 2 two 3 three]
         append_to_aof [formatCommand zadd myzset2 4 four 5 five 6 six]
         append_to_aof [formatCommand zadd myzset3 1 one 2 two 3 three 4 four 5 five]
@@ -416,22 +405,24 @@ tags {"aof external:skip"} {
     }
 
     test {Generate timestamp annotations in AOF} {
-        start_server {overrides {appendonly {yes} appendfilename {appendonly.aof}}} {
+        start_server {overrides {appendonly {yes}}} {
             r config set aof-timestamp-enabled yes
             r config set aof-use-rdb-preamble no
-            set aof [file join [lindex [r config get dir] 1] appendonly.aof]
+            set aof [get_last_incr_aof_path r]
 
             r set foo bar
             assert_match "#TS:*" [exec head -n 1 $aof]
 
             r bgrewriteaof
             waitForBgrewriteaof r
+
+            set aof [get_base_aof_path r]
             assert_match "#TS:*" [exec head -n 1 $aof]
         }
     }
 
     # redis could load AOF which has timestamp annotations inside
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof "#TS:1628217470\r\n"
         append_to_aof [formatCommand set foo1 bar1]
         append_to_aof "#TS:1628217471\r\n"
@@ -453,7 +444,7 @@ tags {"aof external:skip"} {
 
     test {Truncate AOF to specific timestamp} {
         # truncate to timestamp 1628217473
-        exec src/redis-check-aof --truncate-to-timestamp 1628217473 $aof_path
+        exec src/redis-check-aof --truncate-to-timestamp 1628217473 $aof_file
         start_server_aof [list dir $server_path] {
             set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
             wait_done_loading $c
@@ -463,7 +454,7 @@ tags {"aof external:skip"} {
         }
 
         # truncate to timestamp 1628217471
-        exec src/redis-check-aof --truncate-to-timestamp 1628217471 $aof_path
+        exec src/redis-check-aof --truncate-to-timestamp 1628217471 $aof_file
         start_server_aof [list dir $server_path] {
             set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
             wait_done_loading $c
@@ -473,7 +464,7 @@ tags {"aof external:skip"} {
         }
 
         # truncate to timestamp 1628217470
-        exec src/redis-check-aof --truncate-to-timestamp 1628217470 $aof_path
+        exec src/redis-check-aof --truncate-to-timestamp 1628217470 $aof_file
         start_server_aof [list dir $server_path] {
             set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
             wait_done_loading $c
@@ -482,12 +473,12 @@ tags {"aof external:skip"} {
         }
 
         # truncate to timestamp 1628217469
-        catch {exec src/redis-check-aof --truncate-to-timestamp 1628217469 $aof_path} e
+        catch {exec src/redis-check-aof --truncate-to-timestamp 1628217469 $aof_file} e
         assert_match {*aborting*} $e
     }
 
     test {EVAL timeout with slow verbatim Lua script from AOF} {
-        create_aof {
+        create_aof $aof_dirpath $aof_file {
             append_to_aof [formatCommand select 9]
             append_to_aof [formatCommand eval {redis.call('set',KEYS[1],'y'); for i=1,1500000 do redis.call('ping') end return 'ok'} 1 x]
         }
@@ -512,7 +503,10 @@ tags {"aof external:skip"} {
     }
 
     test {EVAL can process writes from AOF in read-only replicas} {
-        create_aof {
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+        }
+        create_aof $aof_dirpath $aof_file {
             append_to_aof [formatCommand select 9]
             append_to_aof [formatCommand eval {redis.call("set",KEYS[1],"100")} 1 foo]
             append_to_aof [formatCommand eval {redis.call("incr",KEYS[1])} 1 foo]