Implement Multi Part AOF mechanism to avoid AOFRW overheads. (#9788)

Implement Multi-Part AOF mechanism to avoid overheads during AOFRW.
Introducing a folder with multiple AOF files tracked by a manifest file.

The main issues with the the original AOFRW mechanism are:
* buffering of commands that are processed during rewrite (consuming a lot of RAM)
* freezes of the main process when the AOFRW completes to drain the remaining part of the buffer and fsync it.
* double disk IO for the data that arrives during AOFRW (had to be written to both the old and new AOF files)

The main modifications of this PR:
1. Remove the AOF rewrite buffer and related code.
2. Divide the AOF into multiple files, they are classified as two types, one is the the `BASE` type,
  it represents the full amount of data (Maybe AOF or RDB format) after each AOFRW, there is only
  one `BASE` file at most. The second is `INCR` type, may have more than one. They represent the
  incremental commands since the last AOFRW.
3. Use a AOF manifest file to record and manage these AOF files mentioned above.
4. The original configuration of `appendfilename` will be the base part of the new file name, for example:
  `appendonly.aof.1.base.rdb` and `appendonly.aof.2.incr.aof`
5. Add manifest-related TCL tests, and modified some existing tests that depend on the `appendfilename`
6. Remove the `aof_rewrite_buffer_length` field in info.
7. Add `aof-disable-auto-gc` configuration. By default we're automatically deleting HISTORY type AOFs.
  It also gives users the opportunity to preserve the history AOFs. just for testing use now.
8. Add AOFRW limiting measure. When the AOFRW failures reaches the threshold (3 times now),
  we will delay the execution of the next AOFRW by 1 minute. If the next AOFRW also fails, it will be
  delayed by 2 minutes. The next is 4, 8, 16, the maximum delay is 60 minutes (1 hour). During the limit
  period, we can still use the 'bgrewriteaof' command to execute AOFRW immediately.
9. Support upgrade (load) data from old version redis.
10. Add `appenddirname` configuration, as the directory name of the append only files. All AOF files and
  manifest file will be placed in this directory.
11. Only the last AOF file (BASE or INCR) can be truncated. Otherwise redis will exit even if
  `aof-load-truncated` is enabled.

Co-authored-by: Oran Agra <oran@redislabs.com>

11 files changed, 1333 insertions, 62 deletions

diff --git a/tests/assets/rdb-preamble.aof b/tests/assets/rdb-preamble.aof
new file mode 100644
index 000000000..73f2301cc
--- /dev/null
+++ b/tests/assets/rdb-preamble.aof
diff --git a/tests/instances.tcl b/tests/instances.tcl
index 52e61418e..d423291ac 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -12,6 +12,7 @@ package require Tcl 8.5
 set tcl_precision 17
 source ../support/redis.tcl
 source ../support/util.tcl
+source ../support/aofmanifest.tcl
 source ../support/server.tcl
 source ../support/test.tcl
 
diff --git a/tests/integration/aof-multi-part.tcl b/tests/integration/aof-multi-part.tcl
new file mode 100644
index 000000000..0ff09ea04
--- /dev/null
+++ b/tests/integration/aof-multi-part.tcl
@@ -0,0 +1,1099 @@
+source tests/support/aofmanifest.tcl
+set defaults {appendonly {yes} appendfilename {appendonly.aof} appenddirname {appendonlydir} auto-aof-rewrite-percentage {0}}
+set server_path [tmpdir server.multi.aof]
+set aof_dirname "appendonlydir"
+set aof_basename "appendonly.aof"
+set aof_dirpath "$server_path/$aof_dirname"
+set aof_base1_file "$server_path/$aof_dirname/${aof_basename}.1$::base_aof_sufix$::aof_format_suffix"
+set aof_base2_file "$server_path/$aof_dirname/${aof_basename}.2$::base_aof_sufix$::aof_format_suffix"
+set aof_incr1_file "$server_path/$aof_dirname/${aof_basename}.1$::incr_aof_sufix$::aof_format_suffix"
+set aof_incr2_file "$server_path/$aof_dirname/${aof_basename}.2$::incr_aof_sufix$::aof_format_suffix"
+set aof_incr3_file "$server_path/$aof_dirname/${aof_basename}.3$::incr_aof_sufix$::aof_format_suffix"
+set aof_manifest_file "$server_path/$aof_dirname/${aof_basename}$::manifest_suffix"
+set aof_old_name_old_path "$server_path/$aof_basename"
+set aof_old_name_new_path "$aof_dirpath/$aof_basename"
+set aof_old_name_old_path2 "$server_path/${aof_basename}2"
+set aof_manifest_file2 "$server_path/$aof_dirname/${aof_basename}2$::manifest_suffix"
+
+tags {"external:skip"} {
+
+    # Test Part 1
+
+    # In order to test the loading logic of redis under different combinations of manifest and AOF.
+    # We will manually construct the manifest file and AOF, and then start redis to verify whether
+    # the redis behavior is as expected.
+
+    test {Multi Part AOF can't load data when some file missing} {
+        create_aof $aof_dirpath $aof_base1_file {
+            append_to_aof [formatCommand set k1 v1]
+        }
+
+        create_aof $aof_dirpath $aof_incr2_file {
+            append_to_aof [formatCommand set k2 v2]
+        }
+
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
+            append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+            append_to_manifest "file appendonly.aof.2.incr.aof seq 2 type i\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            wait_for_condition 100 50 {
+                ! [is_alive $srv]
+            } else {
+                fail "AOF loading didn't fail"
+            }
+
+            assert_equal 1 [count_message_lines $server_path/stdout "appendonly.aof.1.incr.aof doesn't exist"]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can't load data when the sequence not increase monotonically} {
+        create_aof $aof_dirpath $aof_incr1_file {
+            append_to_aof [formatCommand set k1 v1]
+        }
+
+        create_aof $aof_dirpath $aof_incr2_file {
+            append_to_aof [formatCommand set k2 v2]
+        }
+
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof.2.incr.aof seq 2 type i\n"
+            append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            wait_for_condition 100 50 {
+                ! [is_alive $srv]
+            } else {
+                fail "AOF loading didn't fail"
+            }
+
+            assert_equal 1 [count_message_lines $server_path/stdout "Found a non-monotonic sequence number"]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can't load data when there are blank lines in the manifest file} {
+        create_aof $aof_dirpath $aof_incr1_file {
+            append_to_aof [formatCommand set k1 v1]
+        }
+
+        create_aof $aof_dirpath $aof_incr3_file {
+            append_to_aof [formatCommand set k2 v2]
+        }
+
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+            append_to_manifest "\n"
+            append_to_manifest "file appendonly.aof.3.incr.aof seq 3 type i\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            wait_for_condition 100 50 {
+                ! [is_alive $srv]
+            } else {
+                fail "AOF loading didn't fail"
+            }
+
+            assert_equal 1 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can't load data when there is a duplicate base file} {
+        create_aof $aof_dirpath $aof_base1_file {
+            append_to_aof [formatCommand set k1 v1]
+        }
+
+        create_aof $aof_dirpath $aof_base2_file {
+            append_to_aof [formatCommand set k2 v2]
+        }
+
+        create_aof $aof_dirpath $aof_incr1_file {
+            append_to_aof [formatCommand set k3 v3]
+        }
+
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
+            append_to_manifest "file appendonly.aof.2.base.aof seq 2 type b\n"
+            append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            wait_for_condition 100 50 {
+                ! [is_alive $srv]
+            } else {
+                fail "AOF loading didn't fail"
+            }
+
+            assert_equal 1 [count_message_lines $server_path/stdout "Found duplicate base file information"]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can't load data when the manifest format is wrong (type unknown)} {
+        create_aof $aof_dirpath $aof_base1_file {
+            append_to_aof [formatCommand set k1 v1]
+        }
+
+        create_aof $aof_dirpath $aof_incr1_file {
+            append_to_aof [formatCommand set k3 v3]
+        }
+
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof.1.base.aof seq 1 type x\n"
+            append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            wait_for_condition 100 50 {
+                ! [is_alive $srv]
+            } else {
+                fail "AOF loading didn't fail"
+            }
+
+            assert_equal 1 [count_message_lines $server_path/stdout "Unknown AOF file type"]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can't load data when the manifest format is wrong (missing key)} {
+        create_aof $aof_dirpath $aof_base1_file {
+            append_to_aof [formatCommand set k1 v1]
+        }
+
+        create_aof $aof_dirpath $aof_incr1_file {
+            append_to_aof [formatCommand set k3 v3]
+        }
+
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "filx appendonly.aof.1.base.aof seq 1 type b\n"
+            append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            wait_for_condition 100 50 {
+                ! [is_alive $srv]
+            } else {
+                fail "AOF loading didn't fail"
+            }
+
+            assert_equal 1 [count_message_lines $server_path/stdout "Mismatched manifest key"]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can't load data when the manifest format is wrong (line too short)} {
+        create_aof $aof_dirpath $aof_base1_file {
+            append_to_aof [formatCommand set k1 v1]
+        }
+
+        create_aof $aof_dirpath $aof_incr1_file {
+            append_to_aof [formatCommand set k3 v3]
+        }
+
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
+            append_to_manifest "file appendonly.aof.1.incr.aof type i\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            wait_for_condition 100 50 {
+                ! [is_alive $srv]
+            } else {
+                fail "AOF loading didn't fail"
+            }
+
+            assert_equal 2 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can't load data when the manifest format is wrong (line too long)} {
+        create_aof $aof_dirpath $aof_base1_file {
+            append_to_aof [formatCommand set k1 v1]
+        }
+
+        create_aof $aof_dirpath $aof_incr1_file {
+            append_to_aof [formatCommand set k3 v3]
+        }
+
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b\n"
+            append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            wait_for_condition 100 50 {
+                ! [is_alive $srv]
+            } else {
+                fail "AOF loading didn't fail"
+            }
+
+            assert_equal 1 [count_message_lines $server_path/stdout "The AOF manifest file contains too long line"]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can't load data when the manifest format is wrong (odd parameter)} {
+        create_aof $aof_dirpath $aof_base1_file {
+            append_to_aof [formatCommand set k1 v1]
+        }
+
+        create_aof $aof_dirpath $aof_incr1_file {
+            append_to_aof [formatCommand set k3 v3]
+        }
+
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
+            append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i newkey\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            wait_for_condition 100 50 {
+                ! [is_alive $srv]
+            } else {
+                fail "AOF loading didn't fail"
+            }
+
+            assert_equal 3 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can't load data when the manifest file is empty} {
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+        }
+
+        start_server_aof [list dir $server_path] {
+            wait_for_condition 100 50 {
+                ! [is_alive $srv]
+            } else {
+                fail "AOF loading didn't fail"
+            }
+
+            assert_equal 1 [count_message_lines $server_path/stdout "Found an empty AOF manifest"]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can start when no aof and no manifest} {
+        start_server_aof [list dir $server_path] {
+            assert_equal 1 [is_alive $srv]
+
+            set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
+
+            assert_equal OK [$client set k1 v1]
+            assert_equal v1 [$client get k1]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can start when we have en empty AOF dir} {
+        create_aof_dir $aof_dirpath
+
+        start_server_aof [list dir $server_path] {
+            assert_equal 1 [is_alive $srv]
+        }
+    }
+
+    test {Multi Part AOF can load data discontinuously increasing sequence} {
+        create_aof $aof_dirpath $aof_base1_file {
+            append_to_aof [formatCommand set k1 v1]
+        }
+
+        create_aof $aof_dirpath $aof_incr1_file {
+            append_to_aof [formatCommand set k2 v2]
+        }
+
+        create_aof $aof_dirpath $aof_incr3_file {
+            append_to_aof [formatCommand set k3 v3]
+        }
+
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
+            append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+            append_to_manifest "file appendonly.aof.3.incr.aof seq 3 type i\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            assert_equal 1 [is_alive $srv]
+            set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
+            wait_done_loading $client
+
+            assert_equal v1 [$client get k1]
+            assert_equal v2 [$client get k2]
+            assert_equal v3 [$client get k3]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can load data when manifest add new k-v} {
+        create_aof $aof_dirpath $aof_base1_file {
+            append_to_aof [formatCommand set k1 v1]
+        }
+
+        create_aof $aof_dirpath $aof_incr1_file {
+            append_to_aof [formatCommand set k2 v2]
+        }
+
+        create_aof $aof_dirpath $aof_incr3_file {
+            append_to_aof [formatCommand set k3 v3]
+        }
+
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b newkey newvalue\n"
+            append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+            append_to_manifest "file appendonly.aof.3.incr.aof seq 3 type i\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            assert_equal 1 [is_alive $srv]
+            set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
+            wait_done_loading $client
+
+            assert_equal v1 [$client get k1]
+            assert_equal v2 [$client get k2]
+            assert_equal v3 [$client get k3]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can load data when some AOFs are empty} {
+        create_aof $aof_dirpath $aof_base1_file {
+            append_to_aof [formatCommand set k1 v1]
+        }
+
+        create_aof $aof_dirpath $aof_incr1_file {
+        }
+
+        create_aof $aof_dirpath $aof_incr3_file {
+            append_to_aof [formatCommand set k3 v3]
+        }
+
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
+            append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+            append_to_manifest "file appendonly.aof.3.incr.aof seq 3 type i\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            assert_equal 1 [is_alive $srv]
+            set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
+            wait_done_loading $client
+
+            assert_equal v1 [$client get k1]
+            assert_equal "" [$client get k2]
+            assert_equal v3 [$client get k3]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can load data from old version redis (rdb preamble no)} {
+        create_aof $server_path $aof_old_name_old_path {
+            append_to_aof [formatCommand set k1 v1]
+            append_to_aof [formatCommand set k2 v2]
+            append_to_aof [formatCommand set k3 v3]
+        }
+
+        start_server_aof [list dir $server_path] {
+            assert_equal 1 [is_alive $srv]
+
+            set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
+            wait_done_loading $client
+
+            assert_equal v1 [$client get k1]
+            assert_equal v2 [$client get k2]
+            assert_equal v3 [$client get k3]
+
+            assert_equal 0 [check_file_exist $server_path $aof_basename]
+            assert_equal 1 [check_file_exist $aof_dirpath $aof_basename]
+
+            assert_aof_manifest_content $aof_manifest_file  {
+                {file appendonly.aof seq 1 type b}
+                {file appendonly.aof.1.incr.aof seq 1 type i}
+            }
+
+            assert_equal OK [$client set k4 v4]
+
+            $client bgrewriteaof
+            waitForBgrewriteaof $client
+
+            assert_equal OK [$client set k5 v5]
+
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.2.base.rdb seq 2 type b}
+                {file appendonly.aof.2.incr.aof seq 2 type i}
+            }
+
+            set d1 [$client debug digest]
+            $client debug loadaof
+            set d2 [$client debug digest]
+            assert {$d1 eq $d2}
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can load data from old version redis (rdb preamble yes)} {
+        exec cp tests/assets/rdb-preamble.aof $aof_old_name_old_path
+        start_server_aof [list dir $server_path] {
+            assert_equal 1 [is_alive $srv]
+
+            set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
+            wait_done_loading $client
+
+            # k1 k2 in rdb header and k3 in AOF tail
+            assert_equal v1 [$client get k1]
+            assert_equal v2 [$client get k2]
+            assert_equal v3 [$client get k3]
+
+            assert_equal 0 [check_file_exist $server_path $aof_basename]
+            assert_equal 1 [check_file_exist $aof_dirpath $aof_basename]
+
+            assert_aof_manifest_content $aof_manifest_file  {
+                {file appendonly.aof seq 1 type b}
+                {file appendonly.aof.1.incr.aof seq 1 type i}
+            }
+
+            assert_equal OK [$client set k4 v4]
+
+            $client bgrewriteaof
+            waitForBgrewriteaof $client
+
+            assert_equal OK [$client set k5 v5]
+
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.2.base.rdb seq 2 type b}
+                {file appendonly.aof.2.incr.aof seq 2 type i}
+            }
+
+            set d1 [$client debug digest]
+            $client debug loadaof
+            set d2 [$client debug digest]
+            assert {$d1 eq $d2}
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can continue the upgrade from the interrupted upgrade state} {
+        create_aof $server_path $aof_old_name_old_path {
+            append_to_aof [formatCommand set k1 v1]
+            append_to_aof [formatCommand set k2 v2]
+            append_to_aof [formatCommand set k3 v3]
+        }
+
+        # Create a layout of an interrupted upgrade (interrupted before the rename).
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof seq 1 type b\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            assert_equal 1 [is_alive $srv]
+
+            set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
+            wait_done_loading $client
+
+            assert_equal v1 [$client get k1]
+            assert_equal v2 [$client get k2]
+            assert_equal v3 [$client get k3]
+
+            assert_equal 0 [check_file_exist $server_path $aof_basename]
+            assert_equal 1 [check_file_exist $aof_dirpath $aof_basename]
+
+            assert_aof_manifest_content $aof_manifest_file  {
+                {file appendonly.aof seq 1 type b}
+                {file appendonly.aof.1.incr.aof seq 1 type i}
+            }
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can be loaded correctly when both server dir and aof dir contain old AOF} {
+        create_aof $server_path $aof_old_name_old_path {
+            append_to_aof [formatCommand set k1 v1]
+            append_to_aof [formatCommand set k2 v2]
+            append_to_aof [formatCommand set k3 v3]
+        }
+
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof seq 1 type b\n"
+        }
+
+        create_aof $aof_dirpath $aof_old_name_new_path {
+            append_to_aof [formatCommand set k4 v4]
+            append_to_aof [formatCommand set k5 v5]
+            append_to_aof [formatCommand set k6 v6]
+        }
+
+        start_server_aof [list dir $server_path] {
+            assert_equal 1 [is_alive $srv]
+
+            set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
+            wait_done_loading $client
+
+            assert_equal 0 [$client exists k1]
+            assert_equal 0 [$client exists k2]
+            assert_equal 0 [$client exists k3]
+
+            assert_equal v4 [$client get k4]
+            assert_equal v5 [$client get k5]
+            assert_equal v6 [$client get k6]
+
+            assert_equal 1 [check_file_exist $server_path $aof_basename]
+            assert_equal 1 [check_file_exist $aof_dirpath $aof_basename]
+
+            assert_aof_manifest_content $aof_manifest_file  {
+                {file appendonly.aof seq 1 type b}
+                {file appendonly.aof.1.incr.aof seq 1 type i}
+            }
+        }
+
+        clean_aof_persistence $aof_dirpath
+        catch {exec rm -rf $aof_old_name_old_path}
+    }
+
+    test {Multi Part AOF can't load data when the manifest contains the old AOF file name but the file does not exist in server dir and aof dir} {
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof seq 1 type b\n"
+        }
+
+        start_server_aof [list dir $server_path] {
+            wait_for_condition 100 50 {
+                ! [is_alive $srv]
+            } else {
+                fail "AOF loading didn't fail"
+            }
+
+            assert_equal 1 [count_message_lines $server_path/stdout "appendonly.aof doesn't exist"]
+        }
+
+        clean_aof_persistence $aof_dirpath
+    }
+
+    test {Multi Part AOF can upgrade when when two redis share the same server dir} {
+        create_aof $server_path $aof_old_name_old_path {
+            append_to_aof [formatCommand set k1 v1]
+            append_to_aof [formatCommand set k2 v2]
+            append_to_aof [formatCommand set k3 v3]
+        }
+
+        create_aof $server_path $aof_old_name_old_path2 {
+            append_to_aof [formatCommand set k4 v4]
+            append_to_aof [formatCommand set k5 v5]
+            append_to_aof [formatCommand set k6 v6]
+        }
+
+        start_server_aof [list dir $server_path] {
+            set redis1 [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
+
+            start_server [list overrides [list dir $server_path appendonly yes appendfilename appendonly.aof2]] {
+                set redis2 [redis [srv host] [srv port] 0 $::tls]
+
+                test "Multi Part AOF can upgrade when when two redis share the same server dir (redis1)" {
+                    wait_done_loading $redis1
+                    assert_equal v1 [$redis1 get k1]
+                    assert_equal v2 [$redis1 get k2]
+                    assert_equal v3 [$redis1 get k3]
+
+                    assert_equal 0 [$redis1 exists k4]
+                    assert_equal 0 [$redis1 exists k5]
+                    assert_equal 0 [$redis1 exists k6]
+
+                    assert_aof_manifest_content $aof_manifest_file  {
+                        {file appendonly.aof seq 1 type b}
+                        {file appendonly.aof.1.incr.aof seq 1 type i}
+                    }
+
+                    $redis1 bgrewriteaof
+                    waitForBgrewriteaof $redis1
+
+                    assert_equal OK [$redis1 set k v]
+
+                    assert_aof_manifest_content $aof_manifest_file {
+                        {file appendonly.aof.2.base.rdb seq 2 type b}
+                        {file appendonly.aof.2.incr.aof seq 2 type i}
+                    }
+
+                    set d1 [$redis1 debug digest]
+                    $redis1 debug loadaof
+                    set d2 [$redis1 debug digest]
+                    assert {$d1 eq $d2}
+                }
+
+                test "Multi Part AOF can upgrade when when two redis share the same server dir (redis2)" {
+                    wait_done_loading $redis2
+
+                    assert_equal 0 [$redis2 exists k1]
+                    assert_equal 0 [$redis2 exists k2]
+                    assert_equal 0 [$redis2 exists k3]
+
+                    assert_equal v4 [$redis2 get k4]
+                    assert_equal v5 [$redis2 get k5]
+                    assert_equal v6 [$redis2 get k6]
+
+                    assert_aof_manifest_content $aof_manifest_file2  {
+                        {file appendonly.aof2 seq 1 type b}
+                        {file appendonly.aof2.1.incr.aof seq 1 type i}
+                    }
+
+                    $redis2 bgrewriteaof
+                    waitForBgrewriteaof $redis2
+
+                    assert_equal OK [$redis2 set k v]
+
+                    assert_aof_manifest_content $aof_manifest_file2 {
+                        {file appendonly.aof2.2.base.rdb seq 2 type b}
+                        {file appendonly.aof2.2.incr.aof seq 2 type i}
+                    }
+
+                    set d1 [$redis2 debug digest]
+                    $redis2 debug loadaof
+                    set d2 [$redis2 debug digest]
+                    assert {$d1 eq $d2}
+                }
+            }
+        }
+    }
+
+    # Test Part 2
+    #
+    # To test whether the AOFRW behaves as expected during the redis run.
+    # We will start redis first, then perform pressure writing, enable and disable AOF, and manually
+    # and automatically run bgrewrite and other actions, to test whether the correct AOF file is created,
+    # whether the correct manifest is generated, whether the data can be reload correctly under continuous
+    # writing pressure, etc.
+
+
+    start_server {tags {"Multi Part AOF"} overrides {aof-use-rdb-preamble {yes} appendonly {no}}} {
+        set dir [get_redis_dir]
+        set aof_basename "appendonly.aof"
+        set aof_dirname "appendonlydir"
+        set aof_dirpath "$dir/$aof_dirname"
+        set aof_manifest_name "$aof_basename$::manifest_suffix"
+        set aof_manifest_file "$dir/$aof_dirname/$aof_manifest_name"
+
+        set master [srv 0 client]
+        set master_host [srv 0 host]
+        set master_port [srv 0 port]
+
+        catch {exec rm -rf $aof_manifest_file}
+
+        test "Make sure aof manifest $aof_manifest_name not in aof directory" {
+            assert_equal 0 [file exists $aof_manifest_file]
+        }
+
+        test "AOF enable will create manifest file" {
+            r config set appendonly yes ; # Will create manifest and new INCR aof
+            r config set auto-aof-rewrite-percentage 0 ; # Disable auto-rewrite.
+            waitForBgrewriteaof r
+
+            # Start write load
+            set load_handle0 [start_write_load $master_host $master_port 10]
+
+            wait_for_condition 50 100 {
+                [r dbsize] > 0
+            } else {
+                fail "No write load detected."
+            }
+
+            # First AOFRW done
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.1.base.rdb seq 1 type b}
+                {file appendonly.aof.1.incr.aof seq 1 type i}
+            }
+
+            # Check we really have these files
+            assert_equal 1 [check_file_exist $aof_dirpath $aof_manifest_name]
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.1${::base_aof_sufix}${::rdb_format_suffix}"]
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.1${::incr_aof_sufix}${::aof_format_suffix}"]
+
+            r bgrewriteaof
+            waitForBgrewriteaof r
+
+            # The second AOFRW done
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.2.base.rdb seq 2 type b}
+                {file appendonly.aof.2.incr.aof seq 2 type i}
+            }
+
+            assert_equal 1 [check_file_exist $aof_dirpath $aof_manifest_name]
+            # Wait bio delete history
+            wait_for_condition 1000 10 {
+                [check_file_exist $aof_dirpath "${aof_basename}.1${::base_aof_sufix}${::rdb_format_suffix}"] == 0 &&
+                [check_file_exist $aof_dirpath "${aof_basename}.1${::incr_aof_sufix}${::aof_format_suffix}"] == 0
+            } else {
+                fail "Failed to delete history AOF"
+            }
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.2${::base_aof_sufix}${::rdb_format_suffix}"]
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.2${::incr_aof_sufix}${::aof_format_suffix}"]
+
+            stop_write_load $load_handle0
+            wait_load_handlers_disconnected
+
+            set d1 [r debug digest]
+            r debug loadaof
+            set d2 [r debug digest]
+            assert {$d1 eq $d2}
+        }
+
+        test "AOF multiple rewrite failures will open multiple INCR AOFs" {
+            # Start write load
+            r config set rdb-key-save-delay 10000000
+
+            set orig_size [r dbsize]
+            set load_handle0 [start_write_load $master_host $master_port 10]
+
+            wait_for_condition 50 100 {
+                [r dbsize] > $orig_size
+            } else {
+                fail "No write load detected."
+            }
+
+            # Let AOFRW fail three times
+            r bgrewriteaof
+            set pid1 [get_child_pid 0]
+            catch {exec kill -9 $pid1}
+            waitForBgrewriteaof r
+
+            r bgrewriteaof
+            set pid2 [get_child_pid 0]
+            catch {exec kill -9 $pid2}
+            waitForBgrewriteaof r
+
+            r bgrewriteaof
+            set pid3 [get_child_pid 0]
+            catch {exec kill -9 $pid3}
+            waitForBgrewriteaof r
+
+            assert_equal 0 [check_file_exist $dir "temp-rewriteaof-bg-$pid1.aof"]
+            assert_equal 0 [check_file_exist $dir "temp-rewriteaof-bg-$pid2.aof"]
+            assert_equal 0 [check_file_exist $dir "temp-rewriteaof-bg-$pid3.aof"]
+
+            # We will have four INCR AOFs
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.2.base.rdb seq 2 type b}
+                {file appendonly.aof.2.incr.aof seq 2 type i}
+                {file appendonly.aof.3.incr.aof seq 3 type i}
+                {file appendonly.aof.4.incr.aof seq 4 type i}
+                {file appendonly.aof.5.incr.aof seq 5 type i}
+            }
+
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.2${::base_aof_sufix}${::rdb_format_suffix}"]
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.2${::incr_aof_sufix}${::aof_format_suffix}"]
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.3${::incr_aof_sufix}${::aof_format_suffix}"]
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.4${::incr_aof_sufix}${::aof_format_suffix}"]
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.5${::incr_aof_sufix}${::aof_format_suffix}"]
+
+            stop_write_load $load_handle0
+            wait_load_handlers_disconnected
+
+            set d1 [r debug digest]
+            r debug loadaof
+            set d2 [r debug digest]
+            assert {$d1 eq $d2}
+
+            r config set rdb-key-save-delay 0
+            catch {exec kill -9 [get_child_pid 0]}
+            wait_for_condition 1000 10 {
+                [s rdb_bgsave_in_progress] eq 0
+            } else {
+                fail "bgsave did not stop in time"
+            }
+
+            # AOFRW success
+            r bgrewriteaof
+            waitForBgrewriteaof r
+
+            # All previous INCR AOFs have become history
+            # and have be deleted
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.3.base.rdb seq 3 type b}
+                {file appendonly.aof.6.incr.aof seq 6 type i}
+            }
+
+            # Wait bio delete history
+            wait_for_condition 1000 10 {
+                [check_file_exist $aof_dirpath "${aof_basename}.2${::base_aof_sufix}${::rdb_format_suffix}"] == 0 &&
+                [check_file_exist $aof_dirpath "${aof_basename}.2${::incr_aof_sufix}${::aof_format_suffix}"] == 0 &&
+                [check_file_exist $aof_dirpath "${aof_basename}.3${::incr_aof_sufix}${::aof_format_suffix}"] == 0 &&
+                [check_file_exist $aof_dirpath "${aof_basename}.4${::incr_aof_sufix}${::aof_format_suffix}"] == 0 &&
+                [check_file_exist $aof_dirpath "${aof_basename}.5${::incr_aof_sufix}${::aof_format_suffix}"] == 0
+            } else {
+                fail "Failed to delete history AOF"
+            }
+
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.3${::base_aof_sufix}${::rdb_format_suffix}"]
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.6${::incr_aof_sufix}${::aof_format_suffix}"]
+
+            set d1 [r debug digest]
+            r debug loadaof
+            set d2 [r debug digest]
+            assert {$d1 eq $d2}
+        }
+
+        test "AOF rewrite doesn't open new aof when AOF turn off" {
+            r config set appendonly no
+
+            r bgrewriteaof
+            waitForBgrewriteaof r
+
+            # We only have BASE AOF, no INCR AOF
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.4.base.rdb seq 4 type b}
+            }
+
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.4${::base_aof_sufix}${::rdb_format_suffix}"]
+            wait_for_condition 1000 10 {
+                [check_file_exist $aof_dirpath "${aof_basename}.6${::incr_aof_sufix}${::aof_format_suffix}"] == 0 &&
+                [check_file_exist $aof_dirpath "${aof_basename}.7${::incr_aof_sufix}${::aof_format_suffix}"] == 0
+            } else {
+                fail "Failed to delete history AOF"
+            }
+
+            set d1 [r debug digest]
+            r debug loadaof
+            set d2 [r debug digest]
+            assert {$d1 eq $d2}
+
+            # Turn on AOF again
+            r config set appendonly yes
+            waitForBgrewriteaof r
+
+            # A new INCR AOF was created
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.5.base.rdb seq 5 type b}
+                {file appendonly.aof.1.incr.aof seq 1 type i}
+            }
+
+            # Wait bio delete history
+            wait_for_condition 1000 10 {
+                [check_file_exist $aof_dirpath "${aof_basename}.4${::base_aof_sufix}${::rdb_format_suffix}"] == 0
+            } else {
+                fail "Failed to delete history AOF"
+            }
+
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.5${::base_aof_sufix}${::rdb_format_suffix}"]
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.1${::incr_aof_sufix}${::aof_format_suffix}"]
+        }
+
+        test "AOF enable/disable auto gc" {
+            r config set aof-disable-auto-gc yes
+
+            r bgrewriteaof
+            waitForBgrewriteaof r
+
+            r bgrewriteaof
+            waitForBgrewriteaof r
+
+            # We can see four history AOFs (Evolved from two BASE and two INCR)
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.7.base.rdb seq 7 type b}
+                {file appendonly.aof.2.incr.aof seq 2 type h}
+                {file appendonly.aof.6.base.rdb seq 6 type h}
+                {file appendonly.aof.1.incr.aof seq 1 type h}
+                {file appendonly.aof.5.base.rdb seq 5 type h}
+                {file appendonly.aof.3.incr.aof seq 3 type i}
+            }
+
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.5${::base_aof_sufix}${::rdb_format_suffix}"]
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.6${::base_aof_sufix}${::rdb_format_suffix}"]
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.1${::incr_aof_sufix}${::aof_format_suffix}"]
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.2${::incr_aof_sufix}${::aof_format_suffix}"]
+
+            r config set aof-disable-auto-gc no
+
+            # Auto gc success
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.7.base.rdb seq 7 type b}
+                {file appendonly.aof.3.incr.aof seq 3 type i}
+            }
+
+            # wait bio delete history
+            wait_for_condition 1000 10 {
+                [check_file_exist $aof_dirpath "${aof_basename}.5${::base_aof_sufix}${::rdb_format_suffix}"] == 0 &&
+                [check_file_exist $aof_dirpath "${aof_basename}.6${::base_aof_sufix}${::rdb_format_suffix}"] == 0 &&
+                [check_file_exist $aof_dirpath "${aof_basename}.1${::incr_aof_sufix}${::aof_format_suffix}"] == 0 &&
+                [check_file_exist $aof_dirpath "${aof_basename}.2${::incr_aof_sufix}${::aof_format_suffix}"] == 0
+            } else {
+                fail "Failed to delete history AOF"
+            }
+        }
+
+        test "AOF can produce consecutive sequence number after reload" {
+            # Current manifest, BASE seq 7 and INCR seq 3
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.7.base.rdb seq 7 type b}
+                {file appendonly.aof.3.incr.aof seq 3 type i}
+            }
+
+            r debug loadaof
+
+            # Trigger AOFRW
+            r bgrewriteaof
+            waitForBgrewriteaof r
+
+            # Now BASE seq is 8 and INCR seq is 4
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.8.base.rdb seq 8 type b}
+                {file appendonly.aof.4.incr.aof seq 4 type i}
+            }
+        }
+
+        test "AOF enable during BGSAVE will not write data util AOFRW finish" {
+            r config set appendonly no
+            r config set save ""
+            r config set rdb-key-save-delay 10000000
+
+            r set k1 v1
+            r bgsave
+
+            wait_for_condition 1000 10 {
+                [s rdb_bgsave_in_progress] eq 1
+            } else {
+                fail "bgsave did not start in time"
+            }
+
+            # Make server.aof_rewrite_scheduled = 1
+            r config set appendonly yes
+            assert_equal [s aof_rewrite_scheduled] 1
+
+            # Not open new INCR aof
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.8.base.rdb seq 8 type b}
+                {file appendonly.aof.4.incr.aof seq 4 type i}
+            }
+
+            r set k2 v2
+            r debug loadaof
+
+            # Both k1 and k2 lost
+            assert_equal 0 [r exists k1]
+            assert_equal 0 [r exists k2]
+
+            set total_forks [s total_forks]
+            assert_equal [s rdb_bgsave_in_progress] 1
+            r config set rdb-key-save-delay 0
+            catch {exec kill -9 [get_child_pid 0]}
+            wait_for_condition 1000 10 {
+                [s rdb_bgsave_in_progress] eq 0
+            } else {
+                fail "bgsave did not stop in time"
+            }
+
+            # Make sure AOFRW was scheduled
+            wait_for_condition 1000 10 {
+                [s total_forks] == [expr $total_forks + 1]
+            } else {
+                fail "aof rewrite did not scheduled"
+            }
+            waitForBgrewriteaof r
+
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.9.base.rdb seq 9 type b}
+                {file appendonly.aof.5.incr.aof seq 5 type i}
+            }
+
+            r set k3 v3
+            r debug loadaof
+            assert_equal v3 [r get k3]
+        }
+
+        test "AOF will trigger limit when AOFRW fails many times" {
+            r config set rdb-key-save-delay 10000000
+            # Let us trigger AOFRW easily
+            r config set auto-aof-rewrite-percentage 1
+            r config set auto-aof-rewrite-min-size 1mb
+
+            # Let AOFRW fail two times, this will trigger AOFRW limit
+            r bgrewriteaof
+            catch {exec kill -9 [get_child_pid 0]}
+            waitForBgrewriteaof r
+
+            r bgrewriteaof
+            catch {exec kill -9 [get_child_pid 0]}
+            waitForBgrewriteaof r
+
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.9.base.rdb seq 9 type b}
+                {file appendonly.aof.5.incr.aof seq 5 type i}
+                {file appendonly.aof.6.incr.aof seq 6 type i}
+                {file appendonly.aof.7.incr.aof seq 7 type i}
+            }
+
+            set orig_size [r dbsize]
+            set load_handle0 [start_write_load $master_host $master_port 10]
+
+            wait_for_condition 50 100 {
+                [r dbsize] > $orig_size
+            } else {
+                fail "No write load detected."
+            }
+
+            # Make sure we have limit log
+            wait_for_condition 1000 10 {
+                [count_log_message 0 "triggered the limit"] == 1
+            } else {
+                fail "aof rewrite did trigger limit"
+            }
+            assert_equal [status r aof_rewrite_in_progress] 0
+
+            # No new INCR AOF be created
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.9.base.rdb seq 9 type b}
+                {file appendonly.aof.5.incr.aof seq 5 type i}
+                {file appendonly.aof.6.incr.aof seq 6 type i}
+                {file appendonly.aof.7.incr.aof seq 7 type i}
+            }
+
+            # Turn off auto rewrite
+            r config set auto-aof-rewrite-percentage 0
+            r config set rdb-key-save-delay 0
+            catch {exec kill -9 [get_child_pid 0]}
+            wait_for_condition 1000 10 {
+                [s aof_rewrite_in_progress] eq 0
+            } else {
+                fail "aof rewrite did not stop in time"
+            }
+
+            # We can still manually execute AOFRW immediately
+            r bgrewriteaof
+            waitForBgrewriteaof r
+
+            # Can create New INCR AOF
+            assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.8${::incr_aof_sufix}${::aof_format_suffix}"]
+
+            assert_aof_manifest_content $aof_manifest_file {
+                {file appendonly.aof.10.base.rdb seq 10 type b}
+                {file appendonly.aof.8.incr.aof seq 8 type i}
+            }
+
+            stop_write_load $load_handle0
+            wait_load_handlers_disconnected
+
+            set d1 [r debug digest]
+            r debug loadaof
+            set d2 [r debug digest]
+            assert {$d1 eq $d2}
+        }
+    }
+}
diff --git a/tests/integration/aof-race.tcl b/tests/integration/aof-race.tcl
index 2991e7962..927896d72 100644
--- a/tests/integration/aof-race.tcl
+++ b/tests/integration/aof-race.tcl
@@ -1,6 +1,5 @@
-set defaults { appendonly {yes} appendfilename {appendonly.aof} aof-use-rdb-preamble {no} }
+set defaults { appendonly {yes} appendfilename {appendonly.aof} appenddirname {appendonlydir} aof-use-rdb-preamble {no} }
 set server_path [tmpdir server.aof]
-set aof_path "$server_path/appendonly.aof"
 
 proc start_server_aof {overrides code} {
     upvar defaults defaults srv srv server_path server_path
diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl
index aa08b0cd4..bc88f2487 100644
--- a/tests/integration/aof.tcl
+++ b/tests/integration/aof.tcl
@@ -1,36 +1,25 @@
-set defaults { appendonly {yes} appendfilename {appendonly.aof} }
+source tests/support/aofmanifest.tcl
+set defaults { appendonly {yes} appendfilename {appendonly.aof} appenddirname {appendonlydir} auto-aof-rewrite-percentage {0}}
 set server_path [tmpdir server.aof]
-set aof_path "$server_path/appendonly.aof"
-
-proc append_to_aof {str} {
-    upvar fp fp
-    puts -nonewline $fp $str
-}
-
-proc create_aof {code} {
-    upvar fp fp aof_path aof_path
-    set fp [open $aof_path w+]
-    uplevel 1 $code
-    close $fp
-}
-
-proc start_server_aof {overrides code} {
-    upvar defaults defaults srv srv server_path server_path
-    set config [concat $defaults $overrides]
-    set srv [start_server [list overrides $config]]
-    uplevel 1 $code
-    kill_server $srv
-}
+set aof_dirname "appendonlydir"
+set aof_basename "appendonly.aof"
+set aof_dirpath "$server_path/$aof_dirname"
+set aof_file "$server_path/$aof_dirname/${aof_basename}.1$::incr_aof_sufix$::aof_format_suffix"
+set aof_manifest_file "$server_path/$aof_dirname/$aof_basename$::manifest_suffix"
 
 tags {"aof external:skip"} {
-    ## Server can start when aof-load-truncated is set to yes and AOF
-    ## is truncated, with an incomplete MULTI block.
-    create_aof {
+    # Server can start when aof-load-truncated is set to yes and AOF
+    # is truncated, with an incomplete MULTI block.
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand set foo hello]
         append_to_aof [formatCommand multi]
         append_to_aof [formatCommand set bar world]
     }
 
+    create_aof_manifest $aof_dirpath $aof_manifest_file {
+        append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+    }
+
     start_server_aof [list dir $server_path aof-load-truncated yes] {
         test "Unfinished MULTI: Server should start if load-truncated is yes" {
             assert_equal 1 [is_alive $srv]
@@ -38,7 +27,7 @@ tags {"aof external:skip"} {
     }
 
     ## Should also start with truncated AOF without incomplete MULTI block.
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand incr foo]
         append_to_aof [formatCommand incr foo]
         append_to_aof [formatCommand incr foo]
@@ -77,7 +66,7 @@ tags {"aof external:skip"} {
     }
 
     ## Test that the server exits when the AOF contains a format error
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand set foo hello]
         append_to_aof "!!!"
         append_to_aof [formatCommand set foo hello]
@@ -102,7 +91,7 @@ tags {"aof external:skip"} {
     }
 
     ## Test the server doesn't start when the AOF contains an unfinished MULTI
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand set foo hello]
         append_to_aof [formatCommand multi]
         append_to_aof [formatCommand set bar world]
@@ -127,7 +116,7 @@ tags {"aof external:skip"} {
     }
 
     ## Test that the server exits when the AOF contains a short read
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand set foo hello]
         append_to_aof [string range [formatCommand set bar world] 0 end-1]
     }
@@ -153,25 +142,25 @@ tags {"aof external:skip"} {
     ## Test that redis-check-aof indeed sees this AOF is not valid
     test "Short read: Utility should confirm the AOF is not valid" {
         catch {
-            exec src/redis-check-aof $aof_path
+            exec src/redis-check-aof $aof_file
         } result
         assert_match "*not valid*" $result
     }
 
     test "Short read: Utility should show the abnormal line num in AOF" {
-        create_aof {
+        create_aof $aof_dirpath $aof_file {
             append_to_aof [formatCommand set foo hello]
             append_to_aof "!!!"
         }
 
         catch {
-            exec src/redis-check-aof $aof_path
+            exec src/redis-check-aof $aof_file
         } result
         assert_match "*ok_up_to_line=8*" $result
     }
 
     test "Short read: Utility should be able to fix the AOF" {
-        set result [exec src/redis-check-aof --fix $aof_path << "y\n"]
+        set result [exec src/redis-check-aof --fix $aof_file << "y\n"]
         assert_match "*Successfully truncated AOF*" $result
     }
 
@@ -190,7 +179,7 @@ tags {"aof external:skip"} {
     }
 
     ## Test that SPOP (that modifies the client's argc/argv) is correctly free'd
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand sadd set foo]
         append_to_aof [formatCommand sadd set bar]
         append_to_aof [formatCommand spop set]
@@ -209,7 +198,7 @@ tags {"aof external:skip"} {
     }
 
     ## Uses the alsoPropagate() API.
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand sadd set foo]
         append_to_aof [formatCommand sadd set bar]
         append_to_aof [formatCommand sadd set gah]
@@ -229,7 +218,7 @@ tags {"aof external:skip"} {
     }
 
     ## Test that PEXPIREAT is loaded correctly
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand rpush list foo]
         append_to_aof [formatCommand pexpireat list 1000]
         append_to_aof [formatCommand rpush list bar]
@@ -247,14 +236,14 @@ tags {"aof external:skip"} {
         }
     }
 
-    start_server {overrides {appendonly {yes} appendfilename {appendonly.aof}}} {
+    start_server {overrides {appendonly {yes}}} {
         test {Redis should not try to convert DEL into EXPIREAT for EXPIRE -1} {
             r set x 10
             r expire x -1
         }
     }
 
-    start_server {overrides {appendonly {yes} appendfilename {appendonly.aof} appendfsync always}} {
+    start_server {overrides {appendonly {yes} appendfsync always}} {
         test {AOF fsync always barrier issue} {
             set rd [redis_deferring_client]
             # Set a sleep when aof is flushed, so that we have a chance to look
@@ -272,7 +261,7 @@ tags {"aof external:skip"} {
                 r del x
                 r setrange x [expr {int(rand()*5000000)+10000000}] x
                 r debug aof-flush-sleep 500000
-                set aof [file join [lindex [r config get dir] 1] appendonly.aof]
+                set aof [get_last_incr_aof_path r]
                 set size1 [file size $aof]
                 $rd get x
                 after [expr {int(rand()*30)}]
@@ -285,9 +274,9 @@ tags {"aof external:skip"} {
         }
     }
 
-    start_server {overrides {appendonly {yes} appendfilename {appendonly.aof}}} {
+    start_server {overrides {appendonly {yes}}} {
         test {GETEX should not append to AOF} {
-            set aof [file join [lindex [r config get dir] 1] appendonly.aof]
+            set aof [get_last_incr_aof_path r]
             r set foo bar
             set before [file size $aof]
             r getex foo
@@ -297,7 +286,7 @@ tags {"aof external:skip"} {
     }
 
     ## Test that the server exits when the AOF contains a unknown command
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand set foo hello]
         append_to_aof [formatCommand bla foo hello]
         append_to_aof [formatCommand set foo hello]
@@ -322,7 +311,7 @@ tags {"aof external:skip"} {
     }
 
     # Test that LMPOP/BLMPOP work fine with AOF.
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand lpush mylist a b c]
         append_to_aof [formatCommand rpush mylist2 1 2 3]
         append_to_aof [formatCommand lpush mylist3 a b c d e]
@@ -369,7 +358,7 @@ tags {"aof external:skip"} {
     }
 
     # Test that ZMPOP/BZMPOP work fine with AOF.
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof [formatCommand zadd myzset 1 one 2 two 3 three]
         append_to_aof [formatCommand zadd myzset2 4 four 5 five 6 six]
         append_to_aof [formatCommand zadd myzset3 1 one 2 two 3 three 4 four 5 five]
@@ -416,22 +405,24 @@ tags {"aof external:skip"} {
     }
 
     test {Generate timestamp annotations in AOF} {
-        start_server {overrides {appendonly {yes} appendfilename {appendonly.aof}}} {
+        start_server {overrides {appendonly {yes}}} {
             r config set aof-timestamp-enabled yes
             r config set aof-use-rdb-preamble no
-            set aof [file join [lindex [r config get dir] 1] appendonly.aof]
+            set aof [get_last_incr_aof_path r]
 
             r set foo bar
             assert_match "#TS:*" [exec head -n 1 $aof]
 
             r bgrewriteaof
             waitForBgrewriteaof r
+
+            set aof [get_base_aof_path r]
             assert_match "#TS:*" [exec head -n 1 $aof]
         }
     }
 
     # redis could load AOF which has timestamp annotations inside
-    create_aof {
+    create_aof $aof_dirpath $aof_file {
         append_to_aof "#TS:1628217470\r\n"
         append_to_aof [formatCommand set foo1 bar1]
         append_to_aof "#TS:1628217471\r\n"
@@ -453,7 +444,7 @@ tags {"aof external:skip"} {
 
     test {Truncate AOF to specific timestamp} {
         # truncate to timestamp 1628217473
-        exec src/redis-check-aof --truncate-to-timestamp 1628217473 $aof_path
+        exec src/redis-check-aof --truncate-to-timestamp 1628217473 $aof_file
         start_server_aof [list dir $server_path] {
             set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
             wait_done_loading $c
@@ -463,7 +454,7 @@ tags {"aof external:skip"} {
         }
 
         # truncate to timestamp 1628217471
-        exec src/redis-check-aof --truncate-to-timestamp 1628217471 $aof_path
+        exec src/redis-check-aof --truncate-to-timestamp 1628217471 $aof_file
         start_server_aof [list dir $server_path] {
             set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
             wait_done_loading $c
@@ -473,7 +464,7 @@ tags {"aof external:skip"} {
         }
 
         # truncate to timestamp 1628217470
-        exec src/redis-check-aof --truncate-to-timestamp 1628217470 $aof_path
+        exec src/redis-check-aof --truncate-to-timestamp 1628217470 $aof_file
         start_server_aof [list dir $server_path] {
             set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
             wait_done_loading $c
@@ -482,12 +473,12 @@ tags {"aof external:skip"} {
         }
 
         # truncate to timestamp 1628217469
-        catch {exec src/redis-check-aof --truncate-to-timestamp 1628217469 $aof_path} e
+        catch {exec src/redis-check-aof --truncate-to-timestamp 1628217469 $aof_file} e
         assert_match {*aborting*} $e
     }
 
     test {EVAL timeout with slow verbatim Lua script from AOF} {
-        create_aof {
+        create_aof $aof_dirpath $aof_file {
             append_to_aof [formatCommand select 9]
             append_to_aof [formatCommand eval {redis.call('set',KEYS[1],'y'); for i=1,1500000 do redis.call('ping') end return 'ok'} 1 x]
         }
@@ -512,7 +503,10 @@ tags {"aof external:skip"} {
     }
 
     test {EVAL can process writes from AOF in read-only replicas} {
-        create_aof {
+        create_aof_manifest $aof_dirpath $aof_manifest_file {
+            append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
+        }
+        create_aof $aof_dirpath $aof_file {
             append_to_aof [formatCommand select 9]
             append_to_aof [formatCommand eval {redis.call("set",KEYS[1],"100")} 1 foo]
             append_to_aof [formatCommand eval {redis.call("incr",KEYS[1])} 1 foo]
diff --git a/tests/support/aofmanifest.tcl b/tests/support/aofmanifest.tcl
new file mode 100644
index 000000000..ffde3e354
--- /dev/null
+++ b/tests/support/aofmanifest.tcl
@@ -0,0 +1,169 @@
+set ::base_aof_sufix ".base"
+set ::incr_aof_sufix ".incr"
+set ::manifest_suffix ".manifest"
+set ::aof_format_suffix ".aof"
+set ::rdb_format_suffix ".rdb"
+
+proc get_full_path {dir filename} {
+    set _ [format "%s/%s" $dir $filename]
+}
+
+proc join_path {dir1 dir2} {
+    return [format "%s/%s" $dir1 $dir2]
+}
+
+proc get_redis_dir {} {
+    set config [srv config]
+    set _ [dict get $config "dir"]
+}
+
+proc check_file_exist {dir filename} {
+    set file_path [get_full_path $dir $filename]
+    return [file exists $file_path]
+}
+
+proc del_file {dir filename} {
+    set file_path [get_full_path $dir $filename]
+    catch {exec rm -rf $file_path}
+}
+
+proc get_cur_base_aof_name {manifest_filepath} {
+    set fp [open $manifest_filepath r+]
+    set lines {}
+    while {1} {
+        set line [gets $fp]
+        if {[eof $fp]} {
+           close $fp
+           break;
+        }
+
+        lappend lines $line
+    }
+
+    if {[llength $lines] == 0} {
+        return ""
+    }
+
+    set first_line [lindex $lines 0]
+    set aofname [lindex [split $first_line " "] 1]
+    set aoftype [lindex [split $first_line " "] 5]
+    if { $aoftype eq "b" } {
+        return $aofname
+    }
+
+    return ""
+}
+
+proc get_last_incr_aof_name {manifest_filepath} {
+    set fp [open $manifest_filepath r+]
+    set lines {}
+    while {1} {
+        set line [gets $fp]
+        if {[eof $fp]} {
+           close $fp
+           break;
+        }
+
+        lappend lines $line
+    }
+
+    if {[llength $lines] == 0} {
+        return ""
+    }
+
+    set len [llength $lines]
+    set last_line [lindex $lines [expr $len - 1]]
+    set aofname [lindex [split $last_line " "] 1]
+    set aoftype [lindex [split $last_line " "] 5]
+    if { $aoftype eq "i" } {
+        return $aofname
+    }
+
+    return ""
+}
+
+proc get_last_incr_aof_path {r} {
+    set dir [lindex [$r config get dir] 1]
+    set appenddirname [lindex [$r config get appenddirname] 1]
+    set appendfilename [lindex [$r config get appendfilename] 1]
+    set manifest_filepath [file join $dir $appenddirname $appendfilename$::manifest_suffix]
+    set last_incr_aof_name [get_last_incr_aof_name $manifest_filepath]
+    if {$last_incr_aof_name == ""} {
+        return ""
+    }
+    return [file join $dir $appenddirname $last_incr_aof_name]
+}
+
+proc get_base_aof_path {r} {
+    set dir [lindex [$r config get dir] 1]
+    set appenddirname [lindex [$r config get appenddirname] 1]
+    set appendfilename [lindex [$r config get appendfilename] 1]
+    set manifest_filepath [file join $dir $appenddirname $appendfilename$::manifest_suffix]
+    set cur_base_aof_name [get_cur_base_aof_name $manifest_filepath]
+    if {$cur_base_aof_name == ""} {
+        return ""
+    }
+    return [file join $dir $appenddirname $cur_base_aof_name]
+}
+
+proc assert_aof_manifest_content {manifest_path content} {
+    set fp [open $manifest_path r+]
+    set lines {}
+    while {1} {
+        set line [gets $fp]
+        if {[eof $fp]} {
+           close $fp
+           break;
+        }
+
+        lappend lines $line
+    }
+
+    assert_equal [llength $lines] [llength $content]
+
+    for { set i 0 } { $i < [llength $lines] } {incr i} {
+        assert_equal [lindex $lines $i] [lindex $content $i]
+    }
+}
+
+proc clean_aof_persistence {aof_dirpath} {
+    catch {eval exec rm -rf [glob $aof_dirpath]}
+}
+
+proc append_to_manifest {str} {
+    upvar fp fp
+    puts -nonewline $fp $str
+}
+
+proc create_aof_manifest {dir aof_manifest_file code} {
+    create_aof_dir $dir
+    upvar fp fp
+    set fp [open $aof_manifest_file w+]
+    uplevel 1 $code
+    close $fp
+}
+
+proc append_to_aof {str} {
+    upvar fp fp
+    puts -nonewline $fp $str
+}
+
+proc create_aof {dir aof_file code} {
+    create_aof_dir $dir
+    upvar fp fp
+    set fp [open $aof_file w+]
+    uplevel 1 $code
+    close $fp
+}
+
+proc create_aof_dir {dir_path} {
+    file mkdir $dir_path
+}
+
+proc start_server_aof {overrides code} {
+    upvar defaults defaults srv srv server_path server_path
+    set config [concat $defaults $overrides]
+    set srv [start_server [list overrides $config]]
+    uplevel 1 $code
+    kill_server $srv
+}
diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 51d3629a2..a65fa8d7e 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -30,10 +30,16 @@ proc clean_persistence config {
     # we may wanna keep the logs for later, but let's clean the persistence
     # files right away, since they can accumulate and take up a lot of space
     set config [dict get $config "config"]
-    set rdb [format "%s/%s" [dict get $config "dir"] "dump.rdb"]
-    set aof [format "%s/%s" [dict get $config "dir"] "appendonly.aof"]
+    set dir [dict get $config "dir"]
+    set rdb [format "%s/%s" $dir "dump.rdb"]
+    if {[dict exists $config "appenddirname"]} {
+        set aofdir [dict get $config "appenddirname"]
+    } else {
+        set aofdir "appendonlydir"
+    }
+    set aof_dirpath [format "%s/%s" $dir $aofdir]
+    clean_aof_persistence $aof_dirpath
     catch {exec rm -rf $rdb}
-    catch {exec rm -rf $aof}
 }
 
 proc kill_server config {
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index aa14fd3a0..758e06668 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -84,7 +84,7 @@ proc status {r property} {
 
 proc waitForBgsave r {
     while 1 {
-        if {[status r rdb_bgsave_in_progress] eq 1} {
+        if {[status $r rdb_bgsave_in_progress] eq 1} {
             if {$::verbose} {
                 puts -nonewline "\nWaiting for background save to finish... "
                 flush stdout
@@ -98,7 +98,7 @@ proc waitForBgsave r {
 
 proc waitForBgrewriteaof r {
     while 1 {
-        if {[status r aof_rewrite_in_progress] eq 1} {
+        if {[status $r aof_rewrite_in_progress] eq 1} {
             if {$::verbose} {
                 puts -nonewline "\nWaiting for background AOF rewrite to finish... "
                 flush stdout
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index d47e83c56..afa868bd9 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -6,6 +6,7 @@ package require Tcl 8.5
 
 set tcl_precision 17
 source tests/support/redis.tcl
+source tests/support/aofmanifest.tcl
 source tests/support/server.tcl
 source tests/support/tmpfile.tcl
 source tests/support/test.tcl
@@ -46,6 +47,7 @@ set ::all_tests {
     integration/replication-buffer
     integration/shutdown
     integration/aof
+    integration/aof-multi-part
     integration/rdb
     integration/corrupt-dump
     integration/corrupt-dump-fuzzer
diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl
index 7d4397a12..76cb29483 100644
--- a/tests/unit/expire.tcl
+++ b/tests/unit/expire.tcl
@@ -308,14 +308,14 @@ start_server {tags {"expire"}} {
     } {-2}
 
     # Start a new server with empty data and AOF file.
-    start_server {overrides {appendonly {yes} appendfilename {appendonly.aof} appendfsync always} tags {external:skip}} {
+    start_server {overrides {appendonly {yes} appendfsync always} tags {external:skip}} {
         test {All time-to-live(TTL) in commands are propagated as absolute timestamp in milliseconds in AOF} {
             # This test makes sure that expire times are propagated as absolute
             # times to the AOF file and not as relative time, so that when the AOF
             # is reloaded the TTLs are not being shifted forward to the future.
             # We want the time to logically pass when the server is restarted!
 
-            set aof [file join [lindex [r config get dir] 1] [lindex [r config get appendfilename] 1]]
+            set aof [get_last_incr_aof_path r]
 
             # Apply each TTL-related command to a unique key
             # SET commands
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index 22a91ea4f..09ca070d5 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -183,6 +183,7 @@ start_server {tags {"introspection"}} {
             pidfile
             syslog-ident
             appendfilename
+            appenddirname
             supervised
             syslog-facility
             databases