PSYNC2: make partial sync possible after master reboot (#8015)

The main idea is how to allow a master to load replication info from RDB file when rebooting, if master can load replication info it means that replicas may have the chance to psync with master, it can save much traffic.

The key point is we need guarantee safety and consistency, so there
are two differences between master and replica:

1. master would load the replication info as secondary ID and
   offset, in case other masters have the same replid.
2. when master loading RDB, it would propagate expired keys as DEL
   command to replication backlog, then replica can receive these
   commands to delete stale keys.
   p.s. the expired keys when RDB loading is useful for users, so
   we show it as `rdb_last_load_keys_expired` and `rdb_last_load_keys_loaded` in info persistence.

Moreover, after load replication info, master should update
`no_replica_time` in case loading RDB cost too long time.

1 files changed, 159 insertions, 0 deletions

diff --git a/tests/integration/psync2-master-restart.tcl b/tests/integration/psync2-master-restart.tcl
new file mode 100644
index 000000000..301a4f63d
--- /dev/null
+++ b/tests/integration/psync2-master-restart.tcl
@@ -0,0 +1,159 @@
+start_server {tags {"psync2 external:skip"}} {
+start_server {} {
+start_server {} {
+    set master [srv 0 client]
+    set master_host [srv 0 host]
+    set master_port [srv 0 port]
+
+    set replica [srv -1 client]
+    set replica_host [srv -1 host]
+    set replica_port [srv -1 port]
+
+    set sub_replica [srv -2 client]
+
+    # Build replication chain
+    $replica replicaof $master_host $master_port
+    $sub_replica replicaof $replica_host $replica_port
+
+    wait_for_condition 50 100 {
+        [status $replica master_link_status] eq {up} &&
+        [status $sub_replica master_link_status] eq {up}
+    } else {
+        fail "Replication not started."
+    }
+
+    # Avoid PINGs
+    $master config set repl-ping-replica-period 3600
+    $master config rewrite
+
+    # Generate some data
+    createComplexDataset $master 1000
+
+    test "PSYNC2: Partial resync after Master restart using RDB aux fields" {
+        wait_for_condition 500 100 {
+            [status $master master_repl_offset] == [status $replica master_repl_offset] &&
+            [status $master master_repl_offset] == [status $sub_replica master_repl_offset]
+        } else {
+            fail "Replicas and master offsets were unable to match *exactly*."
+        }
+
+        set replid [status $master master_replid]
+        set offset [status $master master_repl_offset]
+        $replica config resetstat
+
+        catch {
+            restart_server 0 true false
+            set master [srv 0 client]
+        }
+        wait_for_condition 50 1000 {
+            [status $replica master_link_status] eq {up} &&
+            [status $sub_replica master_link_status] eq {up}
+        } else {
+            fail "Replicas didn't sync after master restart"
+        }
+
+        # Make sure master restore replication info correctly
+        assert {[status $master master_replid] != $replid}
+        assert {[status $master master_repl_offset] == $offset}
+        assert {[status $master master_replid2] eq $replid}
+        assert {[status $master second_repl_offset] == [expr $offset+1]}
+
+        # Make sure master set replication backlog correctly
+        assert {[status $master repl_backlog_active] == 1}
+        assert {[status $master repl_backlog_first_byte_offset] == [expr $offset+1]}
+        assert {[status $master repl_backlog_histlen] == 0}
+
+        # Partial resync after Master restart
+        assert {[status $master sync_partial_ok] == 1}
+        assert {[status $replica sync_partial_ok] == 1}
+    }
+
+    test "PSYNC2: Partial resync after Master restart using RDB aux fields with expire" {
+        $master debug set-active-expire 0
+        for {set j 0} {$j < 1024} {incr j} {
+            $master select [expr $j%16]
+            $master set $j somevalue px 10
+        }
+
+        after 20
+
+        wait_for_condition 500 100 {
+            [status $master master_repl_offset] == [status $replica master_repl_offset] &&
+            [status $master master_repl_offset] == [status $sub_replica master_repl_offset]
+        } else {
+            show_cluster_status
+            fail "Replicas and master offsets were unable to match *exactly*."
+        }
+
+        set offset [status $master master_repl_offset]
+        $replica config resetstat
+
+        catch {
+            restart_server 0 true false
+            set master [srv 0 client]
+        }
+        wait_for_condition 50 1000 {
+            [status $replica master_link_status] eq {up} &&
+            [status $sub_replica master_link_status] eq {up}
+        } else {
+            fail "Replicas didn't sync after master restart"
+        }
+
+        set expired_offset [status $master repl_backlog_histlen]
+        # Stale keys expired and master_repl_offset grows correctly
+        assert {[status $master rdb_last_load_keys_expired] == 1024}
+        assert {[status $master master_repl_offset] == [expr $offset+$expired_offset]}
+
+        # Partial resync after Master restart
+        assert {[status $master sync_partial_ok] == 1}
+        assert {[status $replica sync_partial_ok] == 1}
+
+        set digest [$master debug digest]
+        assert {$digest eq [$replica debug digest]}
+        assert {$digest eq [$sub_replica debug digest]}
+    }
+
+    test "PSYNC2: Full resync after Master restart when too many key expired" {
+        $master config set repl-backlog-size 16384
+        $master config rewrite
+
+        $master debug set-active-expire 0
+        for {set j 0} {$j < 1024} {incr j} {
+            $master select [expr $j%16]
+            $master set $j somevalue px 10
+        }
+
+        after 20
+
+        wait_for_condition 500 100 {
+            [status $master master_repl_offset] == [status $replica master_repl_offset] &&
+            [status $master master_repl_offset] == [status $sub_replica master_repl_offset]
+        } else {
+            fail "Replicas and master offsets were unable to match *exactly*."
+        }
+
+        $replica config resetstat
+
+        catch {
+            restart_server 0 true false
+            set master [srv 0 client]
+        }
+        wait_for_condition 50 1000 {
+            [status $replica master_link_status] eq {up} &&
+            [status $sub_replica master_link_status] eq {up}
+        } else {
+            fail "Replicas didn't sync after master restart"
+        }
+
+        # Replication backlog is full
+        assert {[status $master repl_backlog_first_byte_offset] > [status $master second_repl_offset]}
+        assert {[status $master sync_partial_ok] == 0}
+        assert {[status $master sync_full] == 1}
+        assert {[status $master rdb_last_load_keys_expired] == 1024}
+        assert {[status $replica sync_full] == 1}
+
+        set digest [$master debug digest]
+        assert {$digest eq [$replica debug digest]}
+        assert {$digest eq [$sub_replica debug digest]}
+    }
+}}}