Import wiredtiger: ed9c74ed0913cf7ff8b9a70ea0e0bd0c86b63a37 from branch mongodb-master

ref: 8a432f1e38..ed9c74ed09
for: 5.3.0

WT-8554       Automated testing for upgrade from 4.4 releases

3 files changed, 106 insertions, 2 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 2fc71015b98..e5851ac0afe 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "8a432f1e385a044307d7c31bb5287878c12585dc"
+    "commit": "ed9c74ed0913cf7ff8b9a70ea0e0bd0c86b63a37"
 }
diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml
index ef3b617498b..49dbec28ad6 100755
--- a/src/third_party/wiredtiger/test/evergreen.yml
+++ b/src/third_party/wiredtiger/test/evergreen.yml
@@ -2029,6 +2029,17 @@ tasks:
             set -o verbose
             test/evergreen/compatibility_test_for_releases.sh -o
 
+  - name: compatibility-test-upgrade-to-latest
+    commands:
+      - func: "get project"
+      - command: shell.exec
+        params:
+          working_dir: "wiredtiger"
+          script: |
+            set -o errexit
+            set -o verbose
+            test/evergreen/compatibility_test_for_releases.sh -u
+
   - name: compatibility-test-for-wt-standalone-releases
     commands:
       - func: "get project"
@@ -4249,6 +4260,14 @@ buildvariants:
     - name: compatibility-test-for-older-releases
     - name: compatibility-test-for-wt-standalone-releases
 
+- name: compatibility-tests-daily
+  display_name: Compatibility tests (daily)
+  batchtime: 1440 # 1 day
+  run_on:
+  - ubuntu2004-test
+  tasks:
+    - name: compatibility-test-upgrade-to-latest
+
 - name: compatibility-tests
   display_name: Compatibility tests
   run_on:
diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh
index b686b6a90ba..655159bb8bc 100755
--- a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh
+++ b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh
@@ -256,10 +256,68 @@ upgrade_downgrade()
         done
 }
 
+#############################################################
+# test_upgrade_to_branch:
+#       arg1: release branch name
+#       arg2: path to test data folder
+#############################################################
+test_upgrade_to_branch()
+{
+        cd $1/test/checkpoint
+
+        for FILE in $2/*; do
+            # Run actual test.
+            echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+            echo "Upgrading $FILE database to $1..."
+
+            # Disable exit on non 0
+            set +e
+
+            output="$(./t -t r -D -v -h $FILE)"
+            test_res=$?
+
+            # Enable exit on non 0
+            set -e
+
+            # Validate test result.
+            if [[ "$FILE" =~ "4.4."[0-6]"_unclean"$ ]]; then
+                echo "Databases generated with unclean shutdown from versions 4.4.[0-6] must fail."
+                if [[ "$test_res" == 0 ]]; then
+                    echo "$output"
+                    echo "Error: Upgrade of $FILE database to $1 has not failed!"
+                    exit 1
+                fi
+            elif [[ "$test_res" != 0 ]]; then
+                echo "$output"
+                echo "Error: Upgrade of $FILE database to $1 failed! Test result is $test_res."
+                exit 1
+            fi
+
+            echo "Success!"
+            echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+        done
+}
+
+#############################################################
+# prepare_test_data_wt_8395:
+#############################################################
+prepare_test_data_wt_8395()
+{
+        echo "Preparing test data..."
+        git clone --quiet --depth 1 --filter=blob:none --no-checkout https://github.com/wiredtiger/mongo-tests.git
+        cd mongo-tests
+        git checkout --quiet master -- WT-8395 &> /dev/null
+        cd WT-8395
+
+        for FILE in *; do tar -zxf $FILE; done
+        rm *.tar.gz; cd ../..
+}
+
 # Only one of below flags will be set by the 1st argument of the script.
 older=false
 newer=false
 wt_standalone=false
+upgrade_to_latest=false
 
 # Branches in below 2 arrays should be put in newer-to-older order.
 #
@@ -276,9 +334,13 @@ older_release_branches=(mongodb-4.2 mongodb-4.0 mongodb-3.6)
 # configuration file. 
 compatible_upgrade_downgrade_release_branches=(mongodb-4.4 mongodb-4.2)
 
+# This array is used to configure the release branches we'd like to run upgrade to latest test.
+upgrade_to_latest_upgrade_downgrade_release_branches=(mongodb-5.0 mongodb-4.4)
+
 declare -A scopes
 scopes[newer]="newer stable release branches"
 scopes[older]="older stable release branches"
+scopes[upgrade_to_latest]="upgrade/downgrade databases to the latest versions of the codebase"
 scopes[wt_standalone]="WiredTiger standalone releases"
 
 #############################################################
@@ -286,9 +348,10 @@ scopes[wt_standalone]="WiredTiger standalone releases"
 #############################################################
 usage()
 {
-    echo -e "Usage: \tcompatibility_test_for_releases [-n|-o|-w]"
+    echo -e "Usage: \tcompatibility_test_for_releases [-n|-o|-u|-w]"
     echo -e "\t-n\trun compatibility tests for ${scopes[newer]}"
     echo -e "\t-o\trun compatibility tests for ${scopes[older]}"
+    echo -e "\t-u\trun compatibility tests for ${scopes[upgrade_to_latest]}"
     echo -e "\t-w\trun compatibility tests for ${scopes[wt_standalone]}"
     exit 1
 }
@@ -311,6 +374,12 @@ case $1 in
     echo "Performing compatibility tests for ${scopes[older]}"
     echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
 ;;
+"-u")
+    upgrade_to_latest=true
+    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+    echo "Performing compatibility tests for ${scopes[upgrade_to_latest]}"
+    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+;;
 "-w")
     wt_standalone=true
     echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
@@ -327,6 +396,22 @@ top="test-compatibility-run"
 rm -rf "$top" && mkdir "$top"
 cd "$top"
 
+if [ "$upgrade_to_latest" = true ]; then
+    test_root=$(pwd)
+    test_data_root="$test_root/mongo-tests"
+    test_data="$test_root/mongo-tests/WT-8395"
+
+    for b in ${upgrade_to_latest_upgrade_downgrade_release_branches[@]}; do
+        # prepare test data and test upgrade to the branch b.
+        (prepare_test_data_wt_8395) && \
+        (build_branch $b) && \
+        (test_upgrade_to_branch $b $test_data)
+
+        # cleanup.
+        cd $test_root
+        rm -rf $test_data_root
+    done
+fi
 
 # Build the branches.
 if [ "$newer" = true ]; then