Import wiredtiger: 934a2c8412d3625abe4935ef93b767e878ab9541 from branch mongodb-master

ref: a01b2e5d7f..934a2c8412
for: 6.1.0-rc0

WT-9348 Merge import_compatibility_test.sh into compatibility_test_for_releases.sh (#8190)

2 files changed, 143 insertions, 173 deletions

diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh
index 6c1b5548469..d06c66b5e17 100755
--- a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh
+++ b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh
@@ -528,13 +528,127 @@ prepare_test_data_wt_8395()
     rm *.tar.gz; cd ../..
 }
 
+#############################################################
+# create_file:
+#       arg1: branch
+#       arg2: file
+#############################################################
+create_file()
+{
+    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+    echo "Branch \"$1\" creating and populating \"$2\""
+    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+
+    wt_cmd="$1/build/wt"
+    test_dir="$1/build/WT_TEST/"
+    uri="file:$2"
+
+    # Make the home directory.
+    mkdir -p $test_dir
+
+    # Create the file and populate with a few key/values.
+    $wt_cmd -h $test_dir create -c "key_format=S,value_format=S" $uri
+    $wt_cmd -h $test_dir write $uri abc 123 def 456 hij 789
+}
+
+#############################################################
+# import_file:
+#       arg1: source branch
+#       arg2: dest branch
+#       arg3: file
+#############################################################
+import_file()
+{
+    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+    echo "Importing file \"$3\" from \"$1\" to \"$2\""
+    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+
+    wt_cmd="$2/build/wt"
+    test_dir="$2/build/WT_TEST/"
+    mkdir -p $test_dir
+
+    # Move the file across to the destination branch's home directory.
+    import_file="$1/build/WT_TEST/$3"
+    cp $import_file $test_dir
+
+    # Run import via the wt tool.
+    uri="file:$3"
+    $wt_cmd -h $test_dir create -c "import=(enabled,repair=true)" $uri
+}
+
+#############################################################
+# verify_file:
+#       arg1: branch
+#       arg2: file
+#############################################################
+verify_file()
+{
+    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+    echo "Branch \"$1\" verifying \"$2\""
+    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+
+    wt_cmd="$1/build/wt"
+    test_dir="$1/build/WT_TEST/"
+    uri="file:$2"
+
+    $wt_cmd -h $test_dir verify $uri
+}
+
+#############################################################
+# cleanup_branch:
+#       arg1: branch
+#############################################################
+cleanup_branch()
+{
+    test_dir="$1/build/WT_TEST/"
+    if [ -d $test_dir ]; then
+        rm -rf $test_dir
+    fi
+}
+
+#############################################################
+# import_compatibility_test:
+#       arg1: older branch
+#       arg2: newer branch
+#############################################################
+import_compatibility_test()
+{
+    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+    echo "Testing import compatibility between \"$2\" and \"$1\""
+    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+
+    # Remove any leftover data files.
+    cleanup_branch $1
+    cleanup_branch $2
+
+    # Create a file in the older branch.
+    create_file $1 test_import
+
+    # Now import it into the newer branch and verify.
+    import_file $1 $2 test_import
+    verify_file $2 test_import
+
+    # Now downgrade by running wt from the older branch and dumping the table contents.
+    #
+    # Before trying this, we must remove the base configuration. The wt tool produces this file
+    # however MongoDB will not so we should emulate this.
+    rm $2/build/WT_TEST/WiredTiger.basecfg
+    $1/build/wt -h $2/build/WT_TEST/ dump file:test_import
+}
+
 # Only one of below flags will be set by the 1st argument of the script.
+import=false
 older=false
 newer=false
 wt_standalone=false
 patch_version=false
 upgrade_to_latest=false
 
+# This array is used to configure the release branches we'd like to use for testing the importing
+# of files created in previous versions of WiredTiger. Go all the way back to mongodb-4.2 since
+# that's the first release where we don't support live import.
+import_release_branches=(develop mongodb-6.0 mongodb-5.0 mongodb-4.4 mongodb-4.2)
+
 # Branches in below 2 arrays should be put in newer-to-older order.
 #
 # An overlap (last element of the 1st array & first element of the 2nd array)
@@ -562,6 +676,7 @@ test_checkpoint_release_branches=(develop mongodb-6.0 mongodb-5.0 mongodb-4.4)
 upgrade_to_latest_upgrade_downgrade_release_branches=(mongodb-6.0 mongodb-5.0 mongodb-4.4)
 
 declare -A scopes
+scopes[import]="import files from previous versions"
 scopes[newer]="newer stable release branches"
 scopes[older]="older stable release branches"
 scopes[patch_version]="patch versions of the same release branch"
@@ -612,7 +727,8 @@ get_build_system()
 #############################################################
 usage()
 {
-    echo -e "Usage: \tcompatibility_test_for_releases [-n|-o|-p|-u|-w|-v]"
+    echo -e "Usage: \tcompatibility_test_for_releases [-i|-n|-o|-p|-u|-w|-v]"
+    echo -e "\t-i\trun compatibility tests for ${scopes[import]}"
     echo -e "\t-n\trun compatibility tests for ${scopes[newer]}"
     echo -e "\t-o\trun compatibility tests for ${scopes[older]}"
     echo -e "\t-p\trun compatibility tests for ${scopes[patch_version]}"
@@ -628,6 +744,12 @@ fi
 
 # Script argument processing
 case $1 in
+"-i")
+    import=true
+    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+    echo "Performing compatibility tests for ${scopes[import]}"
+    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+;;
 "-n")
     newer=true
     echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
@@ -676,6 +798,26 @@ top="test-compatibility-run"
 rm -rf "$top" && mkdir "$top"
 cd "$top"
 
+# Import compatibility testing.
+if [ "$import" = true ]; then
+    for b in ${import_release_branches[@]}; do
+        (build_branch $b)
+    done
+
+    for i in ${!import_release_branches[@]}; do
+        newer=${import_release_branches[$i]}
+
+        # MongoDB v4.2 doesn't support live import so it should only ever be used as the "older" branch
+        # that we're importing from.
+        if [ $newer = mongodb-4.2 ]; then
+            continue
+        fi
+
+        older=${import_release_branches[$i+1]}
+        import_compatibility_test $older $newer
+    done
+fi
+
 if [ "$upgrade_to_latest" = true ]; then
     test_root=$(pwd)
     test_data_root="$test_root/mongo-tests"
diff --git a/src/third_party/wiredtiger/test/evergreen/import_compatibility_test.sh b/src/third_party/wiredtiger/test/evergreen/import_compatibility_test.sh
deleted file mode 100755
index 69427473745..00000000000
--- a/src/third_party/wiredtiger/test/evergreen/import_compatibility_test.sh
+++ /dev/null
@@ -1,172 +0,0 @@
-#!/usr/bin/env bash
-#
-# Test importing of files created in previous versions of WiredTiger.
-# Test that we can downgrade a database after importing a file.
-
-set -e
-
-# build_branch --
-#     1: branch
-build_branch()
-{
-    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
-    echo "Building branch: \"$1\""
-    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
-
-    # Clone if it doesn't already exist.
-    if [ ! -d "$1" ]; then
-        git clone --quiet https://github.com/wiredtiger/wiredtiger.git "$1"
-    fi
-    cd "$1"
-
-    git checkout --quiet "$1"
-
-    if [ "${build_sys[$1]}" == "cmake" ]; then
-            . ./test/evergreen/find_cmake.sh
-            config=""
-            config+="-DENABLE_SNAPPY=1 "
-            config+="-DWT_STANDALONE_BUILD=0 "
-            (mkdir build && cd build &&
-                $CMAKE $config ../. && make -j $(grep -c ^processor /proc/cpuinfo)) > /dev/null
-    else
-        config=""
-        config+="--enable-snappy "
-        config+="--disable-standalone-build "
-        (mkdir build && cd build && sh ../build_posix/reconf &&
-            ../configure $config && make -j $(grep -c ^processor /proc/cpuinfo)) > /dev/null
-    fi
-    cd ..
-}
-
-# create_file --
-#     1: branch
-#     2: file
-create_file()
-{
-    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
-    echo "Branch \"$1\" creating and populating \"$2\""
-    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
-
-    wt_cmd="$1/build/wt"
-    test_dir="$1/build/WT_TEST/"
-    uri="file:$2"
-
-    # Make the home directory.
-    mkdir -p $test_dir
-
-    # Create the file and populate with a few key/values.
-    $wt_cmd -h $test_dir create -c "key_format=S,value_format=S" $uri
-    $wt_cmd -h $test_dir write $uri abc 123 def 456 hij 789
-}
-
-# import_file --
-#     1: dest branch
-#     2: source branch
-#     3: file
-import_file()
-{
-    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
-    echo "Importing file \"$3\" from \"$1\" to \"$2\""
-    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
-
-    wt_cmd="$1/build/wt"
-    test_dir="$1/build/WT_TEST/"
-    mkdir -p $test_dir
-
-    # Move the file across to the destination branch's home directory.
-    import_file="$2/build/WT_TEST/$3"
-    cp $import_file $test_dir
-
-    # Run import via the wt tool.
-    uri="file:$3"
-    $wt_cmd -h $test_dir create -c "import=(enabled,repair=true)" $uri
-}
-
-# verify_file --
-#     1: branch
-#     2: file
-verify_file()
-{
-    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
-    echo "Branch \"$1\" verifying \"$2\""
-    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
-
-    wt_cmd="$1/build/wt"
-    test_dir="$1/build/WT_TEST/"
-    uri="file:$2"
-
-    $wt_cmd -h $test_dir verify $uri
-}
-
-# cleanup_branch --
-#     1: branch
-cleanup_branch()
-{
-    test_dir="$1/build/WT_TEST/"
-    if [ -d $test_dir ]; then
-        rm -rf $test_dir
-    fi
-}
-
-# import_compatibility_test --
-#     1: newer branch
-#     2: older branch
-import_compatibility_test()
-{
-    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
-    echo "Testing import compatibility between \"$1\" and \"$2\""
-    echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
-
-    # Remove any leftover data files.
-    cleanup_branch $1
-    cleanup_branch $2
-
-    # Create a file in the older branch.
-    create_file $2 test_import
-
-    # Now import it into the newer branch and verify.
-    import_file $1 $2 test_import
-    verify_file $1 test_import
-
-    # Now downgrade by running wt from the older branch and dumping the table contents.
-    #
-    # Before trying this, we must remove the base configuration. The wt tool produces this file
-    # however MongoDB will not so we should emulate this.
-    rm $1/build/WT_TEST/WiredTiger.basecfg
-    $2/build/wt -h $1/build/WT_TEST/ dump file:test_import
-}
-
-# The following associative array maps the 'official' build system to use for each branch.
-# CMake build support is reliably mature in newer release branches, whilst earlier revisions
-# primarily use Autoconf (note: some earlier branches may have CMake support, but these aren't
-# considered 'mature' versions.)
-declare -A build_sys
-build_sys['develop']="cmake"
-build_sys['mongodb-6.0']="cmake"
-build_sys['mongodb-5.0']="autoconf"
-build_sys['mongodb-4.4']="autoconf"
-build_sys['mongodb-4.2']="autoconf"
-
-# Release branches.
-#
-# Go all the way back to mongodb-4.2 since that's the first release where we don't support live
-# import.
-release_branches=(develop mongodb-6.0 mongodb-5.0 mongodb-4.4 mongodb-4.2)
-
-# Build each of the release branches.
-for b in ${release_branches[@]}; do
-    build_branch $b
-done
-
-for i in ${!release_branches[@]}; do
-    newer=${release_branches[$i]}
-
-    # MongoDB v4.2 doesn't support live import so it should only ever be used as the "older" branch
-    # that we're importing from.
-    if [ $newer = mongodb-4.2 ]; then
-        continue
-    fi
-
-    older=${release_branches[$i+1]}
-    import_compatibility_test $newer $older
-done