summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCheahuychou Mao <mao.cheahuychou@gmail.com>2021-01-21 01:25:10 +0000
committerEvergreen Agent <no-reply@evergreen.mongodb.com>2021-01-28 02:11:41 +0000
commite9360bd8e7cb8f1447ffd513149d284c394bb4a0 (patch)
tree7404bfc08bba640c65cbaf1aed551a6428aedf06
parent96fe72c36d370a4067240738f051021d4daf72ce (diff)
downloadmongo-e9360bd8e7cb8f1447ffd513149d284c394bb4a0.tar.gz
SERVER-53404 Make tenant migration donor copy the recipient's cluster time signing keys before sending recipientSyncData
-rw-r--r--buildscripts/resmokeconfig/suites/replica_sets_auth.yml2
-rw-r--r--jstests/libs/rs0_tenant_migration.pem88
-rw-r--r--jstests/libs/rs0_tenant_migration_expired.pem88
-rw-r--r--jstests/libs/rs0_tenant_migration_no_find_cluster_time_keys_role.pem55
-rw-r--r--jstests/libs/rs1_tenant_migration.pem88
-rw-r--r--jstests/libs/rs1_tenant_migration_expired.pem88
-rw-r--r--jstests/libs/rs1_tenant_migration_no_advance_cluster_time_role.pem55
-rw-r--r--jstests/libs/rs1_tenant_migration_no_backup_role.pem86
-rw-r--r--jstests/libs/rs1_tenant_migration_no_find_cluster_time_keys_role.pem55
-rw-r--r--jstests/libs/rs2_tenant_migration.pem88
-rw-r--r--jstests/replsets/external_cluster_time_validation.js122
-rw-r--r--jstests/replsets/libs/tenant_migration_test.js92
-rw-r--r--jstests/replsets/tenant_migration_cluster_time_keys_cloning.js101
-rw-r--r--jstests/replsets/tenant_migration_donor_try_abort.js86
-rw-r--r--jstests/replsets/tenant_migration_external_cluster_validation.js148
-rw-r--r--jstests/replsets/tenant_migration_x509.js35
-rw-r--r--jstests/ssl/x509/certs.yml32
-rw-r--r--src/mongo/client/fetcher.cpp29
-rw-r--r--src/mongo/client/fetcher.h5
-rw-r--r--src/mongo/db/catalog/database_impl.cpp3
-rw-r--r--src/mongo/db/commands/tenant_migration_donor_cmds.idl4
-rw-r--r--src/mongo/db/commands/tenant_migration_recipient_cmds.idl6
-rw-r--r--src/mongo/db/logical_time_validator.cpp5
-rw-r--r--src/mongo/db/logical_time_validator.h5
-rw-r--r--src/mongo/db/namespace_string.cpp22
-rw-r--r--src/mongo/db/namespace_string.h15
-rw-r--r--src/mongo/db/repl/SConscript18
-rw-r--r--src/mongo/db/repl/tenant_migration_donor_service.cpp160
-rw-r--r--src/mongo/db/repl/tenant_migration_donor_service.h20
-rw-r--r--src/mongo/db/repl/tenant_migration_pem_payload.idl4
-rw-r--r--src/mongo/db/repl/tenant_migration_state_machine.idl6
-rw-r--r--src/mongo/db/repl/tenant_migration_util.cpp112
-rw-r--r--src/mongo/db/repl/tenant_migration_util.h28
33 files changed, 1203 insertions, 548 deletions
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_auth.yml b/buildscripts/resmokeconfig/suites/replica_sets_auth.yml
index 064f54567ca..5d3668c49b4 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_auth.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_auth.yml
@@ -12,7 +12,7 @@ selector:
# Skip any tests that run with auth explicitly.
- jstests/replsets/*[aA]uth*.js
- jstests/replsets/advance_cluster_time.js
- - jstests/replsets/external_cluster_time_validation.js
+ - jstests/replsets/tenant_migration_external_cluster_validation.js
# Also skip tests that require a Thread, because Threads don't inherit credentials.
- jstests/replsets/interrupted_batch_insert.js
- jstests/replsets/transactions_reaped_with_tickets_exhausted.js
diff --git a/jstests/libs/rs0_tenant_migration.pem b/jstests/libs/rs0_tenant_migration.pem
index 5c2bd303efc..54cd6b28c42 100644
--- a/jstests/libs/rs0_tenant_migration.pem
+++ b/jstests/libs/rs0_tenant_migration.pem
@@ -3,54 +3,54 @@
#
# Client certificate file for tenant migration donor or recipient.
-----BEGIN CERTIFICATE-----
-MIID8DCCAtigAwIBAgIEJJSiyDANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
+MIID+TCCAuGgAwIBAgIEFCAskDANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
UzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENpdHkxEDAO
BgNVBAoMB01vbmdvREIxDzANBgNVBAsMBktlcm5lbDEXMBUGA1UEAwwOS2VybmVs
-IFRlc3QgQ0EwHhcNMjAxMjE1MjMzNTE4WhcNNDAxMjE3MjMzNTE4WjBpMQswCQYD
+IFRlc3QgQ0EwHhcNMjEwMTE0MTc0MjE5WhcNNDEwMTE2MTc0MjE5WjBpMQswCQYD
VQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENp
dHkxEDAOBgNVBAoMB01vbmdvREIxHTAbBgNVBAsMFHJzMF90ZW5hbnRfbWlncmF0
-aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3C6lDdSpY/A4yI3G
-K6HpQkrQ813zMOMhf2BFZSU3zY6rbrXhdTQQpQxyRC2Ht48LceMidB/J/Q+HykIz
-Ygm4BsvjcyR89Uznb5bd82DKftRIfA6LFk3sySZjNmzOf4ZRzItO+N1UZAP+9nfA
-zOTIwblNdPuUeeRutK5RH/tgf7rXdKgOSEjxhinAOV0r50UOLLo0t2ApRjskA0dz
-l61k1GIVrmLgj11+Zq1KaG3jETVjwFn2HwAt0/JI8tvkh9ebsAT7b68Ibz9iCp0Y
-+nPFk3V/SbjpPYDjauPUAb9cjwyzYfnhrVOMKRnDBRN+Vll52UexqVAGLW5cxupA
-46zNhwIDAQABo4GUMIGRMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQM
-MAoGCCsGAQUFBwMCMB0GA1UdDgQWBBQOPwCeDPekwSX1+3pDbC2S8/IKqzBDBgsr
-BgEEAYKOKQIBAQQ0MTIwDwwGYmFja3VwDAVhZG1pbjAfDBZhZHZhbmNlQ2x1c3Rl
-clRpbWVSb2xlDAVhZG1pbjANBgkqhkiG9w0BAQsFAAOCAQEAddk36uq32JPv1ovY
-IGH9c5BHq2iepkNxRbFAPbr1aA/Z+8uZMV9/sUoyXkjjSQYgrnzFmnHrkeMYq6C7
-ntIteUuPnZw5DpC7o1AEPckYxjUdCFcqwHz/7qt1b3fUJ8wuzMXLYeGzdK2x1nPo
-TH8icFJf54tH3xHin6GHJAfaBus5BQwT3PD6OaLE9v7lVCbiUF9kqvP+2NRNkwUQ
-4F/P+C8pZlnRtPgy/30c6fsbUSyzlZkQtzsQtKGbWbhoO5QqCQH7dSPKGqSsnd4o
-Qmq8pgOgt9bi+Z70Ze/JzjviJ1MYgGiz8rEY5bXe6OQ9K3S9psEHvJ1lKOdSXTu9
-FQqLyw==
+aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr5rVhPGtpDf/suM+
+8qVEVDjljiIUC9Lf4Ld70HrvnIU0IfYI3DMXC7b5Hb1opAEWppmIZD04Q+2eymfC
+mj1m4pK/Kv1pLopshWzItEbEevdseAQeRESzQ0lXUdWANBklrNkwBgI2w4XwOYfS
+I4drcBiBIYDpKiHch4n0aKORW2NNhbulc871o57Rs7O4n66CfhUkTjYolPTRbSol
+xfgSfL+QgR1+sIOrLqm6Rf8GBdnaTiXKeW8J2EREFOD/zR158eaAI79Pj5kJUMSx
+wU7b9tlisitc5mRIspXPnu3Y2ro8qIhI3Q8vd1YNpzM9lnuwjYXj1j2vQyqPc4X8
++2Vi1QIDAQABo4GdMIGaMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQM
+MAoGCCsGAQUFBwMCMB0GA1UdDgQWBBS95/pwFPksRZhv2k33VmfctekXRTBMBgsr
+BgEEAYKOKQIBAQQ9MTswDwwGYmFja3VwDAVhZG1pbjAoDB9maW5kSW50ZXJuYWxD
+bHVzdGVyVGltZUtleXNSb2xlDAVhZG1pbjANBgkqhkiG9w0BAQsFAAOCAQEAFLAi
+x9NmQW/501RKMAtstDJWJpM5xZT6o87VfLgtuQwHWfl5CBrsgJFBc3H8d2h8zP35
+UXiixG1XlfcnBcSWLKtEh8YW8Y7At/vUuM1P+pmOjwSMYnezte8kdxvYJgtpSDV8
+yNgFTWjPOHz+0jQR3rMdgoPDR9mWDC1eat/QQKdyEhoVgNSvG19BqGCiXkwYO3yj
+AnhSTpMLLAquG13TNgvxFnl2DN+r/5OluLmgJaKouGEwFLqONpm9kAkhMqiIcTLu
+kV3/RuyjL9iIl/3oUEy3bCvM3AjgJFYYswfWa9T9cklytJbnxx8XEw9akeGxLlBi
+ZWpd7ZyJPoN/Mu33dg==
-----END CERTIFICATE-----
-----BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDcLqUN1Klj8DjI
-jcYroelCStDzXfMw4yF/YEVlJTfNjqtuteF1NBClDHJELYe3jwtx4yJ0H8n9D4fK
-QjNiCbgGy+NzJHz1TOdvlt3zYMp+1Eh8DosWTezJJmM2bM5/hlHMi0743VRkA/72
-d8DM5MjBuU10+5R55G60rlEf+2B/utd0qA5ISPGGKcA5XSvnRQ4sujS3YClGOyQD
-R3OXrWTUYhWuYuCPXX5mrUpobeMRNWPAWfYfAC3T8kjy2+SH15uwBPtvrwhvP2IK
-nRj6c8WTdX9JuOk9gONq49QBv1yPDLNh+eGtU4wpGcMFE35WWXnZR7GpUAYtblzG
-6kDjrM2HAgMBAAECggEAEAJAM/I7Yq3LoyFTwEZkbFJ+QMOa8/n/rIA2a3U6SnLO
-Gh2VR+1CLI3sHPLmhD0dSumFzsWk7QZdDCKmE8G7JJtcbTnA8CDoTt+8QX7PGkua
-/ZIdIMEHr1RgMEvW88SzW9jhJA0Lnr3/orHwTbVwt00eyZulHvux5OcrtNSVHpQ3
-nCniPc3F/WXHRi+klZR7GAK1M2k06mjnV+8YmCr71vPHx4Nu0Ip99yOEWAI/IJKp
-lPrIRgeUKp0h4uCpDRlguKUL8AUgbNoh1m7z1LoVOEcVJrEjyYus2zh4ONMfREb8
-/qkZDvsj7TS2c18buhKEKe4n+KOstCJPnTdq7KxIoQKBgQD2I2mTnLRONGoXira4
-NxNCNVvT02IS/DNPrT4Gm3p5mG5DOMtVTNvwRXaQtNlrfv2qYSekwvqU9Y6YR3sz
-nPSMfz0Kxy7JpgI95oVIGP3dqU0Skuv2pNK4pFxECD8MH4lIJBbvq+zDAyCbFCmx
-gzZ3u8CA2dHr1hVWdOBRQoZysQKBgQDlAQGIfo7mhS0PkVIdqZU0pmpmy0HRHKSB
-SSxD0UuUCrT6aj6cgccaGnjjhmtObW1jG85EowApDnUR9wwGMTCl42AqaLnQixpK
-TyNaReCbVkpNWcluBRKchGBff7Af5oAuSFuggBMWR2jecM17UGkbV5Z0MVhQ1CZ7
-oLDMb0ghtwKBgByWqGCYxuh+dgQd/HMREo+SGwRTfQSvflY2zQl/bY5KAiSUqClU
-MAeMSeUs3EP8EAKGHlCoxOogS1uskcbA6DaZYMGruOd6/K+r4rcpD/N7ApSxs+6F
-3mPL01ujiY8i6pMSfgeAdJOB/XuFsAIKN31YtVdB9Xvq9beBA7zseSlBAoGBAIic
-qQV8OlNabx8yWgJIADKAdEkOSB0vMRPPSxDJ8oRGPuCEMQW8hcIUuLlCnxESRocx
-N0PK2XWkokItVob7IUJU+jFSeEboNK7Ptw9LUEpal8i9H2T6sbedYMCXs8HUB/OU
-RtTgkoJf5zeLo4lE1u7wuVhnd4AAj1SA7/eFC0rRAoGBANEds+itR90vCI7by7I/
-1ttTCDm7rAFwoUSG0KvHvarpbFR3A41ssVmgkHFj4xP4MAgYUieRVSxrd55tifLK
-6L0d4UhO/jfKKdaylv0Rf5VziBa6RtRd0tpPjpgvOdud0HglYxktg4heHVAbZHXh
-+sJU2GNdsKpPnSOhM4JT3agf
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCvmtWE8a2kN/+y
+4z7ypURUOOWOIhQL0t/gt3vQeu+chTQh9gjcMxcLtvkdvWikARammYhkPThD7Z7K
+Z8KaPWbikr8q/WkuimyFbMi0RsR692x4BB5ERLNDSVdR1YA0GSWs2TAGAjbDhfA5
+h9Ijh2twGIEhgOkqIdyHifRoo5FbY02Fu6VzzvWjntGzs7ifroJ+FSRONiiU9NFt
+KiXF+BJ8v5CBHX6wg6suqbpF/wYF2dpOJcp5bwnYREQU4P/NHXnx5oAjv0+PmQlQ
+xLHBTtv22WKyK1zmZEiylc+e7djaujyoiEjdDy93Vg2nMz2We7CNhePWPa9DKo9z
+hfz7ZWLVAgMBAAECggEAFgqXNmKnau9kdu2zHv3Df9bMaXS5k2pak3g0YEyt+rV1
+f7XEEu+OuveyeRsnIT7JuwuWZQ6SG2SeA8XWVUnTuii1Og4ov6C8Ulr/LjGbw9t+
+j+nkaqk83vs7rJfhyahGZFW2g2qPuSb8qhv0MmVzBNTukGf/3Dv31ENqnB+7xP6S
+fulej9rQzu2oNNODrBdKrjQ5ygk3hYvgO5loUgLmSu12/QmpVxUp42JIPmvXRVjS
+0pJbz92MmMjV+5MRRSGpNCTgHiNdCTppDpQmOTxSep4ERkYUFa9wQapAcgYtebKN
+hoJefAgCbugHTNaUiN4sxx5HLAH3SJpucxHivKPz2QKBgQDlP3CfT4+j2s8zRwbk
+j1hYFoFcQ3B9hXDn4rHYRk0fKTdeto9pjFjkrG7fxa84ssOewrl+IotqvhlX07Ee
+QcX99m/3Q0TDvW7sl5cA1+LbGfhbIlD0pRDcOF1eQv7w4cF62b5lcYWrpLKg6bkH
+OYlmhuTir/7JtkFBejbbaf/Z3wKBgQDEGNxf6xENo0PJ6plE26Zu8sif80aVoC8F
+gPVJWLqQTh/hTA9I10hvOpWdC/q6Lez2Chw+U/vRsb5paO1Bk7cjqtuLdaA6Y/Ok
+UTKFXJ0+04cAFXmcO/zo3TstkcC9B5j9xpN1TVWnNbZ228r1li/LgrS537DCPd+P
+5z+jkw5BywKBgQCNlMeebEme/bT3y0bzg6AhDw0niTMBIpefbr89ffWFaDQdF4ys
+uBPybpgvNWzJDIaKF2LuBrGp4+vM9R2vPRnlhfySbgYbCXRnRBvrVmlxvbFHv4LW
+BNfL2je3zpKa/CI9GBthlWHjptz3SkGOt7cnDepwIheuRhQ5NxyIGAf+awKBgQDB
+bBA7ID7y575etOM3UzzoJNuVQb22qoiW8it/jGCBJD432PEelRLie7PFc2J9L/2S
+dkiDhg1WOe8qRA/55fz31ni1J3HDBoH06w5dEwwbQ0Pe5k/MtjLxcf4EJccp7QuQ
+DcW3J2+7/bQbqxKXiOtd3m/rA2RzQ9p3M87Gxq+CXwKBgESxKU+d6cm2hZhKv9XN
+nCZzGqjuKezEuxIDDYEbDVpLPVizrNy04t8hXSwyZVDFNDz6H5Uj55AYOn7F2S6y
+BF/aNKnmlsmv9VxFYR36YGfeeRSHxCUv8oel4Lv1vqxva7qcQHSlNvCSMCX1FE9W
+uw0VUfsne/SS/b4YhWkMLem3
-----END PRIVATE KEY-----
diff --git a/jstests/libs/rs0_tenant_migration_expired.pem b/jstests/libs/rs0_tenant_migration_expired.pem
index 8996b1adf9c..be6350ba5df 100644
--- a/jstests/libs/rs0_tenant_migration_expired.pem
+++ b/jstests/libs/rs0_tenant_migration_expired.pem
@@ -3,54 +3,54 @@
#
# Client certificate file for tenant migration donor or recipient which has passed its expiration date.
-----BEGIN CERTIFICATE-----
-MIID8DCCAtigAwIBAgIEQkaROTANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
+MIID+TCCAuGgAwIBAgIEZI1y8TANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
UzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENpdHkxEDAO
BgNVBAoMB01vbmdvREIxDzANBgNVBAsMBktlcm5lbDEXMBUGA1UEAwwOS2VybmVs
-IFRlc3QgQ0EwHhcNMjAwODI5MDU0ODUzWhcNMjAxMjExMDk0ODUzWjBpMQswCQYD
+IFRlc3QgQ0EwHhcNMjAwOTI3MjM1NTQyWhcNMjEwMTEwMDM1NTQyWjBpMQswCQYD
VQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENp
dHkxEDAOBgNVBAoMB01vbmdvREIxHTAbBgNVBAsMFHJzMF90ZW5hbnRfbWlncmF0
-aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxFwzF0FyOPjnepos
-strY4WUcqgDqh0f/Gyv/HIoetjzc/HYnIAwl6R60bIQ3cdPXRQu0GOZ6McoU10Sz
-UiH+F3dtT8obdvrIpC1N2dwuHl4sOYeELAjeL6jyGua4dl9W17/n9IAJx6en2e0Y
-K/ewcz253/fuVWEa2aBFGTIDfrr0b4z0GoRuvKBSfMqxb9cP4tESKxKfzYs46j/I
-Xndsl5OwQ0ES9lkZ2ioopQqLMAlKKC9OzOrQMd2ZQNe07GPE2hcWGfLoRhbj0+5n
-1X/DCLPK0RZE+HdIIGUhkm1hrnBGZBjf5XXUEl96WO4xBcAJeGsve07LsXGYbJhx
-uvQkXQIDAQABo4GUMIGRMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQM
-MAoGCCsGAQUFBwMCMB0GA1UdDgQWBBRGsYZqjSv40vNnZaEV3yCXgJ6TCDBDBgsr
-BgEEAYKOKQIBAQQ0MTIwDwwGYmFja3VwDAVhZG1pbjAfDBZhZHZhbmNlQ2x1c3Rl
-clRpbWVSb2xlDAVhZG1pbjANBgkqhkiG9w0BAQsFAAOCAQEAJaqf4wVucuyPbZjg
-50AFXsFDMzfGZXzTuxjmgCbEdXfdt8DLHZW1hVtTFIPRwIMyeMZ0ugZzvqxtti82
-Txfb2xbUYzXagZKQmT2zXyUrPY5YisYu3qH9+PuZC9vqBmZYzqVWp0VKyMrb2iS7
-b8qpzVAEKgwodOjgoNjOnmPbvUN+/6P8ZB/xJshmFt0ehoixensW/TuHT40DvHPZ
-L8vfe8xbgTc4R8h6b821a84xw3BYYpMzs9UX7z4iSpHxHUYG92vEg5sh/aXeayDK
-9rmR/+xVI+lSQLaQRGgfu4ggPOS90EPC1/QtoKqZxD8QtPf6HBcb3icM6Dzf/uu5
-tCzCDA==
+aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAziBhBjBwQH8jryzt
+Rg7kwlY80yHZibirC6SWwHGjKWO2Au9asvMrNl+oYs8Y2QMGoqy+dOI23jFbBX9E
+l+Tj8//gesrMl7+MluXYZLLTnA19MScmJrn2JeL2L2FiSEMmWQpHyUpsIdZhseow
+ttsLdrEH9J5WNg82UBk36dxBlndOWSIAYVwiAfXlhuJI+nMOzaA7ROtxZpwc/e9L
+BQApwqwDWOvViVPcwuLQjKH1ZDSUulIVB86dCJ4d/1fUannyWFg8tezf0vhhCEt9
+gZLIrzdEFsqPnbfWpTZJxg327fyF8dclSokRPTvwplWWErSAb9XfMgrKRik4Uhwn
+3zZqXwIDAQABo4GdMIGaMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQM
+MAoGCCsGAQUFBwMCMB0GA1UdDgQWBBQWOH9aWvuIR2YwOiFhTGUvb733VTBMBgsr
+BgEEAYKOKQIBAQQ9MTswDwwGYmFja3VwDAVhZG1pbjAoDB9maW5kSW50ZXJuYWxD
+bHVzdGVyVGltZUtleXNSb2xlDAVhZG1pbjANBgkqhkiG9w0BAQsFAAOCAQEAeYtD
+zNZmUaMgNKKxam+7ojXTbHNN/C9hTZIKSQUYcaH31evZuUTraTK40MQE3tdosh8g
+7pIY0TZME7zQ7VYVIhTqpK8hXjJpD7JbqGDA/4fRlLb59FSMsBIlTJR7R/KT2SdT
+FwbsbKGELjjNozKkdZaXZKLgFsv42YtvtwrBKqJK1LF5x/uxKd5rDWLdoKqk2N4x
+eRue2nclMRlcTav4A6B9vrux58L8Nfe5jTyUz5e42sf4XiHw3CRzjouLcvGKGLDx
+aYgIC7RaRu8khJeUTApEpNgSQMScqvUv3AiMiFUY4Xn/YORvda/BganNdW0i8slf
+FvfQ7v0NQvSdXYBUyw==
-----END CERTIFICATE-----
-----BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDEXDMXQXI4+Od6
-miyy2tjhZRyqAOqHR/8bK/8cih62PNz8dicgDCXpHrRshDdx09dFC7QY5noxyhTX
-RLNSIf4Xd21Pyht2+sikLU3Z3C4eXiw5h4QsCN4vqPIa5rh2X1bXv+f0gAnHp6fZ
-7Rgr97BzPbnf9+5VYRrZoEUZMgN+uvRvjPQahG68oFJ8yrFv1w/i0RIrEp/Nizjq
-P8hed2yXk7BDQRL2WRnaKiilCoswCUooL07M6tAx3ZlA17TsY8TaFxYZ8uhGFuPT
-7mfVf8MIs8rRFkT4d0ggZSGSbWGucEZkGN/lddQSX3pY7jEFwAl4ay97TsuxcZhs
-mHG69CRdAgMBAAECggEAHhvJFbrKSeQYklsLmY0nDkCTyZmEXKP0gPUZFAY1uVnR
-xMBVNXvHYFx969nGKqUB8CtgPSUx/WpwEaYOHPzrWivMzhw+Y9sGYu4zbsFtLOjh
-vRyDbsEZWJN+NBPGmxf5V/GsHnh3h260tRy+xRcgrjr7wlTg7aSLGm2RytA5avf4
-q70dyqPues3K638f0hLiW28qMOBCdRSeJKie8dZRQ4kfyUWNNhEPl8ughaXFbVVI
-WwB/NoLegLXFdcVzwNuZGVfFzwXhAdK3Wv0UjYx+S7CA/Lzrz8AZisgsd7paE+IC
-CZ+zF+st/97WEb94blTCVFMGnUBkJa3kR2BmHUf4HQKBgQD2izoEQVPKI6wAFAAw
-HwyEGn3vhHawNLoidLFFpT82BkGSGFw9clINQOgibTfMsaXO4awS+iwZnYUgvGux
-9kf/E2BVUXOG+j/URLd0CEX+yWOpRH9DcOLfiPCqBP9+jqgV8GwHiBxHMb1W7NHZ
-AGEi3MIM/uc80rScaNOStR9XywKBgQDL5Do86r/cfyqkWwiHczc2TiHtR3+zcP9E
-vZS8l2hIQEzChfWCCxcH47DVTHnig1ANSrhEfA0GBZNj+HAmkVhhDITk8AsCOwIO
-7hk8CIFUjzvV6EB6cXGtruzBIJQUVhFBC6qv3mzwYdj86hjf6R0odZD12KzqtM4a
-1TVocapfdwKBgQC/VJUVsE0CVmSpOtxae/4OlzCcrMQfQwwgqUbZscA4gOpqIpWS
-6iFbP7/m17OyGzt+LgyIPbXzuxcRrKg3V9XP0o20KJ3rZlIavalRVwpbDJdXSV0q
-TXUD3RZIG9DbuoIfZJGx1qN7bNJvnyHLskuv7np582go/6xCpedrtzw3uQKBgATg
-0QPkGfMMl9iW1P1opEmIVQd4TnXcnj7mykg41yXjY/LDgbw6x7JIoFJ6IfBpc+Dd
-iMsarLUYLQ6XQxepIpQv2H4hwCGth78Ts0bcbTu4sZXMmL5VOIMPTFrSjLhv1rnX
-rZ3HlJOrw6VJdI7m5Ouy5GT9aiWzbbr2nvCAx7LBAoGBAOeT74xyNjmB/UzgfArj
-sqRf/9RNAlmmjWyhhEwTT31dsGZXs4l19+mhELDZ18WrvjkS0sIjK1KllLol1uRn
-WziP4rQX6yv6mVTEYtPPu6owUxhiAcYR2HRNi5pWoc8VzaeKRaZ6A/A9dONapkQU
-H3OTS6zJ8U+i4V/NeViCNp0u
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDOIGEGMHBAfyOv
+LO1GDuTCVjzTIdmJuKsLpJbAcaMpY7YC71qy8ys2X6hizxjZAwairL504jbeMVsF
+f0SX5OPz/+B6ysyXv4yW5dhkstOcDX0xJyYmufYl4vYvYWJIQyZZCkfJSmwh1mGx
+6jC22wt2sQf0nlY2DzZQGTfp3EGWd05ZIgBhXCIB9eWG4kj6cw7NoDtE63FmnBz9
+70sFACnCrANY69WJU9zC4tCMofVkNJS6UhUHzp0Inh3/V9RqefJYWDy17N/S+GEI
+S32BksivN0QWyo+dt9alNknGDfbt/IXx1yVKiRE9O/CmVZYStIBv1d8yCspGKThS
+HCffNmpfAgMBAAECggEADnSO2VRESwGfiwVnjmY0/Lv3gAb7roDY3fQo7k4J1zk6
+5nHKton/Gc0TjfpgD7f6myXqJWBH4MUtUPFf6hcPNWlYYMQgHPQ988ciqc0wsdzw
+TqYkMssPzBBim2gE8HRlGBF0f1VMyAK0zzlROUGbSDOaDzmbaice8/cQyMG9h8Ot
+7m5oBvDtALswuq+QckMRIMkquO+8yHp9nnp6ZmGCUmXsEDhX4uCufUt6AusUW6dw
+On+h2tZyB0uvsp5gATVi6+pj5MTzM6bfSAd49pv4itT029wNmJV+Uf5QD//bHOUN
+M5So26RJi6+O6Jq5oUw7RyxW+SGMzh5mfoSsToqUAQKBgQDnxHirncAd+rO1ONKh
+DI9hVuECB0IqiX1Oj7L5xcvWiv+7ergTcLaRGuimQdcLWfDF+sqrdVvgUDMMg9qf
+f0ACbhIZeFTqpWmC15X5Yc8A1wL43fGfpSqiyr9O4Rh5nNThGQEtY6HIG/41OxVq
+nVcmjCg6wYBx6bubvjEiq+4DgQKBgQDjrZi08I/8zm7kAIJyk+d/zCzeqnmAVbJo
+VUa9eySwk61B9K69POZMb/G8w0SaywAVC7wyZvwodbRtJgpRPOrVxlRoWy3nxrsK
+z9wBciZOe+FttgL3SbPO89Q4QlFVZMCXqpyvB15rOFniLXGB548BBjvkv2IiBAUV
+SRpR41vd3wKBgQDctYLcm0s6wWWlbElYyKc54QxBbDwcWfYUfE/KAAyPcKr8FG97
+Q9j38hIfUY/B5ZKeVqfuA8LMqFMU1fO58+o25i8iRi9HjOIfkcHmTLheamMQL/tL
+NnSVtlHcggtWtH0dJzyxxrFn6RB6DJvmfZNiyBH/cgopcJYSSFpuVCBDAQKBgBeY
+fiMCa2m/2y41/07wlSVhe6T39e+e8gZuEA7DHGJQa7xSJ2nlTdjOITd25T1FnJSR
+ysdOdQOcbz6AmsEglxXgoA7QpElJB0lvkVV1BxNLM2mzW0tVkBT6oBvUg6ld77fI
+LsTU9qwWAvq1yvWzkdaBVuO2Ee2EjSJPmTKzZWnxAoGAGplW7A1ZINetvEilXe3s
+oMsrk9tEljaEbHLEu6aH1f874Lq4fEtZ+dQK1t2mg3lAZdjZylPwqJjjTfocc5cM
+GWznvTF5ktA5ieq9iGveeNSo04aui7K2LhlcYbzYlaIj5b6ZSo+ALGqeaEQ8ioIA
+e5cR3e0JTIBgeOdHasTksN4=
-----END PRIVATE KEY-----
diff --git a/jstests/libs/rs0_tenant_migration_no_find_cluster_time_keys_role.pem b/jstests/libs/rs0_tenant_migration_no_find_cluster_time_keys_role.pem
new file mode 100644
index 00000000000..7a4246ad372
--- /dev/null
+++ b/jstests/libs/rs0_tenant_migration_no_find_cluster_time_keys_role.pem
@@ -0,0 +1,55 @@
+# Autogenerated file, do not edit.
+# Generate using jstests/ssl/x509/mkcert.py --config jstests/ssl/x509/certs.yml rs0_tenant_migration_no_find_cluster_time_keys_role.pem
+#
+# Client certificate file for tenant migration donor or recipient without role to run find command against admin.system.keys.
+-----BEGIN CERTIFICATE-----
+MIIDzTCCArWgAwIBAgIEVUE3WDANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
+UzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENpdHkxEDAO
+BgNVBAoMB01vbmdvREIxDzANBgNVBAsMBktlcm5lbDEXMBUGA1UEAwwOS2VybmVs
+IFRlc3QgQ0EwHhcNMjEwMTE0MTc0MjMzWhcNNDEwMTE2MTc0MjMzWjBpMQswCQYD
+VQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENp
+dHkxEDAOBgNVBAoMB01vbmdvREIxHTAbBgNVBAsMFHJzMF90ZW5hbnRfbWlncmF0
+aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqzAqqWyGZYFCLGUg
+nqo4uiTyxfPxacuyadyA8Ma9IGUQ/SFR/srczU+cdBVPHz13zqcpxFSVmHPeXiZZ
+UQVZ5OrnDgT0t46lJKWh7ZXMRc8oFRUWXhjjmmOBYgB5d7kC0iqLCgdutVNT6Eqz
+dcUIIXrdSuJg6ZH0rG8lGOMrRWOoPfJcgW2Bi49Zy491h1AAei3CR8UIWl6gXi9E
+qtmbpwapXhK+qre8rtSawzbe0tPa6lrkUd3c62YyzoMeoKMOeRckpY9gZwaZr3vQ
+hhUvYldwhFqtR8IdyHCG9lv8+/W7qTwYFA3H2Ga6Fav7kI/Aa6ryVsCjRo8MRT/A
+TK9qAwIDAQABo3IwcDAJBgNVHRMEAjAAMAsGA1UdDwQEAwIFoDATBgNVHSUEDDAK
+BggrBgEFBQcDAjAdBgNVHQ4EFgQUyIhfky4fnoNT6O45xDOKHk718w8wIgYLKwYB
+BAGCjikCAQEEEzERMA8MBmJhY2t1cAwFYWRtaW4wDQYJKoZIhvcNAQELBQADggEB
+AJTYjvg5ebxI1tegTjxKh9kQLBI8LdlvxI1X3do1CJ9oFxMVNQa487mvxp/pS39O
+jKig3K+DVsK7RUMYNGyPdxe3/xCUvJF2VHcahhAmhAE6uYdHq0JWcXj1TGqa9Mh3
+c8SCHWVyOlrJ4+XMChq4s1RdUwg6xgp/Vfg1M09EGEbwKtfYAov/nIEWhnrzJ0tD
+veXy4xUMowe1i3pTMU+QJ6AKl0ogQBBr+zM1sP6HBL49nsFxtjCLBW9zE/N0iQri
++2FVRAa3GNaT9AtlyxJwrPThNZiQIAlnAbfs8Z+30TrzrwcfKdhdqLZgNua+1F+1
+4VcbXVm4s+sF1Q1y9Fq774o=
+-----END CERTIFICATE-----
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCrMCqpbIZlgUIs
+ZSCeqji6JPLF8/Fpy7Jp3IDwxr0gZRD9IVH+ytzNT5x0FU8fPXfOpynEVJWYc95e
+JllRBVnk6ucOBPS3jqUkpaHtlcxFzygVFRZeGOOaY4FiAHl3uQLSKosKB261U1Po
+SrN1xQghet1K4mDpkfSsbyUY4ytFY6g98lyBbYGLj1nLj3WHUAB6LcJHxQhaXqBe
+L0Sq2ZunBqleEr6qt7yu1JrDNt7S09rqWuRR3dzrZjLOgx6gow55FySlj2BnBpmv
+e9CGFS9iV3CEWq1Hwh3IcIb2W/z79bupPBgUDcfYZroVq/uQj8BrqvJWwKNGjwxF
+P8BMr2oDAgMBAAECggEAQzbSSVt3GXu7A7thJ2smgs/nEppJe4SjNLWDWwZtU77b
+AMiIbUYxk9tiTqHfQyDbXaR83b1oT0ePU+AyyYEUXrFsrzB69WdoqA7wmVxrrC0+
+S6jP83pKQ2A2eMg+6ScXWHWjyfhJG6DwHRDXjbIjEyIuqSxhaNQKAXpSUoc/giqO
+NyTtuA+PTev7b53EMmsB4qjnU6V8XErTKaJJjzR9Q8m8KXOHPFQIK44ALuzabjLR
+QHpiSXFGGuTaKc5nNY9Mtrkrr1gkkSSuSia11DQF/js4EjSo91EU0BxWyTnZHlui
+uT7sXcNHthTCvdPGHC8vnTylGT/fB7uYuIQLoPQJwQKBgQDfEfh5hkxzxxqE+Rpi
+ZuknTs6371ckL7vqCxBUN62oapoJA3A5sjlrfcnqK8c3McZPS6f2IjaDHlbjOerO
+rG1WWrUtxlYRGtSivyKgR0oEQbDu7X16P4iawSJoiZEOBeXRQdsDUQpRSbauZ361
+JtyOcpcsew+0LladQlxR1gc8+wKBgQDEdYaDTe3se/UTlR8eqkx2GoKjEoqfgLhJ
+Ung/MGAoAE3nwQerxaSYROrleDoJpzVs3ZQP2rI4Igg3hJeBnkSUcUGqsxfX3FHO
+XBpQF4maxPN8ez07gjcVQzmEH/t/5E5h+nv3YumfInDsQavSutQltkJZcDIa6aJn
+F0unFOJomQKBgBKjicN9uT2jUU/zEIVQXALwzs0HrpxTcLwpmH+w902vOpeuNd5c
+Sff1Vr2+UrMF8bAGfk5OpYdnLZNm73u9ZJVmluTwQZjESSggGGCUFFcutf4A0g84
+580mfh2GARzO2VkWy5oHV3lbU2xXMXspeBxe7srErZc8l/gj51rfK1OpAoGAApGq
+3aKyPspsLQJhMsx1TYIRPUKrz2QmhMbwTZs7VqqSWjp6+DMQcPFNxQxqbd/i4faP
+zhlFBVjklJ1GQeqo6OwGOM44vJjj9PfdXDjCkuSKbpFJ/rI4OC313ZVxe32nzL4y
+ysxfU7ZmlaBSN+I7jfaxLWw7K4Dsn759PcUXJNkCgYBerue/IIjNIyAcyhMV1uRF
+9DooGZ8qqK6bGuyXJim3vyiZmg9dysJ3pZjolarcuJ62lO1YhJ0gYNLXN3ZXc6ts
+dc2FJelqQER8ww0fhmrKsNChWmJqEmWqJcANyO7ObcqwU6xHfyj5FRyzuht2cSvc
+4Sy+l6oxalWyPkzQjThKow==
+-----END PRIVATE KEY-----
diff --git a/jstests/libs/rs1_tenant_migration.pem b/jstests/libs/rs1_tenant_migration.pem
index c27df3a1a74..a600e57db4a 100644
--- a/jstests/libs/rs1_tenant_migration.pem
+++ b/jstests/libs/rs1_tenant_migration.pem
@@ -3,54 +3,54 @@
#
# Client certificate file for tenant migration donor or recipient.
-----BEGIN CERTIFICATE-----
-MIID8DCCAtigAwIBAgIEO9xRljANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
+MIID+TCCAuGgAwIBAgIEPmBxvjANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
UzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENpdHkxEDAO
BgNVBAoMB01vbmdvREIxDzANBgNVBAsMBktlcm5lbDEXMBUGA1UEAwwOS2VybmVs
-IFRlc3QgQ0EwHhcNMjAxMjE1MjMzNTIzWhcNNDAxMjE3MjMzNTIzWjBpMQswCQYD
+IFRlc3QgQ0EwHhcNMjEwMTE0MTc0MjQ1WhcNNDEwMTE2MTc0MjQ1WjBpMQswCQYD
VQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENp
dHkxEDAOBgNVBAoMB01vbmdvREIxHTAbBgNVBAsMFHJzMV90ZW5hbnRfbWlncmF0
-aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArsz3SKweyy9/qrNE
-Ddr5QNypH+UVvoGilSjFn7n2imSDqV4yzAwZS9wraSEGQlPcgPlJOfDPs2Cllo44
-gt5iRoBSR1WPFGoWEq2OxLCGOjW5rMu6iF9uZodvMHc0xL2IzkwhGO8FI9Yycpb2
-pjUm+H9CBsRvH9JjGd7xzNgzcFKpWCnVgi4Oydhu0oflIegF7UN1yTaSv9LH1ZKL
-8RkrLZ4iv88ISwE3QAAThkQokAvFKk08S9cInEuC4gXunZTpHrLV7cwMxyn+Om64
-6UfUfB9NDRhcdScrxFRX1jfRthgAEw3Ud7P3D27oVBlBhTrDF7BM46j9cv5Juzb4
-w3wVywIDAQABo4GUMIGRMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQM
-MAoGCCsGAQUFBwMCMB0GA1UdDgQWBBQAAJhR5WYHSv9WN9eBnD5A9tUZ+zBDBgsr
-BgEEAYKOKQIBAQQ0MTIwDwwGYmFja3VwDAVhZG1pbjAfDBZhZHZhbmNlQ2x1c3Rl
-clRpbWVSb2xlDAVhZG1pbjANBgkqhkiG9w0BAQsFAAOCAQEANRu1l5gg4MNdCQ6J
-htER4a1rM7DEpJAOLf/TqbRK9BbKuTko3yvJ4XMAvrAPzjgV7cdhLY2XZQYP6tuW
-M1rmgk+KOQPh/35WGGPG7lhZW7hnCwv7yl2Rn1xHPqWODgIroBbZS8ONFeXMM9lO
-PweCkib0s5KfBY7QrUNyILg+gYyuRgKrZg0n4hzXb8cKA+KDCO6DCL0UJgbqlxpy
-5Piv5Qn3wl3k2+EhfStQJKj0xUYqATkyWnUs87xKt3PFO44nTinGrENLXC1zJaIH
-3qqRiDIfbZxULrS81F0mvaWsyxPENC3Nm0tK7ob7WX40v/351M5eExX5zGpR9806
-Ql0dNw==
+aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3W6KXQqRIFnaEMg+
+Zs3jhsViWNitSspcKMB8as5O9eUqv4xVTC8JJNruISrRkS7sBrW+tmpQQmdFH7jC
+/Pd9mjgsfb41mlDdzBfB4VAn+7DjytYvY984RD5l3LyUwV01ahuhDxqKc2zGWD6h
+fMWOYP9DzZI5PVvefFH4MuRLCrjVWVMb+RjrGk1XL9nc0BEUl7I7m26cdq4w5Rg8
+9wiXxUHKeYNx5ndFaeiKqVnrZ/HdDqut8YlSt+VFBv64qD537fwqiSm+VfEtoPRQ
+yCihbYyko9MPWJQKZreZlRwuW3TnU3IRQuchgSj4hDHfH/NDa1bvJSn4m1u/iyU1
+mDjHGQIDAQABo4GdMIGaMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQM
+MAoGCCsGAQUFBwMCMB0GA1UdDgQWBBRBpKUiPCF3ngcDu5SvgzR4nScXZTBMBgsr
+BgEEAYKOKQIBAQQ9MTswDwwGYmFja3VwDAVhZG1pbjAoDB9maW5kSW50ZXJuYWxD
+bHVzdGVyVGltZUtleXNSb2xlDAVhZG1pbjANBgkqhkiG9w0BAQsFAAOCAQEAd53W
+D7MBNchKIABSx72ruzDQwrGg9FaJJdXfbLjMk9QUZ+V85FCrFRh66NXQEoE4HPTe
+Z+xhaOdyWTtnf8OXq+KcCiw0460O1sWRSzcnCInkMEYQafBBxr2TB7ASKVk2+Zec
+KNHw3/2CFcxE7ZJCwjwmSPVnKKpqjcY9XS/K8YLmF7Lz1gfqLBc+61orqCNmL/9v
+a1SDNqCc3dtg7mcJEnhAkz9SFLpM3Cphj3yXiowMd4yQhXSusU5zm5cGt078qEs1
+J7K1NB4VDZilCy/J845QIbmEb3rhvenMqr2XaR1RhxSYlhvt+UAY/SDUi9aOHTNc
+0LDg+v09WIC/vGACyQ==
-----END CERTIFICATE-----
-----BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCuzPdIrB7LL3+q
-s0QN2vlA3Kkf5RW+gaKVKMWfufaKZIOpXjLMDBlL3CtpIQZCU9yA+Uk58M+zYKWW
-jjiC3mJGgFJHVY8UahYSrY7EsIY6Nbmsy7qIX25mh28wdzTEvYjOTCEY7wUj1jJy
-lvamNSb4f0IGxG8f0mMZ3vHM2DNwUqlYKdWCLg7J2G7Sh+Uh6AXtQ3XJNpK/0sfV
-kovxGSstniK/zwhLATdAABOGRCiQC8UqTTxL1wicS4LiBe6dlOkestXtzAzHKf46
-brjpR9R8H00NGFx1JyvEVFfWN9G2GAATDdR3s/cPbuhUGUGFOsMXsEzjqP1y/km7
-NvjDfBXLAgMBAAECggEAfEaZIP/OPOMePzyIi9k9ENJRI635kaIU9zsvlene0HG5
-8bZZIx0O9xInguzJyGpbAQ868oi1AnEaCTpADzsQX9Nnq/O2b+skmk7ujAR8yfHc
-em+1xLMkiJyNfsWhrKacFx+lAFcfLmVhJHsUDQMJG+9OAAW0g8lOT9xjySGFILtW
-Sr4QHT7BzAY9aBkOD/JsLnHtx1fXaqyWFgCfhVnUEj4gG0+bT9hgX0c1xM6GlaQF
-8ldja+buzEcFmjw3X8FZTIzp+WNXC0D1DpjgA81XxCK5dwh09CcmNeGeH9gjmOPr
-BRIrI1vGG4uMFaiwOMQZE+JAuYxFFoGHE9Nudfl8wQKBgQDnd2RajHDQd3jmKqcJ
-g0378tTsPefb/GTf726fwFzSchDm7hYSVGplJCU5lCo2CiP5o5dfFSRuBcESAaec
-jhB0ZuVqaYCDOmXvpAJUXj2zrJ0PU4E8ZZSQTCWidUNo424XU+dcgjHdEG4AqyUv
-d6UBEmzqrWUqURfXscMFG5NO8QKBgQDBVAGa2jrp5HzSRvInKQWM1gPhnD4u1+PU
-p6I0nbDP03YwVAxT0cHclTFQfW819XUv091WFJv5gBpp1jzts9M3SueiwSdDAtwT
-g03A2qex+NhOOFfzwz7n1YrCbNImd+xKmuzuc3b+vL6aI+OQ/uQ9KIHUAqhEeZXY
-yIekRFeoewKBgQC5EKWFHErK8usQJEQfgANnEVWaeTibk+ZXgYlSYywT+q125h9x
-klrq+QZmTged42e2Hk1V8YKKEUG4EuifbJrNZsPA3A+rXJLKErjmGd99e+KrsVlg
-H9uzr55bkSRPhZ+tOSF/vnz9wjghgrUdiay8+a1pyS9csS+9/lvuBok2EQKBgEOr
-wBQQJ2cPj5GbO8/xT/wIPbuPEUUcVKdvNhvsxqM4hbpM8TO1zIIFxwlBntfoX2iq
-ZGNpO1+OM7CwSQt7UoecNJCHw9J1AcviviivYNuvGyfmGQM+aJqF6Ng9dyH4AEHt
-ENo2mIR6VM1nmpR6ZDH0J92qQHQXixgiuTgKpBRZAoGAUxyIpK6IbUajxSWaBN1C
-75yixyUlBYI0oxm6fMyc9//ihVWIp3J1f1QdhM9ea68IzfW8foWM647R2AutOlK5
-z76pP1gPSJ6cu4VFF+VvVfA6rvFSFZ5IwNdyL4R1ioY7uBldItdYiY4FWv+Im7j1
-Qr5pw9+r22voo8l2Q5RjAnA=
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDdbopdCpEgWdoQ
+yD5mzeOGxWJY2K1KylwowHxqzk715Sq/jFVMLwkk2u4hKtGRLuwGtb62alBCZ0Uf
+uML8932aOCx9vjWaUN3MF8HhUCf7sOPK1i9j3zhEPmXcvJTBXTVqG6EPGopzbMZY
+PqF8xY5g/0PNkjk9W958Ufgy5EsKuNVZUxv5GOsaTVcv2dzQERSXsjubbpx2rjDl
+GDz3CJfFQcp5g3Hmd0Vp6IqpWetn8d0Oq63xiVK35UUG/rioPnft/CqJKb5V8S2g
+9FDIKKFtjKSj0w9YlApmt5mVHC5bdOdTchFC5yGBKPiEMd8f80NrVu8lKfibW7+L
+JTWYOMcZAgMBAAECggEBAK382H1I+0WHNGAxZ07SAjB4UdFr16ZHoOhobpphpwtx
+WQiPVMp6vt2VpLys72P7kXuoPMUKfivfByJow9WPBcMy1kqrNWUem1htZmsK14/d
+qu9LIot/7q84tEMgbmYNPhY0xXWfriAJ3UOjWrI2bKxshrM1bQJ4eDYJFlOurAQs
+R1a2l1dcvZlrzw8zOAEReexpaNz2lA4SVm1gqn4skli+xsF4A/KjwSBxTQSfBosv
+mccTbQCHOxH1986I4zSYbxCIWArGX6U+UGt3bq5HHyYwOnjznKbhLtnphayHBLTE
+8C2aL8WKguZYwTaAD03UdPsPLR70mfROzK5dCs9ZkG0CgYEA7o2EnV/u1JjvVb8A
++S3YcMnF/n82j2hfydOXX+fv0jFETrHTCTTq6uc7orOnsSQcM3VtrHgxnsXnk+8Q
+mVamqwfpf4z1X5ZtiLvVVbPfMLhAdtXobkrLcr4GPZ7f/e4/hdRFlMCl+1CEYaSf
+bHh+2rmotcmYQgVfcgwb94qwD7cCgYEA7aB2Mq+YQiIW0YHHPWeuXx7lBNj4Hogt
+VzI9QjjOXi1heZJQEmrY2+JRNQau4EcD/Fu+2l1T7TqxIQp78kz585nUhenGEEWt
+KqP3l4yVsZuS6wSs85NvMFqL09nQvfjcVEZ2tPR3C+MkJYe0+tmWSimS8LsBdbcQ
+8Z/wGzoAP68CgYEArbbIV4HAfXhQ8PQCVXIZJykH0Wf55zxuZJ58oQ5ZCYtu3o+H
+yjCK0TxJpRWGi+xrzN06XFm6aJBdLtDsGX2MKQe99XBETrNQWD3QAHTXlrbV2rOy
+an504L2X9c939YZFI8OMV+Jh/DyJ6kDjBy5H0idYv+hpw8n90yMWSpDn3MkCgYAt
+qQStTqJyKry5uzc+rFqMOpKFMO/UZhRYSfYNcH7vB8dQJfYu48sRzVYPW15lCCBS
+8IomyYIj9oAsuigA8huhXHDC3p3JaKoSpHkh7Mii3SMI99gH4c+eufdeWN9McbJc
+m0GaGdN6PEYo0//Jp9/CFmT+zuBD705ZL4Nip+1+TwKBgQCCnybXzKQAHxmgzxYU
+haaGpC2cMTANnrcYM1vk6cIDl0KX7+Le9U1y28G4st1C90zDLHrW9WB2hjV0kAu/
+h52/7M3RSzAc+crhMRfb2xyiEX9COVSy2s/FLa4C64S9n9SCxKKObjKbLL2h+tXJ
+CBGDNZpCAQUb82lafp34yLUBcw==
-----END PRIVATE KEY-----
diff --git a/jstests/libs/rs1_tenant_migration_expired.pem b/jstests/libs/rs1_tenant_migration_expired.pem
index 9560efc359f..f2958fb7123 100644
--- a/jstests/libs/rs1_tenant_migration_expired.pem
+++ b/jstests/libs/rs1_tenant_migration_expired.pem
@@ -3,54 +3,54 @@
#
# Client certificate file for tenant migration donor or recipient which has passed its expiration date.
-----BEGIN CERTIFICATE-----
-MIID8DCCAtigAwIBAgIEPS2aVjANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
+MIID+TCCAuGgAwIBAgIELjPfpzANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
UzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENpdHkxEDAO
BgNVBAoMB01vbmdvREIxDzANBgNVBAsMBktlcm5lbDEXMBUGA1UEAwwOS2VybmVs
-IFRlc3QgQ0EwHhcNMjAwODI5MDU1ODMzWhcNMjAxMjExMDk1ODMzWjBpMQswCQYD
+IFRlc3QgQ0EwHhcNMjAwOTI3MjM1NjEzWhcNMjEwMTEwMDM1NjEzWjBpMQswCQYD
VQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENp
dHkxEDAOBgNVBAoMB01vbmdvREIxHTAbBgNVBAsMFHJzMV90ZW5hbnRfbWlncmF0
-aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0lQ4/uAOxdeFurA+
-nt8tB79RqmTXcMbMqUb14DjOxtzlM0XEqs2N+fHf8gRRki9+CMegP6Ug/LGDF0p5
-gjuO59Kg1e1sQaxFOIUKuNUsTe12pG/krcME24QYieOLKM7yV3LcMquuA5a1oikx
-WkjYTGsSRrOPzovswOmwCHqax/PjJ4Zk1sEUOHecwzLBdUwNuN5whiCHTYTPim5T
-dr1lWcXxb5b8RqzoNzDQBw9BakY2ni/lZrqa43CYejBJdvgekaaQb6dUfPjNrjwT
-aFQqvBRZ787PZ/M297JCeYcZwJmsmjlGTstRMlSjrph5G14FEoUeEHVbQ3h5VSDr
-KPSjRQIDAQABo4GUMIGRMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQM
-MAoGCCsGAQUFBwMCMB0GA1UdDgQWBBTWNGjVGjdVOvMAFvW0/o5ec6LjxTBDBgsr
-BgEEAYKOKQIBAQQ0MTIwDwwGYmFja3VwDAVhZG1pbjAfDBZhZHZhbmNlQ2x1c3Rl
-clRpbWVSb2xlDAVhZG1pbjANBgkqhkiG9w0BAQsFAAOCAQEAQJnEt3QtQYOESTT3
-MMJKpIYwMFS7td/xXDf0UiJPZrbrcOOUMjZHxQrqA4ACuSpxAXRvwaL9o5iS8LXE
-WHYe54L2c5XYVUjmGG7k5ZqcKfqmU9NR1D6/fVnE+U5Jy2hhLOHqK4rnc0u27OjY
-pKevxw5VizJppKVcOD3DSLD87kKrQg9GmNz1FWFIYOWPS94ky7h4TRWhuMifGGgo
-QHGRa0I3ywAl6cBqvQ5GMfJ2efJ6o4/rE+8FRhfjaaYMQErw1sRp53cOM6hAKdzN
-3xth5yESH+qD8dJln2uxee+QkIZ60vmGdb+H1c+o+ITfQc5xvAP6ltJgYu9vDiik
-vOL0dQ==
+aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz/Q1ocWz/Qe9pk81
+yXx5FlsYVuDXCBKvIl/pLp0nAVZe1fYvEaeqkNZIOExuq0Ir1hKWSAEMBdy0KDI6
+rGmZ59rmCSGE7GzGm93CTYd5fUKEebP6trgJeF03e+gwFPMjNyhs3eKuKtMR45Id
+lGP7IcdlJBxEKsjFSK60Ua6g1RElmCjoiwtLX4ZRT6DuDNVv7I40cjdw10EgLJL7
+74wXecv+ohZcbhmHE0lADXADAb7t9b8BO9QJWb3/QT60qMouPv0KEQvL3LfumBMz
+gBcQQT5UGRMWa9QZYxZHIgUiiiecdvZZQDW5OFfgzQ5dSVX6ZVjiPHCAskI2dft1
+T4lrhQIDAQABo4GdMIGaMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQM
+MAoGCCsGAQUFBwMCMB0GA1UdDgQWBBRFrtzeWTHsPf+XDIqsbrj8Fn0W3jBMBgsr
+BgEEAYKOKQIBAQQ9MTswDwwGYmFja3VwDAVhZG1pbjAoDB9maW5kSW50ZXJuYWxD
+bHVzdGVyVGltZUtleXNSb2xlDAVhZG1pbjANBgkqhkiG9w0BAQsFAAOCAQEAIwrw
+vEo6rXKZpJsfc3YiXTKxExKypUkUhXx0iJgq9AeZ53wXxmHJqTrqghcH/tIwGtWt
+2saVb8KUV++FpboUbzS7kjYTw9L074zN0XkzaJ81pzjzuY8p5ZtcNyrlG3+sxEYf
+5Xp3EreXep2xo2ZpuWPe86S5JjLGIxRBsfex6HyMXV+VRXQ12jl6OeWgeuqTP4by
+VRpDKMG9oZ860S3y39Tsi1cOAOwBbVsv6c+FuSuu7PJHR3vk/pMshXgUQ34J5rN+
+x9KB0Yph6Yzgl0QFwLhad29OKOPyYDi3xsXrxhf9iFx5AaO0Jj1InQpJQN2b8ftq
+n9IAWCmu9Bz8M+enCA==
-----END CERTIFICATE-----
-----BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDSVDj+4A7F14W6
-sD6e3y0Hv1GqZNdwxsypRvXgOM7G3OUzRcSqzY358d/yBFGSL34Ix6A/pSD8sYMX
-SnmCO47n0qDV7WxBrEU4hQq41SxN7Xakb+StwwTbhBiJ44sozvJXctwyq64DlrWi
-KTFaSNhMaxJGs4/Oi+zA6bAIeprH8+MnhmTWwRQ4d5zDMsF1TA243nCGIIdNhM+K
-blN2vWVZxfFvlvxGrOg3MNAHD0FqRjaeL+VmuprjcJh6MEl2+B6RppBvp1R8+M2u
-PBNoVCq8FFnvzs9n8zb3skJ5hxnAmayaOUZOy1EyVKOumHkbXgUShR4QdVtDeHlV
-IOso9KNFAgMBAAECggEBAKgvKsg79/Qhq9Cln91ATmy0gqQDMPcAMKVnlbLvGgGQ
-n0JJlsKRzebAj7eh1mxTx69TmYNrEp6df1q/dbw6zcH3/h4yVO+Kxms0j2dPJrdF
-nCpPQwfjOt4hmdGg/yTpnoVo1kU6XNKHz8PUtp8gQr2QnLOmX+bkorN76gRNrbkW
-i9y/HAi7uYHuVtH/E/3dbKmb9md2aqwrYi8/8vQvqfAvAhNLDiTt5ExzjOzN+Uhz
-D7MxbUR6kyAFPwSH4tJajDssr/BD74jtUllKjyIOPG9IpOLE+nF5An4EwEkLGM3k
-hfVe5tRW4W1Z1N+ZYpyHhaGfGQM+NuaecXmgXD3ERH0CgYEA8wsFRc7LXKTftRAc
-FRDxfDxobw2tRStZoz3l0Tyg8aQxfk6pcg/pOZbfdOqEni7C8IAPLftYKHMLJ9VN
-H7phiKOjz2A0+WkJiSsw1tXGJ+zbFLKeKw52a4cUfLPhzcsx4QC6yZlbg1oVoCZR
-UR8UI1wXsXKrSWmePWAjHV4mdNsCgYEA3Yq690cJWU6Rk0Zbqiyq06dbtaXbglRl
-cKCunubGPskeyaGalIa7biHk2kggVETOrHFzfbaL4P/AIqLZy/Uj8iBSc+5K6P5Z
-MDnrmp23GLiey5NI0SShFfN9bWIhgRVXmOc8MkE7qZO9+sG4Qh2umHc8a2D4N8i4
-EB5PT639sl8CgYEAsgS5ePdOZ25wWUwSda+yYaBRkj4/UvG8t2AILGkvLa9pfS2a
-2WwkFUWYKSf5uW9g5A24kKKQYRha7HNFCV9YP2A1BRlf0+uGy4zyOfndKbNIiPGH
-/tuaC4qmi+yqETWrNDZ3mmCU8jISsvo+B6Au/PNge14VbjFJcYUwUzt6CY8CgYB2
-/mn4Iaaq2mGkyji4Ce/jaVQAGEr0EOK/gP7qCKhY1uv7fxqpRkjsGbZzbCQ5PxAH
-XZSb5G3zd0s7D1+Ohmg9SBEhhc+kn+FF3wZmcOoElD1uuPh2pYcEZFltZDT8wXrg
-fOs/Tvx91AZJ5r1SfeDmsby5LEeGpv/+YFlXQewvXwKBgQDj8zf0uXK0CJtHQvg6
-UotsE0st9N4qg9zkCajEcnrdbeX7sNb4L060YEP7sLYnF4EZyn0ss2AP3CJtUyD5
-dOmTC6KmYWDH9V8/MHsJdVutE3zaHSCAk6bXFOie3qDSaT2aCli00DrS1Sl/fCUn
-ovvC2Q0JUouoo8LFcUv5FEsctw==
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDP9DWhxbP9B72m
+TzXJfHkWWxhW4NcIEq8iX+kunScBVl7V9i8Rp6qQ1kg4TG6rQivWEpZIAQwF3LQo
+MjqsaZnn2uYJIYTsbMab3cJNh3l9QoR5s/q2uAl4XTd76DAU8yM3KGzd4q4q0xHj
+kh2UY/shx2UkHEQqyMVIrrRRrqDVESWYKOiLC0tfhlFPoO4M1W/sjjRyN3DXQSAs
+kvvvjBd5y/6iFlxuGYcTSUANcAMBvu31vwE71AlZvf9BPrSoyi4+/QoRC8vct+6Y
+EzOAFxBBPlQZExZr1BljFkciBSKKJ5x29llANbk4V+DNDl1JVfplWOI8cICyQjZ1
++3VPiWuFAgMBAAECggEAV6MgUwvBfZoch5EMJtyJHy3GxR4+IM3W9tKar6VVAdBH
+aT41uegCDsXva6JyZUQeuzncEX43deT8G5CHf4+FZ9utZB23Kffxf1BmiUvVnD9D
+GxMJTqWigBIwxswkL0AEEEO+FYhpQ+zttg0m4Bl+hU7FxgEPjURNOANARMgmSYQL
+72kB7TMQuZmQ5GQTrG+QGt0XaksLxe9nHJ35tk53G9aQciE8is4q7GYnrk3vQk8M
+/IuGdGaRm6Sukseesxz+Tcwi5CgqKssX3+8uMsHdx2ywBCnj1zGh+LZskDNELhvU
+/6WBivcz4VeuroTq88JAHJiUcm69M4HGZbc3Gg5rQQKBgQD/QAQ0QbSitl2ZwjLd
+aQw85dBmEgroNPxN4Hz1iNASOijnjeM0pu6IrDpsGR9Cs1SXNvucqqp0E9uojWzR
+hhNlmSBKOpQofYgjGyFvvA6MzVGjItNxua5oRnwZYZCMlm455t6NDFzlBF4zS1P6
+wf7QY3aY+QuDDvbyNb0glxaYMQKBgQDQkJ6r8UYkZ5/pWueuxevg+41Mip8igiZy
+U3Q9vAkF+l99JHqma3Hd7ZZSOK/fn7qyu/pSIbij9lzkIi8QWieVWvShD+D+5OYL
+bzwsw+5CPitfkCbSKzIcWcO8/4ONN5nN+gYCPH9PfnSjzN27fs+nHLrRhsTc/X/9
+TTbhX/GHlQKBgB0CEXg+32B+iTSoNU+kWJezJnPzKPoqy2GuWYUt/zeTFIL8RQ3+
+yq0NpbYTRNI3vy1QOlgbrTNz7krBh9RoAFxTgq/s6FmRqp8SvkIMhTrLOrPniVAr
+Yf7Hw+K+i8bucahy7GfrN8NvM6VwsiKVWXglVZ0c8In8cWQlZ1EE2rSBAoGBALzs
+U57C5luxtkvmeXl6dWmN0/eilfXnWTY0XFZTjwToQDkWozc+mp2CSqleCgoSIUco
+Od7OBvgXARvxzxoL4FFNW8zNmmyqIav37dVLAfblY9qjtFaqpDDEKyd3eM2hsOPA
++FJsRJJYpQV/XNcrTqQ8xQlo/8iCpL5phC3w/9NlAoGBAPmoXu1DSjfrg0X3tvfy
+jTaNRhU3MZlol3ZGQxa3zYPTYQGIJYgSN2sd9cUdDPHjmqpCxpws+CH8DQd/FQXJ
+TFznHRmP305T6OFPYkakZ1JsucoeWzFOEK/G2B7XgeSVbY4rpY6IHwPiyVKXQx8t
+blby+PKlmXzhpgQ67yb1aU29
-----END PRIVATE KEY-----
diff --git a/jstests/libs/rs1_tenant_migration_no_advance_cluster_time_role.pem b/jstests/libs/rs1_tenant_migration_no_advance_cluster_time_role.pem
deleted file mode 100644
index a58f2c22685..00000000000
--- a/jstests/libs/rs1_tenant_migration_no_advance_cluster_time_role.pem
+++ /dev/null
@@ -1,55 +0,0 @@
-# Autogenerated file, do not edit.
-# Generate using jstests/ssl/x509/mkcert.py --config jstests/ssl/x509/certs.yml rs1_tenant_migration_no_advance_cluster_time_role.pem
-#
-# Client certificate file for tenant migration donor or recipient without role to advance cluster time.
------BEGIN CERTIFICATE-----
-MIIDzTCCArWgAwIBAgIEfwFHxTANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
-UzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENpdHkxEDAO
-BgNVBAoMB01vbmdvREIxDzANBgNVBAsMBktlcm5lbDEXMBUGA1UEAwwOS2VybmVs
-IFRlc3QgQ0EwHhcNMjAxMjE1MjM0NTM3WhcNNDAxMjE3MjM0NTM3WjBpMQswCQYD
-VQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENp
-dHkxEDAOBgNVBAoMB01vbmdvREIxHTAbBgNVBAsMFHJzMV90ZW5hbnRfbWlncmF0
-aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwEt2mL4vQ/7S+dXG
-DNPhcaPclsI/4Q2RARWuOilMpaX0z8AAIygaUWqHCpKqU5lkr3EH6cLU7aode6ss
-UiNtnqEjUFKQ7ceiQEF7CxT3Bcis5U5bLY7NrvH6dbV5EYDaDchOqEZI7lrNnRMA
-4JtJVXdxGRRW7C3oL7zWkN1nO9m55UhO/jUcl4QJF44fV4YlRapNNCwUDXRIzLnD
-y6EiB8MXJ2czFg2cIl+HgGepzkiCvXLlmK67I5QLjjXFQT7VMWLhPqR8ujZmBvYF
-6OK4+EgRp2xJKNOyKCxlvknvHet82qik3PtgUPDs8EMhLGsHd4n3lPHsb/kUy6+s
-IaOPuQIDAQABo3IwcDAJBgNVHRMEAjAAMAsGA1UdDwQEAwIFoDATBgNVHSUEDDAK
-BggrBgEFBQcDAjAdBgNVHQ4EFgQUm9Z/cnSFdVUXEAPNJjyCBsF0lQowIgYLKwYB
-BAGCjikCAQEEEzERMA8MBmJhY2t1cAwFYWRtaW4wDQYJKoZIhvcNAQELBQADggEB
-ACTRrBVPYFfI7G9yCkJ+r5aece+R+sxSPvG1tBIyQGZPfOOmAkh2k4xE+GXEEQMs
-3OGE6qmUnWl9QaxLIrGnbbXimoTpYOh9prng99WG/RQtos7KtcizECNl4UFHtnOJ
-hlqih6EllZ7Dle197rMO0zP4ajg2jgyeFZGKkwP+ng2YFWtMwsWqaKJ5OG1eqPvC
-D3Jnwif+tXL+aza8Bl8Kgf1i+psAmA41fww6Zyrbp1vwiiV6hLX584bVoq2n37WK
-VMDUcBPO+2TZ/Rhjt2u8R7A+eZkZYQXwYtZ5Xl7OxCGRd37DE0I3I6dTQxlNhaUL
-U4gbGa3l6wFBBsEvO7VIaxU=
------END CERTIFICATE-----
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDAS3aYvi9D/tL5
-1cYM0+Fxo9yWwj/hDZEBFa46KUylpfTPwAAjKBpRaocKkqpTmWSvcQfpwtTtqh17
-qyxSI22eoSNQUpDtx6JAQXsLFPcFyKzlTlstjs2u8fp1tXkRgNoNyE6oRkjuWs2d
-EwDgm0lVd3EZFFbsLegvvNaQ3Wc72bnlSE7+NRyXhAkXjh9XhiVFqk00LBQNdEjM
-ucPLoSIHwxcnZzMWDZwiX4eAZ6nOSIK9cuWYrrsjlAuONcVBPtUxYuE+pHy6NmYG
-9gXo4rj4SBGnbEko07IoLGW+Se8d63zaqKTc+2BQ8OzwQyEsawd3ifeU8exv+RTL
-r6who4+5AgMBAAECggEABGp3OcC8UyJp8+TN3IONHM/3g9rIHsHx/4Vg7+L86Jzs
-B01Qc347RH1exoZhtPPD1SeSV/Lqct7COJHQvb4kiU0IA5Mx0tQPzMv0KGf92Dso
-hUeMXnp2DLtv1f4p3pYH6+Mas1CR4i9ho2bJUV8fiv/oL/xAlsHjIB2FC+SZt00Z
-fgzKy4B2i5qtbMxTRdVSAj6Oprs9Gjbr7SGEug6algO2n/a5fcYFtAz/HDjjpRVq
-htaFNVPQVX/DkG2kPreErhIy5xI7JCEFQkeNFWQaRFaK4R9xqwOf54F3p9FxAqk+
-1vFCbfY4RPQ7XJ6qBgUKrfjtpt8WpLpjlN+MqcDeUQKBgQDmrfmaP1dELzWJTztZ
-N0D/NCVwuTwTtu73fRamb2yuFc8fL3n8HYrA86AFJ7l8zNLfpeBbuAbQt35i6GmZ
-VtZh2SlIvi35Wt2cEo5b6lxd7ONw1daMPjrIaNJtreMNgvfybdXH7np1QArFFGwj
-Whp5XMAl11uIRM5+vPCrVRqYXQKBgQDVZuMPMEd4Hp2XLOUMLh4Nel/MGf/ZTty5
-ksA51spfFnk3z422Ht3HRL3zbOwdXHPVDuOrEqlViCJnzpYiq1dfg0fjeEaCSmWm
-hd9aOjK5i4Ky5/upyYxW7eVA0XLRQh2j6Hgi+Z6ldza6rhQeMbq/metkkhYMbat4
-E1EHCofvDQKBgQCRUtLnaLYKX49jpIWoC9YnsGtR23BMnzdmtOg7DAE0nsqwQ9Ge
-keHi8thBxyfkxoimmrKANawdeyvlFbPJBVrjo9KNYDpIWAU6mLoUj82G4oDaieyy
-x1VzAg5nSF+9GGAp2yYN4WBGU4U1aUtC7jW1dUgbztN8QpSwZUldNq0dhQKBgEnJ
-8rc8xWmmhG1Si7yjcabNarSDRPFaEH5mXOti9sTcW3vN271efOc7QMccIaShZDgG
-lvh/UmgN03+7cjR4tJSFK2VfIFpK6U0kOy/SsdlwgPfluSJjW5BvX/cCtUXgK6mS
-Z4AIei/SwezY5oHuRhmcWkWDMfaIMWMqorjhjlE9AoGAMO17ncqvijF+wGceQTpF
-R/1QmgFDwLxR+TGKnfgtUGizU0FcZZlGRU47+vWWKeNv2YF7XLQRuRvFc+p67AE7
-4NRaOwp7HnfwKdsVAyf32eyPj2TuzbuM/1Bm6W1lhwaLjdLbCjssNaRkpTUhLhVZ
-G2r1bs3pSapWeKOfSpzAjdc=
------END PRIVATE KEY-----
diff --git a/jstests/libs/rs1_tenant_migration_no_backup_role.pem b/jstests/libs/rs1_tenant_migration_no_backup_role.pem
index e7e6825d5ed..6109688d873 100644
--- a/jstests/libs/rs1_tenant_migration_no_backup_role.pem
+++ b/jstests/libs/rs1_tenant_migration_no_backup_role.pem
@@ -3,53 +3,53 @@
#
# Client certificate file for tenant migration donor or recipient without backup role.
-----BEGIN CERTIFICATE-----
-MIID3zCCAsegAwIBAgIEfOC8ojANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
+MIID6DCCAtCgAwIBAgIEDt1dbzANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
UzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENpdHkxEDAO
BgNVBAoMB01vbmdvREIxDzANBgNVBAsMBktlcm5lbDEXMBUGA1UEAwwOS2VybmVs
-IFRlc3QgQ0EwHhcNMjAxMjE1MjM0NTIyWhcNNDAxMjE3MjM0NTIyWjBpMQswCQYD
+IFRlc3QgQ0EwHhcNMjEwMTE0MTc0MzA0WhcNNDEwMTE2MTc0MzA0WjBpMQswCQYD
VQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENp
dHkxEDAOBgNVBAoMB01vbmdvREIxHTAbBgNVBAsMFHJzMV90ZW5hbnRfbWlncmF0
-aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxtxdF/ecGQqxQJy3
-FHgt2bBBykTusfQWfyOBv+cxcpaW/s6mbi+Tc/6pE4FVfhspsyKsmRMTwDnYoHZ9
-opCT4g25GiqGl5FbxNKCxZ+RHiIVCVpYYZ7jM1YerLJhUm77oxyCveAE9VXfWGF0
-QI+Fy/8Kwo8dBB/v89IeroPar8KpYh8h8w/TpbdVht7y8nMMlelVGLv9HEC/1Xdo
-H5WIbLM0y3z6Tyvn9WOzcsEV51HshjZ4wTDgMP+5T7sIlUu+me4P4UrWC1Jz++XG
-Yt5B6RYL+16BRjAClYU3HJ4josHj3/iTuVVmeCx6DASVh/vKYUxzPOzXh9xe46Le
-UCLtOwIDAQABo4GDMIGAMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQM
-MAoGCCsGAQUFBwMCMB0GA1UdDgQWBBSuyDkwKio/+O2UT2Q1QCLhKuUJFDAyBgsr
-BgEEAYKOKQIBAQQjMSEwHwwWYWR2YW5jZUNsdXN0ZXJUaW1lUm9sZQwFYWRtaW4w
-DQYJKoZIhvcNAQELBQADggEBAJd6ain1QgOZvM4MlZqa3YO81jSOF0vg6Rr+QxEU
-urv9e2doqtCGpqFMoAi12YtSzv8WkD3sPwqh0GKy/evnSuaCqS8JYLiXz4TNPbiq
-VN0TpVODS6c/ubnk0+K2pBJv/Nh+vH5W5VHNJSeHAcZbrgDQz0Y3fzScLw+bZP13
-wFVhEyTbsgaX4Kp0kwjsxRroYGGn83pvHDw3QdM1YxiF1+tWXz24EJU5/fwZrQU5
-coWGp7KPY3TT5FPdM2b+O6rpqvaHtxjDngLL3iIAGVANvW/bHP6xRrnqCm9PrQz2
-GwtfQJ+oFoOx+EDWj4maL0xWXK7NUseZJcU5Uh58PysJHnM=
+aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz1z5qz9ZSjTTLsbx
+1wEC8E4G21+xCiw7V1T/MI8cqbuzCejzyVSV3AzxluXGhyWYTy8YVREqq4e35//S
+9wluY43DkkTnmKpKsO/Ackk03H7gYQfTT4tyCq94n1go02gA+O78H9WQJrCib2iJ
+pwWkTGUJWxSSe53OhPFgYXKlKFi5pFjQczdmUYPxhM+/5eJpMTwmXOfoesW9Elyz
+kWsjhcRoTGjMT1SjKcbXWI9ZxEVfXspzZ4B9AywF4tZJRBf6tv49rKYYkKAh5x1K
+NardoCYjWRIdDefAQkAd9BoKW4jG3sALGbbZgfK360GPzeKj/vP0i4BvEMH00AWE
+cC3s3QIDAQABo4GMMIGJMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQM
+MAoGCCsGAQUFBwMCMB0GA1UdDgQWBBR+8pc/xJVDV/JX2UJVg4ReStNihzA7Bgsr
+BgEEAYKOKQIBAQQsMSowKAwfZmluZEludGVybmFsQ2x1c3RlclRpbWVLZXlzUm9s
+ZQwFYWRtaW4wDQYJKoZIhvcNAQELBQADggEBAGxQAbAczDQ04yImvUKbG29sYFEM
+J65FJSLqaXxsIKWfCnwW2uvoeB0YKw/8V31OpHroB8yjm4XkWIM0D3bOR/VUgdMm
+h/greYF4x+Z/7MPI5NVaa4Uy/34XyHMMSV8QcZ4ltvhP9ogGfsshDhWyPAbue+Eg
+lo/2bUDtAYZmfJABU5QIPDKFFhc/WpY7bo/OsMvCz7XSAqh9cVUuHCtDLVY1tElk
+44fpusp5QGNa9WB2zhYBeDi0pZSWMwi3UXtzfZ4fOk5tOX7YWfIMqLDxww7m5Arv
+GECc8YvcNiibO4gpIckgsCpMr5AIfeOJuaAJ7bAgwppdIrcZ6LYf69YpNbE=
-----END CERTIFICATE-----
-----BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDG3F0X95wZCrFA
-nLcUeC3ZsEHKRO6x9BZ/I4G/5zFylpb+zqZuL5Nz/qkTgVV+GymzIqyZExPAOdig
-dn2ikJPiDbkaKoaXkVvE0oLFn5EeIhUJWlhhnuMzVh6ssmFSbvujHIK94AT1Vd9Y
-YXRAj4XL/wrCjx0EH+/z0h6ug9qvwqliHyHzD9Olt1WG3vLycwyV6VUYu/0cQL/V
-d2gflYhsszTLfPpPK+f1Y7NywRXnUeyGNnjBMOAw/7lPuwiVS76Z7g/hStYLUnP7
-5cZi3kHpFgv7XoFGMAKVhTccniOiwePf+JO5VWZ4LHoMBJWH+8phTHM87NeH3F7j
-ot5QIu07AgMBAAECggEAPdbxLC7X4AGvNUSGulrVB0kEbH4bLEOXa1oDU0X13oht
-1Dc3EjTBSeT7dxFVIK4ZvZcezUtdbP2QG/Y9A3yqQDW8RiLLT/7n5sD/4azJV/GJ
-UStDKLhcNu6lA1dFZHkWz5ZiIeG0KQbXOYi0ug7nsJRTKEWQ5GN6HvxMhkCClP0L
-WqnWuvXQn2BWuseXMlavrNpWg/LZslLJ0s+u6I46B7mRIhvi/JgRg93EaJBvO1qS
-DMlc0e/GJCevEncKc0+QP5PvpAADlsopRmmATD98qk4GS5WFfQfXV1DLe5Fa3iYQ
-+lOKc9ABtHGo+XsE2700hoWH4MufDtjRqc7/Io2d4QKBgQDuoyIdsYWFK/7ocXQm
-Qix+WuL1jP86MIthp0ieFL6aVJsWtYvRZbSp2yhF7oc1MgfG92kM9cRfBwULYpC8
-HbGiwURqpO6nT32LvBZmQq1IAnE9Givenh1eQCbQUiejEWzioLbodZJgiV3UDfJC
-PRFfavTyDnTbwlbZe03896WK5wKBgQDVVFpOAZIzRxx9BcMBbATPNB2x62MnPvV6
-pkANUV6pFS9ezkeEja3u6qioX/A4fUsvKap6+j665ZKH0D9j5vh5nWvNokMQptt9
-EZmHWGsmGCbiwbI+vJtBHAmAgw3CpYTZcqanAICuJgjqSWOxhKwCgXIBBtTWtbIh
-8GM9Ev20jQKBgQCiPW6YWXS5UWwAhAG/cQXm8cuTvcRXv2FDhwOr1dav/g1ipxLj
-zg3B3SnIQZ7S36zB7LsTAJP9NzyKvPxBqL54NIVbK77FKRnlwn5ID9TXPDHYpM5m
-j4DP7zV0NCaWiuZIPZOc/zIBR+LupHyr4/DHumPFnWZ9ceax/U7MSWHWyQKBgQCm
-imrReSKEy6Au7j+VrZl3lU0VaBkJO3py3LUmcH7A61wG3G0yy1SdLy6p+/PSdeYr
-FJEV3qT4TV7+jhHe0SLr9nyBlYpbxpeVZRcBdYJ4w+QfpG2d0qo1w0ibvjQQ4Bz9
-kScPOzI6QlE1bkPZAyUx85JnwMiKMmC1ZDBlLp1HxQKBgQDkPRB5J2P86KJy7Aix
-0bITp+dwBMueOKtxGHcmE1v6lxLtCEVUPy9wJQadwExotqFvx+UXMBTlGtMm6JdK
-zUxVu9JmU8AbeRRCoRXxVsimqEbcN6vTxHGph4zSeohOcQNef97TzMrJK5Mvdti/
-X/pikEIfdwzSTPl+2H3BxvRvBg==
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDPXPmrP1lKNNMu
+xvHXAQLwTgbbX7EKLDtXVP8wjxypu7MJ6PPJVJXcDPGW5caHJZhPLxhVESqrh7fn
+/9L3CW5jjcOSROeYqkqw78BySTTcfuBhB9NPi3IKr3ifWCjTaAD47vwf1ZAmsKJv
+aImnBaRMZQlbFJJ7nc6E8WBhcqUoWLmkWNBzN2ZRg/GEz7/l4mkxPCZc5+h6xb0S
+XLORayOFxGhMaMxPVKMpxtdYj1nERV9eynNngH0DLAXi1klEF/q2/j2sphiQoCHn
+HUo1qt2gJiNZEh0N58BCQB30GgpbiMbewAsZttmB8rfrQY/N4qP+8/SLgG8QwfTQ
+BYRwLezdAgMBAAECggEBAKBmJOIoIGNcNTbrwMo1CBbl5Dvw6zsZN9W386WWLe/F
+NxqlTV4gxcLvaDj3B/gmRoiJv2Kd4snIM6OLqZ5Zd0Aeunb8QcE/iIMREWXV5sPg
+iKPFr9jJ3q8RSFbJlNriik4EXpYpVlX3L87JqCDAJnTJetEvOpfQW51OMOVwDv0D
+L0ePRCcmXzsPBGz9GNM2VAAUzG96nOyBMm5sfHIheKk48xgvDVyx40+zc9rOve5H
+XJMyf0s+U9h1mc8uSZNNhquTjvaSU7mEDt1INfyTADufSWr9tDvLQttG6eOepVIb
+YC6fwbjcO6zwUMmkVnPaKqKxSXWjZxtrDpGVmzSKmmkCgYEA95wgysC5nMf0Rk1H
+rdYC965ugL4frB6nEHODIXtU0cSyshRDcGcfAGG7YrMOLltSgOOT7ct/pPu0jhI9
+pr2lGKpSBll30bEypr4Gb0l984RblAHkfoUX3Pj7Rndxzg6XG10XWSH0FZzic7Fi
+sT76rJoSSZS1QHyHSGPXAjZVqCMCgYEA1mO7Ahl/KK4zu5lKQCLDhXnpU/OlRz3m
+i366U9kdciF0JJrFFwHlql5TfWGr09RNnycTe7FTb0UZOa1EP0GVKXMamCGInkHU
+zN8P81mcr1//ascebt2nLqsjJJN9enDgKl7I3IFPdojNQXBQINJ4gzD9MvvRrCAr
+4JrnDGvE5v8CgYEA0iNThR7op5De06cUVQiSficwKtyvvoBJImIWAJ1Nj7USPqWJ
+0mQB4G4Q/KOfPmJMc4wwaBhTXO4s/oGmhtLdwUAS2salNHXg2bfdeCbz2G4U1liA
+1YsYhHVdsyD9mXWycgugHAn/vzPkz3DUpD4hucgMCxxEctT8bsgkmgEI0ucCgYB9
+0rRLjCk799k2zcVtFLoJmT4eXk1vHV40GmFCmpcUh5vBa/AmC5/9uDRpzqvou53+
+vRFLNAxSrowadJ0sPtJLLFm8VVn3/4zj3DphSQOTTdT6lvg+euoGkrwiWEl5VcvK
+MizSdTXszGcic8cjLEM8Q8EmbfJlgdAelurpOEqsgwKBgQDkvbPsfrlVsbGUtFE2
+qzJ6GzVRl5uQlWNxuOUnZO+S93aa5Br7ttVJTiHh1eLR/UbgDA6ivxe3BI5A96Yc
+wc1UO0n9gnGarN+2gbt9DSnEvhDbCGjdIzuMSJ1i5VokLJVyWWOEswYigmZoks8C
+DvcQTVD+FXbCHY3IhjQrvZET0w==
-----END PRIVATE KEY-----
diff --git a/jstests/libs/rs1_tenant_migration_no_find_cluster_time_keys_role.pem b/jstests/libs/rs1_tenant_migration_no_find_cluster_time_keys_role.pem
new file mode 100644
index 00000000000..9d1cc06e7b5
--- /dev/null
+++ b/jstests/libs/rs1_tenant_migration_no_find_cluster_time_keys_role.pem
@@ -0,0 +1,55 @@
+# Autogenerated file, do not edit.
+# Generate using jstests/ssl/x509/mkcert.py --config jstests/ssl/x509/certs.yml rs1_tenant_migration_no_find_cluster_time_keys_role.pem
+#
+# Client certificate file for tenant migration donor or recipient without role to run find command against admin.system.keys.
+-----BEGIN CERTIFICATE-----
+MIIDzTCCArWgAwIBAgIEXQe3zzANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
+UzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENpdHkxEDAO
+BgNVBAoMB01vbmdvREIxDzANBgNVBAsMBktlcm5lbDEXMBUGA1UEAwwOS2VybmVs
+IFRlc3QgQ0EwHhcNMjEwMTE0MTc0MzE2WhcNNDEwMTE2MTc0MzE2WjBpMQswCQYD
+VQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENp
+dHkxEDAOBgNVBAoMB01vbmdvREIxHTAbBgNVBAsMFHJzMV90ZW5hbnRfbWlncmF0
+aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqDUXnFUiQ6/Huy1m
+0ayXrfXdvB2Xgt1AzpromAi7H0yIogiikHlZZbkWiftWWb9PP5BPGlSq7bQv3Zyk
+P1F+wQ3ToUi8uUoZ9aFEzmBMlkb6RRaFxB4sVOnAVhlnPlm4Gzoxf9QFb7c1DCo9
+GSzyLBKDS3HvsKy4OJJ1r3apy8TtXbbkLP66/ecrFJokRTuAfUNyjFOgI1RO5Y8P
+c7xuQz9xn8zK1lq+u34sqTH6VbRNaTd8secEONd2z+2CSATmbgugU7VaYZym6H/C
+4zIzQF1ACV55kRkiby7JmB9XuX4a81TxTcsU1m4Rk0c6Khp+qCO+h4Q1coERMnnV
+fX0JKwIDAQABo3IwcDAJBgNVHRMEAjAAMAsGA1UdDwQEAwIFoDATBgNVHSUEDDAK
+BggrBgEFBQcDAjAdBgNVHQ4EFgQU7BaV1XLbR9Mj+hdJ0nebjmWduiMwIgYLKwYB
+BAGCjikCAQEEEzERMA8MBmJhY2t1cAwFYWRtaW4wDQYJKoZIhvcNAQELBQADggEB
+ACuPNj3mlHSQKJM13dsrB1HCRYmirNvLh2T0lEBijfquatXK1tiIjwYMnZtwHTLQ
+VFvtWvpM561yONXXJGJP8H3RhIvmhxeL/CkJgYGnFxKjQ2ldxWr2F0mJjmrcPA0G
+FLFAgfk95s9NehHM+Zjmoa7nzDfWUIzRsWlob5Vwhw/x3jk7u2l6GYGJoOlc8OoP
+ZW1PTsJCZxBKtb2xRGOGYJKYTCr1UkYs5kmXry25lhDrGDdt76nm7YauBEX3KGMj
+Mi+R86ny9SyfY9TSbzQ3RWmWs6s6Hxjnrxj10/O3HJoWUFwHD0vDRVBJQOpfYvyL
+cmkAlJrtse4TWCZp3qu/Gpk=
+-----END CERTIFICATE-----
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCoNRecVSJDr8e7
+LWbRrJet9d28HZeC3UDOmuiYCLsfTIiiCKKQeVlluRaJ+1ZZv08/kE8aVKrttC/d
+nKQ/UX7BDdOhSLy5Shn1oUTOYEyWRvpFFoXEHixU6cBWGWc+WbgbOjF/1AVvtzUM
+Kj0ZLPIsEoNLce+wrLg4knWvdqnLxO1dtuQs/rr95ysUmiRFO4B9Q3KMU6AjVE7l
+jw9zvG5DP3GfzMrWWr67fiypMfpVtE1pN3yx5wQ413bP7YJIBOZuC6BTtVphnKbo
+f8LjMjNAXUAJXnmRGSJvLsmYH1e5fhrzVPFNyxTWbhGTRzoqGn6oI76HhDVygREy
+edV9fQkrAgMBAAECggEBAJrDdrzdS9BxTFj49hBXN5vUizaJF51Ni1JND1LLPbgX
+h+NmBITfUhSZDpbYDOQ4dGbslQuJ+VcKIbxKYojrp69i1GQh85qcBnSM9CzaR71B
+AwZMKbl/dZSPGeteJjMlA5LxoReW8YWPUJynxF2EkDFiuzDQXRP2Zu2OO/+8M5AK
+W+iYfofcv/WCFDLGNplfrHJdDAcGeFEcEx102oiMsV2LYaP0tHzppcW62agJ1zBz
+o+EjoxEk4eChdTLM5aMYw/GwBQVElDzMf+B8/xrJk/9opjoK+6uRXIQyQmYKT43H
+E4bIUMpZrIWEg2m4TGrQ0KGGQnygfTpK28a2D1jEjXkCgYEA0S2HJdj4/mARJM+p
+TakGUJ8hLiVUqesqBV5iAfmxtg90EeNw1Rhob5UNWQLruFvdd9NNJ6HZUvUVqbRg
+dmbtqQS5/hOJu+ZBlNKp3iQwzYCr1WIzGaABQSANmm3oJHl5HA53XSYYFTe5M4R4
+lUOL+/134Dp82uT74IIzNHx7W4cCgYEAzdvX7tyCluk8ctt5795UlmjLWw9wTX8C
+IvA22waPDWz5WiBDWd+7UYR+PJcqtBGK5/2m9rHHJQvJfH4nC9HhtC6OkzhoNGBg
+a96QUvgNwLenTivS4Nq1+EVMSDmRkU4CkfjIKouRkM+zFafyvj3X56c3SnXBiXQy
+CHkCL9LYdj0CgYA1TWCu6EcqqtgzhScjPvr4+FPAKosUcXRXweE/l50NQ4rUNBzA
+cGPMazLnh7MBW4dnzVRP0RJL7WADyo76HOhatXWfpOLoGpiDj9rxXE+DTlJ7IXbc
+BMXNDpnozDDneny+BYGtHtPsNDLhSdRgVMILCF/Pp5fBP7BfWU4rBb15aQKBgHL2
+He232XnbgLWH2d3Z5JUmqsIMqiOtFQ1b5taEIClYPRxWviYXqq1Id5LwvJlY3qDV
+54MHHvaSUZAjhzyzlf8grOCMdK0jUVWMMj8TZeLE7TrbgGWv3kVqtcHz3FYwIYFR
+VL9luAIN663hX89cDBHDMT506oMPj81pfo4kilvlAoGAbWraKp2fj45llFQpCGQP
+DaM73bXXDQKUiO14rt8sEgjlfsGLlevNdApq6UZpDrrb4vjRUDdh0bpd4dy2x23b
++e0p0gy4uODERQugBFkRSRfWRl3fGqD9CusnU9Fn6BabKnn9FB0bLcQsOmIqPS/F
+k0Esutw6hOz1ZjzkPefU5mA=
+-----END PRIVATE KEY-----
diff --git a/jstests/libs/rs2_tenant_migration.pem b/jstests/libs/rs2_tenant_migration.pem
index 545741f45e2..a7736c1cbc6 100644
--- a/jstests/libs/rs2_tenant_migration.pem
+++ b/jstests/libs/rs2_tenant_migration.pem
@@ -3,54 +3,54 @@
#
# Client certificate file for tenant migration donor or recipient.
-----BEGIN CERTIFICATE-----
-MIID8DCCAtigAwIBAgIEAZxBAjANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
+MIID+TCCAuGgAwIBAgIELoFGSzANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJV
UzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENpdHkxEDAO
BgNVBAoMB01vbmdvREIxDzANBgNVBAsMBktlcm5lbDEXMBUGA1UEAwwOS2VybmVs
-IFRlc3QgQ0EwHhcNMjAxMjE1MjMzNTI2WhcNNDAxMjE3MjMzNTI2WjBpMQswCQYD
+IFRlc3QgQ0EwHhcNMjEwMTE0MTc0MzM2WhcNNDEwMTE2MTc0MzM2WjBpMQswCQYD
VQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZb3JrIENp
dHkxEDAOBgNVBAoMB01vbmdvREIxHTAbBgNVBAsMFHJzMl90ZW5hbnRfbWlncmF0
-aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA8g9nUBxKuUz/4k8m
-WJyfO+Ao2IOe2lJgPHa10KHIaDdIB2VpW+hHf98ckUpsB7vElssFGAfbgrlYNWGx
-EP/ajueilVQyiUgcLVaQF8C16fjqQVbdOCffbWu/y3c5R781Rq2pS/WP6Sayh04l
-tnvpbnK/UWJLOxYzB2AnSc3/sQwGxPgnWuFZBMaNC2rvCJ8wnZvVklYYe7oDp6BB
-jTw2DC6WKq3Syn5iwxYaFr32M650b1vkor/yyCJL5AN9u5W+oEEb17qyZESjYeDX
-KYwbNMtiG2Ny9G/dVQrXJMAThA9AazKqEJ4tV0ny/IDdMXOCMEnxW08+HGVDXGEb
-RDEBBQIDAQABo4GUMIGRMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQM
-MAoGCCsGAQUFBwMCMB0GA1UdDgQWBBRETiI/ygcC02rJrko5AnGM72fHQjBDBgsr
-BgEEAYKOKQIBAQQ0MTIwDwwGYmFja3VwDAVhZG1pbjAfDBZhZHZhbmNlQ2x1c3Rl
-clRpbWVSb2xlDAVhZG1pbjANBgkqhkiG9w0BAQsFAAOCAQEApQNRXHfJub+85buh
-LAX0Trx9ZU2KdBaJHQMU8TWnJmQSxZrDENm3fa3p9CJFaClL3g/+kRej9dRImKJi
-WVIlw+6aQ7AcY1GSlGSRnjR2VG5FN8NhulYmVxAxZhCrhtNnKmsEMYYn5sKGyqf4
-sJRYAXiLrqB67u3IZXKxFDDC78tMBnFt0zA2RMf4TR94XYttoQYqsa1tRPfbv4k7
-c1T2VeTLFGvF04Lyf/EUPt8fIYc324r3thxCWnzmOsdWWVScjpt3f0wLfGMtYodl
-vp2mwEPljQ4T3VpHHItcId9SjvmloG9zWl7p3NMo4B3qNzoIODUqP9Mq8DVnTmgV
-OZ/h/w==
+aW9uMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA67x7CAZHMV5KI1oo
+MTrRXrIa1t+zBXn9ZBrXw6/hkVvyYJOdSB/9/X+ptRCAvV+Q+gO6sLJcCsRy6DFX
+z5jVbvKxP9nHbIpj4KBNN5QjZx5FB/QHxJjemBlSswQ6dj/INKquOvGuMEaP0GRw
+0ID2lwAFj8gWmQt7CJKug/tebo7idMqIEEdzIZgA4q0whwVTz/b3cGCb7x78p/PU
+5AoD14Cb3RLwLsGq0KWGlkXVSt8DWruHLWwe3/PS1DA0IhKOONvPZeISSgEOPH52
+C/4vt6VkBV1txo3HX6C4uBTokTNY4P2rzqg9S3ImxdI+YrfXhDw36E4yNEEmkHDA
+NmYwQQIDAQABo4GdMIGaMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQM
+MAoGCCsGAQUFBwMCMB0GA1UdDgQWBBSnZLEE3GyGLRPiRgvLU0i0Dob1AzBMBgsr
+BgEEAYKOKQIBAQQ9MTswDwwGYmFja3VwDAVhZG1pbjAoDB9maW5kSW50ZXJuYWxD
+bHVzdGVyVGltZUtleXNSb2xlDAVhZG1pbjANBgkqhkiG9w0BAQsFAAOCAQEAOziG
+HI4acGVf9AUSkPtiE5cEGjZ6ioHlvYBbUr85vsRm5mGDgvWf+rsQRUvtV6iKmHYZ
+M8HR/vvpoGvOyuDJXZoHsTGtMIPUMlV/kaVIT6uinyjbRAQxGBnzEi25SzDFhQ6k
+A9H0ep4Ryy7SSY9xkrf8XhbgF6tCo+zh2su6YegnkxqkQg8n/pLoDhxdmp6a77l9
+chW/M+8SrTYL3PTMIOEVvimb4Xi6yxSWOe41bSFAg0AVdm2mumsi1KU2IC/lYVMO
+KWx+9JqXV4YcE3eQmSf5cSs8WyTPJPPIrzBiVj37trYmpOBgPkce+TsHOC4qG4dr
+P8x7MAxb+/vmGy/m6g==
-----END CERTIFICATE-----
-----BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDyD2dQHEq5TP/i
-TyZYnJ874CjYg57aUmA8drXQochoN0gHZWlb6Ed/3xyRSmwHu8SWywUYB9uCuVg1
-YbEQ/9qO56KVVDKJSBwtVpAXwLXp+OpBVt04J99ta7/LdzlHvzVGralL9Y/pJrKH
-TiW2e+lucr9RYks7FjMHYCdJzf+xDAbE+Cda4VkExo0Lau8InzCdm9WSVhh7ugOn
-oEGNPDYMLpYqrdLKfmLDFhoWvfYzrnRvW+Siv/LIIkvkA327lb6gQRvXurJkRKNh
-4NcpjBs0y2IbY3L0b91VCtckwBOED0BrMqoQni1XSfL8gN0xc4IwSfFbTz4cZUNc
-YRtEMQEFAgMBAAECggEBAJCyNIfm4aQzKyMVPU7rESSxsrj6tRK8+1opfDXi//GM
-WjpRnNGb4GHH9UPEwR8+Vx3s7naW+9kBAoGmjDolN3kFbmLlmfAGcxGHFUudnyDl
-8uJsEoFnFGBMcLIn6s3AtxAw72rAt+4fe8j2JgUXvUqQvVzg7Se89XQLPnEX+yjR
-Ze9tbQGTSPyLYlOH4Kd8WLIVxIpgU7flqWowNGjBdsVhiMp3trhNK4LtA9Lt0TkE
-A1EBuG1uI0qeVGMXi3AQ2xxWipTO/F6gwNZk+2fh8ExwvIhkMByClAjFcZ9Ac4Zd
-eSdXFdms0jX+paKrDSnPahGbm2Fb9VshDukpnn5inWECgYEA/cjDhvLUr8p4N5dp
-IgbcZhe1I66BwblByuiWxPXIIbGFcqTZq6LqxvO8eX3eGxZj1yHnEGuUYvZ/2L9s
-0PUWQuDEX39hAwUD1/HTa6D6Ix0S+kSfSGe4RMy8hjf69buD9vHWQKGIUOUkrvYd
-goQ9FJA+LLXiWvVM5Y3WCU+0KzcCgYEA9CxvaLzx+6tnl0bgrqNeUE9Vh+khQCyv
-8LWrPZd3oy6kPJbrg1GU8WL6BWMs4oudIaKHvmvtS5ezf6C4dJSLpKWZcVuvkXw7
-ujEgEjZ15S3TorICymhrX1805NZ8ibDNyU/Z/9RhbZm2rSUsuddRYLTDR56iN9jm
-ODSIMnii66MCgYBAy+rIYOXXAG4TW5BXRMMvOexg8ORmbwOSaDJelZaEq8uzys1c
-2lrxczfzGSUFa4EQY7KYaa9YWTGpSK7i50DI0JSzAVXmrHgfwbnGaDKVlo3K4ox8
-sybEQjX8Wvyzky4Ndg/LLrMcXHUyStqcAWGLB40oY0QpDyqUDdPRSs1onwKBgEJx
-3RdZY1gi+puV9ApR9pZQIGxNqni2MGKGbUTjdPD4/kRmpUwk5S0SzAqvREWZzHac
-refJotdRPs1aRRekXKO/VJQeaRZkAjTWrW2HWGm3IMJDQrMl2yIm3FPxcg5eCIHh
-sucFOkYC8E0JsrQXvdNdHU7eAjLEWVGuKeniHiOVAoGBAKBuJWlof2bknYsEPaOl
-mTp0GVuLcqbkp41qjvvirWpVop8K+LeUrW7eznSt5ZzmMq6UNYR8mbpkdQuUvSmB
-T3ApNeYnKn9+TlfyJJTHtGOdGrNqFoNtg/S8W/p7rZU8316ETXxOFSpoLrjzYsCs
-A4vlN5gooGeJP0v7yU8kIFOO
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDrvHsIBkcxXkoj
+WigxOtFeshrW37MFef1kGtfDr+GRW/Jgk51IH/39f6m1EIC9X5D6A7qwslwKxHLo
+MVfPmNVu8rE/2cdsimPgoE03lCNnHkUH9AfEmN6YGVKzBDp2P8g0qq468a4wRo/Q
+ZHDQgPaXAAWPyBaZC3sIkq6D+15ujuJ0yogQR3MhmADirTCHBVPP9vdwYJvvHvyn
+89TkCgPXgJvdEvAuwarQpYaWRdVK3wNau4ctbB7f89LUMDQiEo44289l4hJKAQ48
+fnYL/i+3pWQFXW3GjcdfoLi4FOiRM1jg/avOqD1LcibF0j5it9eEPDfoTjI0QSaQ
+cMA2ZjBBAgMBAAECggEARxOdRN4fuRLk56PXWoCRgCvTS9ZG8+/EJtqxE5+EM3+1
+pMjdM1HyEuzbDmzbZD3cSIW3VG9ysQxqWPgHseAy+jDSoJ3ZMzDGzIOIxKk1w3Gs
+/qjlbrFo7BM2jCNKPT0d8ENGJR+ZlZSw5/Rgz30S3JR1GNULezmq7vrLA6BW6J44
+GLAVQRCQGLLsGT8RNB63KFay1aIkuwiOyKDxhZaGgsQ/JhQZ3tK6lfKkU6dujb0W
+Z6m+F6c896YwjCCJCbxjw1X1i5EnAdv4+vLCh684R8DajYz0RE3BBnOVwM5zaYRB
+wpjTvb+H5wlOaSKSdK37Heqsp/cuOk6rRwXypc1UAQKBgQD8LypYDFWb46hrYgn2
+CMoumsqdVTb2bz20xDoqo/tLad2MWuqA9C0QPfKMReLiWUz2+gjYtA6PMxxSi9cu
+jaI6n5EcRT7FcPa3rEaZhSiHePghnInAKuIEIwY5h7/FTZ6h/Jtcqa4iHlbBUplS
+zsruePRxSFXsAf5rz4RW7NhgWQKBgQDvTZqeEfcCB8t0LckG1lRJzmLuXTkPC6B6
+3ey7yPhiSJm1NGIPoaV6vuMkhOKcBfDurJAyf4PZe+GNT8vKyM/rISV67s1Yn8pa
+aYK5kS0iCh1fPjxVs7CrIwwjt9gWuTiIXvYF4wRltvAArUAGbmaXN/q8Oe9H1dnF
+C1LkraySKQKBgQCws9fGxDXEOnvLanGyXKxWhY3NOmV9vaqOHCTpteciC3b6nsXM
+P8vHorBuS5XTpmEAeWhzd47wFg8VY/kOHJ3818wU9Awsyp+FSYhaj+w7KK8s1gPL
+jAC8h1IYQ59C/8VUmay+/AIQ1BNT+K2IUW/uMwCM0/OooCkGn+yKAAEJWQKBgGO+
+8CTVDiVthFNvxiwwaD+/ZCNA/kiUKxcrq8J2OXoWEuOh6wHi+evY5yrQrPglzdBe
+Y1h5KEcdyN0zFpn27Btw3IOKSFOZfyoCcrfQmIRovOd3NITwekJLQIMe3VZTbAoD
+sv6vKqUPBKjzjg1gNSjw73VHnMJPlA768llZPuZJAoGAMg5jWT5fhXkM/uFCyuk3
+zL8SjtdOp777a0MnTucdsV+7rjm1T7tPUSz25PHqottNiGkfxG6TTUjgyqxaWpFn
+ZeOriEQMpde3Mm/KOv86Av5PLWH4ElNZJ+mS/NAfEYyoyEcaU7DsIq+V7oCFMXIY
+z0pfN/kycPYsBy37Kb2ARa0=
-----END PRIVATE KEY-----
diff --git a/jstests/replsets/external_cluster_time_validation.js b/jstests/replsets/external_cluster_time_validation.js
deleted file mode 100644
index 372a4c98553..00000000000
--- a/jstests/replsets/external_cluster_time_validation.js
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Test that a replica set can validate a cluster time signed by a different replica set if it has
- * the key document for that cluster time in its admin.system.external_validation_keys collection.
- *
- * @tags: [requires_fcv_47]
- */
-
-(function() {
-"use strict";
-
-// Multiple users cannot be authenticated on one connection within a session.
-TestData.disableImplicitSessions = true;
-
-const kDbName = "testDb";
-const kCollName = "testColl";
-
-const kSystemUser = {
- name: "system",
- pwd: "pwd",
-};
-const kAdminUser = {
- name: "admin",
- pwd: "pwd",
-};
-const kTestUser = {
- name: "testUser",
- pwd: "pwd",
-};
-
-function createUsers(primary) {
- const adminDB = primary.getDB("admin");
- assert.commandWorked(
- adminDB.runCommand({createUser: kAdminUser.name, pwd: kAdminUser.pwd, roles: ["root"]}));
- assert.eq(1, adminDB.auth(kAdminUser.name, kAdminUser.pwd));
-
- assert.commandWorked(adminDB.runCommand(
- {createUser: kSystemUser.name, pwd: kSystemUser.pwd, roles: ["__system"]}));
- const testDB = primary.getDB(kDbName);
- assert.commandWorked(
- testDB.runCommand({createUser: kTestUser.name, pwd: kTestUser.pwd, roles: ["readWrite"]}));
-
- adminDB.logout();
-}
-
-const rst1 = new ReplSetTest({
- nodes: [
- {setParameter: {"failpoint.alwaysValidateClientsClusterTime": tojson({mode: "alwaysOn"})}}
- ],
- name: "rst1",
- keyFile: "jstests/libs/key1"
-});
-const rst2 = new ReplSetTest({nodes: 1, name: "rst2", keyFile: "jstests/libs/key2"});
-
-rst1.startSet();
-rst1.initiate();
-
-rst2.startSet();
-rst2.initiate();
-
-const rst1Primary = rst1.getPrimary();
-const rst2Primary = rst2.getPrimary();
-
-const rst1AdminDB = rst1Primary.getDB("admin");
-const rst2AdminDB = rst2Primary.getDB("admin");
-
-const rst1TestDB = rst1Primary.getDB(kDbName);
-const rst2TestDB = rst2Primary.getDB(kDbName);
-
-createUsers(rst1Primary);
-createUsers(rst2Primary);
-
-jsTest.log(
- "Run commands on rst1 and rst2 such that rst2's clusterTime is greater than rst1's clusterTime");
-assert.eq(1, rst1TestDB.auth(kTestUser.name, kTestUser.pwd));
-assert.eq(1, rst2TestDB.auth(kTestUser.name, kTestUser.pwd));
-
-const rst1ClusterTime = assert.commandWorked(rst1TestDB.runCommand({find: kCollName})).$clusterTime;
-const rst2ClusterTime =
- assert.commandWorked(rst2TestDB.runCommand({insert: kCollName, documents: [{_id: 0}]}))
- .$clusterTime;
-jsTest.log("rst1's clusterTime " + tojson(rst1ClusterTime));
-jsTest.log("rst2's clusterTime " + tojson(rst2ClusterTime));
-
-jsTest.log("Verify that rst1 fails to validate the clusterTime from rst2 at first");
-// A key's keyId is generated as the node's current clusterTime's Timestamp so it is possible
-// for the keyId for rst2ClusterTime to match the key on rst1, and in that case the command
-// would fail with TimeProofMismatch instead of KeyNotFound.
-assert.commandFailedWithCode(
- rst1TestDB.runCommand({find: kCollName, $clusterTime: rst2ClusterTime}),
- [ErrorCodes.TimeProofMismatch, ErrorCodes.KeyNotFound]);
-
-rst1TestDB.logout();
-rst2TestDB.logout();
-
-// Copy the admin.system.keys doc for rst2's clusterTime into the
-// admin.system.external_validation_keys collection on rst1. Authenticate as the __system user to
-// get access to the system collections.
-assert.eq(1, rst1AdminDB.auth(kSystemUser.name, kSystemUser.pwd));
-assert.eq(1, rst2AdminDB.auth(kSystemUser.name, kSystemUser.pwd));
-
-const rst2KeyDoc = rst2AdminDB.system.keys.findOne({_id: rst2ClusterTime.signature.keyId});
-assert.commandWorked(rst1AdminDB.system.external_validation_keys.insert({
- keyId: rst2KeyDoc._id,
- purpose: rst2KeyDoc.purpose,
- key: rst2KeyDoc.key,
- expiresAt: rst2KeyDoc.expiresAt,
- replicaSetName: rst2.name,
- ttlExpiresAt: new Date(),
-},
- {w: "majority"}));
-
-rst1AdminDB.logout();
-rst2AdminDB.logout();
-
-jsTest.log("Verify that rst1 can now validate the clusterTime from rst2 using the key doc in " +
- "admin.system.external_validation_keys");
-assert.eq(1, rst1TestDB.auth(kTestUser.name, kTestUser.pwd));
-assert.commandWorked(rst1TestDB.runCommand({find: kCollName, $clusterTime: rst2ClusterTime}));
-
-rst1.stopSet();
-rst2.stopSet();
-})();
diff --git a/jstests/replsets/libs/tenant_migration_test.js b/jstests/replsets/libs/tenant_migration_test.js
index 0b273ac93d8..175c3ac9623 100644
--- a/jstests/replsets/libs/tenant_migration_test.js
+++ b/jstests/replsets/libs/tenant_migration_test.js
@@ -35,7 +35,7 @@ function TenantMigrationTest(
recipientRst.getPrimary();
recipientRst.awaitReplication();
- createAdvanceClusterTimeRoleIfNotExist(donorRst);
+ createFindInternalClusterTimeKeysRoleIfNotExist(recipientRst);
/**
* Creates a ReplSetTest instance. The repl set will have 2 nodes.
@@ -63,23 +63,84 @@ function TenantMigrationTest(
return rst;
}
- function createAdvanceClusterTimeRoleIfNotExist(rst) {
+ /**
+ * Returns true if the given database role already exists.
+ */
+ function roleExists(db, roleName) {
+ const roles = db.getRoles({rolesInfo: 1, showPrivileges: false, showBuiltinRoles: false});
+ const fullRoleName = `${db.getName()}.${roleName}`;
+ for (let role of roles) {
+ if (role._id == fullRoleName) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Creates a role for running find command against admin.system.keys if it doesn't exist.
+ */
+ function createFindInternalClusterTimeKeysRoleIfNotExist(rst) {
+ const adminDB = rst.getPrimary().getDB("admin");
+
+ if (roleExists(adminDB, "findInternalClusterTimeKeysRole")) {
+ return;
+ }
+
+ assert.commandWorked(adminDB.runCommand({
+ createRole: "findInternalClusterTimeKeysRole",
+ privileges: [{resource: {db: "admin", collection: "system.keys"}, actions: ["find"]}],
+ roles: []
+ }));
+ }
+
+ /**
+ * Creates a role for running find command against admin.system.external_validation_keys if it
+ * doesn't exist.
+ */
+ function createFindExternalClusterTimeKeysRoleIfNotExist(rst) {
const adminDB = rst.getPrimary().getDB("admin");
- const roles =
- adminDB.getRoles({rolesInfo: 1, showPrivileges: false, showBuiltinRoles: false});
- if (roles.filter(role => role._id == "admin.advanceClusterTimeRole").length > 0) {
+ if (roleExists(adminDB, "findExternalClusterTimeKeysRole")) {
return;
}
assert.commandWorked(adminDB.runCommand({
- createRole: "advanceClusterTimeRole",
- privileges: [{resource: {cluster: true}, actions: ["advanceClusterTime"]}],
+ createRole: "findExternalClusterTimeKeysRole",
+ privileges: [{
+ resource: {db: "admin", collection: "system.external_validation_keys"},
+ actions: ["find"]
+ }],
roles: []
}));
}
/**
+ * Gives the current admin database user the privilege to run find commands against
+ * admin.system.external_validation_keys if it does not have that privilege. Used by
+ * 'assertNoDuplicatedExternalKeyDocs' below.
+ */
+ function grantFindExternalClusterTimeKeysPrivilegeIfNeeded(rst) {
+ const adminDB = rst.getPrimary().getDB("admin");
+ const users = assert.commandWorked(adminDB.runCommand({connectionStatus: 1}))
+ .authInfo.authenticatedUsers;
+
+ if (users.length === 0 || users[0].user === "__system" || users[0].db != "admin") {
+ return;
+ }
+
+ const userRoles = adminDB.getUser(users[0].user).roles;
+
+ if (userRoles.includes("findExternalClusterTimeKeysRole")) {
+ return;
+ }
+
+ createFindExternalClusterTimeKeysRoleIfNotExist(rst);
+ userRoles.push("findExternalClusterTimeKeysRole");
+ assert.commandWorked(adminDB.runCommand({updateUser: users[0].user, roles: userRoles}));
+ }
+
+ /**
* Returns whether tenant migration commands are supported.
*/
this.isFeatureFlagEnabled = function() {
@@ -229,6 +290,8 @@ function TenantMigrationTest(
this.checkTenantDBHashes(tenantId);
}
+ this.assertNoDuplicatedExternalKeyDocs(this.getDonorRst());
+
return stateRes;
};
@@ -543,6 +606,21 @@ function TenantMigrationTest(
};
/**
+ * Asserts that there are no admin.system.external_validation_keys docs with the same keyId
+ * and replicaSetName.
+ */
+ this.assertNoDuplicatedExternalKeyDocs = function(rst) {
+ grantFindExternalClusterTimeKeysPrivilegeIfNeeded(rst);
+
+ let aggRes = rst.getPrimary().getDB("admin").system.external_validation_keys.aggregate([
+ {$group: {_id: {keyId: "$keyId", replicaSetName: "$replicaSetName"}, count: {$sum: 1}}}
+ ]);
+ aggRes.forEach(doc => {
+ assert.eq(1, doc.count);
+ });
+ };
+
+ /**
* Shuts down the donor and recipient sets, only if they were not passed in as parameters.
* If they were passed in, the test that initialized them should be responsible for shutting
* them down.
diff --git a/jstests/replsets/tenant_migration_cluster_time_keys_cloning.js b/jstests/replsets/tenant_migration_cluster_time_keys_cloning.js
new file mode 100644
index 00000000000..efea0480167
--- /dev/null
+++ b/jstests/replsets/tenant_migration_cluster_time_keys_cloning.js
@@ -0,0 +1,101 @@
+/**
+ * Test that tenant migration donor correctly copies recipient's cluster time keys into its
+ * admin.system.external_validation_keys collection.
+ *
+ * @tags: [requires_fcv_47, requires_majority_read_concern, incompatible_with_eft]
+ */
+
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");
+load("jstests/libs/uuid_util.js");
+load("jstests/replsets/libs/tenant_migration_test.js");
+
+const kInternalKeysNs = "admin.system.keys";
+const kExternalKeysNs = "admin.system.external_validation_keys";
+
+/**
+ * Asserts that the donor has copied all the recipient's keys into
+ * admin.system.external_validation_keys.
+ */
+function assertDonorCopiedExternalKeys(tenantMigrationTest) {
+ const donorPrimary = tenantMigrationTest.getDonorPrimary();
+ const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
+
+ recipientPrimary.getCollection(kInternalKeysNs).find().forEach(internalKeyDoc => {
+ assert.neq(null, donorPrimary.getCollection(kExternalKeysNs).findOne({
+ keyId: internalKeyDoc._id,
+ key: internalKeyDoc.key,
+ expiresAt: internalKeyDoc.expiresAt,
+ replicaSetName: tenantMigrationTest.getRecipientRst().name
+ }));
+ });
+}
+
+const kTenantId = "testTenantId";
+const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest();
+
+(() => {
+ jsTest.log("Test that the donor correctly copies the recipient's cluster time keys " +
+ "when there is no failover.");
+ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
+ if (!tenantMigrationTest.isFeatureFlagEnabled()) {
+ jsTestLog("Skipping test because the tenant migrations feature flag is disabled");
+ tenantMigrationTest.stop();
+ return;
+ }
+
+ const migrationId = UUID();
+ const migrationOpts = {
+ migrationIdString: extractUUIDFromObject(migrationId),
+ tenantId: kTenantId,
+ };
+ assert.commandWorked(tenantMigrationTest.runMigration(migrationOpts));
+ assertDonorCopiedExternalKeys(tenantMigrationTest);
+
+ tenantMigrationTest.stop();
+})();
+
+(() => {
+ jsTest.log("Test that the donor correctly copies the recipient's cluster time keys " +
+ "when there is donor failover.");
+ const donorRst =
+ new ReplSetTest({nodes: 3, name: "donorRst", nodeOptions: migrationX509Options.donor});
+ donorRst.startSet();
+ donorRst.initiate();
+
+ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName(), donorRst});
+ if (!tenantMigrationTest.isFeatureFlagEnabled()) {
+ jsTestLog("Skipping test because the tenant migrations feature flag is disabled");
+ donorRst.stopSet();
+ tenantMigrationTest.stop();
+ return;
+ }
+
+ let donorPrimary = donorRst.getPrimary();
+ const fp =
+ configureFailPoint(donorPrimary, "pauseTenantMigrationAfterPersitingInitialDonorStateDoc");
+
+ const migrationId = UUID();
+ const migrationOpts = {
+ migrationIdString: extractUUIDFromObject(migrationId),
+ tenantId: kTenantId,
+ };
+ assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts));
+ fp.wait();
+
+ assert.commandWorked(
+ donorPrimary.adminCommand({replSetStepDown: ReplSetTest.kForeverSecs, force: true}));
+ assert.commandWorked(donorPrimary.adminCommand({replSetFreeze: 0}));
+
+ fp.off();
+ assert.commandWorked(tenantMigrationTest.waitForMigrationToComplete(
+ migrationOpts, true /* retryOnRetryableErrors */));
+
+ assertDonorCopiedExternalKeys(tenantMigrationTest);
+
+ donorRst.stopSet();
+ tenantMigrationTest.stop();
+})();
+})();
diff --git a/jstests/replsets/tenant_migration_donor_try_abort.js b/jstests/replsets/tenant_migration_donor_try_abort.js
index e6ccfd3fffc..b7bb70039fc 100644
--- a/jstests/replsets/tenant_migration_donor_try_abort.js
+++ b/jstests/replsets/tenant_migration_donor_try_abort.js
@@ -22,7 +22,8 @@ const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest();
(() => {
jsTestLog(
- "Test sending donorAbortMigration during a tenant migration while recipientSyncData command repeatedly fails.");
+ "Test sending donorAbortMigration during a tenant migration while recipientSyncData " +
+ "command repeatedly fails with retryable errors.");
const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
if (!tenantMigrationTest.isFeatureFlagEnabled()) {
@@ -61,8 +62,49 @@ const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest();
})();
(() => {
- jsTestLog(
- "Test sending donorAbortMigration during a tenant migration while waiting for the response of recipientSyncData.");
+ jsTestLog("Test sending donorAbortMigration during a tenant migration while find command " +
+ "against admin.system.keys repeatedly fails with retryable errors.");
+
+ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
+ if (!tenantMigrationTest.isFeatureFlagEnabled()) {
+ jsTestLog("Skipping test because the tenant migrations feature flag is disabled");
+ return;
+ }
+
+ const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
+ let fp = configureFailPoint(recipientPrimary, "failCommand", {
+ failInternalCommands: true,
+ errorCode: ErrorCodes.ShutdownInProgress,
+ failCommands: ["find"],
+ namespace: "admin.system.keys"
+ });
+
+ const tenantId = kTenantId;
+ const migrationId = UUID();
+ const migrationOpts = {
+ migrationIdString: extractUUIDFromObject(migrationId),
+ tenantId: tenantId,
+ recipientConnString: tenantMigrationTest.getRecipientConnString(),
+ };
+
+ assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts));
+
+ fp.wait();
+
+ assert.commandWorked(tenantMigrationTest.tryAbortMigration(
+ {migrationIdString: migrationOpts.migrationIdString}));
+
+ const stateRes =
+ assert.commandWorked(tenantMigrationTest.waitForMigrationToComplete(migrationOpts));
+ assert.eq(stateRes.state, TenantMigrationTest.State.kAborted);
+
+ fp.off();
+ tenantMigrationTest.stop();
+})();
+
+(() => {
+ jsTestLog("Test sending donorAbortMigration during a tenant migration while waiting for the " +
+ "response of recipientSyncData.");
const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
if (!tenantMigrationTest.isFeatureFlagEnabled()) {
@@ -121,6 +163,44 @@ const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest();
})();
(() => {
+ jsTestLog("Test sending donorAbortMigration during a tenant migration while waiting for the " +
+ "response of find against admin.system.keys.");
+ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
+ if (!tenantMigrationTest.isFeatureFlagEnabled()) {
+ jsTestLog("Skipping test because the tenant migrations feature flag is disabled");
+ return;
+ }
+ const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
+ configureFailPoint(recipientPrimary, "failCommand", {
+ failInternalCommands: true,
+ blockConnection: true,
+ blockTimeMS: kDelayMS,
+ failCommands: ["find"],
+ namespace: "admin.system.keys"
+ });
+ const tenantId = kTenantId;
+ const migrationId = UUID();
+ const migrationOpts = {
+ migrationIdString: extractUUIDFromObject(migrationId),
+ tenantId: tenantId,
+ recipientConnString: tenantMigrationTest.getRecipientConnString(),
+ };
+ assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts));
+ // Wait for donorAbortMigration command to start.
+ assert.soon(() => {
+ const res = assert.commandWorked(recipientPrimary.adminCommand(
+ {currentOp: true, $and: [{"command.find": "system.keys"}, {"command.$db": "admin"}]}));
+ return res.inprog.length == 1;
+ });
+ assert.commandWorked(tenantMigrationTest.tryAbortMigration(
+ {migrationIdString: migrationOpts.migrationIdString}));
+ const stateRes =
+ assert.commandWorked(tenantMigrationTest.waitForMigrationToComplete(migrationOpts));
+ assert.eq(stateRes.state, TenantMigrationTest.State.kAborted);
+ tenantMigrationTest.stop();
+})();
+
+(() => {
jsTestLog("Test sending donorAbortMigration during a tenant migration while in data sync.");
const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
diff --git a/jstests/replsets/tenant_migration_external_cluster_validation.js b/jstests/replsets/tenant_migration_external_cluster_validation.js
new file mode 100644
index 00000000000..8c98e4d4754
--- /dev/null
+++ b/jstests/replsets/tenant_migration_external_cluster_validation.js
@@ -0,0 +1,148 @@
+/**
+ * Verify that after a tenant migration the donor can validate the recipient's cluster times.
+ *
+ * @tags: [requires_fcv_47, requires_majority_read_concern, incompatible_with_eft]
+ */
+
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");
+load("jstests/libs/uuid_util.js");
+load("jstests/replsets/libs/tenant_migration_test.js");
+
+// Multiple users cannot be authenticated on one connection within a session.
+TestData.disableImplicitSessions = true;
+
+// User that runs the tenant migration.
+const kAdminUser = {
+ name: "admin",
+ pwd: "pwd",
+};
+
+// User that runs commands against the tenant database.
+const kTestUser = {
+ name: "testUser",
+ pwd: "pwd",
+};
+
+function createUsers(primary) {
+ const adminDB = primary.getDB("admin");
+ assert.commandWorked(
+ adminDB.runCommand({createUser: kAdminUser.name, pwd: kAdminUser.pwd, roles: ["root"]}));
+ assert.eq(1, adminDB.auth(kAdminUser.name, kAdminUser.pwd));
+
+ const testDB = primary.getDB(kDbName);
+ assert.commandWorked(
+ testDB.runCommand({createUser: kTestUser.name, pwd: kTestUser.pwd, roles: ["readWrite"]}));
+
+ adminDB.logout();
+}
+
+const kTenantId = "testTenantId";
+const kDbName = kTenantId + "_" +
+ "testDb";
+const kCollName = "testColl";
+
+const x509Options = TenantMigrationUtil.makeX509OptionsForTest();
+const donorRst = new ReplSetTest({
+ nodes: 1,
+ name: "donor",
+ keyFile: "jstests/libs/key1",
+ nodeOptions: Object.assign(
+ x509Options.donor,
+ {setParameter: {"failpoint.alwaysValidateClientsClusterTime": tojson({mode: "alwaysOn"})}}),
+});
+
+const recipientRst = new ReplSetTest({
+ nodes: 1,
+ name: "recipient",
+ keyFile: "jstests/libs/key1",
+ nodeOptions: Object.assign(
+ x509Options.recipient,
+ {setParameter: {"failpoint.alwaysValidateClientsClusterTime": tojson({mode: "alwaysOn"})}}),
+});
+
+donorRst.startSet();
+donorRst.initiate();
+
+recipientRst.startSet();
+recipientRst.initiate();
+
+const donorPrimary = donorRst.getPrimary();
+const recipientPrimary = recipientRst.getPrimary();
+
+const donorAdminDB = donorPrimary.getDB("admin");
+const recipientAdminDB = recipientPrimary.getDB("admin");
+
+const donorTestDB = donorPrimary.getDB(kDbName);
+const recipientTestDB = recipientPrimary.getDB(kDbName);
+
+createUsers(donorPrimary);
+createUsers(recipientPrimary);
+
+assert.eq(1, donorAdminDB.auth(kAdminUser.name, kAdminUser.pwd));
+assert.eq(1, recipientAdminDB.auth(kAdminUser.name, kAdminUser.pwd));
+
+const tenantMigrationTest = new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst});
+if (!tenantMigrationTest.isFeatureFlagEnabled()) {
+ jsTestLog("Skipping test because the tenant migrations feature flag is disabled");
+ donorRst.stopSet();
+ recipientRst.stopSet();
+ return;
+}
+
+donorAdminDB.logout();
+recipientAdminDB.logout();
+
+assert.eq(1, donorTestDB.auth(kTestUser.name, kTestUser.pwd));
+assert.eq(1, recipientTestDB.auth(kTestUser.name, kTestUser.pwd));
+
+const donorClusterTime =
+ assert.commandWorked(donorTestDB.runCommand({find: kCollName})).$clusterTime;
+const recipientClusterTime =
+ assert.commandWorked(recipientTestDB.runCommand({find: kCollName})).$clusterTime;
+jsTest.log("donor's clusterTime " + tojson(donorClusterTime));
+jsTest.log("recipient's clusterTime " + tojson(recipientClusterTime));
+
+jsTest.log("Verify that prior to the migration, the donor and recipient fail to validate each " +
+ "other's clusterTime");
+
+assert.commandFailedWithCode(
+ donorTestDB.runCommand({find: kCollName, $clusterTime: recipientClusterTime}),
+ [ErrorCodes.TimeProofMismatch, ErrorCodes.KeyNotFound]);
+assert.commandFailedWithCode(
+ recipientTestDB.runCommand({find: kCollName, $clusterTime: donorClusterTime}),
+ [ErrorCodes.TimeProofMismatch, ErrorCodes.KeyNotFound]);
+
+donorTestDB.logout();
+recipientTestDB.logout();
+
+assert.eq(1, donorAdminDB.auth(kAdminUser.name, kAdminUser.pwd));
+assert.eq(1, recipientAdminDB.auth(kAdminUser.name, kAdminUser.pwd));
+
+const migrationId = UUID();
+const migrationOpts = {
+ migrationIdString: extractUUIDFromObject(migrationId),
+ tenantId: kTenantId,
+};
+assert.commandWorked(tenantMigrationTest.runMigration(migrationOpts));
+
+donorAdminDB.logout();
+recipientAdminDB.logout();
+
+jsTest.log("Verify that after the migration, the donor can validate the recipient's clusterTime");
+
+assert.eq(1, donorTestDB.auth(kTestUser.name, kTestUser.pwd));
+assert.eq(1, recipientTestDB.auth(kTestUser.name, kTestUser.pwd));
+
+assert.commandWorked(donorTestDB.runCommand({find: kCollName, $clusterTime: recipientClusterTime}));
+// TODO (SERVER-53405): Test that the recipient can validate the donor's clusterTime.
+assert.commandFailedWithCode(
+ recipientTestDB.runCommand({find: kCollName, $clusterTime: donorClusterTime}),
+ [ErrorCodes.TimeProofMismatch, ErrorCodes.KeyNotFound]);
+
+tenantMigrationTest.stop();
+donorRst.stopSet();
+recipientRst.stopSet();
+})();
diff --git a/jstests/replsets/tenant_migration_x509.js b/jstests/replsets/tenant_migration_x509.js
index df4bd2edf3c..9a546116e17 100644
--- a/jstests/replsets/tenant_migration_x509.js
+++ b/jstests/replsets/tenant_migration_x509.js
@@ -315,6 +315,27 @@ if (!TestData.auth) {
}
(() => {
+ jsTest.log("Test donor certificate without findInternalClusterTimeKeysRole role");
+ const migrationId = UUID();
+ const tenantId = "donorCertificateNoFindInternalClusterTimeKeysRole";
+ const migrationOpts = {
+ migrationIdString: extractUUIDFromObject(migrationId),
+ tenantId: tenantId,
+ donorCertificateForRecipient: TenantMigrationUtil.getCertificateAndPrivateKey(
+ "jstests/libs/rs0_tenant_migration_no_find_cluster_time_keys_role.pem"),
+ recipientCertificateForDonor: kRecipientCertificateAndPrivateKey,
+ };
+ const {dbName, collName} = makeTestNs(tenantId);
+
+ tenantMigrationTest.insertDonorDB(dbName, collName);
+ const stateRes = assert.commandWorked(tenantMigrationTest.runMigration(migrationOpts));
+ assert.eq(stateRes.state, TenantMigrationTest.State.kAborted);
+ assert.eq(stateRes.abortReason.code, ErrorCodes.Unauthorized);
+ tenantMigrationTest.verifyRecipientDB(
+ tenantId, dbName, collName, false /* migrationCommitted */);
+})();
+
+(() => {
jsTest.log("Test recipient certificate without backup role");
const migrationId = UUID();
const tenantId = "recipientCertificateNoBackupRole";
@@ -336,24 +357,26 @@ if (!TestData.auth) {
})();
(() => {
- jsTest.log("Test recipient certificate without advanceClusterTime role");
+ jsTest.log("Test recipient certificate without findInternalClusterTimeKeysRole role");
const migrationId = UUID();
- const tenantId = "recipientCertificateNoAdvanceClusterTimeRole";
+ const tenantId = "recipientCertificateNoFindInternalClusterTimeKeysRole";
const migrationOpts = {
migrationIdString: extractUUIDFromObject(migrationId),
tenantId: tenantId,
donorCertificateForRecipient: kDonorCertificateAndPrivateKey,
recipientCertificateForDonor: TenantMigrationUtil.getCertificateAndPrivateKey(
- "jstests/libs/rs1_tenant_migration_no_advance_cluster_time_role.pem"),
+ "jstests/libs/rs1_tenant_migration_no_find_cluster_time_keys_role.pem"),
};
const {dbName, collName} = makeTestNs(tenantId);
tenantMigrationTest.insertDonorDB(dbName, collName);
const stateRes = assert.commandWorked(tenantMigrationTest.runMigration(migrationOpts));
- assert.eq(stateRes.state, TenantMigrationTest.State.kAborted);
- assert.eq(stateRes.abortReason.code, ErrorCodes.KeyNotFound);
+ // TODO (SERVER-53405): Make tenant migration recipient copy the donor's cluster time signing
+ // keys before starting to clone. Right now the recipient doesn't copy the keys so it doesn't
+ // need the findInternalClusterTimeKeysRole role.
+ assert.eq(stateRes.state, TenantMigrationTest.State.kCommitted);
tenantMigrationTest.verifyRecipientDB(
- tenantId, dbName, collName, false /* migrationCommitted */);
+ tenantId, dbName, collName, true /* migrationCommitted */);
})();
tenantMigrationTest.stop();
diff --git a/jstests/ssl/x509/certs.yml b/jstests/ssl/x509/certs.yml
index 2953a0448db..1141b36636b 100644
--- a/jstests/ssl/x509/certs.yml
+++ b/jstests/ssl/x509/certs.yml
@@ -322,7 +322,7 @@ certs:
extendedKeyUsage: [clientAuth]
mongoRoles:
- {role: backup, db: admin}
- - {role: advanceClusterTimeRole, db: admin}
+ - {role: findInternalClusterTimeKeysRole, db: admin}
- name: 'rs0_tenant_migration_expired.pem'
description:
@@ -339,7 +339,21 @@ certs:
extendedKeyUsage: [clientAuth]
mongoRoles:
- {role: backup, db: admin}
- - {role: advanceClusterTimeRole, db: admin}
+ - {role: findInternalClusterTimeKeysRole, db: admin}
+
+- name: 'rs0_tenant_migration_no_find_cluster_time_keys_role.pem'
+ description:
+ Client certificate file for tenant migration donor or recipient without role to run find command
+ against admin.system.keys.
+ Subject:
+ OU: 'rs0_tenant_migration'
+ extensions:
+ basicConstraints: {CA: false}
+ subjectKeyIdentifier: hash
+ keyUsage: [digitalSignature, keyEncipherment]
+ extendedKeyUsage: [clientAuth]
+ mongoRoles:
+ - {role: backup, db: admin}
- name: 'rs1.pem'
description: General purpose server certificate file.
@@ -363,7 +377,7 @@ certs:
extendedKeyUsage: [clientAuth]
mongoRoles:
- {role: backup, db: admin}
- - {role: advanceClusterTimeRole, db: admin}
+ - {role: findInternalClusterTimeKeysRole, db: admin}
- name: 'rs1_tenant_migration_expired.pem'
description:
@@ -380,7 +394,7 @@ certs:
extendedKeyUsage: [clientAuth]
mongoRoles:
- {role: backup, db: admin}
- - {role: advanceClusterTimeRole, db: admin}
+ - {role: findInternalClusterTimeKeysRole, db: admin}
- name: 'rs1_tenant_migration_no_backup_role.pem'
description:
@@ -393,12 +407,12 @@ certs:
keyUsage: [digitalSignature, keyEncipherment]
extendedKeyUsage: [clientAuth]
mongoRoles:
- - {role: advanceClusterTimeRole, db: admin}
+ - {role: findInternalClusterTimeKeysRole, db: admin}
-- name: 'rs1_tenant_migration_no_advance_cluster_time_role.pem'
+- name: 'rs1_tenant_migration_no_find_cluster_time_keys_role.pem'
description:
- Client certificate file for tenant migration donor or recipient without role to advance
- cluster time.
+ Client certificate file for tenant migration donor or recipient without role to run find command
+ against admin.system.keys.
Subject:
OU: 'rs1_tenant_migration'
extensions:
@@ -431,7 +445,7 @@ certs:
extendedKeyUsage: [clientAuth]
mongoRoles:
- {role: backup, db: admin}
- - {role: advanceClusterTimeRole, db: admin}
+ - {role: findInternalClusterTimeKeysRole, db: admin}
###
# Certificates not based on the primary root ca.pem
diff --git a/src/mongo/client/fetcher.cpp b/src/mongo/client/fetcher.cpp
index d8c78fd25c0..cf0f72cb113 100644
--- a/src/mongo/client/fetcher.cpp
+++ b/src/mongo/client/fetcher.cpp
@@ -157,7 +157,8 @@ Fetcher::Fetcher(executor::TaskExecutor* executor,
const BSONObj& metadata,
Milliseconds findNetworkTimeout,
Milliseconds getMoreNetworkTimeout,
- std::unique_ptr<RemoteCommandRetryScheduler::RetryPolicy> firstCommandRetryPolicy)
+ std::unique_ptr<RemoteCommandRetryScheduler::RetryPolicy> firstCommandRetryPolicy,
+ transport::ConnectSSLMode sslMode)
: _executor(executor),
_source(source),
_dbname(dbname),
@@ -168,9 +169,15 @@ Fetcher::Fetcher(executor::TaskExecutor* executor,
_getMoreNetworkTimeout(getMoreNetworkTimeout),
_firstRemoteCommandScheduler(
_executor,
- RemoteCommandRequest(_source, _dbname, _cmdObj, _metadata, nullptr, _findNetworkTimeout),
+ [&] {
+ RemoteCommandRequest request(
+ _source, _dbname, _cmdObj, _metadata, nullptr, _findNetworkTimeout);
+ request.sslMode = sslMode;
+ return request;
+ }(),
[this](const auto& x) { return this->_callback(x, kFirstBatchFieldName); },
- std::move(firstCommandRetryPolicy)) {
+ std::move(firstCommandRetryPolicy)),
+ _sslMode(sslMode) {
uassert(ErrorCodes::BadValue, "callback function cannot be null", _work);
}
@@ -297,11 +304,14 @@ Status Fetcher::_scheduleGetMore(const BSONObj& cmdObj) {
return Status(ErrorCodes::CallbackCanceled,
"fetcher was shut down after previous batch was processed");
}
+
+ RemoteCommandRequest request(
+ _source, _dbname, cmdObj, _metadata, nullptr, _getMoreNetworkTimeout);
+ request.sslMode = _sslMode;
+
StatusWith<executor::TaskExecutor::CallbackHandle> scheduleResult =
_executor->scheduleRemoteCommand(
- RemoteCommandRequest(
- _source, _dbname, cmdObj, _metadata, nullptr, _getMoreNetworkTimeout),
- [this](const auto& x) { return this->_callback(x, kNextBatchFieldName); });
+ request, [this](const auto& x) { return this->_callback(x, kNextBatchFieldName); });
if (!scheduleResult.isOK()) {
return scheduleResult.getStatus();
@@ -405,9 +415,12 @@ void Fetcher::_sendKillCursors(const CursorId id, const NamespaceString& nss) {
"error"_attr = redact(status));
}
};
+
auto cmdObj = BSON("killCursors" << nss.coll() << "cursors" << BSON_ARRAY(id));
- auto scheduleResult = _executor->scheduleRemoteCommand(
- RemoteCommandRequest(_source, _dbname, cmdObj, nullptr), logKillCursorsResult);
+ RemoteCommandRequest request(_source, _dbname, cmdObj, nullptr);
+ request.sslMode = _sslMode;
+
+ auto scheduleResult = _executor->scheduleRemoteCommand(request, logKillCursorsResult);
if (!scheduleResult.isOK()) {
LOGV2_WARNING(23920,
"Failed to schedule killCursors command: {error}",
diff --git a/src/mongo/client/fetcher.h b/src/mongo/client/fetcher.h
index 3d0d10abec8..4ce57087c20 100644
--- a/src/mongo/client/fetcher.h
+++ b/src/mongo/client/fetcher.h
@@ -131,7 +131,8 @@ public:
Milliseconds findNetworkTimeout = RemoteCommandRequest::kNoTimeout,
Milliseconds getMoreNetworkTimeout = RemoteCommandRequest::kNoTimeout,
std::unique_ptr<RemoteCommandRetryScheduler::RetryPolicy> firstCommandRetryPolicy =
- RemoteCommandRetryScheduler::makeNoRetryPolicy());
+ RemoteCommandRetryScheduler::makeNoRetryPolicy(),
+ transport::ConnectSSLMode sslMode = transport::kGlobalSSLMode);
virtual ~Fetcher();
@@ -259,6 +260,8 @@ private:
// First remote command scheduler.
RemoteCommandRetryScheduler _firstRemoteCommandScheduler;
+
+ const transport::ConnectSSLMode _sslMode;
};
/**
diff --git a/src/mongo/db/catalog/database_impl.cpp b/src/mongo/db/catalog/database_impl.cpp
index 87ebf3d6a47..789ea8e882e 100644
--- a/src/mongo/db/catalog/database_impl.cpp
+++ b/src/mongo/db/catalog/database_impl.cpp
@@ -348,7 +348,8 @@ Status DatabaseImpl::dropCollection(OperationContext* opCtx,
"turn off profiling before dropping system.profile collection");
} else if (!(nss.isSystemDotViews() || nss.isHealthlog() ||
nss == NamespaceString::kLogicalSessionsNamespace ||
- nss == NamespaceString::kSystemKeysNamespace ||
+ nss == NamespaceString::kKeysCollectionNamespace ||
+ nss == NamespaceString::kExternalKeysCollectionNamespace ||
nss.isTemporaryReshardingCollection())) {
return Status(ErrorCodes::IllegalOperation,
str::stream() << "can't drop system collection " << nss);
diff --git a/src/mongo/db/commands/tenant_migration_donor_cmds.idl b/src/mongo/db/commands/tenant_migration_donor_cmds.idl
index 43823f99cb2..f81b718cffa 100644
--- a/src/mongo/db/commands/tenant_migration_donor_cmds.idl
+++ b/src/mongo/db/commands/tenant_migration_donor_cmds.idl
@@ -66,12 +66,12 @@ commands:
description: "The URI string that the donor will utilize to create a connection with the recipient."
type: string
validator:
- callback: "validateConnectionString"
+ callback: "tenant_migration_util::validateConnectionString"
tenantId:
description: "The prefix from which the migrating database will be matched. The prefixes 'admin', 'local', 'config', the empty string, are not allowed."
type: string
validator:
- callback: "validateDatabasePrefix"
+ callback: "tenant_migration_util::validateDatabasePrefix"
readPreference:
description: "The read preference settings that the donor will pass on to the recipient."
type: readPreference
diff --git a/src/mongo/db/commands/tenant_migration_recipient_cmds.idl b/src/mongo/db/commands/tenant_migration_recipient_cmds.idl
index 4c96e219c4e..a3c4fb991d2 100644
--- a/src/mongo/db/commands/tenant_migration_recipient_cmds.idl
+++ b/src/mongo/db/commands/tenant_migration_recipient_cmds.idl
@@ -62,14 +62,14 @@ structs:
donor.
type: string
validator:
- callback: "validateConnectionString"
+ callback: "tenant_migration_util::validateConnectionString"
tenantId:
description: >-
The prefix from which the migrating database will be matched. The prefixes 'admin',
'local', 'config', the empty string, are not allowed.
type: string
validator:
- callback: "validateDatabasePrefix"
+ callback: "tenant_migration_util::validateDatabasePrefix"
readPreference:
description: >-
The read preference settings that the donor will pass on to the recipient.
@@ -98,7 +98,7 @@ commands:
type: timestamp
optional: true
validator:
- callback: "validateTimestampNotNull"
+ callback: "tenant_migration_util::validateTimestampNotNull"
recipientForgetMigration:
description: "Parser for the 'recipientForgetMigration' command."
diff --git a/src/mongo/db/logical_time_validator.cpp b/src/mongo/db/logical_time_validator.cpp
index d2b8866a79c..b848b49cc5e 100644
--- a/src/mongo/db/logical_time_validator.cpp
+++ b/src/mongo/db/logical_time_validator.cpp
@@ -220,6 +220,11 @@ bool LogicalTimeValidator::shouldGossipLogicalTime() {
return _getKeyManagerCopy()->hasSeenKeys();
}
+void LogicalTimeValidator::refreshKeyManagerCache(OperationContext* opCtx) {
+ invariant(_keyManager);
+ _keyManager->refreshNow(opCtx);
+}
+
void LogicalTimeValidator::resetKeyManagerCache() {
LOGV2(20716, "Resetting key manager cache");
invariant(_keyManager);
diff --git a/src/mongo/db/logical_time_validator.h b/src/mongo/db/logical_time_validator.h
index 1b78c51d1f5..825b7dc70df 100644
--- a/src/mongo/db/logical_time_validator.h
+++ b/src/mongo/db/logical_time_validator.h
@@ -109,6 +109,11 @@ public:
void stopKeyManager();
/**
+ * Forces the key manager cache to refresh.
+ */
+ void refreshKeyManagerCache(OperationContext* opCtx);
+
+ /**
* Reset the key manager cache of keys.
*/
void resetKeyManagerCache();
diff --git a/src/mongo/db/namespace_string.cpp b/src/mongo/db/namespace_string.cpp
index eea72c9e3f8..11445d700a5 100644
--- a/src/mongo/db/namespace_string.cpp
+++ b/src/mongo/db/namespace_string.cpp
@@ -79,8 +79,10 @@ const NamespaceString NamespaceString::kShardConfigCollectionsNamespace(Namespac
"cache.collections");
const NamespaceString NamespaceString::kShardConfigDatabasesNamespace(NamespaceString::kConfigDb,
"cache.databases");
-const NamespaceString NamespaceString::kSystemKeysNamespace(NamespaceString::kAdminDb,
- "system.keys");
+const NamespaceString NamespaceString::kKeysCollectionNamespace(NamespaceString::kAdminDb,
+ "system.keys");
+const NamespaceString NamespaceString::kExternalKeysCollectionNamespace(
+ NamespaceString::kAdminDb, "system.external_validation_keys");
const NamespaceString NamespaceString::kRsOplogNamespace(NamespaceString::kLocalDb, "oplog.rs");
const NamespaceString NamespaceString::kSystemReplSetNamespace(NamespaceString::kLocalDb,
"system.replset");
@@ -108,12 +110,6 @@ const NamespaceString NamespaceString::kReshardingApplierProgressNamespace(
const NamespaceString NamespaceString::kReshardingTxnClonerProgressNamespace(
NamespaceString::kConfigDb, "localReshardingOperations.recipient.progress_txn_cloner");
-const NamespaceString NamespaceString::kKeysCollectionNamespace(NamespaceString::kAdminDb,
- "system.keys");
-
-const NamespaceString NamespaceString::kExternalKeysCollectionNamespace(
- NamespaceString::kAdminDb, "system.external_validation_keys");
-
bool NamespaceString::isListCollectionsCursorNS() const {
return coll() == listCollectionsCursorCol;
}
@@ -128,16 +124,12 @@ bool NamespaceString::isLegalClientSystemNS() const {
return true;
if (coll() == kServerConfigurationNamespace.coll())
return true;
- if (coll() == kSystemKeysNamespace.coll())
+ if (coll() == kKeysCollectionNamespace.coll())
return true;
- if (coll() == "system.backup_users")
+ if (coll() == kExternalKeysCollectionNamespace.coll())
return true;
- if (coll() == kExternalKeysCollectionNamespace.coll()) {
- // TODO (SERVER-53404): This was added to allow client in an integration test to
- // manually insert the key document into this system collection. Remove this when the
- // tenant migration donor does the copying by itself.
+ if (coll() == "system.backup_users")
return true;
- }
} else if (db() == kConfigDb) {
if (coll() == "system.sessions")
return true;
diff --git a/src/mongo/db/namespace_string.h b/src/mongo/db/namespace_string.h
index 03accfec6f7..a60adb43d51 100644
--- a/src/mongo/db/namespace_string.h
+++ b/src/mongo/db/namespace_string.h
@@ -100,8 +100,12 @@ public:
// of a specific database
static const NamespaceString kShardConfigDatabasesNamespace;
- // Name for causal consistency's key collection.
- static const NamespaceString kSystemKeysNamespace;
+ // Namespace for storing keys for signing and validating cluster times created by the cluster
+ // that this node is in.
+ static const NamespaceString kKeysCollectionNamespace;
+
+ // Namespace for storing keys for validating cluster times created by other clusters.
+ static const NamespaceString kExternalKeysCollectionNamespace;
// Namespace of the the oplog collection.
static const NamespaceString kRsOplogNamespace;
@@ -148,13 +152,6 @@ public:
// Namespace for storing config.transactions cloner progress for resharding.
static const NamespaceString kReshardingTxnClonerProgressNamespace;
- // Namespace for storing keys for signing and validating cluster times created by the cluster
- // that this node is in.
- static const NamespaceString kKeysCollectionNamespace;
-
- // Namespace for storing keys for validating cluster times created by other clusters.
- static const NamespaceString kExternalKeysCollectionNamespace;
-
/**
* Constructs an empty NamespaceString.
*/
diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript
index ca18216794f..f8d16936ad9 100644
--- a/src/mongo/db/repl/SConscript
+++ b/src/mongo/db/repl/SConscript
@@ -1284,10 +1284,16 @@ env.Library(
)
env.Library(
- target='tenant_migration_recipient_utils',
+ target='tenant_migration_utils',
source=[
+ "tenant_migration_util.cpp",
"tenant_migration_recipient_entry_helpers.cpp",
],
+ LIBDEPS=[
+ '$BUILD_DIR/mongo/util/future_util',
+ "repl_server_parameters",
+ 'wait_for_majority_service',
+ ],
LIBDEPS_PRIVATE=[
"$BUILD_DIR/mongo/base",
"$BUILD_DIR/mongo/db/catalog_raii",
@@ -1306,10 +1312,8 @@ env.Library(
],
LIBDEPS=[
'$BUILD_DIR/mongo/client/read_preference',
- '$BUILD_DIR/mongo/util/future_util',
'primary_only_service',
- 'tenant_migration_recipient_utils',
- 'wait_for_majority_service',
+ 'tenant_migration_utils',
],
LIBDEPS_PRIVATE=[
'$BUILD_DIR/mongo/client/clientdriver_network',
@@ -1363,10 +1367,10 @@ env.Library(
'tenant_migration_donor_service.cpp',
],
LIBDEPS=[
+ '$BUILD_DIR/mongo/client/fetcher',
'primary_only_service',
- 'repl_server_parameters',
'tenant_migration_donor',
- 'wait_for_majority_service',
+ 'tenant_migration_utils',
],
)
@@ -1573,7 +1577,7 @@ env.CppUnitTest(
'task_executor_mock',
'task_runner',
'tenant_migration_recipient_service',
- 'tenant_migration_recipient_utils',
+ 'tenant_migration_utils',
'tenant_oplog_processing',
'wait_for_majority_service',
],
diff --git a/src/mongo/db/repl/tenant_migration_donor_service.cpp b/src/mongo/db/repl/tenant_migration_donor_service.cpp
index 6959f19d995..5d10481b025 100644
--- a/src/mongo/db/repl/tenant_migration_donor_service.cpp
+++ b/src/mongo/db/repl/tenant_migration_donor_service.cpp
@@ -39,6 +39,7 @@
#include "mongo/db/db_raii.h"
#include "mongo/db/dbhelpers.h"
#include "mongo/db/persistent_task_store.h"
+#include "mongo/db/query/find_command_gen.h"
#include "mongo/db/repl/repl_server_parameters_gen.h"
#include "mongo/db/repl/tenant_migration_access_blocker.h"
#include "mongo/db/repl/tenant_migration_donor_util.h"
@@ -58,13 +59,18 @@ namespace mongo {
namespace {
MONGO_FAIL_POINT_DEFINE(abortTenantMigrationBeforeLeavingBlockingState);
+MONGO_FAIL_POINT_DEFINE(pauseTenantMigrationAfterPersitingInitialDonorStateDoc);
MONGO_FAIL_POINT_DEFINE(pauseTenantMigrationBeforeLeavingBlockingState);
MONGO_FAIL_POINT_DEFINE(pauseTenantMigrationBeforeLeavingDataSyncState);
const std::string kTTLIndexName = "TenantMigrationDonorTTLIndex";
-const Seconds kRecipientSyncDataTimeout(30);
const Backoff kExponentialBackoff(Seconds(1), Milliseconds::max());
+const ReadPreferenceSetting kPrimaryOnlyReadPreference(ReadPreference::PrimaryOnly);
+
+const Seconds kRecipientSyncDataTimeout(30);
+const int kMaxRecipientKeyDocsFindAttempts = 10;
+
std::shared_ptr<TenantMigrationAccessBlocker> getTenantMigrationAccessBlocker(
ServiceContext* serviceContext, StringData tenantId) {
return TenantMigrationAccessBlockerRegistry::get(serviceContext)
@@ -261,6 +267,11 @@ TenantMigrationDonorService::Instance::getDurableState(OperationContext* opCtx)
void TenantMigrationDonorService::Instance::onReceiveDonorAbortMigration() {
_instanceCancelationSource.cancel();
+
+ stdx::lock_guard<Latch> lg(_mutex);
+ if (auto fetcher = _recipientKeysFetcher.lock()) {
+ fetcher->shutdown();
+ }
}
void TenantMigrationDonorService::Instance::onReceiveDonorForgetMigration() {
@@ -287,7 +298,96 @@ void TenantMigrationDonorService::Instance::interrupt(Status status) {
}
}
-ExecutorFuture<repl::OpTime> TenantMigrationDonorService::Instance::_insertStateDocument(
+ExecutorFuture<void>
+TenantMigrationDonorService::Instance::_fetchAndStoreRecipientClusterTimeKeyDocs(
+ std::shared_ptr<executor::ScopedTaskExecutor> executor,
+ std::shared_ptr<RemoteCommandTargeter> recipientTargeterRS,
+ const CancelationToken& token) {
+ return recipientTargeterRS
+ ->findHost(kPrimaryOnlyReadPreference, _instanceCancelationSource.token())
+ .thenRunOn(**executor)
+ .then([this, self = shared_from_this(), executor](HostAndPort host) {
+ const auto nss = NamespaceString::kKeysCollectionNamespace;
+
+ const auto cmdObj = [&] {
+ FindCommand request(NamespaceStringOrUUID{nss});
+ request.setReadConcern(
+ repl::ReadConcernArgs(repl::ReadConcernLevel::kMajorityReadConcern)
+ .toBSONInner());
+ return request.toBSON(BSONObj());
+ }();
+
+ std::vector<ExternalKeysCollectionDocument> keyDocs;
+ boost::optional<Status> fetchStatus;
+
+ auto fetcherCallback = [this, self = shared_from_this(), &keyDocs, &fetchStatus](
+ const Fetcher::QueryResponseStatus& dataStatus,
+ Fetcher::NextAction* nextAction,
+ BSONObjBuilder* getMoreBob) {
+ // Throw out any accumulated results on error
+ if (!dataStatus.isOK()) {
+ fetchStatus = dataStatus.getStatus();
+ keyDocs.clear();
+ return;
+ }
+
+ const auto& data = dataStatus.getValue();
+ for (const BSONObj& doc : data.documents) {
+ keyDocs.push_back(tenant_migration_util::makeExternalClusterTimeKeyDoc(
+ _serviceContext, _recipientUri.getSetName(), doc.getOwned()));
+ }
+ fetchStatus = Status::OK();
+
+ if (!getMoreBob) {
+ return;
+ }
+ getMoreBob->append("getMore", data.cursorId);
+ getMoreBob->append("collection", data.nss.coll());
+ };
+
+ auto fetcher = std::make_shared<Fetcher>(
+ _recipientCmdExecutor.get(),
+ host,
+ nss.db().toString(),
+ cmdObj,
+ fetcherCallback,
+ kPrimaryOnlyReadPreference.toContainingBSON(),
+ executor::RemoteCommandRequest::kNoTimeout, /* findNetworkTimeout */
+ executor::RemoteCommandRequest::kNoTimeout, /* getMoreNetworkTimeout */
+ RemoteCommandRetryScheduler::makeRetryPolicy<ErrorCategory::RetriableError>(
+ kMaxRecipientKeyDocsFindAttempts, executor::RemoteCommandRequest::kNoTimeout),
+ transport::kEnableSSL);
+ uassertStatusOK(fetcher->schedule());
+
+ {
+ stdx::lock_guard<Latch> lg(_mutex);
+ _recipientKeysFetcher = fetcher;
+ }
+
+ fetcher->join();
+
+ {
+ stdx::lock_guard<Latch> lg(_mutex);
+ _recipientKeysFetcher.reset();
+ }
+
+ if (!fetchStatus) {
+ // The callback never got invoked.
+ uasserted(5340400, "Internal error running cursor callback in command");
+ }
+ uassertStatusOK(fetchStatus.get());
+
+ return keyDocs;
+ })
+ .then([this, self = shared_from_this(), executor, token](auto keyDocs) {
+ checkIfReceivedDonorAbortMigration(token, _instanceCancelationSource.token());
+
+ return tenant_migration_util::storeExternalClusterTimeKeyDocsAndRefreshCache(
+ executor, std::move(keyDocs), _instanceCancelationSource.token());
+ });
+}
+
+ExecutorFuture<repl::OpTime> TenantMigrationDonorService::Instance::_insertStateDoc(
std::shared_ptr<executor::ScopedTaskExecutor> executor) {
invariant(_stateDoc.getState() == TenantMigrationDonorStateEnum::kUninitialized);
_stateDoc.setState(TenantMigrationDonorStateEnum::kDataSync);
@@ -299,7 +399,7 @@ ExecutorFuture<repl::OpTime> TenantMigrationDonorService::Instance::_insertState
AutoGetCollection collection(opCtx, _stateDocumentsNS, MODE_IX);
writeConflictRetry(
- opCtx, "tenantMigrationInsertStateDoc", _stateDocumentsNS.ns(), [&] {
+ opCtx, "TenantMigrationDonorInsertStateDoc", _stateDocumentsNS.ns(), [&] {
const auto filter =
BSON(TenantMigrationDonorDocument::kIdFieldName << _stateDoc.getId());
const auto updateMod = BSON("$setOnInsert" << _stateDoc.toBSON());
@@ -321,7 +421,7 @@ ExecutorFuture<repl::OpTime> TenantMigrationDonorService::Instance::_insertState
.on(**executor, CancelationToken::uncancelable());
}
-ExecutorFuture<repl::OpTime> TenantMigrationDonorService::Instance::_updateStateDocument(
+ExecutorFuture<repl::OpTime> TenantMigrationDonorService::Instance::_updateStateDoc(
std::shared_ptr<executor::ScopedTaskExecutor> executor,
const TenantMigrationDonorStateEnum nextState) {
const auto originalStateDocBson = _stateDoc.toBSON();
@@ -332,15 +432,14 @@ ExecutorFuture<repl::OpTime> TenantMigrationDonorService::Instance::_updateState
auto opCtxHolder = cc().makeOperationContext();
auto opCtx = opCtxHolder.get();
- uassertStatusOK(writeConflictRetry(
- opCtx, "updateStateDoc", _stateDocumentsNS.ns(), [&]() -> Status {
- AutoGetCollection collection(opCtx, _stateDocumentsNS, MODE_IX);
- if (!collection) {
- return Status(ErrorCodes::NamespaceNotFound,
- str::stream()
- << _stateDocumentsNS.ns() << " does not exist");
- }
+ AutoGetCollection collection(opCtx, _stateDocumentsNS, MODE_IX);
+ uassert(ErrorCodes::NamespaceNotFound,
+ str::stream() << _stateDocumentsNS.ns() << " does not exist",
+ collection);
+
+ writeConflictRetry(
+ opCtx, "TenantMigrationDonorUpdateStateDoc", _stateDocumentsNS.ns(), [&] {
WriteUnitOfWork wuow(opCtx);
const auto originalRecordId = Helpers::findOne(opCtx,
@@ -403,8 +502,7 @@ ExecutorFuture<repl::OpTime> TenantMigrationDonorService::Instance::_updateState
wuow.commit();
updateOpTime = oplogSlot;
- return Status::OK();
- }));
+ });
invariant(updateOpTime);
return updateOpTime.get();
@@ -418,11 +516,10 @@ ExecutorFuture<repl::OpTime> TenantMigrationDonorService::Instance::_updateState
}
ExecutorFuture<repl::OpTime>
-TenantMigrationDonorService::Instance::_markStateDocumentAsGarbageCollectable(
+TenantMigrationDonorService::Instance::_markStateDocAsGarbageCollectable(
std::shared_ptr<executor::ScopedTaskExecutor> executor) {
_stateDoc.setExpireAt(_serviceContext->getFastClockSource()->now() +
Milliseconds{repl::tenantMigrationGarbageCollectionDelayMS.load()});
-
return AsyncTry([this, self = shared_from_this()] {
auto opCtxHolder = cc().makeOperationContext();
auto opCtx = opCtxHolder.get();
@@ -431,7 +528,7 @@ TenantMigrationDonorService::Instance::_markStateDocumentAsGarbageCollectable(
writeConflictRetry(
opCtx,
- "tenantMigrationDonorMarkStateDocAsGarbageCollectable",
+ "TenantMigrationDonorMarkStateDocAsGarbageCollectable",
_stateDocumentsNS.ns(),
[&] {
const auto filter =
@@ -486,7 +583,7 @@ ExecutorFuture<void> TenantMigrationDonorService::Instance::_sendCommandToRecipi
const BSONObj& cmdObj) {
return AsyncTry([this, self = shared_from_this(), executor, recipientTargeterRS, cmdObj] {
return recipientTargeterRS
- ->findHost(ReadPreferenceSetting(), _instanceCancelationSource.token())
+ ->findHost(kPrimaryOnlyReadPreference, _instanceCancelationSource.token())
.thenRunOn(**executor)
.then([this, self = shared_from_this(), executor, cmdObj](auto recipientHost) {
executor::RemoteCommandRequest request(std::move(recipientHost),
@@ -495,7 +592,6 @@ ExecutorFuture<void> TenantMigrationDonorService::Instance::_sendCommandToRecipi
rpc::makeEmptyMetadata(),
nullptr,
kRecipientSyncDataTimeout);
-
request.sslMode = transport::kEnableSSL;
return (_recipientCmdExecutor)
@@ -526,7 +622,7 @@ ExecutorFuture<void> TenantMigrationDonorService::Instance::_sendRecipientSyncDa
auto opCtxHolder = cc().makeOperationContext();
auto opCtx = opCtxHolder.get();
- BSONObj cmdObj = BSONObj([&]() {
+ const auto cmdObj = [&] {
auto donorConnString =
repl::ReplicationCoordinator::get(opCtx)->getConfig().getConnectionString();
RecipientSyncData request;
@@ -538,7 +634,7 @@ ExecutorFuture<void> TenantMigrationDonorService::Instance::_sendRecipientSyncDa
_stateDoc.getRecipientCertificateForDonor()});
request.setReturnAfterReachingDonorTimestamp(_stateDoc.getBlockTimestamp());
return request.toBSON(BSONObj());
- }());
+ }();
return _sendCommandToRecipient(executor, recipientTargeterRS, cmdObj);
}
@@ -576,15 +672,25 @@ SemiFuture<void> TenantMigrationDonorService::Instance::run(
}
// Enter "dataSync" state.
- return _insertStateDocument(executor).then(
- [this, self = shared_from_this(), executor](repl::OpTime opTime) {
+ return _insertStateDoc(executor)
+ .then([this, self = shared_from_this(), executor](repl::OpTime opTime) {
// TODO (SERVER-53389): TenantMigration{Donor, Recipient}Service should
// use its base PrimaryOnlyService's cancelation source to pass tokens
// in calls to WaitForMajorityService::waitUntilMajority.
return _waitForMajorityWriteConcern(executor, std::move(opTime));
+ })
+ .then([this, self = shared_from_this()] {
+ auto opCtxHolder = cc().makeOperationContext();
+ auto opCtx = opCtxHolder.get();
+ pauseTenantMigrationAfterPersitingInitialDonorStateDoc.pauseWhileSet(opCtx);
});
})
.then([this, self = shared_from_this(), executor, recipientTargeterRS, token] {
+ checkIfReceivedDonorAbortMigration(token, _instanceCancelationSource.token());
+
+ return _fetchAndStoreRecipientClusterTimeKeyDocs(executor, recipientTargeterRS, token);
+ })
+ .then([this, self = shared_from_this(), executor, recipientTargeterRS, token] {
if (_stateDoc.getState() > TenantMigrationDonorStateEnum::kDataSync) {
return ExecutorFuture<void>(**executor, Status::OK());
}
@@ -601,7 +707,7 @@ SemiFuture<void> TenantMigrationDonorService::Instance::run(
checkIfReceivedDonorAbortMigration(token, _instanceCancelationSource.token());
// Enter "blocking" state.
- return _updateStateDocument(executor, TenantMigrationDonorStateEnum::kBlocking)
+ return _updateStateDoc(executor, TenantMigrationDonorStateEnum::kBlocking)
.then([this, self = shared_from_this(), executor, token](
repl::OpTime opTime) {
// TODO (SERVER-53389): TenantMigration{Donor, Recipient}Service should
@@ -691,7 +797,7 @@ SemiFuture<void> TenantMigrationDonorService::Instance::run(
checkIfReceivedDonorAbortMigration(token, _instanceCancelationSource.token());
// Enter "commit" state.
- return _updateStateDocument(executor, TenantMigrationDonorStateEnum::kCommitted)
+ return _updateStateDoc(executor, TenantMigrationDonorStateEnum::kCommitted)
.then([this, self = shared_from_this(), executor, token](
repl::OpTime opTime) {
// TODO (SERVER-53389): TenantMigration{Donor, Recipient}Service should
@@ -728,7 +834,7 @@ SemiFuture<void> TenantMigrationDonorService::Instance::run(
} else {
// Enter "abort" state.
_abortReason.emplace(status);
- return _updateStateDocument(executor, TenantMigrationDonorStateEnum::kAborted)
+ return _updateStateDoc(executor, TenantMigrationDonorStateEnum::kAborted)
.then([this, self = shared_from_this(), executor](repl::OpTime opTime) {
return _waitForMajorityWriteConcern(executor, std::move(opTime))
.then([this, self = shared_from_this()] {
@@ -766,7 +872,7 @@ SemiFuture<void> TenantMigrationDonorService::Instance::run(
return _sendRecipientForgetMigrationCommand(executor, recipientTargeterRS);
})
.then([this, self = shared_from_this(), executor] {
- return _markStateDocumentAsGarbageCollectable(executor);
+ return _markStateDocAsGarbageCollectable(executor);
})
.then([this, self = shared_from_this(), executor](repl::OpTime opTime) {
return _waitForMajorityWriteConcern(executor, std::move(opTime));
diff --git a/src/mongo/db/repl/tenant_migration_donor_service.h b/src/mongo/db/repl/tenant_migration_donor_service.h
index 6bbc131c525..e0f11201602 100644
--- a/src/mongo/db/repl/tenant_migration_donor_service.h
+++ b/src/mongo/db/repl/tenant_migration_donor_service.h
@@ -30,6 +30,7 @@
#pragma once
#include "mongo/base/string_data.h"
+#include "mongo/client/fetcher.h"
#include "mongo/client/remote_command_targeter_rs.h"
#include "mongo/db/repl/primary_only_service.h"
#include "mongo/db/repl/repl_server_parameters_gen.h"
@@ -149,10 +150,19 @@ public:
const NamespaceString _stateDocumentsNS = NamespaceString::kTenantMigrationDonorsNamespace;
/**
+ * Fetches all key documents from the recipient's admin.system.keys collection, stores
+ * them in admin.system.external_validation_keys, and refreshes the keys cache.
+ */
+ ExecutorFuture<void> _fetchAndStoreRecipientClusterTimeKeyDocs(
+ std::shared_ptr<executor::ScopedTaskExecutor> executor,
+ std::shared_ptr<RemoteCommandTargeter> recipientTargeterRS,
+ const CancelationToken& token);
+
+ /**
* Inserts the state document to _stateDocumentsNS and returns the opTime for the insert
* oplog entry.
*/
- ExecutorFuture<repl::OpTime> _insertStateDocument(
+ ExecutorFuture<repl::OpTime> _insertStateDoc(
std::shared_ptr<executor::ScopedTaskExecutor> executor);
/**
@@ -161,7 +171,7 @@ public:
* commitOrAbortTimestamp depending on the state. Returns the opTime for the update oplog
* entry.
*/
- ExecutorFuture<repl::OpTime> _updateStateDocument(
+ ExecutorFuture<repl::OpTime> _updateStateDoc(
std::shared_ptr<executor::ScopedTaskExecutor> executor,
const TenantMigrationDonorStateEnum nextState);
@@ -169,7 +179,7 @@ public:
* Sets the "expireAt" time for the state document to be garbage collected, and returns the
* the opTime for the write.
*/
- ExecutorFuture<repl::OpTime> _markStateDocumentAsGarbageCollectable(
+ ExecutorFuture<repl::OpTime> _markStateDocAsGarbageCollectable(
std::shared_ptr<executor::ScopedTaskExecutor> executor);
/**
@@ -216,6 +226,10 @@ public:
return recipientCmdThreadPoolLimits;
}
+ // Weak pointer to the Fetcher used for fetching admin.system.keys documents from the
+ // recipient. It is only not null when the instance is actively fetching the documents.
+ std::weak_ptr<Fetcher> _recipientKeysFetcher;
+
boost::optional<Status> _abortReason;
// Protects the durable state and the promises below.
diff --git a/src/mongo/db/repl/tenant_migration_pem_payload.idl b/src/mongo/db/repl/tenant_migration_pem_payload.idl
index 2e4aa666c62..6886c4be34a 100644
--- a/src/mongo/db/repl/tenant_migration_pem_payload.idl
+++ b/src/mongo/db/repl/tenant_migration_pem_payload.idl
@@ -42,9 +42,9 @@ structs:
type: string
description: "Certificate PEM blob."
validator:
- callback: "validateCertificatePEMPayload"
+ callback: "tenant_migration_util::validateCertificatePEMPayload"
privateKey:
type: string
description: "Private key PEM blob."
validator:
- callback: "validatePrivateKeyPEMPayload"
+ callback: "tenant_migration_util::validatePrivateKeyPEMPayload"
diff --git a/src/mongo/db/repl/tenant_migration_state_machine.idl b/src/mongo/db/repl/tenant_migration_state_machine.idl
index 02d6d7f42b8..a473cd06a88 100644
--- a/src/mongo/db/repl/tenant_migration_state_machine.idl
+++ b/src/mongo/db/repl/tenant_migration_state_machine.idl
@@ -72,7 +72,7 @@ structs:
The URI string that the donor will utilize to create a connection with
the recipient.
validator:
- callback: "validateConnectionString"
+ callback: "tenant_migration_util::validateConnectionString"
readPreference:
type: readPreference
description: >-
@@ -132,12 +132,12 @@ structs:
The URI string that the recipient will utilize to create a connection
with the donor.
validator:
- callback: "validateConnectionString"
+ callback: "tenant_migration_util::validateConnectionString"
tenantId:
type: string
description: "The tenantId for the migration."
validator:
- callback: "validateDatabasePrefix"
+ callback: "tenant_migration_util::validateDatabasePrefix"
readPreference:
type: readPreference
description: >-
diff --git a/src/mongo/db/repl/tenant_migration_util.cpp b/src/mongo/db/repl/tenant_migration_util.cpp
new file mode 100644
index 00000000000..69e3e935759
--- /dev/null
+++ b/src/mongo/db/repl/tenant_migration_util.cpp
@@ -0,0 +1,112 @@
+/**
+ * Copyright (C) 2021-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/db/repl/tenant_migration_util.h"
+
+#include "mongo/db/concurrency/write_conflict_exception.h"
+#include "mongo/db/db_raii.h"
+#include "mongo/db/dbhelpers.h"
+#include "mongo/db/logical_time_validator.h"
+#include "mongo/db/repl/repl_client_info.h"
+#include "mongo/db/repl/repl_server_parameters_gen.h"
+#include "mongo/db/repl/wait_for_majority_service.h"
+#include "mongo/util/future_util.h"
+
+namespace mongo {
+
+namespace tenant_migration_util {
+
+ExternalKeysCollectionDocument makeExternalClusterTimeKeyDoc(ServiceContext* serviceContext,
+ std::string rsName,
+ BSONObj keyDoc) {
+ auto originalKeyDoc = KeysCollectionDocument::parse(IDLParserErrorContext("keyDoc"), keyDoc);
+
+ ExternalKeysCollectionDocument externalKeyDoc(
+ OID::gen(),
+ originalKeyDoc.getKeyId(),
+ rsName,
+ serviceContext->getFastClockSource()->now() +
+ Seconds{repl::tenantMigrationExternalKeysRemovalDelaySecs.load()});
+ externalKeyDoc.setKeysCollectionDocumentBase(originalKeyDoc.getKeysCollectionDocumentBase());
+
+ return externalKeyDoc;
+}
+
+ExecutorFuture<void> storeExternalClusterTimeKeyDocsAndRefreshCache(
+ std::shared_ptr<executor::ScopedTaskExecutor> executor,
+ std::vector<ExternalKeysCollectionDocument> keyDocs,
+ const CancelationToken& token) {
+ auto opCtxHolder = cc().makeOperationContext();
+ auto opCtx = opCtxHolder.get();
+ auto nss = NamespaceString::kExternalKeysCollectionNamespace;
+
+ for (auto& keyDoc : keyDocs) {
+ AutoGetCollection collection(opCtx, nss, MODE_IX);
+
+ writeConflictRetry(opCtx, "CloneExternalKeyDocs", nss.ns(), [&] {
+ const auto filter = BSON(ExternalKeysCollectionDocument::kKeyIdFieldName
+ << keyDoc.getKeyId()
+ << ExternalKeysCollectionDocument::kReplicaSetNameFieldName
+ << keyDoc.getReplicaSetName()
+ << ExternalKeysCollectionDocument::kTTLExpiresAtFieldName
+ << BSON("$lt" << keyDoc.getTTLExpiresAt()));
+
+ // Remove _id since updating _id is not allowed.
+ const auto updateMod = keyDoc.toBSON().removeField("_id");
+
+ Helpers::upsert(opCtx,
+ nss.ns(),
+ filter,
+ updateMod,
+ /*fromMigrate=*/false);
+ });
+ }
+
+ const auto opTime = repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp();
+
+ return WaitForMajorityService::get(opCtx->getServiceContext())
+ .waitUntilMajority(opTime)
+ .thenRunOn(**executor)
+ .then([] {
+ auto opCtxHolder = cc().makeOperationContext();
+ auto opCtx = opCtxHolder.get();
+
+ auto validator = LogicalTimeValidator::get(opCtx);
+ if (validator) {
+ // Refresh the keys cache to avoid validation errors for external cluster times with
+ // a keyId that matches the keyId of an internal key since the LogicalTimeValidator
+ // only refreshes the cache when it cannot find a matching internal key.
+ validator->refreshKeyManagerCache(opCtx);
+ }
+ });
+}
+
+} // namespace tenant_migration_util
+
+} // namespace mongo
diff --git a/src/mongo/db/repl/tenant_migration_util.h b/src/mongo/db/repl/tenant_migration_util.h
index 009d9649097..e7d253ed6fd 100644
--- a/src/mongo/db/repl/tenant_migration_util.h
+++ b/src/mongo/db/repl/tenant_migration_util.h
@@ -1,5 +1,5 @@
/**
- * Copyright (C) 2018-present MongoDB, Inc.
+ * Copyright (C) 2020-present MongoDB, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
@@ -26,6 +26,7 @@
* exception statement from all source files in the program, then also delete
* it in the license file.
*/
+
#pragma once
#include <set>
@@ -34,7 +35,9 @@
#include "mongo/bson/timestamp.h"
#include "mongo/client/mongo_uri.h"
#include "mongo/config.h"
+#include "mongo/db/keys_collection_document_gen.h"
#include "mongo/db/repl/replication_coordinator.h"
+#include "mongo/executor/scoped_task_executor.h"
#include "mongo/util/net/ssl_util.h"
#include "mongo/util/str.h"
@@ -46,6 +49,8 @@ const std::set<std::string> kUnsupportedTenantIds{"", "admin", "local", "config"
} // namespace
+namespace tenant_migration_util {
+
inline Status validateDatabasePrefix(const std::string& tenantId) {
const bool isPrefixSupported =
kUnsupportedTenantIds.find(tenantId) == kUnsupportedTenantIds.end();
@@ -122,4 +127,25 @@ inline Status validatePrivateKeyPEMPayload(const StringData& payload) {
#endif
}
+/*
+ * Creates an ExternalKeysCollectionDocument representing an admin.system.external_validation_keys
+ * document from the given the admin.system.keys document BSONObj.
+ */
+ExternalKeysCollectionDocument makeExternalClusterTimeKeyDoc(ServiceContext* serviceContext,
+ std::string rsName,
+ BSONObj keyDoc);
+
+/*
+ * For each given ExternalKeysCollectionDocument, inserts it if there is not an existing document in
+ * admin.system.external_validation_keys for it with the same keyId and replicaSetName. Otherwise,
+ * updates the ttlExpiresAt of the existing document if it is less than the new ttlExpiresAt. Waits
+ * for the writes to be majority-committed, and refreshes the logical validator's cache.
+ */
+ExecutorFuture<void> storeExternalClusterTimeKeyDocsAndRefreshCache(
+ std::shared_ptr<executor::ScopedTaskExecutor> executor,
+ std::vector<ExternalKeysCollectionDocument> keyDocs,
+ const CancelationToken& token);
+
+} // namespace tenant_migration_util
+
} // namespace mongo