diff options
author | Will Korteland <will.korteland@mongodb.com> | 2022-07-07 07:16:29 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-07-07 07:51:41 +0000 |
commit | 4e09aad80ae03639496b17351f02f0117ec0cad6 (patch) | |
tree | fb7ce644a42f1993f5283bf966cb407c672397d0 /src/third_party | |
parent | e756e14a6679b3a97b5f7438f984d5b40736d1e8 (diff) | |
download | mongo-4e09aad80ae03639496b17351f02f0117ec0cad6.tar.gz |
Import wiredtiger: 051701d1e5e30ad3cef6e3607d7e370e7e3af49a from branch mongodb-master
ref: 6cb26d60ec..051701d1e5
for: 6.1.0-rc0
WT-9446 tiered structure in work unit has empty bstorage field
Diffstat (limited to 'src/third_party')
-rw-r--r-- | src/third_party/wiredtiger/dist/stat_data.py | 1 | ||||
-rw-r--r-- | src/third_party/wiredtiger/import.data | 2 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/extern.h | 1 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/stat.h | 1 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/wiredtiger.in | 186 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/schema/schema_drop.c | 20 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/support/stat.c | 3 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/tiered/tiered_handle.c | 7 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/tiered/tiered_work.c | 30 | ||||
-rw-r--r-- | src/third_party/wiredtiger/test/csuite/schema_abort/main.c | 23 | ||||
-rw-r--r-- | src/third_party/wiredtiger/test/suite/test_tiered07.py | 5 |
11 files changed, 175 insertions, 104 deletions
diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index cbba9d1e652..b74177f8027 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -546,6 +546,7 @@ conn_stats = [ StorageStat('tiered_retention', 'tiered storage local retention time (secs)', 'no_clear,no_scale,size'), StorageStat('tiered_work_units_created', 'tiered operations scheduled'), StorageStat('tiered_work_units_dequeued', 'tiered operations dequeued and processed'), + StorageStat('tiered_work_units_removed', 'tiered operations removed without processing'), ########################################## # Thread Count statistics diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 4f416cbae26..d879335c857 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "6cb26d60ec92eabdcb50badee7cc1f2205ba7384" + "commit": "051701d1e5e30ad3cef6e3607d7e370e7e3af49a" } diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 19cd98b8d30..f7d925483b8 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -1890,6 +1890,7 @@ extern void __wt_tiered_get_flush_finish(WT_SESSION_IMPL *session, WT_TIERED_WOR extern void __wt_tiered_pop_work( WT_SESSION_IMPL *session, uint32_t type, uint64_t maxval, WT_TIERED_WORK_UNIT **entryp); extern void __wt_tiered_push_work(WT_SESSION_IMPL *session, WT_TIERED_WORK_UNIT *entry); +extern void __wt_tiered_remove_work(WT_SESSION_IMPL *session, WT_TIERED *tiered, bool locked); extern void __wt_tiered_work_free(WT_SESSION_IMPL *session, WT_TIERED_WORK_UNIT *entry); extern void __wt_timestamp_to_hex_string(wt_timestamp_t ts, char *hex_timestamp); extern void __wt_txn_bump_snapshot(WT_SESSION_IMPL *session); diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index fcba3ff2535..60fd4c3b7ed 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -787,6 +787,7 @@ struct __wt_connection_stats { int64_t session_table_verify_fail; int64_t session_table_verify_success; int64_t tiered_work_units_dequeued; + int64_t tiered_work_units_removed; int64_t tiered_work_units_created; int64_t tiered_retention; int64_t thread_fsync_active; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index ea5a86c23dc..4cda1f00b98 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -6311,232 +6311,234 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1453 /*! session: tiered operations dequeued and processed */ #define WT_STAT_CONN_TIERED_WORK_UNITS_DEQUEUED 1454 +/*! session: tiered operations removed without processing */ +#define WT_STAT_CONN_TIERED_WORK_UNITS_REMOVED 1455 /*! session: tiered operations scheduled */ -#define WT_STAT_CONN_TIERED_WORK_UNITS_CREATED 1455 +#define WT_STAT_CONN_TIERED_WORK_UNITS_CREATED 1456 /*! session: tiered storage local retention time (secs) */ -#define WT_STAT_CONN_TIERED_RETENTION 1456 +#define WT_STAT_CONN_TIERED_RETENTION 1457 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1457 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1458 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1458 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1459 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1459 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1460 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1460 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1461 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1461 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1462 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1462 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1463 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1463 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1464 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1464 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1465 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1465 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1466 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1466 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1467 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1467 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1468 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1468 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1469 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1469 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1470 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1470 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1471 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1471 +#define WT_STAT_CONN_PAGE_SLEEP 1472 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1472 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1473 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1473 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1474 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES 1474 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES 1475 /*! transaction: Number of prepared updates committed */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COMMITTED 1475 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COMMITTED 1476 /*! transaction: Number of prepared updates repeated on the same key */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_KEY_REPEATED 1476 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_KEY_REPEATED 1477 /*! transaction: Number of prepared updates rolled back */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_ROLLEDBACK 1477 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_ROLLEDBACK 1478 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1478 +#define WT_STAT_CONN_TXN_PREPARE 1479 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1479 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1480 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1480 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1481 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1481 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1482 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1482 +#define WT_STAT_CONN_TXN_QUERY_TS 1483 /*! transaction: race to read prepared update retry */ -#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1483 +#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1484 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1484 +#define WT_STAT_CONN_TXN_RTS 1485 /*! * transaction: rollback to stable history store records with stop * timestamps older than newer records */ -#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1485 +#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1486 /*! transaction: rollback to stable inconsistent checkpoint */ -#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1486 +#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1487 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1487 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1488 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1488 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1489 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1489 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1490 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1490 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1491 /*! transaction: rollback to stable restored updates from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1491 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1492 /*! transaction: rollback to stable skipping delete rle */ -#define WT_STAT_CONN_TXN_RTS_DELETE_RLE_SKIPPED 1492 +#define WT_STAT_CONN_TXN_RTS_DELETE_RLE_SKIPPED 1493 /*! transaction: rollback to stable skipping stable rle */ -#define WT_STAT_CONN_TXN_RTS_STABLE_RLE_SKIPPED 1493 +#define WT_STAT_CONN_TXN_RTS_STABLE_RLE_SKIPPED 1494 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1494 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1495 /*! transaction: rollback to stable tree walk skipping pages */ -#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1495 +#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1496 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1496 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1497 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1497 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1498 /*! transaction: sessions scanned in each walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1498 +#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1499 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1499 +#define WT_STAT_CONN_TXN_SET_TS 1500 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1500 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1501 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1501 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1502 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1502 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1503 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1503 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1504 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1504 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1505 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1505 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1506 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1506 +#define WT_STAT_CONN_TXN_BEGIN 1507 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1507 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1508 /*! * transaction: transaction checkpoint currently running for history * store file */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1508 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1509 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1509 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1510 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1510 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1511 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1511 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1512 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1512 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1513 /*! * transaction: transaction checkpoint most recent duration for gathering * all handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1513 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1514 /*! * transaction: transaction checkpoint most recent duration for gathering * applied handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1514 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1515 /*! * transaction: transaction checkpoint most recent duration for gathering * skipped handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1515 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1516 /*! transaction: transaction checkpoint most recent handles applied */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1516 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1517 /*! transaction: transaction checkpoint most recent handles skipped */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1517 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1518 /*! transaction: transaction checkpoint most recent handles walked */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1518 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1519 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1519 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1520 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1520 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1521 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1521 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1522 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1522 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1523 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1523 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1524 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1524 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1525 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1525 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1526 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1526 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1527 /*! transaction: transaction checkpoint stop timing stress active */ -#define WT_STAT_CONN_TXN_CHECKPOINT_STOP_STRESS_ACTIVE 1527 +#define WT_STAT_CONN_TXN_CHECKPOINT_STOP_STRESS_ACTIVE 1528 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1528 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1529 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1529 +#define WT_STAT_CONN_TXN_CHECKPOINT 1530 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1530 +#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1531 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1531 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1532 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1532 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1533 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1533 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1534 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1534 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1535 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1535 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1536 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1536 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1537 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1537 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1538 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1538 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1539 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1539 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1540 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1540 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1541 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1541 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1542 /*! transaction: transaction rollback to stable currently running */ -#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1542 +#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1543 /*! transaction: transaction walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_WALK_SESSIONS 1543 +#define WT_STAT_CONN_TXN_WALK_SESSIONS 1544 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1544 +#define WT_STAT_CONN_TXN_COMMIT 1545 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1545 +#define WT_STAT_CONN_TXN_ROLLBACK 1546 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1546 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1547 /*! * @} diff --git a/src/third_party/wiredtiger/src/schema/schema_drop.c b/src/third_party/wiredtiger/src/schema/schema_drop.c index 5411f39fe95..4dee7dc525c 100644 --- a/src/third_party/wiredtiger/src/schema/schema_drop.c +++ b/src/third_party/wiredtiger/src/schema/schema_drop.c @@ -177,13 +177,16 @@ static int __drop_tiered(WT_SESSION_IMPL *session, const char *uri, bool force, const char *cfg[]) { WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *tier; WT_DECL_RET; WT_TIERED *tiered; u_int i; const char *filename, *name; - bool exist, remove_files, remove_shared; + bool exist, locked, remove_files, remove_shared; + conn = S2C(session); + locked = false; WT_RET(__wt_config_gets(session, cfg, "remove_files", &cval)); remove_files = cval.val != 0; WT_RET(__wt_config_gets(session, cfg, "remove_shared", &cval)); @@ -262,6 +265,14 @@ __drop_tiered(WT_SESSION_IMPL *session, const char *uri, bool force, const char } /* + * We are about to close the dhandle. If that is successful we need to remove any tiered work + * from the queue relating to that dhandle. But if closing the dhandle has an error we don't + * remove the work. So hold the tiered lock for the duration so that the worker thread cannot + * race and process work for this handle. + */ + __wt_spin_lock(session, &conn->tiered_lock); + locked = true; + /* * Close all btree handles associated with this table. This must be done after we're done using * the tiered structure because that is from the dhandle. */ @@ -270,11 +281,18 @@ __drop_tiered(WT_SESSION_IMPL *session, const char *uri, bool force, const char session, ret = __wt_conn_dhandle_close_all(session, uri, true, force)); WT_ERR(ret); + /* If everything is successful, remove any tiered work associated with this tiered handle. */ + __wt_tiered_remove_work(session, tiered, locked); + __wt_spin_unlock(session, &conn->tiered_lock); + locked = false; + __wt_verbose(session, WT_VERB_TIERED, "DROP_TIERED: remove tiered table %s from metadata", uri); ret = __wt_metadata_remove(session, uri); err: __wt_free(session, name); + if (locked) + __wt_spin_unlock(session, &conn->tiered_lock); return (ret); } diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index a82a78d6eef..26d4708cec6 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -1631,6 +1631,7 @@ static const char *const __stats_connection_desc[] = { "session: table verify failed calls", "session: table verify successful calls", "session: tiered operations dequeued and processed", + "session: tiered operations removed without processing", "session: tiered operations scheduled", "session: tiered storage local retention time (secs)", "thread-state: active filesystem fsync calls", @@ -2220,6 +2221,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing session_table_verify_fail */ /* not clearing session_table_verify_success */ stats->tiered_work_units_dequeued = 0; + stats->tiered_work_units_removed = 0; stats->tiered_work_units_created = 0; /* not clearing tiered_retention */ /* not clearing thread_fsync_active */ @@ -2825,6 +2827,7 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->session_table_verify_fail += WT_STAT_READ(from, session_table_verify_fail); to->session_table_verify_success += WT_STAT_READ(from, session_table_verify_success); to->tiered_work_units_dequeued += WT_STAT_READ(from, tiered_work_units_dequeued); + to->tiered_work_units_removed += WT_STAT_READ(from, tiered_work_units_removed); to->tiered_work_units_created += WT_STAT_READ(from, tiered_work_units_created); to->tiered_retention += WT_STAT_READ(from, tiered_retention); to->thread_fsync_active += WT_STAT_READ(from, thread_fsync_active); diff --git a/src/third_party/wiredtiger/src/tiered/tiered_handle.c b/src/third_party/wiredtiger/src/tiered/tiered_handle.c index 216b40e2eb7..3beeb195a06 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_handle.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_handle.c @@ -59,9 +59,14 @@ __tiered_name_check(WT_SESSION_IMPL *session, WT_TIERED *tiered) */ len = strlen(obj_files[i]); if (len == obj_len) { + /* + * While we want to return EEXIST if we find the object exists, we only want to return + * the error, and not give a message. Otherwise cases that are commonly handled can give + * a lot of spurious error messages. + */ __wt_verbose( session, WT_VERB_TIERED, "EEXIST %s already exists on shared storage", obj_files[i]); - WT_ERR_MSG(session, EEXIST, "%s already exists on shared storage", obj_files[i]); + WT_ERR(EEXIST); } } diff --git a/src/third_party/wiredtiger/src/tiered/tiered_work.c b/src/third_party/wiredtiger/src/tiered/tiered_work.c index a434b6e6762..2d05db12b87 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_work.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_work.c @@ -44,6 +44,33 @@ __wt_tiered_work_free(WT_SESSION_IMPL *session, WT_TIERED_WORK_UNIT *entry) } /* + * __wt_tiered_remove_work -- + * Remove all work on the queue that applies to the given tiered handle. + */ +void +__wt_tiered_remove_work(WT_SESSION_IMPL *session, WT_TIERED *tiered, bool locked) +{ + WT_CONNECTION_IMPL *conn; + WT_TIERED_WORK_UNIT *entry, *entry_tmp; + + conn = S2C(session); + if (!locked) + __wt_spin_lock(session, &conn->tiered_lock); + TAILQ_FOREACH_SAFE(entry, &conn->tieredqh, q, entry_tmp) + { + /* Remove and free any entry for this tiered handle. */ + if (entry->tiered == tiered) { + TAILQ_REMOVE(&conn->tieredqh, entry, q); + WT_STAT_CONN_INCR(session, tiered_work_units_removed); + __wt_tiered_work_free(session, entry); + } + } + if (!locked) + __wt_spin_unlock(session, &conn->tiered_lock); + return; +} + +/* * __wt_tiered_push_work -- * Push a work unit to the queue. Assumes it is passed an already filled out structure. */ @@ -55,6 +82,7 @@ __wt_tiered_push_work(WT_SESSION_IMPL *session, WT_TIERED_WORK_UNIT *entry) conn = S2C(session); __wt_spin_lock(session, &conn->tiered_lock); TAILQ_INSERT_TAIL(&conn->tieredqh, entry, q); + WT_ASSERT(session, entry->tiered != NULL); WT_STAT_CONN_INCR(session, tiered_work_units_created); __wt_spin_unlock(session, &conn->tiered_lock); __tiered_flush_state(session, entry->type, true); @@ -86,6 +114,8 @@ __wt_tiered_pop_work( if (FLD_ISSET(type, entry->type) && (maxval == 0 || entry->op_val < maxval)) { TAILQ_REMOVE(&conn->tieredqh, entry, q); WT_STAT_CONN_INCR(session, tiered_work_units_dequeued); + WT_ASSERT(session, entry->tiered != NULL); + WT_ASSERT(session, entry->tiered->bstorage != NULL); *entryp = entry; break; } diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c index 88ea6e3a31c..e13c979faac 100644 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -97,8 +97,9 @@ static pthread_rwlock_t flush_lock; "create," \ "eviction_updates_trigger=95,eviction_updates_target=80," \ "log=(enabled,file_max=10M,remove=false)" -#define ENV_CONFIG_TIER \ - ",tiered_storage=(bucket=./bucket,bucket_prefix=pfx-,local_retention=2,name=dir_store)" +#define ENV_CONFIG_TIER \ + ",tiered_storage=(bucket=./" \ + "bucket,bucket_prefix=pfx-,local_retention=2,name=dir_store)" #define ENV_CONFIG_TIER_EXT \ ",extensions=(../../../../ext/storage_sources/dir_store/" \ "libwiredtiger_dir_store.so=(early_load=true))" @@ -353,7 +354,7 @@ test_create_unique(THREAD_DATA *td, int force) WT_DECL_RET; WT_SESSION *session; uint64_t my_uid; - char new_uri[64]; + char dropconf[128], new_uri[64]; testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); @@ -370,7 +371,12 @@ test_create_unique(THREAD_DATA *td, int force) __wt_yield(); if (use_txn) testutil_check(session->begin_transaction(session, NULL)); - while ((ret = session->drop(session, new_uri, force ? "force" : NULL)) != 0) + + testutil_check(__wt_snprintf(dropconf, sizeof(dropconf), "force=%s", force ? "true" : "false")); + /* For testing we want to remove objects too. */ + if (tiered) + strcat(dropconf, ",remove_shared=true"); + while ((ret = session->drop(session, new_uri, dropconf)) != 0) if (ret != EBUSY) testutil_die(ret, "session.drop: %s", new_uri); if (use_txn && (ret = session->commit_transaction(session, NULL)) != 0 && ret != EINVAL) @@ -388,12 +394,17 @@ test_drop(THREAD_DATA *td, int force) { WT_DECL_RET; WT_SESSION *session; + char dropconf[128]; testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); if (use_txn) testutil_check(session->begin_transaction(session, NULL)); - if ((ret = session->drop(session, uri, force ? "force" : NULL)) != 0) + testutil_check(__wt_snprintf(dropconf, sizeof(dropconf), "force=%s", force ? "true" : "false")); + /* For testing we want to remove objects too. */ + if (tiered) + strcat(dropconf, ",remove_shared=true"); + if ((ret = session->drop(session, uri, dropconf)) != 0) if (ret != ENOENT && ret != EBUSY) testutil_die(ret, "session.drop"); @@ -619,7 +630,7 @@ thread_flush_run(void *arg) if (ret != EBUSY) testutil_die(ret, "session.flush_tier"); } else - printf("Flush tier %" PRIu32 " completed.\n", ++i); + printf("Flush tier %" PRIu32 " completed.\n", i); testutil_check(pthread_rwlock_unlock(&flush_lock)); fflush(stdout); } diff --git a/src/third_party/wiredtiger/test/suite/test_tiered07.py b/src/third_party/wiredtiger/test/suite/test_tiered07.py index faa278547bc..734cd0c7fed 100644 --- a/src/third_party/wiredtiger/test/suite/test_tiered07.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered07.py @@ -136,10 +136,9 @@ class test_tiered07(wttest.WiredTigerTestCase, TieredConfigMixin): # If we didn't do a checkpoint before the flush_tier then creating with the same name # will succeed because no bucket objects were created. if (self.first_ckpt): - msg = "/already exists/" self.pr('check cannot create with same name') - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda:self.assertEquals(self.session.create(self.uri, 'key_format=S'), 0), msg) + self.assertRaises(wiredtiger.WiredTigerError, + lambda:self.session.create(self.uri, 'key_format=S')) else: self.session.create(self.uri, 'key_format=S') |