diff options
28 files changed, 398 insertions, 36 deletions
diff --git a/src/third_party/wiredtiger/dist/docs_data.py b/src/third_party/wiredtiger/dist/docs_data.py index 47ae7c3c51a..a1301c87057 100644 --- a/src/third_party/wiredtiger/dist/docs_data.py +++ b/src/third_party/wiredtiger/dist/docs_data.py @@ -11,15 +11,68 @@ class ArchDocPage: # List of all architecture subsections ########################################## arch_doc_pages = [ + ArchDocPage('arch-block', + ['WT_BLOCK', 'WT_BLOCK_CKPT', 'WT_BLOCK_DESC', 'WT_BLOCK_HEADER', + 'WT_BM', 'WT_EXTLIST'], + ['src/include/block.h', 'src/include/block_inline.h', + 'src/block/']), + ArchDocPage('arch-cache', + ['WT_CACHE', 'WT_CACHE_POOL'], + ['src/include/cache.h', 'src/include/cache_inline.h']), + ArchDocPage('arch-column', + ['WT_BTREE'], + ['src/include/btree.h']), + ArchDocPage('arch-cursor', + ['WT_CURSOR', 'WT_CURSOR_BACKUP', 'WT_CURSOR_BTREE', 'WT_CURSOR_BULK', + 'WT_CURSOR_DATA_SOURCE', 'WT_CURSOR_DUMP', 'WT_CURSOR_INDEX', + 'WT_CURSOR_LOG', 'WT_CURSOR_METADATA', 'WT_CURSOR_STAT'], + ['src/include/cursor.h', 'src/include/cursor_inline.h', + 'src/cursor/']), + ArchDocPage('arch-data-file', + ['WT_CELL'], + ['src/include/block.h', 'src/include/btmem.h', + 'src/include/cell.h', 'src/include/cell_inline.h', + 'src/reconcile/rec_col.c', 'src/reconcile/rec_row.c']), ArchDocPage('arch-dhandle', ['WT_BTREE', 'WT_DHANDLE'], ['src/include/btree.h', 'src/include/dhandle.h', 'src/conn/conn_dhandle.c', 'src/session/session_dhandle.c']), + ArchDocPage('arch-eviction', + ['WT_EVICT_ENTRY', 'WT_EVICT_QUEUE'], + ['src/include/cache.h', + 'src/evict/']), + ArchDocPage('arch-fs-os', + ['WT_FILE_SYSTEM'], + ['src/include/os.h', 'src/include/os_fhandle_inline.h', + 'src/include/os_fs_inline.h', 'src/include/os_fstream_inline.h', + 'src/include/os_windows.h', + 'src/os_posix/', 'src/os_win/']), + ArchDocPage('arch-hs', + ['WT_CURSOR_HS'], + ['src/history/']), + ArchDocPage('arch-log-file', + ['WT_LOGSLOT', 'WT_LOG_RECORD', 'WT_LSN'], + ['src/include/log.h', 'src/log/']), + ArchDocPage('arch-logging', + ['WT_CURSOR_LOG', 'WT_LOG', 'WT_LOGSLOT', 'WT_LOG_RECORD', 'WT_LSN'], + ['src/include/log.h', 'src/include/log_inline.h', 'src/log/']), + ArchDocPage('arch-metadata', + [], + ['src/include/meta.h', 'src/meta/']), + ArchDocPage('arch-python', + [], + ['lang/python/']), + ArchDocPage('arch-row', + ['WT_BTREE'], + ['src/include/btree.h']), ArchDocPage('arch-schema', ['WT_COLGROUP', 'WT_INDEX', 'WT_LSM_TREE', 'WT_TABLE'], ['src/include/intpack_inline.h', 'src/include/packing_inline.h', 'src/include/schema.h', 'src/lsm/', 'src/packing/', 'src/schema/']), + ArchDocPage('arch-snapshot', + ['WT_TXN'], + ['src/include/txn.h']), ArchDocPage('arch-transaction', ['WT_TXN', 'WT_TXN_GLOBAL', 'WT_TXN_OP', 'WT_TXN_SHARED'], ['src/include/txn.h', 'src/include/txn_inline.h', 'src/txn/']), diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index 37686c3187b..6cb43b3c49e 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -96,7 +96,8 @@ spellchk() type aspell > /dev/null 2>&1 || return (cd ../src/docs && - cat *.dox | + # Separate quoted newlines "line\nline" so "nline" is not reported. + sed -e 's/\("[^"]*\)\\n\([^"]*"\)/\1 \2/' *.dox | \ aspell --encoding=iso-8859-1 --lang=en_US --personal=./spell.ok list) | sort -u > $t test -s $t && { diff --git a/src/third_party/wiredtiger/examples/python/ex_access.py b/src/third_party/wiredtiger/examples/python/ex_access.py index 2b765b8141f..c043a419a73 100755 --- a/src/third_party/wiredtiger/examples/python/ex_access.py +++ b/src/third_party/wiredtiger/examples/python/ex_access.py @@ -27,6 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. # +#! [python simple example] import os from wiredtiger import wiredtiger_open @@ -53,3 +54,4 @@ for key, value in cursor: print('Got record: %s : %s' % (key, value)) conn.close() +#! [python simple example] diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 518309b2758..bffc336840c 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "0ab3e59875ecfabbe3cd9d19c0c3e05b72bad1cf" + "commit": "4f4873f25fc826f5c2594302620a4d5db3748ce3" } diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 7199a6b4c24..cdabd131e40 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -397,7 +397,8 @@ __ckpt_add_blkmod_entry( * Add the checkpoint's allocated blocks to all valid incremental backup source identifiers. */ static int -__ckpt_add_blk_mods_alloc(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_BLOCK_CKPT *ci) +__ckpt_add_blk_mods_alloc( + WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_BLOCK_CKPT *ci, WT_BLOCK *block) { WT_BLOCK_MODS *blk_mod; WT_CKPT *ckpt; @@ -417,10 +418,13 @@ __ckpt_add_blk_mods_alloc(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_BLOCK_ if (!F_ISSET(blk_mod, WT_BLOCK_MODS_VALID)) continue; + if (block->created_during_backup) + WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, 0, block->allocsize)); WT_EXT_FOREACH (ext, ci->alloc.off) { WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ext->off, ext->size)); } } + block->created_during_backup = false; return (0); } @@ -595,7 +599,7 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) * Record the checkpoint's allocated blocks. Do so before skipping any processing and before * possibly merging in blocks from any previous checkpoint. */ - WT_ERR(__ckpt_add_blk_mods_alloc(session, ckptbase, ci)); + WT_ERR(__ckpt_add_blk_mods_alloc(session, ckptbase, ci, block)); /* Skip the additional processing if we aren't deleting checkpoints. */ if (!deleting) diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index 20568ce0941..d97f3a86f7d 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -207,6 +207,13 @@ __wt_block_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[ /* Set the file's size. */ WT_ERR(__wt_filesize(session, block->fh, &block->size)); + /* + * If we're opening a file and it only contains a header and we're doing incremental backup + * indicate this so that the first checkpoint is sure to set all the bits as dirty to cover the + * header so that the header gets copied. + */ + if (block->size == allocsize && F_ISSET(conn, WT_CONN_INCR_BACKUP)) + block->created_during_backup = true; /* Initialize the live checkpoint's lock. */ WT_ERR(__wt_spin_init(session, &block->live_lock, "block manager")); diff --git a/src/third_party/wiredtiger/src/docs/Doxyfile b/src/third_party/wiredtiger/src/docs/Doxyfile index 3653e6cd242..7a855b2f94d 100644 --- a/src/third_party/wiredtiger/src/docs/Doxyfile +++ b/src/third_party/wiredtiger/src/docs/Doxyfile @@ -796,7 +796,8 @@ EXCLUDE_SYMBOLS = __F \ EXAMPLE_PATH = ../../examples/c \ ../../ext/compressors/nop \ ../../ext/encryptors/nop \ - ../../ext/encryptors/rotn + ../../ext/encryptors/rotn \ + ../../examples/python # If the value of the EXAMPLE_PATH tag contains directories, you can use the # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp diff --git a/src/third_party/wiredtiger/src/docs/arch-block.dox b/src/third_party/wiredtiger/src/docs/arch-block.dox new file mode 100644 index 00000000000..7c8fdf5d72b --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-block.dox @@ -0,0 +1,9 @@ +/*! @arch_page arch-block Block Manager + +The Block Manager manages the reading and writing of disk blocks +in WiredTiger. It does compression and encryption when these +are configured. + +The state of the block manager is represented by the \c WT_BM structure. +Individual blocks being tracked are in \c WT_BLOCK structures. +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-cache.dox b/src/third_party/wiredtiger/src/docs/arch-cache.dox new file mode 100644 index 00000000000..94888260cfe --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-cache.dox @@ -0,0 +1,13 @@ +/*! @arch_page arch-cache Cache + +Cache in WiredTiger is represented by the various shared data structures +that make up in-memory Btrees and subordinate data structures. + +Memory used to read in and write out the on-disk representations of Btrees +is not cached, it only exists temporarily during the I/O operation and +while the data is transferred to or from the on-disk format. + +Internally, the current cache state is represented by the WT_CACHE structure, +which contains various counters that drive statistics and information +used for eviction. +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-column.dox b/src/third_party/wiredtiger/src/docs/arch-column.dox new file mode 100644 index 00000000000..4b3b399349a --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-column.dox @@ -0,0 +1,23 @@ +/*! @arch_page arch-column Column Store + +Column Stores are Btrees stored in WiredTiger that have as their +key a record id, that is, a 64 bit unsigned integer. Thus, they implement +a specialized version of a Btree, where the key is a predictable length. + +A particular kind of column store is the fixed length column store. +As its name implies, the value is fixed length, and furthermore the +value is restricted to 1 and 8 bits in length. The bit length is specified +when the column store is created. The fixed length column store +has specialized use cases like bitmaps. + +The more general case is the variable length column store which allows +for values that have any length, and may have arbitrary types, including +aggregates of various types. + +Internally, row stores and both kinds of column stores all use a common +\c WT_BTREE structure. Column stores are distinguished in that +<code>WT_BTREE->type == BTREE_COL_VAR</code> for variable length column stores +and <code>WT_BTREE->type == BTREE_COL_FIX</code> for fixed length column stores. +Internal functions that navigate, access and manipulate Btrees have +code sprinkled throughout that is conditional on <code>WT_BTREE->type</code>. +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-cursor.dox b/src/third_party/wiredtiger/src/docs/arch-cursor.dox new file mode 100644 index 00000000000..60e47c5a8ad --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-cursor.dox @@ -0,0 +1,15 @@ +/*! @arch_page arch-cursor Cursor + +Cursors are used in WiredTiger to get and modify data. +A caller of WiredTiger uses WT_SESSION::open_cursor to create +a WT_CURSOR. Methods on the WT_CURSOR can then be used to +position, iterate, get, and set data. + +Depending on the <code>uri</code> used when creating a cursor, the cursor will +be internally implemented as one of the many cursor structures that include +WT_CURSOR_BTREE, WT_CURSOR_BACKUP, WT_CURSOR_INDEX, WT_CURSOR_LOG, +WT_CURSOR_METADATA, WT_CURSOR_STAT. Each of these structures starts +with the common \c %WT_CURSOR structure, which contain all +of the data and method pointers that make up the public part of the API. + +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-data-file.dox b/src/third_party/wiredtiger/src/docs/arch-data-file.dox new file mode 100644 index 00000000000..834329c981f --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-data-file.dox @@ -0,0 +1,17 @@ +/*! @arch_page arch-data-file Data File Format + +The format of the WiredTiger data file is given by structures +in \c block.h , defining the overall structure of the file and +its blocks. The \c WT_BLOCK_DESC starts the file. Following that, +individual pages appear, each with a \c WT_PAGE_HEADER defined in \c btmem.h +and a \c WT_BLOCK_HEADER defined in \c block.h . Individual cells then +sequentially appear as defined in \c cell.h . Each cell encodes +a key or value. Ordering is important, values are all associated with +the key that precedes them. Multiple values may be present, which +can represent multiple versions. Extremely large values may be represented +as a reference to another page. + +The exact encoding is rather complex, and beyond what can be described here. +The encoding strikes a balance between data that can be compacted efficiently +in time and space, extensibility, and compatibility with previous versions. +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-dhandle.dox b/src/third_party/wiredtiger/src/docs/arch-dhandle.dox index d88c82f9675..118285ec014 100644 --- a/src/third_party/wiredtiger/src/docs/arch-dhandle.dox +++ b/src/third_party/wiredtiger/src/docs/arch-dhandle.dox @@ -1,7 +1,7 @@ /*! @arch_page arch-dhandle Data Handles and Btrees An internal structure called a Data Handle (dhandle) is used to represent and -access Btrees and other data sources in WiredTiger. A dhandle gets created +access Btrees and other data sources in WiredTiger. A dhandle is created when a table is accessed for the first time. It is kept in a global list and is shared across the sessions. When a dhandle is not needed anymore and has been idle for a while, it is closed and destroyed, releasing all the diff --git a/src/third_party/wiredtiger/src/docs/arch-eviction.dox b/src/third_party/wiredtiger/src/docs/arch-eviction.dox new file mode 100644 index 00000000000..5c877e6322f --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-eviction.dox @@ -0,0 +1,15 @@ +/*! @arch_page arch-eviction Eviction + +Eviction represents the process or removing old data from the cache, +writing it to disk if it is dirty. The general strategy uses a dedicated +set of eviction threads that are tasked with identifying candidate data. +If the data needs to be written, it is reconciled (converting the +in-memory format to on-disk format), and then written. Clean memory +can be freed if needed. + +Eviction cannot be triggered directly by API calls, but happens as a result +of enough data being dirtied. + +Eviction is managed using \c WT_EVICT_QUEUE structures, +each of which contains a list of \c WT_EVICT_ENTRY structures. +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-fs-os.dox b/src/third_party/wiredtiger/src/docs/arch-fs-os.dox new file mode 100644 index 00000000000..cedade13248 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-fs-os.dox @@ -0,0 +1,10 @@ +/*! @arch_page arch-fs-os File System and Operating System Interface + +Internally, a layer of abstraction is above all operating system calls, +allowing main line WiredTiger code to make a call to single set of interfaces. +There are currently OS specific APIs for POSIX and Windows. + +Additionally, a program can use WT_CONNECTION::set_file_system to register +a set of functions that will be called for each file system operation. +WiredTiger will then call these functions at the appropriate time. +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-hs.dox b/src/third_party/wiredtiger/src/docs/arch-hs.dox new file mode 100644 index 00000000000..6ed50c5fbf6 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-hs.dox @@ -0,0 +1,8 @@ +/*! @arch_page arch-hs History Store + +The History Store in WiredTiger tracks old (all but the latest committed) +versions of records. By having these records in storage separate from +the current version, they can be used to service long running transactions, +and be evicted as necessary, without interfering with activity that uses +the most recent committed versions. +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-index.dox b/src/third_party/wiredtiger/src/docs/arch-index.dox index 7be364d330e..21af5d1f62f 100644 --- a/src/third_party/wiredtiger/src/docs/arch-index.dox +++ b/src/third_party/wiredtiger/src/docs/arch-index.dox @@ -28,46 +28,50 @@ skinparam fileShadowing false ' with most nesting within the "engine" rectangle. together { - rectangle "Python API" as python_api + rectangle "[[arch-python.html Python API]]" as python_api ' "storage" displays as an oval. storage " C/C++ \n applications " as application - rectangle "wt Utility" as utility + rectangle "[[command_line.html wt Utility]]" as utility } ' Trailing spaces for this label puts the text to the left. rectangle "**WiredTiger Engine** " as wt_engine { ' Leading and trailing spaces make a wide rectangle. together { + ' Putting two invisible file boxes on either side centers the middle box. file "____" as SPACE_api - rectangle " C API " as c_api + rectangle " [[modules.html C API]] " as c_api file "____" as SPACE_api2 + ' Influence the ordering of the invisible boxes using (hidden) arrows. + SPACE_api -[hidden]right-> c_api + c_api -[hidden]right-> SPACE_api2 } rectangle "[[arch-schema.html Schema]]" as schema - rectangle "Cursor" as cursor + rectangle "[[arch-cursor.html Cursor]]" as cursor rectangle "[[arch-transaction.html Transactions]]" as txn - rectangle "Metadata" as meta - rectangle "[[arch-dhandle.html dhandle/\n Btree]]" as btree - rectangle " Row\n storage" as row - rectangle " Column\n storage" as column - rectangle "History\n Store" as history - rectangle "Snapshots" as snapshot - rectangle "Cache" as cache - rectangle "Eviction" as evict + rectangle "[[arch-metadata.html Metadata]]" as meta + rectangle "[[arch-dhandle.html dhandle/\nBtree]]" as btree + rectangle "[[arch-row.html Row\nStorage]]" as row + rectangle "[[arch-column.html Column\nStorage]]" as column + rectangle "[[arch-hs.html History\nStore]]" as history + rectangle "[[arch-snapshot.html Snapshots]]" as snapshot + rectangle "[[arch-cache.html Cache]]" as cache + rectangle "[[arch-eviction.html Eviction]]" as evict together { - rectangle " Block\n Manager" as block + rectangle "[[arch-block.html Block\nManager]]" as block file "__________" as SPACE_log - rectangle "Logging" as log + rectangle "[[arch-logging.html Logging]]" as log file "___" as SPACE_log2 } - rectangle " File System & OS \n interface" as os + rectangle " [[arch-fs-os.html File System & OS interface]] " as os } together { - database "Database\n Files" as wt_file - database " Log \n Files" as log_file + database "[[arch-data-file.html Database\nFiles]]" as wt_file + database " [[arch-log-file.html Log\nFiles]]" as log_file } -' Influence the ordering at the top using (hidden) directed labels +' Influence the ordering at the top using (hidden) arrows. python_api -[hidden]right-> application application -[hidden]right-> utility @@ -78,8 +82,6 @@ utility -down-> c_api c_api -down-> schema c_api -down-> cursor c_api -down-> txn -SPACE_api -[hidden]right-> c_api -c_api -[hidden]right-> SPACE_api2 schema -down-> meta schema -down-> btree @@ -121,15 +123,76 @@ wt_file -[hidden]right-> log_file We go into some detail for some of the internal components. +@subpage arch-block + +The Block Manager manages the reading and writing of disk blocks. + +@subpage arch-cache + +Cache is represented by the various shared data structures that +make up in-memory Btrees and subordinate data structures. + +@subpage arch-column + +Column Stores are Btrees that have as their key a record id. + +@subpage arch-cursor + +Cursors are used to get and modify data. + +@subpage arch-data-file + +The format of the data file is given by structures in \c block.h . + @subpage arch-dhandle An internal structure called a Data Handle (dhandle) is used to represent and access Btrees and other data sources in WiredTiger. +@subpage arch-eviction + +Eviction represents the process or removing old data from the cache, +writing it to disk if it is dirty. + +@subpage arch-fs-os + +A layer of abstraction is above all operating system calls and +a set of functions can be registered to be called for each file system +operation. + +@subpage arch-hs + +The History Store tracks old versions of records. + +@subpage arch-logging + +WiredTiger writes all changes into a write-ahead log when configured. + +@subpage arch-log-file + +The format of a log file is defined in \c log.h . + +@subpage arch-metadata + +Metadata is stored as <code>uri, config</code> K/V pairs in a designated table. + +@subpage arch-python + +WiredTiger has a Python API that is useful for scripting and experimentation. + +@subpage arch-row + +Row Stores are Btrees that have a variable size key and data. + @subpage arch-schema A schema defines the format of the application data in WiredTiger. +@subpage arch-snapshot + +Snapshots are implemented by storing transaction ids committed before +the transaction started. + @subpage arch-transaction Transactions provide a powerful abstraction for multiple threads to operate on data concurrently. diff --git a/src/third_party/wiredtiger/src/docs/arch-log-file.dox b/src/third_party/wiredtiger/src/docs/arch-log-file.dox new file mode 100644 index 00000000000..a684e023d76 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-log-file.dox @@ -0,0 +1,7 @@ +/*! @arch_page arch-log-file Log File Format + +The format of a WiredTiger log file is defined in \c log.h . +Each file begins with a fixed length header, followed by a set of +variable length log records. Though the records may have varying +length, they all begin on boundaries aligned to 128 bytes. +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-logging.dox b/src/third_party/wiredtiger/src/docs/arch-logging.dox new file mode 100644 index 00000000000..41c96f3ad78 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-logging.dox @@ -0,0 +1,8 @@ +/*! @arch_page arch-logging Logging + +WiredTiger writes all changes into a write-ahead log when configured. +Log files are shared among all writers, so changes made to +different tables, by different threads, are interleaved. + +Internally, the current logging state is represented by the WT_LOG structure. +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-metadata.dox b/src/third_party/wiredtiger/src/docs/arch-metadata.dox new file mode 100644 index 00000000000..d709dc78810 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-metadata.dox @@ -0,0 +1,19 @@ +/*! @arch_page arch-metadata Metadata + +Metadata in WiredTiger is stored as a table in the \c "WiredTiger.wt" file. +The table's key is a \c uri string, and the value is its complete +configuration string. The configuration itself is a list of key/value +pairs in string form. The configuration's keys are dependent on the +type of \c uri. Thus, a metadata entry with a \c uri key beginning with +\c "table:" will be a configuration string +having configuration entries like \c key_format and \c value_format to +describe the data encoding for the uri. A metadata key beginning with +\c "file:" will have a different set of configuration entries associated +with it. + +A caller of WiredTiger can use WT_SESSION::open_cursor with a \c uri equal to +\c "meta:" to read the metadata. Using this interface, metadata can only +be queried, not changed. Changes to the metadata are affected by API calls +such as WT_SESSION::create, WT_SESSION::drop, WT_SESSION::rename. + +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-python.dox b/src/third_party/wiredtiger/src/docs/arch-python.dox new file mode 100644 index 00000000000..c70cef8968a --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-python.dox @@ -0,0 +1,18 @@ +/*! @arch_page arch-python Python API + +WiredTiger includes a Python API, which is useful for scripting and +experimentation. + +The following simple example show some highlights of the API: + +@snippet ex_access.py python simple example + +The API is implemented using SWIG. SWIG imports the \c \<wiredtiger.h\> file, +and from it creates the classes and C support files needed to allow +Python to import the WiredTiger library. + +Not every facility that is present in the C API is available in Python. +In particular, setting up custom collators, extractors, compressors, +encryptors and file system implementations must all be done within +the C framework. +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-row.dox b/src/third_party/wiredtiger/src/docs/arch-row.dox new file mode 100644 index 00000000000..ae3ffe22589 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-row.dox @@ -0,0 +1,12 @@ +/*! @arch_page arch-row Row Store + +Row Stores are Btrees stored in WiredTiger that do not have a record id +as key. Thus, they implement a generalized version of a Btree, where +the key and data can be arbitrary length. + +Internally, a row store and column store both use a common \c WT_BTREE +structure. The fundamental difference is that +<code>WT_BTREE->type == BTREE_ROW</code> for row stores. +Internal functions that navigate, access and manipulate Btrees have +code sprinkled throughout that is conditional on <code>WT_BTREE->type</code>. +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-snapshot.dox b/src/third_party/wiredtiger/src/docs/arch-snapshot.dox new file mode 100644 index 00000000000..8f8142d1f33 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-snapshot.dox @@ -0,0 +1,9 @@ +/*! @arch_page arch-snapshot Snapshot + +When the snapshot isolation level is used in WiredTiger, the transaction +will see versions of records committed before the transaction started. + +Snapshots are implemented by storing the set of transaction ids committed +before the transaction started in the \c WT_TXN structure. Transaction +ids older than a certain point need not be stored. +*/ diff --git a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx index 2f29589a7a7..0400be53354 100644 --- a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx +++ b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx @@ -1,5 +1,21 @@ <map id="wt_diagram_map" name="wt_diagram_map"> -<area shape="rect" id="id1" href="arch-schema.html" title="arch-schema.html" alt="" coords="98,224,151,241"/> -<area shape="rect" id="id2" href="arch-transaction.html" title="arch-transaction.html" alt="" coords="341,329,429,345"/> -<area shape="rect" id="id3" href="arch-dhandle.html" title="arch-dhandle.html" alt="" coords="181,320,244,353"/> +<area shape="rect" id="id1" href="modules.html" title="modules.html" alt="" coords="248,128,284,145"/> +<area shape="rect" id="id2" href="arch-cache.html" title="arch-cache.html" alt="" coords="198,546,240,563"/> +<area shape="rect" id="id3" href="arch-cursor.html" title="arch-cursor.html" alt="" coords="206,224,253,241"/> +<area shape="rect" id="id4" href="arch-eviction.html" title="arch-eviction.html" alt="" coords="295,546,348,563"/> +<area shape="rect" id="id5" href="arch-schema.html" title="arch-schema.html" alt="" coords="98,224,151,241"/> +<area shape="rect" id="id6" href="arch-logging.html" title="arch-logging.html" alt="" coords="388,651,444,667"/> +<area shape="rect" id="id7" href="command_line.html" title="command_line.html" alt="" coords="374,24,433,40"/> +<area shape="rect" id="id8" href="arch-log-file.html" title="arch-log-file.html" alt="" coords="307,865,338,898"/> +<area shape="rect" id="id9" href="arch-metadata.html" title="arch-metadata.html" alt="" coords="25,329,88,345"/> +<area shape="rect" id="id10" href="arch-snapshot.html" title="arch-snapshot.html" alt="" coords="381,442,452,458"/> +<area shape="rect" id="id11" href="arch-python.html" title="arch-python.html" alt="" coords="84,24,157,40"/> +<area shape="rect" id="id12" href="arch-transaction.html" title="arch-transaction.html" alt="" coords="340,329,428,345"/> +<area shape="rect" id="id13" href="arch-hs.html" title="arch-hs.html" alt="" coords="93,642,141,675"/> +<area shape="rect" id="id14" href="arch-row.html" title="arch-row.html" alt="" coords="163,433,216,466"/> +<area shape="rect" id="id15" href="arch-column.html" title="arch-column.html" alt="" coords="272,433,326,466"/> +<area shape="rect" id="id16" href="arch-block.html" title="arch-block.html" alt="" coords="196,642,256,675"/> +<area shape="rect" id="id17" href="arch-dhandle.html" title="arch-dhandle.html" alt="" coords="181,320,244,353"/> +<area shape="rect" id="id18" href="arch-data-file.html" title="arch-data-file.html" alt="" coords="181,865,244,898"/> +<area shape="rect" id="id19" href="arch-fs-os.html" title="arch-fs-os.html" alt="" coords="177,755,354,772"/> </map> diff --git a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.png b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.png Binary files differindex 0dcf1f285ed..99ae107fa95 100644 --- a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.png +++ b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.png diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index b153ba12668..be45196ff97 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -142,6 +142,7 @@ bool boolean booleans br +btmem btree btrees bufs @@ -238,6 +239,7 @@ eq erlang errno exe +extensibility fadvise failchk fallocate @@ -264,6 +266,7 @@ fnv forw fput freelist +fs fsync ftruncate fvisibility diff --git a/src/third_party/wiredtiger/src/include/block.h b/src/third_party/wiredtiger/src/include/block.h index cd34baec52c..f35d09f05f3 100644 --- a/src/third_party/wiredtiger/src/include/block.h +++ b/src/third_party/wiredtiger/src/include/block.h @@ -231,6 +231,8 @@ struct __wt_block { wt_off_t extend_size; /* File extended size */ wt_off_t extend_len; /* File extend chunk size */ + bool created_during_backup; /* Created during incremental backup */ + /* Configuration information, set when the file is opened. */ uint32_t allocfirst; /* Allocation is first-fit */ uint32_t allocsize; /* Allocation size */ diff --git a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c index c9520f5621b..fb57e88d8ae 100644 --- a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c +++ b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c @@ -47,9 +47,13 @@ #define URI_MAX_LEN 32 #define URI_FORMAT "table:t%d-%d" #define KEY_FORMAT "key-%d-%d" +#define TABLE_FORMAT "key_format=S,value_format=u" #define CONN_CONFIG_COMMON "timing_stress_for_test=[backup_rename]" +#define NUM_ALLOC 5 +static const char *alloc_sizes[] = {"512B", "8K", "64K", "1M", "16M"}; + static int verbose_level = 0; static uint64_t seed = 0; @@ -404,17 +408,31 @@ table_changes(WT_SESSION *session, TABLE *table) * Create a table for the given slot. */ static void -create_table(WT_SESSION *session, TABLE_INFO *tinfo, uint32_t slot) +create_table(WT_SESSION *session, WT_RAND_STATE *rand, TABLE_INFO *tinfo, uint32_t slot) { - char *uri; + uint32_t alloc; + char buf[4096], *uri; + const char *allocstr; testutil_assert(!TABLE_VALID(&tinfo->table[slot])); uri = dcalloc(1, URI_MAX_LEN); testutil_check( __wt_snprintf(uri, URI_MAX_LEN, URI_FORMAT, (int)slot, (int)tinfo->table[slot].name_index++)); - VERBOSE(3, "create %s\n", uri); - testutil_check(session->create(session, uri, "key_format=S,value_format=u")); + /* + * A quarter of the time use a non-default allocation size on the table. This is set + * independently of the granularity to stress mismatched values. + */ + if (__wt_random(rand) % 4 == 0) { + alloc = __wt_random(rand) % NUM_ALLOC; + allocstr = alloc_sizes[alloc]; + testutil_check(__wt_snprintf(buf, sizeof(buf), + "%s,allocation_size=%s,internal_page_max=%s,leaf_page_max=%s", TABLE_FORMAT, allocstr, + allocstr, allocstr)); + } else + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", TABLE_FORMAT)); + VERBOSE(3, "create %s: %s\n", uri, buf); + testutil_check(session->create(session, uri, buf)); tinfo->table[slot].name = uri; tinfo->tables_in_use++; } @@ -753,11 +771,11 @@ main(int argc, char *argv[]) WT_RAND_STATE rnd; WT_SESSION *session; uint32_t file_max, iter, max_value_size, next_checkpoint, rough_size, slot; - int ch, ncheckpoints, status; + int ch, ncheckpoints, nreopens, status; const char *backup_verbose, *working_dir; char conf[1024], home[1024], backup_check[1024], backup_dir[1024], command[4096]; - ncheckpoints = 0; + ncheckpoints = nreopens = 0; (void)testutil_set_progname(argv); custom_die = die; /* Set our own abort handler */ WT_CLEAR(tinfo); @@ -859,7 +877,7 @@ main(int argc, char *argv[]) */ slot = __wt_random(&rnd) % tinfo.table_count; if (!TABLE_VALID(&tinfo.table[slot])) - create_table(session, &tinfo, slot); + create_table(session, &rnd, &tinfo, slot); else if (__wt_random(&rnd) % 3 == 0 && do_rename) rename_table(session, &tinfo, slot); else if (do_drop) @@ -877,6 +895,15 @@ main(int argc, char *argv[]) } } + /* Close and reopen the connection once in a while. */ + if (__wt_random(&rnd) % 10 == 0) { + VERBOSE(2, "Close and reopen the connection %d\n", nreopens); + testutil_check(conn->close(conn, NULL)); + testutil_check(wiredtiger_open(home, NULL, conf, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + nreopens++; + } + if (iter == 0) { base_backup(conn, &rnd, home, backup_dir, &tinfo, &active); check_backup(backup_dir, backup_check, &tinfo); |
