diff options
Diffstat (limited to 'subversion/libsvn_fs_fs/caching.c')
-rw-r--r-- | subversion/libsvn_fs_fs/caching.c | 356 |
1 files changed, 253 insertions, 103 deletions
diff --git a/subversion/libsvn_fs_fs/caching.c b/subversion/libsvn_fs_fs/caching.c index 42898cb..b54d69b 100644 --- a/subversion/libsvn_fs_fs/caching.c +++ b/subversion/libsvn_fs_fs/caching.c @@ -25,6 +25,7 @@ #include "id.h" #include "dag.h" #include "tree.h" +#include "index.h" #include "temp_serializer.h" #include "../libsvn_fs/fs-loader.h" @@ -65,32 +66,18 @@ normalize_key_part(const char *original, return normalized->data; } -/* Return a memcache in *MEMCACHE_P for FS if it's configured to use - memcached, or NULL otherwise. Also, sets *FAIL_STOP to a boolean - indicating whether cache errors should be returned to the caller or - just passed to the FS warning handler. - - *CACHE_TXDELTAS, *CACHE_FULLTEXTS and *CACHE_REVPROPS flags will be set - according to FS->CONFIG. *CACHE_NAMESPACE receives the cache prefix - to use. +/* *CACHE_TXDELTAS, *CACHE_FULLTEXTS flags will be set according to + FS->CONFIG. *CACHE_NAMESPACE receives the cache prefix to use. Use FS->pool for allocating the memcache and CACHE_NAMESPACE, and POOL for temporary allocations. */ static svn_error_t * -read_config(svn_memcache_t **memcache_p, - svn_boolean_t *fail_stop, - const char **cache_namespace, +read_config(const char **cache_namespace, svn_boolean_t *cache_txdeltas, svn_boolean_t *cache_fulltexts, - svn_boolean_t *cache_revprops, svn_fs_t *fs, apr_pool_t *pool) { - fs_fs_data_t *ffd = fs->fsap_data; - - SVN_ERR(svn_cache__make_memcache_from_config(memcache_p, ffd->config, - fs->pool)); - /* No cache namespace by default. I.e. all FS instances share the * cached data. If you specify different namespaces, the data will * share / compete for the same cache memory but keys will not match @@ -116,7 +103,8 @@ read_config(svn_memcache_t **memcache_p, *cache_txdeltas = svn_hash__get_bool(fs->config, SVN_FS_CONFIG_FSFS_CACHE_DELTAS, - FALSE); + TRUE); + /* by default, cache fulltexts. * Most SVN tools care about reconstructed file content. * Thus, this is a reasonable default. @@ -129,13 +117,7 @@ read_config(svn_memcache_t **memcache_p, SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS, TRUE); - /* For now, always disable revprop caching. - */ - *cache_revprops = FALSE; - - return svn_config_get_bool(ffd->config, fail_stop, - CONFIG_SECTION_CACHES, CONFIG_OPTION_FAIL_STOP, - FALSE); + return SVN_NO_ERROR; } @@ -196,9 +178,10 @@ dump_cache_statistics(void *baton_void) TRUE, baton->pool); - if (! err) + /* skip unused caches */ + if (! err && (info.gets > 0 || info.sets > 0)) { - text_stats = svn_cache__format_info(&info, baton->pool); + text_stats = svn_cache__format_info(&info, TRUE, baton->pool); lines = svn_cstring_split(text_stats->data, "\n", FALSE, baton->pool); for (i = 0; i < lines->nelts; ++i) @@ -219,6 +202,29 @@ dump_cache_statistics(void *baton_void) return result; } + +static apr_status_t +dump_global_cache_statistics(void *baton_void) +{ + apr_pool_t *pool = baton_void; + + svn_cache__info_t *info = svn_cache__membuffer_get_global_info(pool); + svn_string_t *text_stats = svn_cache__format_info(info, FALSE, pool); + apr_array_header_t *lines = svn_cstring_split(text_stats->data, "\n", + FALSE, pool); + + int i; + for (i = 0; i < lines->nelts; ++i) + { + const char *line = APR_ARRAY_IDX(lines, i, const char *); +#ifdef SVN_DEBUG + SVN_DBG(("%s\n", line)); +#endif + } + + return APR_SUCCESS; +} + #endif /* SVN_DEBUG_CACHE_DUMP_STATS */ /* This function sets / registers the required callbacks for a given @@ -267,12 +273,13 @@ init_callbacks(svn_cache__t *cache, * Creates memcache if MEMCACHE is not NULL. Creates membuffer cache if * MEMBUFFER is not NULL. Fallbacks to inprocess cache if MEMCACHE and * MEMBUFFER are NULL and pages is non-zero. Sets *CACHE_P to NULL - * otherwise. + * otherwise. Use the given PRIORITY class for the new cache. If it + * is 0, then use the default priority class. * * Unless NO_HANDLER is true, register an error handler that reports errors * as warnings to the FS warning callback. * - * Cache is allocated in POOL. + * Cache is allocated in RESULT_POOL, temporaries in SCRATCH_POOL. * */ static svn_error_t * create_cache(svn_cache__t **cache_p, @@ -284,19 +291,23 @@ create_cache(svn_cache__t **cache_p, svn_cache__deserialize_func_t deserializer, apr_ssize_t klen, const char *prefix, + apr_uint32_t priority, svn_fs_t *fs, svn_boolean_t no_handler, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { svn_cache__error_handler_t error_handler = no_handler ? NULL : warn_and_fail_on_cache_errors; + if (priority == 0) + priority = SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY; if (memcache) { SVN_ERR(svn_cache__create_memcache(cache_p, memcache, serializer, deserializer, klen, - prefix, pool)); + prefix, result_pool)); error_handler = no_handler ? NULL : warn_and_continue_on_cache_errors; @@ -305,20 +316,20 @@ create_cache(svn_cache__t **cache_p, { SVN_ERR(svn_cache__create_membuffer_cache( cache_p, membuffer, serializer, deserializer, - klen, prefix, FALSE, pool)); + klen, prefix, priority, FALSE, result_pool, scratch_pool)); } else if (pages) { SVN_ERR(svn_cache__create_inprocess( cache_p, serializer, deserializer, klen, pages, - items_per_page, FALSE, prefix, pool)); + items_per_page, FALSE, prefix, result_pool)); } else { *cache_p = NULL; } - SVN_ERR(init_callbacks(*cache_p, fs, error_handler, pool)); + SVN_ERR(init_callbacks(*cache_p, fs, error_handler, result_pool)); return SVN_NO_ERROR; } @@ -332,29 +343,45 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, "fsfs:", fs->uuid, "/", normalize_key_part(fs->path, pool), ":", - (char *)NULL); - svn_memcache_t *memcache; + SVN_VA_NULL); svn_membuffer_t *membuffer; - svn_boolean_t no_handler; + svn_boolean_t no_handler = ffd->fail_stop; svn_boolean_t cache_txdeltas; svn_boolean_t cache_fulltexts; - svn_boolean_t cache_revprops; const char *cache_namespace; /* Evaluating the cache configuration. */ - SVN_ERR(read_config(&memcache, - &no_handler, - &cache_namespace, + SVN_ERR(read_config(&cache_namespace, &cache_txdeltas, &cache_fulltexts, - &cache_revprops, fs, pool)); - prefix = apr_pstrcat(pool, "ns:", cache_namespace, ":", prefix, NULL); + prefix = apr_pstrcat(pool, "ns:", cache_namespace, ":", prefix, SVN_VA_NULL); membuffer = svn_cache__get_global_membuffer_cache(); + /* General rules for assigning cache priorities: + * + * - Data that can be reconstructed from other elements has low prio + * (e.g. fulltexts, directories etc.) + * - Index data required to find any of the other data has high prio + * (e.g. noderevs, L2P and P2L index pages) + * - everthing else should use default prio + */ + +#ifdef SVN_DEBUG_CACHE_DUMP_STATS + + /* schedule printing the global access statistics upon pool cleanup, + * i.e. when the repo instance gets closed / cleaned up. + */ + if (membuffer) + apr_pool_cleanup_register(fs->pool, + fs->pool, + dump_global_cache_statistics, + apr_pool_cleanup_null); +#endif + /* Make the cache for revision roots. For the vast majority of * commands, this is only going to contain a few entries (svnadmin * dump/verify is an exception here), so to reduce overhead let's @@ -370,10 +397,11 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, svn_fs_fs__serialize_id, svn_fs_fs__deserialize_id, sizeof(svn_revnum_t), - apr_pstrcat(pool, prefix, "RRI", (char *)NULL), + apr_pstrcat(pool, prefix, "RRI", SVN_VA_NULL), + 0, fs, no_handler, - fs->pool)); + fs->pool, pool)); /* Rough estimate: revision DAG nodes have size around 320 bytes, so * let's put 16 on a page. */ @@ -384,13 +412,14 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, svn_fs_fs__dag_serialize, svn_fs_fs__dag_deserialize, APR_HASH_KEY_STRING, - apr_pstrcat(pool, prefix, "DAG", (char *)NULL), + apr_pstrcat(pool, prefix, "DAG", SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_LOW_PRIORITY, fs, no_handler, - fs->pool)); + fs->pool, pool)); /* 1st level DAG node cache */ - ffd->dag_node_cache = svn_fs_fs__create_dag_cache(pool); + ffd->dag_node_cache = svn_fs_fs__create_dag_cache(fs->pool); /* Very rough estimate: 1K per directory. */ SVN_ERR(create_cache(&(ffd->dir_cache), @@ -399,11 +428,12 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, 1024, 8, svn_fs_fs__serialize_dir_entries, svn_fs_fs__deserialize_dir_entries, - APR_HASH_KEY_STRING, - apr_pstrcat(pool, prefix, "DIR", (char *)NULL), + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "DIR", SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, fs, no_handler, - fs->pool)); + fs->pool, pool)); /* Only 16 bytes per entry (a revision number + the corresponding offset). Since we want ~8k pages, that means 512 entries per page. */ @@ -415,51 +445,69 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, svn_fs_fs__deserialize_manifest, sizeof(svn_revnum_t), apr_pstrcat(pool, prefix, "PACK-MANIFEST", - (char *)NULL), + SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, fs, no_handler, - fs->pool)); + fs->pool, pool)); /* initialize node revision cache, if caching has been enabled */ SVN_ERR(create_cache(&(ffd->node_revision_cache), NULL, membuffer, - 0, 0, /* Do not use inprocess cache */ + 32, 32, /* ~200 byte / entry; 1k entries total */ svn_fs_fs__serialize_node_revision, svn_fs_fs__deserialize_node_revision, sizeof(pair_cache_key_t), - apr_pstrcat(pool, prefix, "NODEREVS", (char *)NULL), + apr_pstrcat(pool, prefix, "NODEREVS", SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool, pool)); + + /* initialize representation header cache, if caching has been enabled */ + SVN_ERR(create_cache(&(ffd->rep_header_cache), + NULL, + membuffer, + 1, 1000, /* ~8 bytes / entry; 1k entries total */ + svn_fs_fs__serialize_rep_header, + svn_fs_fs__deserialize_rep_header, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "REPHEADER", SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY, fs, no_handler, - fs->pool)); + fs->pool, pool)); /* initialize node change list cache, if caching has been enabled */ SVN_ERR(create_cache(&(ffd->changes_cache), NULL, membuffer, - 0, 0, /* Do not use inprocess cache */ + 1, 8, /* 1k / entry; 8 entries total, rarely used */ svn_fs_fs__serialize_changes, svn_fs_fs__deserialize_changes, sizeof(svn_revnum_t), - apr_pstrcat(pool, prefix, "CHANGES", (char *)NULL), + apr_pstrcat(pool, prefix, "CHANGES", SVN_VA_NULL), + 0, fs, no_handler, - fs->pool)); + fs->pool, pool)); /* if enabled, cache fulltext and other derived information */ if (cache_fulltexts) { SVN_ERR(create_cache(&(ffd->fulltext_cache), - memcache, + ffd->memcache, membuffer, 0, 0, /* Do not use inprocess cache */ /* Values are svn_stringbuf_t */ NULL, NULL, sizeof(pair_cache_key_t), - apr_pstrcat(pool, prefix, "TEXT", (char *)NULL), + apr_pstrcat(pool, prefix, "TEXT", SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY, fs, no_handler, - fs->pool)); + fs->pool, pool)); SVN_ERR(create_cache(&(ffd->properties_cache), NULL, @@ -469,10 +517,11 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, svn_fs_fs__deserialize_properties, sizeof(pair_cache_key_t), apr_pstrcat(pool, prefix, "PROP", - (char *)NULL), + SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY, fs, no_handler, - fs->pool)); + fs->pool, pool)); SVN_ERR(create_cache(&(ffd->mergeinfo_cache), NULL, @@ -482,10 +531,11 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, svn_fs_fs__deserialize_mergeinfo, APR_HASH_KEY_STRING, apr_pstrcat(pool, prefix, "MERGEINFO", - (char *)NULL), + SVN_VA_NULL), + 0, fs, no_handler, - fs->pool)); + fs->pool, pool)); SVN_ERR(create_cache(&(ffd->mergeinfo_existence_cache), NULL, @@ -495,10 +545,11 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, NULL, NULL, APR_HASH_KEY_STRING, apr_pstrcat(pool, prefix, "HAS_MERGEINFO", - (char *)NULL), + SVN_VA_NULL), + 0, fs, no_handler, - fs->pool)); + fs->pool, pool)); } else { @@ -508,42 +559,36 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, ffd->mergeinfo_existence_cache = NULL; } - /* initialize revprop cache, if full-text caching has been enabled */ - if (cache_revprops) + /* if enabled, cache text deltas and their combinations */ + if (cache_txdeltas) { - SVN_ERR(create_cache(&(ffd->revprop_cache), + SVN_ERR(create_cache(&(ffd->raw_window_cache), NULL, membuffer, 0, 0, /* Do not use inprocess cache */ - svn_fs_fs__serialize_properties, - svn_fs_fs__deserialize_properties, - sizeof(pair_cache_key_t), - apr_pstrcat(pool, prefix, "REVPROP", - (char *)NULL), + svn_fs_fs__serialize_raw_window, + svn_fs_fs__deserialize_raw_window, + sizeof(window_cache_key_t), + apr_pstrcat(pool, prefix, "RAW_WINDOW", + SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_LOW_PRIORITY, fs, no_handler, - fs->pool)); - } - else - { - ffd->revprop_cache = NULL; - } + fs->pool, pool)); - /* if enabled, cache text deltas and their combinations */ - if (cache_txdeltas) - { SVN_ERR(create_cache(&(ffd->txdelta_window_cache), NULL, membuffer, 0, 0, /* Do not use inprocess cache */ svn_fs_fs__serialize_txdelta_window, svn_fs_fs__deserialize_txdelta_window, - APR_HASH_KEY_STRING, + sizeof(window_cache_key_t), apr_pstrcat(pool, prefix, "TXDELTA_WINDOW", - (char *)NULL), + SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_LOW_PRIORITY, fs, no_handler, - fs->pool)); + fs->pool, pool)); SVN_ERR(create_cache(&(ffd->combined_window_cache), NULL, @@ -551,12 +596,13 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, 0, 0, /* Do not use inprocess cache */ /* Values are svn_stringbuf_t */ NULL, NULL, - APR_HASH_KEY_STRING, + sizeof(window_cache_key_t), apr_pstrcat(pool, prefix, "COMBINED_WINDOW", - (char *)NULL), + SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_LOW_PRIORITY, fs, no_handler, - fs->pool)); + fs->pool, pool)); } else { @@ -564,6 +610,61 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, ffd->combined_window_cache = NULL; } + SVN_ERR(create_cache(&(ffd->l2p_header_cache), + NULL, + membuffer, + 64, 16, /* entry size varies but we must cover + a reasonable number of revisions (1k) */ + svn_fs_fs__serialize_l2p_header, + svn_fs_fs__deserialize_l2p_header, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "L2P_HEADER", + (char *)NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool, pool)); + SVN_ERR(create_cache(&(ffd->l2p_page_cache), + NULL, + membuffer, + 64, 16, /* entry size varies but we must cover + a reasonable number of revisions (1k) */ + svn_fs_fs__serialize_l2p_page, + svn_fs_fs__deserialize_l2p_page, + sizeof(svn_fs_fs__page_cache_key_t), + apr_pstrcat(pool, prefix, "L2P_PAGE", + (char *)NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool, pool)); + SVN_ERR(create_cache(&(ffd->p2l_header_cache), + NULL, + membuffer, + 4, 1, /* Large entries. Rarely used. */ + svn_fs_fs__serialize_p2l_header, + svn_fs_fs__deserialize_p2l_header, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "P2L_HEADER", + (char *)NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool, pool)); + SVN_ERR(create_cache(&(ffd->p2l_page_cache), + NULL, + membuffer, + 4, 16, /* Variably sized entries. Rarely used. */ + svn_fs_fs__serialize_p2l_page, + svn_fs_fs__deserialize_p2l_page, + sizeof(svn_fs_fs__page_cache_key_t), + apr_pstrcat(pool, prefix, "P2L_PAGE", + (char *)NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + fs, + no_handler, + fs->pool, pool)); + return SVN_NO_ERROR; } @@ -575,12 +676,44 @@ struct txn_cleanup_baton_t /* the position where to reset it */ svn_cache__t **to_reset; + + /* pool that TXN_CACHE was allocated in */ + apr_pool_t *txn_pool; + + /* pool that the FS containing the TO_RESET pointer was allocator */ + apr_pool_t *fs_pool; }; +/* Forward declaration. */ +static apr_status_t +remove_txn_cache_fs(void *baton_void); + +/* APR pool cleanup handler that will reset the cache pointer given in + BATON_VOID when the TXN_POOL gets cleaned up. */ +static apr_status_t +remove_txn_cache_txn(void *baton_void) +{ + struct txn_cleanup_baton_t *baton = baton_void; + + /* be careful not to hurt performance by resetting newer txn's caches. */ + if (*baton->to_reset == baton->txn_cache) + { + /* This is equivalent to calling svn_fs_fs__reset_txn_caches(). */ + *baton->to_reset = NULL; + } + + /* It's cleaned up now. Prevent double cleanup. */ + apr_pool_cleanup_kill(baton->fs_pool, + baton, + remove_txn_cache_fs); + + return APR_SUCCESS; +} + /* APR pool cleanup handler that will reset the cache pointer given in - BATON_VOID. */ + BATON_VOID when the FS_POOL gets cleaned up. */ static apr_status_t -remove_txn_cache(void *baton_void) +remove_txn_cache_fs(void *baton_void) { struct txn_cleanup_baton_t *baton = baton_void; @@ -588,19 +721,25 @@ remove_txn_cache(void *baton_void) if (*baton->to_reset == baton->txn_cache) { /* This is equivalent to calling svn_fs_fs__reset_txn_caches(). */ - *baton->to_reset = NULL; + *baton->to_reset = NULL; } + /* It's cleaned up now. Prevent double cleanup. */ + apr_pool_cleanup_kill(baton->txn_pool, + baton, + remove_txn_cache_txn); + return APR_SUCCESS; } /* This function sets / registers the required callbacks for a given - * transaction-specific *CACHE object, if CACHE is not NULL and a no-op - * otherwise. In particular, it will ensure that *CACHE gets reset to NULL - * upon POOL destruction latest. + * transaction-specific *CACHE object in FS, if CACHE is not NULL and + * a no-op otherwise. In particular, it will ensure that *CACHE gets + * reset to NULL upon POOL or FS->POOL destruction latest. */ static void -init_txn_callbacks(svn_cache__t **cache, +init_txn_callbacks(svn_fs_t *fs, + svn_cache__t **cache, apr_pool_t *pool) { if (*cache != NULL) @@ -610,10 +749,20 @@ init_txn_callbacks(svn_cache__t **cache, baton = apr_palloc(pool, sizeof(*baton)); baton->txn_cache = *cache; baton->to_reset = cache; + baton->txn_pool = pool; + baton->fs_pool = fs->pool; + /* If any of these pools gets cleaned, we must reset the cache. + * We don't know which one will get cleaned up first, so register + * cleanup actions for both and during the cleanup action, unregister + * the respective other action. */ apr_pool_cleanup_register(pool, baton, - remove_txn_cache, + remove_txn_cache_txn, + apr_pool_cleanup_null); + apr_pool_cleanup_register(fs->pool, + baton, + remove_txn_cache_fs, apr_pool_cleanup_null); } } @@ -635,7 +784,7 @@ svn_fs_fs__initialize_txn_caches(svn_fs_t *fs, "/", fs->path, ":", txn_id, ":", svn_uuid_generate(pool), ":", - (char *)NULL); + SVN_VA_NULL); /* We don't support caching for concurrent transactions in the SAME * FSFS session. Maybe, you forgot to clean POOL. */ @@ -656,13 +805,14 @@ svn_fs_fs__initialize_txn_caches(svn_fs_t *fs, svn_fs_fs__deserialize_dir_entries, APR_HASH_KEY_STRING, apr_pstrcat(pool, prefix, "TXNDIR", - (char *)NULL), + SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, fs, TRUE, - pool)); + pool, pool)); /* reset the transaction-specific cache if the pool gets cleaned up. */ - init_txn_callbacks(&(ffd->txn_dir_cache), pool); + init_txn_callbacks(fs, &(ffd->txn_dir_cache), pool); return SVN_NO_ERROR; } |