diff options
Diffstat (limited to 'subversion/libsvn_fs_fs')
| -rw-r--r-- | subversion/libsvn_fs_fs/caching.c | 347 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/dag.c | 178 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/dag.h | 143 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/fs.c | 151 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/fs.h | 134 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/fs_fs.c | 6174 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/fs_fs.h | 65 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/id.c | 40 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/key-gen.c | 14 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/lock.c | 58 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/rep-cache-db.h | 46 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/rep-cache-db.sql | 20 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/rep-cache.c | 215 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/rep-cache.h | 25 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/structure | 95 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/temp_serializer.c | 535 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/temp_serializer.h | 96 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/tree.c | 955 | ||||
| -rw-r--r-- | subversion/libsvn_fs_fs/tree.h | 13 |
19 files changed, 7414 insertions, 1890 deletions
diff --git a/subversion/libsvn_fs_fs/caching.c b/subversion/libsvn_fs_fs/caching.c index 4f2a34c..42898cb 100644 --- a/subversion/libsvn_fs_fs/caching.c +++ b/subversion/libsvn_fs_fs/caching.c @@ -24,6 +24,7 @@ #include "fs_fs.h" #include "id.h" #include "dag.h" +#include "tree.h" #include "temp_serializer.h" #include "../libsvn_fs/fs-loader.h" @@ -32,25 +33,79 @@ #include "svn_private_config.h" #include "svn_hash.h" +#include "svn_pools.h" + #include "private/svn_debug.h" +#include "private/svn_subr_private.h" + +/* Take the ORIGINAL string and replace all occurrences of ":" without + * limiting the key space. Allocate the result in POOL. + */ +static const char * +normalize_key_part(const char *original, + apr_pool_t *pool) +{ + apr_size_t i; + apr_size_t len = strlen(original); + svn_stringbuf_t *normalized = svn_stringbuf_create_ensure(len, pool); + + for (i = 0; i < len; ++i) + { + char c = original[i]; + switch (c) + { + case ':': svn_stringbuf_appendbytes(normalized, "%_", 2); + break; + case '%': svn_stringbuf_appendbytes(normalized, "%%", 2); + break; + default : svn_stringbuf_appendbyte(normalized, c); + } + } + + return normalized->data; +} /* Return a memcache in *MEMCACHE_P for FS if it's configured to use memcached, or NULL otherwise. Also, sets *FAIL_STOP to a boolean indicating whether cache errors should be returned to the caller or - just passed to the FS warning handler. Use FS->pool for allocating - the memcache, and POOL for temporary allocations. */ + just passed to the FS warning handler. + + *CACHE_TXDELTAS, *CACHE_FULLTEXTS and *CACHE_REVPROPS flags will be set + according to FS->CONFIG. *CACHE_NAMESPACE receives the cache prefix + to use. + + Use FS->pool for allocating the memcache and CACHE_NAMESPACE, and POOL + for temporary allocations. */ static svn_error_t * read_config(svn_memcache_t **memcache_p, svn_boolean_t *fail_stop, + const char **cache_namespace, svn_boolean_t *cache_txdeltas, svn_boolean_t *cache_fulltexts, + svn_boolean_t *cache_revprops, svn_fs_t *fs, apr_pool_t *pool) { fs_fs_data_t *ffd = fs->fsap_data; SVN_ERR(svn_cache__make_memcache_from_config(memcache_p, ffd->config, - fs->pool)); + fs->pool)); + + /* No cache namespace by default. I.e. all FS instances share the + * cached data. If you specify different namespaces, the data will + * share / compete for the same cache memory but keys will not match + * across namespaces and, thus, cached data will not be shared between + * namespaces. + * + * Since the namespace will be concatenated with other elements to form + * the complete key prefix, we must make sure that the resulting string + * is unique and cannot be created by any other combination of elements. + */ + *cache_namespace + = normalize_key_part(svn_hash__get_cstring(fs->config, + SVN_FS_CONFIG_FSFS_CACHE_NS, + ""), + pool); /* don't cache text deltas by default. * Once we reconstructed the fulltexts from the deltas, @@ -74,24 +129,44 @@ read_config(svn_memcache_t **memcache_p, SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS, TRUE); + /* For now, always disable revprop caching. + */ + *cache_revprops = FALSE; + return svn_config_get_bool(ffd->config, fail_stop, CONFIG_SECTION_CACHES, CONFIG_OPTION_FAIL_STOP, FALSE); } -/* Implements svn_cache__error_handler_t */ +/* Implements svn_cache__error_handler_t + * This variant clears the error after logging it. + */ static svn_error_t * -warn_on_cache_errors(svn_error_t *err, - void *baton, - apr_pool_t *pool) +warn_and_continue_on_cache_errors(svn_error_t *err, + void *baton, + apr_pool_t *pool) { svn_fs_t *fs = baton; (fs->warning)(fs->warning_baton, err); svn_error_clear(err); + return SVN_NO_ERROR; } +/* Implements svn_cache__error_handler_t + * This variant logs the error and passes it on to the callers. + */ +static svn_error_t * +warn_and_fail_on_cache_errors(svn_error_t *err, + void *baton, + apr_pool_t *pool) +{ + svn_fs_t *fs = baton; + (fs->warning)(fs->warning_baton, err); + return err; +} + #ifdef SVN_DEBUG_CACHE_DUMP_STATS /* Baton to be used for the dump_cache_statistics() pool cleanup function, */ struct dump_cache_baton_t @@ -150,13 +225,12 @@ dump_cache_statistics(void *baton_void) * not transaction-specific CACHE object in FS, if CACHE is not NULL. * * All these svn_cache__t instances shall be handled uniformly. Unless - * NO_HANDLER is true, register an error handler that reports errors - * as warnings for the given CACHE. + * ERROR_HANDLER is NULL, register it for the given CACHE in FS. */ static svn_error_t * init_callbacks(svn_cache__t *cache, svn_fs_t *fs, - svn_boolean_t no_handler, + svn_cache__error_handler_t error_handler, apr_pool_t *pool) { if (cache != NULL) @@ -178,9 +252,9 @@ init_callbacks(svn_cache__t *cache, apr_pool_cleanup_null); #endif - if (! no_handler) + if (error_handler) SVN_ERR(svn_cache__set_error_handler(cache, - warn_on_cache_errors, + error_handler, fs, pool)); @@ -195,6 +269,9 @@ init_callbacks(svn_cache__t *cache, * MEMBUFFER are NULL and pages is non-zero. Sets *CACHE_P to NULL * otherwise. * + * Unless NO_HANDLER is true, register an error handler that reports errors + * as warnings to the FS warning callback. + * * Cache is allocated in POOL. * */ static svn_error_t * @@ -207,32 +284,43 @@ create_cache(svn_cache__t **cache_p, svn_cache__deserialize_func_t deserializer, apr_ssize_t klen, const char *prefix, + svn_fs_t *fs, + svn_boolean_t no_handler, apr_pool_t *pool) { - if (memcache) - { - SVN_ERR(svn_cache__create_memcache(cache_p, memcache, - serializer, deserializer, klen, - prefix, pool)); - } - else if (membuffer) - { - SVN_ERR(svn_cache__create_membuffer_cache( - cache_p, membuffer, serializer, deserializer, - klen, prefix, pool)); - } - else if (pages) - { - SVN_ERR(svn_cache__create_inprocess( - cache_p, serializer, deserializer, klen, pages, - items_per_page, FALSE, prefix, pool)); - } - else + svn_cache__error_handler_t error_handler = no_handler + ? NULL + : warn_and_fail_on_cache_errors; + + if (memcache) + { + SVN_ERR(svn_cache__create_memcache(cache_p, memcache, + serializer, deserializer, klen, + prefix, pool)); + error_handler = no_handler + ? NULL + : warn_and_continue_on_cache_errors; + } + else if (membuffer) + { + SVN_ERR(svn_cache__create_membuffer_cache( + cache_p, membuffer, serializer, deserializer, + klen, prefix, FALSE, pool)); + } + else if (pages) + { + SVN_ERR(svn_cache__create_inprocess( + cache_p, serializer, deserializer, klen, pages, + items_per_page, FALSE, prefix, pool)); + } + else { *cache_p = NULL; } - return SVN_NO_ERROR; + SVN_ERR(init_callbacks(*cache_p, fs, error_handler, pool)); + + return SVN_NO_ERROR; } svn_error_t * @@ -241,23 +329,30 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, { fs_fs_data_t *ffd = fs->fsap_data; const char *prefix = apr_pstrcat(pool, - "fsfs:", ffd->uuid, - "/", fs->path, ":", + "fsfs:", fs->uuid, + "/", normalize_key_part(fs->path, pool), + ":", (char *)NULL); svn_memcache_t *memcache; svn_membuffer_t *membuffer; svn_boolean_t no_handler; svn_boolean_t cache_txdeltas; svn_boolean_t cache_fulltexts; + svn_boolean_t cache_revprops; + const char *cache_namespace; /* Evaluating the cache configuration. */ SVN_ERR(read_config(&memcache, &no_handler, + &cache_namespace, &cache_txdeltas, &cache_fulltexts, + &cache_revprops, fs, pool)); + prefix = apr_pstrcat(pool, "ns:", cache_namespace, ":", prefix, NULL); + membuffer = svn_cache__get_global_membuffer_cache(); /* Make the cache for revision roots. For the vast majority of @@ -276,10 +371,10 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, svn_fs_fs__deserialize_id, sizeof(svn_revnum_t), apr_pstrcat(pool, prefix, "RRI", (char *)NULL), + fs, + no_handler, fs->pool)); - SVN_ERR(init_callbacks(ffd->rev_root_id_cache, fs, no_handler, pool)); - /* Rough estimate: revision DAG nodes have size around 320 bytes, so * let's put 16 on a page. */ SVN_ERR(create_cache(&(ffd->rev_node_cache), @@ -289,10 +384,13 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, svn_fs_fs__dag_serialize, svn_fs_fs__dag_deserialize, APR_HASH_KEY_STRING, - apr_pstrcat(pool, prefix, "DAG", NULL), + apr_pstrcat(pool, prefix, "DAG", (char *)NULL), + fs, + no_handler, fs->pool)); - SVN_ERR(init_callbacks(ffd->rev_node_cache, fs, no_handler, pool)); + /* 1st level DAG node cache */ + ffd->dag_node_cache = svn_fs_fs__create_dag_cache(pool); /* Very rough estimate: 1K per directory. */ SVN_ERR(create_cache(&(ffd->dir_cache), @@ -303,10 +401,10 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, svn_fs_fs__deserialize_dir_entries, APR_HASH_KEY_STRING, apr_pstrcat(pool, prefix, "DIR", (char *)NULL), + fs, + no_handler, fs->pool)); - SVN_ERR(init_callbacks(ffd->dir_cache, fs, no_handler, pool)); - /* Only 16 bytes per entry (a revision number + the corresponding offset). Since we want ~8k pages, that means 512 entries per page. */ SVN_ERR(create_cache(&(ffd->packed_offset_cache), @@ -318,28 +416,120 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, sizeof(svn_revnum_t), apr_pstrcat(pool, prefix, "PACK-MANIFEST", (char *)NULL), + fs, + no_handler, fs->pool)); - SVN_ERR(init_callbacks(ffd->packed_offset_cache, fs, no_handler, pool)); + /* initialize node revision cache, if caching has been enabled */ + SVN_ERR(create_cache(&(ffd->node_revision_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_fs__serialize_node_revision, + svn_fs_fs__deserialize_node_revision, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "NODEREVS", (char *)NULL), + fs, + no_handler, + fs->pool)); + + /* initialize node change list cache, if caching has been enabled */ + SVN_ERR(create_cache(&(ffd->changes_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_fs__serialize_changes, + svn_fs_fs__deserialize_changes, + sizeof(svn_revnum_t), + apr_pstrcat(pool, prefix, "CHANGES", (char *)NULL), + fs, + no_handler, + fs->pool)); - /* initialize fulltext cache as configured */ - ffd->fulltext_cache = NULL; + /* if enabled, cache fulltext and other derived information */ if (cache_fulltexts) { SVN_ERR(create_cache(&(ffd->fulltext_cache), memcache, membuffer, 0, 0, /* Do not use inprocess cache */ - /* Values are svn_string_t */ + /* Values are svn_stringbuf_t */ NULL, NULL, - APR_HASH_KEY_STRING, + sizeof(pair_cache_key_t), apr_pstrcat(pool, prefix, "TEXT", (char *)NULL), + fs, + no_handler, + fs->pool)); + + SVN_ERR(create_cache(&(ffd->properties_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_fs__serialize_properties, + svn_fs_fs__deserialize_properties, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "PROP", + (char *)NULL), + fs, + no_handler, + fs->pool)); + + SVN_ERR(create_cache(&(ffd->mergeinfo_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_fs__serialize_mergeinfo, + svn_fs_fs__deserialize_mergeinfo, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "MERGEINFO", + (char *)NULL), + fs, + no_handler, + fs->pool)); + + SVN_ERR(create_cache(&(ffd->mergeinfo_existence_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + /* Values are svn_stringbuf_t */ + NULL, NULL, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "HAS_MERGEINFO", + (char *)NULL), + fs, + no_handler, fs->pool)); } + else + { + ffd->fulltext_cache = NULL; + ffd->properties_cache = NULL; + ffd->mergeinfo_cache = NULL; + ffd->mergeinfo_existence_cache = NULL; + } - SVN_ERR(init_callbacks(ffd->fulltext_cache, fs, no_handler, pool)); + /* initialize revprop cache, if full-text caching has been enabled */ + if (cache_revprops) + { + SVN_ERR(create_cache(&(ffd->revprop_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_fs__serialize_properties, + svn_fs_fs__deserialize_properties, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "REVPROP", + (char *)NULL), + fs, + no_handler, + fs->pool)); + } + else + { + ffd->revprop_cache = NULL; + } - /* initialize txdelta window cache, if that has been enabled */ + /* if enabled, cache text deltas and their combinations */ if (cache_txdeltas) { SVN_ERR(create_cache(&(ffd->txdelta_window_cache), @@ -351,28 +541,29 @@ svn_fs_fs__initialize_caches(svn_fs_t *fs, APR_HASH_KEY_STRING, apr_pstrcat(pool, prefix, "TXDELTA_WINDOW", (char *)NULL), + fs, + no_handler, + fs->pool)); + + SVN_ERR(create_cache(&(ffd->combined_window_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + /* Values are svn_stringbuf_t */ + NULL, NULL, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "COMBINED_WINDOW", + (char *)NULL), + fs, + no_handler, fs->pool)); } else { ffd->txdelta_window_cache = NULL; + ffd->combined_window_cache = NULL; } - SVN_ERR(init_callbacks(ffd->txdelta_window_cache, fs, no_handler, pool)); - - /* initialize node revision cache, if caching has been enabled */ - SVN_ERR(create_cache(&(ffd->node_revision_cache), - NULL, - membuffer, - 0, 0, /* Do not use inprocess cache */ - svn_fs_fs__serialize_node_revision, - svn_fs_fs__deserialize_node_revision, - APR_HASH_KEY_STRING, - apr_pstrcat(pool, prefix, "NODEREVS", (char *)NULL), - fs->pool)); - - SVN_ERR(init_callbacks(ffd->node_revision_cache, fs, no_handler, pool)); - return SVN_NO_ERROR; } @@ -440,7 +631,7 @@ svn_fs_fs__initialize_txn_caches(svn_fs_t *fs, to start a new transaction later that receives the same id. Therefore, throw in a uuid as well - just to be sure. */ const char *prefix = apr_pstrcat(pool, - "fsfs:", ffd->uuid, + "fsfs:", fs->uuid, "/", fs->path, ":", txn_id, ":", svn_uuid_generate(pool), ":", @@ -457,24 +648,18 @@ svn_fs_fs__initialize_txn_caches(svn_fs_t *fs, } /* create a txn-local directory cache */ - if (svn_cache__get_global_membuffer_cache()) - SVN_ERR(svn_cache__create_membuffer_cache(&(ffd->txn_dir_cache), - svn_cache__get_global_membuffer_cache(), - svn_fs_fs__serialize_dir_entries, - svn_fs_fs__deserialize_dir_entries, - APR_HASH_KEY_STRING, - apr_pstrcat(pool, prefix, "TXNDIR", - (char *)NULL), - pool)); - else - SVN_ERR(svn_cache__create_inprocess(&(ffd->txn_dir_cache), - svn_fs_fs__serialize_dir_entries, - svn_fs_fs__deserialize_dir_entries, - APR_HASH_KEY_STRING, - 1024, 8, FALSE, - apr_pstrcat(pool, prefix, "TXNDIR", - (char *)NULL), - pool)); + SVN_ERR(create_cache(&ffd->txn_dir_cache, + NULL, + svn_cache__get_global_membuffer_cache(), + 1024, 8, + svn_fs_fs__serialize_dir_entries, + svn_fs_fs__deserialize_dir_entries, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "TXNDIR", + (char *)NULL), + fs, + TRUE, + pool)); /* reset the transaction-specific cache if the pool gets cleaned up. */ init_txn_callbacks(&(ffd->txn_dir_cache), pool); diff --git a/subversion/libsvn_fs_fs/dag.c b/subversion/libsvn_fs_fs/dag.c index 2bd7f14..3c51ffd 100644 --- a/subversion/libsvn_fs_fs/dag.c +++ b/subversion/libsvn_fs_fs/dag.c @@ -70,6 +70,9 @@ struct dag_node_t things for you. */ node_revision_t *node_revision; + /* The pool to allocate NODE_REVISION in. */ + apr_pool_t *node_pool; + /* the path at which this node was created. */ const char *created_path; }; @@ -139,7 +142,7 @@ copy_node_revision(node_revision_t *noderev, /* Set *NODEREV_P to the cached node-revision for NODE. If the node-revision was not already cached in NODE, read it in, - allocating the cache in POOL. + allocating the cache in NODE->NODE_POOL. If you plan to change the contents of NODE, be careful! We're handing you a pointer directly to our cached node-revision, not @@ -150,16 +153,15 @@ copy_node_revision(node_revision_t *noderev, the structure at all. */ static svn_error_t * get_node_revision(node_revision_t **noderev_p, - dag_node_t *node, - apr_pool_t *pool) + dag_node_t *node) { - node_revision_t *noderev; - /* If we've already got a copy, there's no need to read it in. */ if (! node->node_revision) { + node_revision_t *noderev; + SVN_ERR(svn_fs_fs__get_node_revision(&noderev, node->fs, - node->id, pool)); + node->id, node->node_pool)); node->node_revision = noderev; } @@ -190,7 +192,8 @@ svn_fs_fs__dag_get_node(dag_node_t **node, new_node->id = svn_fs_fs__id_copy(id, pool); /* Grab the contents so we can inspect the node's kind and created path. */ - SVN_ERR(get_node_revision(&noderev, new_node, pool)); + new_node->node_pool = pool; + SVN_ERR(get_node_revision(&noderev, new_node)); /* Initialize the KIND and CREATED_PATH attributes */ new_node->kind = noderev->kind; @@ -227,12 +230,11 @@ svn_fs_fs__dag_get_revision(svn_revnum_t *rev, svn_error_t * svn_fs_fs__dag_get_predecessor_id(const svn_fs_id_t **id_p, - dag_node_t *node, - apr_pool_t *pool) + dag_node_t *node) { node_revision_t *noderev; - SVN_ERR(get_node_revision(&noderev, node, pool)); + SVN_ERR(get_node_revision(&noderev, node)); *id_p = noderev->predecessor_id; return SVN_NO_ERROR; } @@ -240,44 +242,40 @@ svn_fs_fs__dag_get_predecessor_id(const svn_fs_id_t **id_p, svn_error_t * svn_fs_fs__dag_get_predecessor_count(int *count, - dag_node_t *node, - apr_pool_t *pool) + dag_node_t *node) { node_revision_t *noderev; - SVN_ERR(get_node_revision(&noderev, node, pool)); + SVN_ERR(get_node_revision(&noderev, node)); *count = noderev->predecessor_count; return SVN_NO_ERROR; } svn_error_t * svn_fs_fs__dag_get_mergeinfo_count(apr_int64_t *count, - dag_node_t *node, - apr_pool_t *pool) + dag_node_t *node) { node_revision_t *noderev; - SVN_ERR(get_node_revision(&noderev, node, pool)); + SVN_ERR(get_node_revision(&noderev, node)); *count = noderev->mergeinfo_count; return SVN_NO_ERROR; } svn_error_t * svn_fs_fs__dag_has_mergeinfo(svn_boolean_t *has_mergeinfo, - dag_node_t *node, - apr_pool_t *pool) + dag_node_t *node) { node_revision_t *noderev; - SVN_ERR(get_node_revision(&noderev, node, pool)); + SVN_ERR(get_node_revision(&noderev, node)); *has_mergeinfo = noderev->has_mergeinfo; return SVN_NO_ERROR; } svn_error_t * svn_fs_fs__dag_has_descendants_with_mergeinfo(svn_boolean_t *do_they, - dag_node_t *node, - apr_pool_t *pool) + dag_node_t *node) { node_revision_t *noderev; @@ -287,7 +285,7 @@ svn_fs_fs__dag_has_descendants_with_mergeinfo(svn_boolean_t *do_they, return SVN_NO_ERROR; } - SVN_ERR(get_node_revision(&noderev, node, pool)); + SVN_ERR(get_node_revision(&noderev, node)); if (noderev->mergeinfo_count > 1) *do_they = TRUE; else if (noderev->mergeinfo_count == 1 && !noderev->has_mergeinfo) @@ -309,15 +307,13 @@ static svn_error_t * dir_entry_id_from_node(const svn_fs_id_t **id_p, dag_node_t *parent, const char *name, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { svn_fs_dirent_t *dirent; - apr_pool_t *subpool = svn_pool_create(pool); - - SVN_ERR(svn_fs_fs__dag_dir_entry(&dirent, parent, name, subpool, pool)); - *id_p = dirent ? svn_fs_fs__id_copy(dirent->id, pool) : NULL; - svn_pool_destroy(subpool); + SVN_ERR(svn_fs_fs__dag_dir_entry(&dirent, parent, name, scratch_pool)); + *id_p = dirent ? svn_fs_fs__id_copy(dirent->id, result_pool) : NULL; return SVN_NO_ERROR; } @@ -342,7 +338,7 @@ set_entry(dag_node_t *parent, node_revision_t *parent_noderev; /* Get the parent's node-revision. */ - SVN_ERR(get_node_revision(&parent_noderev, parent, pool)); + SVN_ERR(get_node_revision(&parent_noderev, parent)); /* Set the new entry. */ return svn_fs_fs__set_entry(parent->fs, txn_id, parent_noderev, name, id, @@ -355,7 +351,7 @@ set_entry(dag_node_t *parent, will be a file. The new node will be allocated in POOL. PARENT must be mutable, and must not have an entry named NAME. - Use POOL for all allocations including caching the node_revision in PARENT. + Use POOL for all allocations, except caching the node_revision in PARENT. */ static svn_error_t * make_entry(dag_node_t **child_p, @@ -392,7 +388,7 @@ make_entry(dag_node_t **child_p, new_noderev.kind = is_dir ? svn_node_dir : svn_node_file; new_noderev.created_path = svn_fspath__join(parent_path, name, pool); - SVN_ERR(get_node_revision(&parent_noderev, parent, pool)); + SVN_ERR(get_node_revision(&parent_noderev, parent)); new_noderev.copyroot_path = apr_pstrdup(pool, parent_noderev->copyroot_path); new_noderev.copyroot_rev = parent_noderev->copyroot_rev; @@ -419,12 +415,11 @@ make_entry(dag_node_t **child_p, svn_error_t * svn_fs_fs__dag_dir_entries(apr_hash_t **entries, dag_node_t *node, - apr_pool_t *pool, - apr_pool_t *node_pool) + apr_pool_t *pool) { node_revision_t *noderev; - SVN_ERR(get_node_revision(&noderev, node, node_pool)); + SVN_ERR(get_node_revision(&noderev, node)); if (noderev->kind != svn_node_dir) return svn_error_create(SVN_ERR_FS_NOT_DIRECTORY, NULL, @@ -437,11 +432,10 @@ svn_error_t * svn_fs_fs__dag_dir_entry(svn_fs_dirent_t **dirent, dag_node_t *node, const char* name, - apr_pool_t *pool, - apr_pool_t *node_pool) + apr_pool_t *pool) { node_revision_t *noderev; - SVN_ERR(get_node_revision(&noderev, node, node_pool)); + SVN_ERR(get_node_revision(&noderev, node)); if (noderev->kind != svn_node_dir) return svn_error_create(SVN_ERR_FS_NOT_DIRECTORY, NULL, @@ -449,7 +443,7 @@ svn_fs_fs__dag_dir_entry(svn_fs_dirent_t **dirent, /* Get a dirent hash for this directory. */ return svn_fs_fs__rep_contents_dir_entry(dirent, node->fs, - noderev, name, pool); + noderev, name, pool, pool); } @@ -488,7 +482,7 @@ svn_fs_fs__dag_get_proplist(apr_hash_t **proplist_p, node_revision_t *noderev; apr_hash_t *proplist = NULL; - SVN_ERR(get_node_revision(&noderev, node, pool)); + SVN_ERR(get_node_revision(&noderev, node)); SVN_ERR(svn_fs_fs__get_proplist(&proplist, node->fs, noderev, pool)); @@ -517,7 +511,7 @@ svn_fs_fs__dag_set_proplist(dag_node_t *node, } /* Go get a fresh NODE-REVISION for this node. */ - SVN_ERR(get_node_revision(&noderev, node, pool)); + SVN_ERR(get_node_revision(&noderev, node)); /* Set the new proplist. */ return svn_fs_fs__set_proplist(node->fs, noderev, proplist, pool); @@ -545,7 +539,7 @@ svn_fs_fs__dag_increment_mergeinfo_count(dag_node_t *node, return SVN_NO_ERROR; /* Go get a fresh NODE-REVISION for this node. */ - SVN_ERR(get_node_revision(&noderev, node, pool)); + SVN_ERR(get_node_revision(&noderev, node)); noderev->mergeinfo_count += increment; if (noderev->mergeinfo_count < 0) @@ -594,7 +588,7 @@ svn_fs_fs__dag_set_has_mergeinfo(dag_node_t *node, } /* Go get a fresh NODE-REVISION for this node. */ - SVN_ERR(get_node_revision(&noderev, node, pool)); + SVN_ERR(get_node_revision(&noderev, node)); noderev->has_mergeinfo = has_mergeinfo; @@ -658,6 +652,7 @@ svn_fs_fs__dag_clone_child(dag_node_t **child_p, dag_node_t *cur_entry; /* parent's current entry named NAME */ const svn_fs_id_t *new_node_id; /* node id we'll put into NEW_NODE */ svn_fs_t *fs = svn_fs_fs__dag_get_fs(parent); + apr_pool_t *subpool = svn_pool_create(pool); /* First check that the parent is mutable. */ if (! svn_fs_fs__dag_check_mutable(parent)) @@ -672,7 +667,7 @@ svn_fs_fs__dag_clone_child(dag_node_t **child_p, "Attempted to make a child clone with an illegal name '%s'", name); /* Find the node named NAME in PARENT's entries list if it exists. */ - SVN_ERR(svn_fs_fs__dag_open(&cur_entry, parent, name, pool)); + SVN_ERR(svn_fs_fs__dag_open(&cur_entry, parent, name, pool, subpool)); /* Check for mutability in the node we found. If it's mutable, we don't need to clone it. */ @@ -686,11 +681,11 @@ svn_fs_fs__dag_clone_child(dag_node_t **child_p, node_revision_t *noderev, *parent_noderev; /* Go get a fresh NODE-REVISION for current child node. */ - SVN_ERR(get_node_revision(&noderev, cur_entry, pool)); + SVN_ERR(get_node_revision(&noderev, cur_entry)); if (is_parent_copyroot) { - SVN_ERR(get_node_revision(&parent_noderev, parent, pool)); + SVN_ERR(get_node_revision(&parent_noderev, parent)); noderev->copyroot_rev = parent_noderev->copyroot_rev; noderev->copyroot_path = apr_pstrdup(pool, parent_noderev->copyroot_path); @@ -714,6 +709,7 @@ svn_fs_fs__dag_clone_child(dag_node_t **child_p, } /* Initialize the youngster. */ + svn_pool_destroy(subpool); return svn_fs_fs__dag_get_node(child_p, fs, new_node_id, pool); } @@ -753,7 +749,6 @@ svn_fs_fs__dag_delete(dag_node_t *parent, apr_pool_t *pool) { node_revision_t *parent_noderev; - apr_hash_t *entries; svn_fs_t *fs = parent->fs; svn_fs_dirent_t *dirent; svn_fs_id_t *id; @@ -778,15 +773,13 @@ svn_fs_fs__dag_delete(dag_node_t *parent, "Attempted to delete a node with an illegal name '%s'", name); /* Get a fresh NODE-REVISION for the parent node. */ - SVN_ERR(get_node_revision(&parent_noderev, parent, pool)); + SVN_ERR(get_node_revision(&parent_noderev, parent)); subpool = svn_pool_create(pool); - /* Get a dirent hash for this directory. */ - SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, parent_noderev, subpool)); - - /* Find name in the ENTRIES hash. */ - dirent = apr_hash_get(entries, name, APR_HASH_KEY_STRING); + /* Search this directory for a dirent with that NAME. */ + SVN_ERR(svn_fs_fs__rep_contents_dir_entry(&dirent, fs, parent_noderev, + name, subpool, subpool)); /* If we never found ID in ENTRIES (perhaps because there are no ENTRIES, perhaps because ID just isn't in the existing ENTRIES @@ -851,7 +844,7 @@ svn_fs_fs__dag_delete_if_mutable(svn_fs_t *fs, apr_hash_index_t *hi; /* Loop over hash entries */ - SVN_ERR(svn_fs_fs__dag_dir_entries(&entries, node, pool, pool)); + SVN_ERR(svn_fs_fs__dag_dir_entries(&entries, node, pool)); if (entries) { for (hi = apr_hash_first(pool, entries); @@ -912,7 +905,7 @@ svn_fs_fs__dag_get_contents(svn_stream_t **contents_p, "Attempted to get textual contents of a *non*-file node"); /* Go get a fresh node-revision for FILE. */ - SVN_ERR(get_node_revision(&noderev, file, pool)); + SVN_ERR(get_node_revision(&noderev, file)); /* Get a stream to the contents. */ SVN_ERR(svn_fs_fs__get_contents(&contents, file->fs, @@ -942,10 +935,10 @@ svn_fs_fs__dag_get_file_delta_stream(svn_txdelta_stream_t **stream_p, /* Go get fresh node-revisions for the nodes. */ if (source) - SVN_ERR(get_node_revision(&src_noderev, source, pool)); + SVN_ERR(get_node_revision(&src_noderev, source)); else src_noderev = NULL; - SVN_ERR(get_node_revision(&tgt_noderev, target, pool)); + SVN_ERR(get_node_revision(&tgt_noderev, target)); /* Get the delta stream. */ return svn_fs_fs__get_file_delta_stream(stream_p, target->fs, @@ -954,6 +947,24 @@ svn_fs_fs__dag_get_file_delta_stream(svn_txdelta_stream_t **stream_p, svn_error_t * +svn_fs_fs__dag_try_process_file_contents(svn_boolean_t *success, + dag_node_t *node, + svn_fs_process_contents_func_t processor, + void* baton, + apr_pool_t *pool) +{ + node_revision_t *noderev; + + /* Go get fresh node-revisions for the nodes. */ + SVN_ERR(get_node_revision(&noderev, node)); + + return svn_fs_fs__try_process_file_contents(success, node->fs, + noderev, + processor, baton, pool); +} + + +svn_error_t * svn_fs_fs__dag_file_length(svn_filesize_t *length, dag_node_t *file, apr_pool_t *pool) @@ -967,7 +978,7 @@ svn_fs_fs__dag_file_length(svn_filesize_t *length, "Attempted to get length of a *non*-file node"); /* Go get a fresh node-revision for FILE, and . */ - SVN_ERR(get_node_revision(&noderev, file, pool)); + SVN_ERR(get_node_revision(&noderev, file)); return svn_fs_fs__file_length(length, noderev, pool); } @@ -986,7 +997,7 @@ svn_fs_fs__dag_file_checksum(svn_checksum_t **checksum, (SVN_ERR_FS_NOT_FILE, NULL, "Attempted to get checksum of a *non*-file node"); - SVN_ERR(get_node_revision(&noderev, file, pool)); + SVN_ERR(get_node_revision(&noderev, file)); return svn_fs_fs__file_checksum(checksum, noderev, kind, pool); } @@ -1013,7 +1024,7 @@ svn_fs_fs__dag_get_edit_stream(svn_stream_t **contents, "Attempted to set textual contents of an immutable node"); /* Get the node revision. */ - SVN_ERR(get_node_revision(&noderev, file, pool)); + SVN_ERR(get_node_revision(&noderev, file)); SVN_ERR(svn_fs_fs__set_contents(&ws, file->fs, noderev, pool)); @@ -1066,11 +1077,13 @@ svn_fs_fs__dag_dup(const dag_node_t *node, new_node->node_revision->is_fresh_txn_root = node->node_revision->is_fresh_txn_root; } + new_node->node_pool = pool; + return new_node; } svn_error_t * -svn_fs_fs__dag_serialize(char **data, +svn_fs_fs__dag_serialize(void **data, apr_size_t *data_len, void *in, apr_pool_t *pool) @@ -1082,7 +1095,7 @@ svn_fs_fs__dag_serialize(char **data, svn_temp_serializer__context_t *context = svn_temp_serializer__init(node, sizeof(*node), - 503, + 1024 - SVN_TEMP_SERIALIZER__OVERHEAD, pool); /* for mutable nodes, we will _never_ cache the noderev */ @@ -1092,6 +1105,10 @@ svn_fs_fs__dag_serialize(char **data, svn_temp_serializer__set_null(context, (const void * const *)&node->node_revision); + /* The deserializer will use its own pool. */ + svn_temp_serializer__set_null(context, + (const void * const *)&node->node_pool); + /* serialize other sub-structures */ svn_fs_fs__id_serialize(context, (const svn_fs_id_t **)&node->id); svn_fs_fs__id_serialize(context, &node->fresh_root_predecessor_id); @@ -1107,7 +1124,7 @@ svn_fs_fs__dag_serialize(char **data, svn_error_t * svn_fs_fs__dag_deserialize(void **out, - char *data, + void *data, apr_size_t data_len, apr_pool_t *pool) { @@ -1124,6 +1141,7 @@ svn_fs_fs__dag_deserialize(void **out, svn_fs_fs__id_deserialize(node, (svn_fs_id_t **)&node->fresh_root_predecessor_id); svn_fs_fs__noderev_deserialize(node, &node->node_revision); + node->node_pool = pool; svn_temp_deserializer__resolve(node, (void**)&node->created_path); @@ -1137,12 +1155,14 @@ svn_error_t * svn_fs_fs__dag_open(dag_node_t **child_p, dag_node_t *parent, const char *name, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { const svn_fs_id_t *node_id; /* Ensure that NAME exists in PARENT's entry list. */ - SVN_ERR(dir_entry_id_from_node(&node_id, parent, name, pool)); + SVN_ERR(dir_entry_id_from_node(&node_id, parent, name, + scratch_pool, scratch_pool)); if (! node_id) return svn_error_createf (SVN_ERR_FS_NOT_FOUND, NULL, @@ -1156,7 +1176,7 @@ svn_fs_fs__dag_open(dag_node_t **child_p, /* Now get the node that was requested. */ return svn_fs_fs__dag_get_node(child_p, svn_fs_fs__dag_get_fs(parent), - node_id, pool); + node_id, result_pool); } @@ -1180,7 +1200,7 @@ svn_fs_fs__dag_copy(dag_node_t *to_node, svn_fs_t *fs = svn_fs_fs__dag_get_fs(from_node); /* Make a copy of the original node revision. */ - SVN_ERR(get_node_revision(&from_noderev, from_node, pool)); + SVN_ERR(get_node_revision(&from_noderev, from_node)); to_noderev = copy_node_revision(from_noderev, pool); /* Reserve a copy ID for this new copy. */ @@ -1222,8 +1242,7 @@ svn_error_t * svn_fs_fs__dag_things_different(svn_boolean_t *props_changed, svn_boolean_t *contents_changed, dag_node_t *node1, - dag_node_t *node2, - apr_pool_t *pool) + dag_node_t *node2) { node_revision_t *noderev1, *noderev2; @@ -1233,8 +1252,8 @@ svn_fs_fs__dag_things_different(svn_boolean_t *props_changed, return SVN_NO_ERROR; /* The node revision skels for these two nodes. */ - SVN_ERR(get_node_revision(&noderev1, node1, pool)); - SVN_ERR(get_node_revision(&noderev2, node2, pool)); + SVN_ERR(get_node_revision(&noderev1, node1)); + SVN_ERR(get_node_revision(&noderev2, node2)); /* Compare property keys. */ if (props_changed != NULL) @@ -1253,13 +1272,12 @@ svn_fs_fs__dag_things_different(svn_boolean_t *props_changed, svn_error_t * svn_fs_fs__dag_get_copyroot(svn_revnum_t *rev, const char **path, - dag_node_t *node, - apr_pool_t *pool) + dag_node_t *node) { node_revision_t *noderev; /* Go get a fresh node-revision for NODE. */ - SVN_ERR(get_node_revision(&noderev, node, pool)); + SVN_ERR(get_node_revision(&noderev, node)); *rev = noderev->copyroot_rev; *path = noderev->copyroot_path; @@ -1269,13 +1287,12 @@ svn_fs_fs__dag_get_copyroot(svn_revnum_t *rev, svn_error_t * svn_fs_fs__dag_get_copyfrom_rev(svn_revnum_t *rev, - dag_node_t *node, - apr_pool_t *pool) + dag_node_t *node) { node_revision_t *noderev; /* Go get a fresh node-revision for NODE. */ - SVN_ERR(get_node_revision(&noderev, node, pool)); + SVN_ERR(get_node_revision(&noderev, node)); *rev = noderev->copyfrom_rev; @@ -1284,13 +1301,12 @@ svn_fs_fs__dag_get_copyfrom_rev(svn_revnum_t *rev, svn_error_t * svn_fs_fs__dag_get_copyfrom_path(const char **path, - dag_node_t *node, - apr_pool_t *pool) + dag_node_t *node) { node_revision_t *noderev; /* Go get a fresh node-revision for NODE. */ - SVN_ERR(get_node_revision(&noderev, node, pool)); + SVN_ERR(get_node_revision(&noderev, node)); *path = noderev->copyfrom_path; @@ -1309,8 +1325,8 @@ svn_fs_fs__dag_update_ancestry(dag_node_t *target, (SVN_ERR_FS_NOT_MUTABLE, NULL, _("Attempted to update ancestry of non-mutable node")); - SVN_ERR(get_node_revision(&source_noderev, source, pool)); - SVN_ERR(get_node_revision(&target_noderev, target, pool)); + SVN_ERR(get_node_revision(&source_noderev, source)); + SVN_ERR(get_node_revision(&target_noderev, target)); target_noderev->predecessor_id = source->id; target_noderev->predecessor_count = source_noderev->predecessor_count; diff --git a/subversion/libsvn_fs_fs/dag.h b/subversion/libsvn_fs_fs/dag.h index 0dc4d1a..867b025 100644 --- a/subversion/libsvn_fs_fs/dag.h +++ b/subversion/libsvn_fs_fs/dag.h @@ -81,7 +81,7 @@ svn_fs_fs__dag_dup(const dag_node_t *node, /* Serialize a DAG node, except don't try to preserve the 'fs' member. Implements svn_cache__serialize_func_t */ svn_error_t * -svn_fs_fs__dag_serialize(char **data, +svn_fs_fs__dag_serialize(void **data, apr_size_t *data_len, void *in, apr_pool_t *pool); @@ -90,7 +90,7 @@ svn_fs_fs__dag_serialize(char **data, Implements svn_cache__deserialize_func_t */ svn_error_t * svn_fs_fs__dag_deserialize(void **out, - char *data, + void *data, apr_size_t data_len, apr_pool_t *pool); @@ -121,58 +121,39 @@ const char *svn_fs_fs__dag_get_created_path(dag_node_t *node); /* Set *ID_P to the node revision ID of NODE's immediate predecessor, - or NULL if NODE has no predecessor, allocating from POOL. - - Use POOL for all allocations, including to cache the node_revision in - NODE. + or NULL if NODE has no predecessor. */ svn_error_t *svn_fs_fs__dag_get_predecessor_id(const svn_fs_id_t **id_p, - dag_node_t *node, - apr_pool_t *pool); + dag_node_t *node); /* Set *COUNT to the number of predecessors NODE has (recursively), or - -1 if not known, allocating from POOL. - - Use POOL for all allocations, including to cache the node_revision in - NODE. + -1 if not known. */ +/* ### This function is currently only used by 'verify'. */ svn_error_t *svn_fs_fs__dag_get_predecessor_count(int *count, - dag_node_t *node, - apr_pool_t *pool); + dag_node_t *node); /* Set *COUNT to the number of node under NODE (inclusive) with - svn:mergeinfo properties, allocating from POOL. - - Use POOL for all allocations, including to cache the node_revision in - NODE. + svn:mergeinfo properties. */ svn_error_t *svn_fs_fs__dag_get_mergeinfo_count(apr_int64_t *count, - dag_node_t *node, - apr_pool_t *pool); + dag_node_t *node); /* Set *DO_THEY to a flag indicating whether or not NODE is a directory with at least one descendant (not including itself) with svn:mergeinfo. - - Use POOL for all allocations, including to cache the node_revision in - NODE. */ svn_error_t * svn_fs_fs__dag_has_descendants_with_mergeinfo(svn_boolean_t *do_they, - dag_node_t *node, - apr_pool_t *pool); + dag_node_t *node); /* Set *HAS_MERGEINFO to a flag indicating whether or not NODE itself has svn:mergeinfo set on it. - - Use POOL for all allocations, including to cache the node_revision in - NODE. */ svn_error_t * svn_fs_fs__dag_has_mergeinfo(svn_boolean_t *has_mergeinfo, - dag_node_t *node, - apr_pool_t *pool); + dag_node_t *node); /* Return non-zero IFF NODE is currently mutable. */ svn_boolean_t svn_fs_fs__dag_check_mutable(const dag_node_t *node); @@ -188,8 +169,7 @@ svn_node_kind_t svn_fs_fs__dag_node_kind(dag_node_t *node); If properties do not exist on NODE, *PROPLIST_P will be set to NULL. - Use POOL for all allocations, including to cache the node_revision in - NODE. + Use POOL for all allocations. */ svn_error_t *svn_fs_fs__dag_get_proplist(apr_hash_t **proplist_p, dag_node_t *node, @@ -198,8 +178,7 @@ svn_error_t *svn_fs_fs__dag_get_proplist(apr_hash_t **proplist_p, /* Set the property list of NODE to PROPLIST, allocating from POOL. The node being changed must be mutable. - Use POOL for all allocations, including to cache the node_revision in - NODE. + Use POOL for all allocations. */ svn_error_t *svn_fs_fs__dag_set_proplist(dag_node_t *node, apr_hash_t *proplist, @@ -208,8 +187,7 @@ svn_error_t *svn_fs_fs__dag_set_proplist(dag_node_t *node, /* Increment the mergeinfo_count field on NODE by INCREMENT. The node being changed must be mutable. - Use POOL for all allocations, including to cache the node_revision in - NODE. + Use POOL for all allocations. */ svn_error_t *svn_fs_fs__dag_increment_mergeinfo_count(dag_node_t *node, apr_int64_t increment, @@ -218,8 +196,7 @@ svn_error_t *svn_fs_fs__dag_increment_mergeinfo_count(dag_node_t *node, /* Set the has-mergeinfo flag on NODE to HAS_MERGEINFO. The node being changed must be mutable. - Use POOL for all allocations, including to cache the node_revision in - NODE. + Use POOL for all allocations. */ svn_error_t *svn_fs_fs__dag_set_has_mergeinfo(dag_node_t *node, svn_boolean_t has_mergeinfo, @@ -274,40 +251,35 @@ svn_error_t *svn_fs_fs__dag_clone_root(dag_node_t **root_p, /* Open the node named NAME in the directory PARENT. Set *CHILD_P to - the new node, allocated in POOL. NAME must be a single path + the new node, allocated in RESULT_POOL. NAME must be a single path component; it cannot be a slash-separated directory path. - - Use POOL for all allocations, including to cache the node_revision in - PARENT. */ -svn_error_t *svn_fs_fs__dag_open(dag_node_t **child_p, - dag_node_t *parent, - const char *name, - apr_pool_t *pool); +svn_error_t * +svn_fs_fs__dag_open(dag_node_t **child_p, + dag_node_t *parent, + const char *name, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); /* Set *ENTRIES_P to a hash table of NODE's entries. The keys of the table are entry names, and the values are svn_fs_dirent_t's. The returned table (and its keys and values) is allocated in POOL, - which is also used for temporary allocations. NODE_POOL is used - for any allocation of memory that needs to live as long as NODE - lives. */ + which is also used for temporary allocations. */ svn_error_t *svn_fs_fs__dag_dir_entries(apr_hash_t **entries_p, dag_node_t *node, - apr_pool_t *pool, - apr_pool_t *node_pool); + apr_pool_t *pool); /* Fetches the NODE's entries and returns a copy of the entry selected by the key value given in NAME and set *DIRENT to a copy of that entry. If such entry was found, the copy will be allocated in POOL. - Otherwise, the *DIRENT will be set to NULL. NODE_POOL is used for - any allocation of memory that needs to live as long as NODE lives. + Otherwise, the *DIRENT will be set to NULL. */ +/* ### This function is currently only called from dag.c. */ svn_error_t * svn_fs_fs__dag_dir_entry(svn_fs_dirent_t **dirent, dag_node_t *node, const char* name, - apr_pool_t *pool, - apr_pool_t *node_pool); + apr_pool_t *pool); /* Set ENTRY_NAME in NODE to point to ID (with kind KIND), allocating from POOL. NODE must be a mutable directory. ID can refer to a @@ -343,8 +315,7 @@ svn_error_t *svn_fs_fs__dag_set_entry(dag_node_t *node, TXN_ID is the Subversion transaction under which this occurs. - Use POOL for all allocations, including to cache the node_revision in - FILE. + Use POOL for all allocations. */ svn_error_t *svn_fs_fs__dag_clone_child(dag_node_t **child_p, dag_node_t *parent, @@ -366,8 +337,7 @@ svn_error_t *svn_fs_fs__dag_clone_child(dag_node_t **child_p, If return SVN_ERR_FS_NO_SUCH_ENTRY, then there is no entry NAME in PARENT. - Use POOL for all allocations, including to cache the node_revision in - FILE. + Use POOL for all allocations. */ svn_error_t *svn_fs_fs__dag_delete(dag_node_t *parent, const char *name, @@ -408,8 +378,7 @@ svn_error_t *svn_fs_fs__dag_delete_if_mutable(svn_fs_t *fs, not currently have an entry named NAME. TXN_ID is the Subversion transaction under which this occurs. - Use POOL for all allocations, including to cache the node_revision in - PARENT. + Use POOL for all allocations. */ svn_error_t *svn_fs_fs__dag_make_dir(dag_node_t **child_p, dag_node_t *parent, @@ -428,20 +397,31 @@ svn_error_t *svn_fs_fs__dag_make_dir(dag_node_t **child_p, If FILE is not a file, return SVN_ERR_FS_NOT_FILE. - Use POOL for all allocations, including to cache the node_revision in - FILE. + Use POOL for all allocations. */ svn_error_t *svn_fs_fs__dag_get_contents(svn_stream_t **contents, dag_node_t *file, apr_pool_t *pool); +/* Attempt to fetch the contents of NODE and pass it along with the BATON + to the PROCESSOR. Set *SUCCESS only of the data could be provided + and the processor had been called. + + Use POOL for all allocations. + */ +svn_error_t * +svn_fs_fs__dag_try_process_file_contents(svn_boolean_t *success, + dag_node_t *node, + svn_fs_process_contents_func_t processor, + void* baton, + apr_pool_t *pool); + /* Set *STREAM_P to a delta stream that will turn the contents of SOURCE into the contents of TARGET, allocated in POOL. If SOURCE is null, the empty string will be used. - Use POOL for all allocations, including to cache the node_revision in - SOURCE and TARGET. + Use POOL for all allocations. */ svn_error_t * svn_fs_fs__dag_get_file_delta_stream(svn_txdelta_stream_t **stream_p, @@ -455,8 +435,7 @@ svn_fs_fs__dag_get_file_delta_stream(svn_txdelta_stream_t **stream_p, Any previous edits on the file will be deleted, and a new edit stream will be constructed. - Use POOL for all allocations, including to cache the node_revision in - FILE. + Use POOL for all allocations. */ svn_error_t *svn_fs_fs__dag_get_edit_stream(svn_stream_t **contents, dag_node_t *file, @@ -482,8 +461,7 @@ svn_error_t *svn_fs_fs__dag_finalize_edits(dag_node_t *file, /* Set *LENGTH to the length of the contents of FILE. - Use POOL for all allocations, including to cache the node_revision in - FILE. + Use POOL for all allocations. */ svn_error_t *svn_fs_fs__dag_file_length(svn_filesize_t *length, dag_node_t *file, @@ -495,8 +473,7 @@ svn_error_t *svn_fs_fs__dag_file_length(svn_filesize_t *length, If no stored checksum is available, do not calculate the checksum, just put NULL into CHECKSUM. - Use POOL for all allocations, including to cache the node_revision in - FILE. + Use POOL for all allocations. */ svn_error_t * svn_fs_fs__dag_file_checksum(svn_checksum_t **checksum, @@ -512,8 +489,7 @@ svn_fs_fs__dag_file_checksum(svn_checksum_t **checksum, canonicalized absolute path of the parent directory. TXN_ID is the Subversion transaction under which this occurs. - Use POOL for all allocations, including to cache the node_revision in - PARENT. + Use POOL for all allocations. */ svn_error_t *svn_fs_fs__dag_make_file(dag_node_t **child_p, dag_node_t *parent, @@ -538,8 +514,7 @@ svn_error_t *svn_fs_fs__dag_make_file(dag_node_t **child_p, If PRESERVE_HISTORY is false, FROM_PATH and FROM_REV are ignored. - Use POOL for all allocations, including to cache the node_revision in - FROM_NODE. + Use POOL for all allocations. */ svn_error_t *svn_fs_fs__dag_copy(dag_node_t *to_node, const char *entry, @@ -569,39 +544,29 @@ svn_error_t *svn_fs_fs__dag_copy(dag_node_t *to_node, may leave us with a slight chance of a false positive, though I don't really see how that would happen in practice. Nevertheless, it should probably be fixed. - - Use POOL for all allocations, including to cache the node_revision in NODE1 - and NODE2. */ svn_error_t *svn_fs_fs__dag_things_different(svn_boolean_t *props_changed, svn_boolean_t *contents_changed, dag_node_t *node1, - dag_node_t *node2, - apr_pool_t *pool); + dag_node_t *node2); /* Set *REV and *PATH to the copyroot revision and path of node NODE, or to SVN_INVALID_REVNUM and NULL if no copyroot exists. - Use POOL for all allocations, including to cache the node_revision in NODE. */ svn_error_t *svn_fs_fs__dag_get_copyroot(svn_revnum_t *rev, const char **path, - dag_node_t *node, - apr_pool_t *pool); + dag_node_t *node); /* Set *REV to the copyfrom revision associated with NODE. - Use POOL for all allocations, including to cache the node_revision in NODE. */ svn_error_t *svn_fs_fs__dag_get_copyfrom_rev(svn_revnum_t *rev, - dag_node_t *node, - apr_pool_t *pool); + dag_node_t *node); /* Set *PATH to the copyfrom path associated with NODE. - Use POOL for all allocations, including to cache the node_revision in NODE. */ svn_error_t *svn_fs_fs__dag_get_copyfrom_path(const char **path, - dag_node_t *node, - apr_pool_t *pool); + dag_node_t *node); /* Update *TARGET so that SOURCE is it's predecessor. */ diff --git a/subversion/libsvn_fs_fs/fs.c b/subversion/libsvn_fs_fs/fs.c index 482878c..d0ba734 100644 --- a/subversion/libsvn_fs_fs/fs.c +++ b/subversion/libsvn_fs_fs/fs.c @@ -38,8 +38,10 @@ #include "tree.h" #include "lock.h" #include "id.h" +#include "rep-cache.h" #include "svn_private_config.h" #include "private/svn_fs_util.h" +#include "private/svn_subr_private.h" #include "../libsvn_fs/fs-loader.h" @@ -73,7 +75,8 @@ fs_serialized_init(svn_fs_t *fs, apr_pool_t *common_pool, apr_pool_t *pool) know of a better way of associating such data with the repository. */ - key = apr_pstrcat(pool, SVN_FSFS_SHARED_USERDATA_PREFIX, ffd->uuid, + SVN_ERR_ASSERT(fs->uuid); + key = apr_pstrcat(pool, SVN_FSFS_SHARED_USERDATA_PREFIX, fs->uuid, (char *) NULL); status = apr_pool_userdata_get(&val, key, common_pool); if (status) @@ -85,33 +88,21 @@ fs_serialized_init(svn_fs_t *fs, apr_pool_t *common_pool, apr_pool_t *pool) ffsd = apr_pcalloc(common_pool, sizeof(*ffsd)); ffsd->common_pool = common_pool; -#if SVN_FS_FS__USE_LOCK_MUTEX /* POSIX fcntl locks are per-process, so we need a mutex for intra-process synchronization when grabbing the repository write lock. */ - status = apr_thread_mutex_create(&ffsd->fs_write_lock, - APR_THREAD_MUTEX_DEFAULT, common_pool); - if (status) - return svn_error_wrap_apr(status, - _("Can't create FSFS write-lock mutex")); + SVN_ERR(svn_mutex__init(&ffsd->fs_write_lock, + SVN_FS_FS__USE_LOCK_MUTEX, common_pool)); /* ... not to mention locking the txn-current file. */ - status = apr_thread_mutex_create(&ffsd->txn_current_lock, - APR_THREAD_MUTEX_DEFAULT, common_pool); - if (status) - return svn_error_wrap_apr(status, - _("Can't create FSFS txn-current mutex")); -#endif -#if APR_HAS_THREADS - /* We also need a mutex for synchronising access to the active - transaction list and free transaction pointer. */ - status = apr_thread_mutex_create(&ffsd->txn_list_lock, - APR_THREAD_MUTEX_DEFAULT, common_pool); - if (status) - return svn_error_wrap_apr(status, - _("Can't create FSFS txn list mutex")); -#endif + SVN_ERR(svn_mutex__init(&ffsd->txn_current_lock, + SVN_FS_FS__USE_LOCK_MUTEX, common_pool)); + /* We also need a mutex for synchronizing access to the active + transaction list and free transaction pointer. This one is + enabled unconditionally. */ + SVN_ERR(svn_mutex__init(&ffsd->txn_list_lock, + TRUE, common_pool)); key = apr_pstrdup(common_pool, key); status = apr_pool_userdata_set(ffsd, key, NULL, common_pool); @@ -137,6 +128,46 @@ fs_set_errcall(svn_fs_t *fs, return SVN_NO_ERROR; } +struct fs_freeze_baton_t { + svn_fs_t *fs; + svn_fs_freeze_func_t freeze_func; + void *freeze_baton; +}; + +static svn_error_t * +fs_freeze_body(void *baton, + apr_pool_t *pool) +{ + struct fs_freeze_baton_t *b = baton; + svn_boolean_t exists; + + SVN_ERR(svn_fs_fs__exists_rep_cache(&exists, b->fs, pool)); + if (exists) + SVN_ERR(svn_fs_fs__lock_rep_cache(b->fs, pool)); + + SVN_ERR(b->freeze_func(b->freeze_baton, pool)); + + return SVN_NO_ERROR; +} + +static svn_error_t * +fs_freeze(svn_fs_t *fs, + svn_fs_freeze_func_t freeze_func, + void *freeze_baton, + apr_pool_t *pool) +{ + struct fs_freeze_baton_t b; + + b.fs = fs; + b.freeze_func = freeze_func; + b.freeze_baton = freeze_baton; + + SVN_ERR(svn_fs__check_fs(fs, TRUE)); + SVN_ERR(svn_fs_fs__with_write_lock(fs, fs_freeze_body, &b, pool)); + + return SVN_NO_ERROR; +} + /* The vtable associated with a specific open filesystem. */ @@ -145,7 +176,6 @@ static fs_vtable_t fs_vtable = { svn_fs_fs__revision_prop, svn_fs_fs__revision_proplist, svn_fs_fs__change_rev_prop, - svn_fs_fs__get_uuid, svn_fs_fs__set_uuid, svn_fs_fs__revision_root, svn_fs_fs__begin_txn, @@ -158,7 +188,9 @@ static fs_vtable_t fs_vtable = { svn_fs_fs__unlock, svn_fs_fs__get_lock, svn_fs_fs__get_locks, - fs_set_errcall, + svn_fs_fs__verify_root, + fs_freeze, + fs_set_errcall }; @@ -255,19 +287,40 @@ fs_upgrade(svn_fs_t *fs, const char *path, apr_pool_t *pool, } static svn_error_t * +fs_verify(svn_fs_t *fs, const char *path, + svn_revnum_t start, + svn_revnum_t end, + svn_fs_progress_notify_func_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool, + apr_pool_t *common_pool) +{ + SVN_ERR(svn_fs__check_fs(fs, FALSE)); + SVN_ERR(initialize_fs_struct(fs)); + SVN_ERR(svn_fs_fs__open(fs, path, pool)); + SVN_ERR(svn_fs_fs__initialize_caches(fs, pool)); + SVN_ERR(fs_serialized_init(fs, common_pool, pool)); + return svn_fs_fs__verify(fs, start, end, notify_func, notify_baton, + cancel_func, cancel_baton, pool); +} + +static svn_error_t * fs_pack(svn_fs_t *fs, const char *path, svn_fs_pack_notify_t notify_func, void *notify_baton, svn_cancel_func_t cancel_func, void *cancel_baton, - apr_pool_t *pool) + apr_pool_t *pool, + apr_pool_t *common_pool) { SVN_ERR(svn_fs__check_fs(fs, FALSE)); SVN_ERR(initialize_fs_struct(fs)); SVN_ERR(svn_fs_fs__open(fs, path, pool)); SVN_ERR(svn_fs_fs__initialize_caches(fs, pool)); - SVN_ERR(fs_serialized_init(fs, pool, pool)); + SVN_ERR(fs_serialized_init(fs, common_pool, pool)); return svn_fs_fs__pack(fs, notify_func, notify_baton, cancel_func, cancel_baton, pool); } @@ -276,16 +329,36 @@ fs_pack(svn_fs_t *fs, /* This implements the fs_library_vtable_t.hotcopy() API. Copy a - possibly live Subversion filesystem from SRC_PATH to DEST_PATH. + possibly live Subversion filesystem SRC_FS from SRC_PATH to a + DST_FS at DEST_PATH. If INCREMENTAL is TRUE, make an effort not to + re-copy data which already exists in DST_FS. The CLEAN_LOGS argument is ignored and included for Subversion 1.0.x compatibility. Perform all temporary allocations in POOL. */ static svn_error_t * -fs_hotcopy(const char *src_path, - const char *dest_path, +fs_hotcopy(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + const char *src_path, + const char *dst_path, svn_boolean_t clean_logs, + svn_boolean_t incremental, + svn_cancel_func_t cancel_func, + void *cancel_baton, apr_pool_t *pool) { - return svn_fs_fs__hotcopy(src_path, dest_path, pool); + SVN_ERR(svn_fs__check_fs(src_fs, FALSE)); + SVN_ERR(initialize_fs_struct(src_fs)); + SVN_ERR(svn_fs_fs__open(src_fs, src_path, pool)); + SVN_ERR(svn_fs_fs__initialize_caches(src_fs, pool)); + SVN_ERR(fs_serialized_init(src_fs, pool, pool)); + + SVN_ERR(svn_fs__check_fs(dst_fs, FALSE)); + SVN_ERR(initialize_fs_struct(dst_fs)); + /* In INCREMENTAL mode, svn_fs_fs__hotcopy() will open DST_FS. + Otherwise, it's not an FS yet --- possibly just an empty dir --- so + can't be opened. + */ + return svn_fs_fs__hotcopy(src_fs, dst_fs, src_path, dst_path, + incremental, cancel_func, cancel_baton, pool); } @@ -331,6 +404,17 @@ fs_get_description(void) return _("Module for working with a plain file (FSFS) repository."); } +static svn_error_t * +fs_set_svn_fs_open(svn_fs_t *fs, + svn_error_t *(*svn_fs_open_)(svn_fs_t **, + const char *, + apr_hash_t *, + apr_pool_t *)) +{ + fs_fs_data_t *ffd = fs->fsap_data; + ffd->svn_fs_open_ = svn_fs_open_; + return SVN_NO_ERROR; +} /* Base FS library vtable, used by the FS loader library. */ @@ -341,12 +425,15 @@ static fs_library_vtable_t library_vtable = { fs_open, fs_open_for_recovery, fs_upgrade, + fs_verify, fs_delete_fs, fs_hotcopy, fs_get_description, svn_fs_fs__recover, fs_pack, - fs_logfiles + fs_logfiles, + NULL /* parse_id */, + fs_set_svn_fs_open }; svn_error_t * @@ -366,7 +453,7 @@ svn_fs_fs__init(const svn_version_t *loader_version, return svn_error_createf(SVN_ERR_VERSION_MISMATCH, NULL, _("Unsupported FS loader version (%d) for fsfs"), loader_version->major); - SVN_ERR(svn_ver_check_list(fs_version(), checklist)); + SVN_ERR(svn_ver_check_list2(fs_version(), checklist, svn_ver_equal)); *vtable = &library_vtable; return SVN_NO_ERROR; diff --git a/subversion/libsvn_fs_fs/fs.h b/subversion/libsvn_fs_fs/fs.h index 838b9ad..5cdc270 100644 --- a/subversion/libsvn_fs_fs/fs.h +++ b/subversion/libsvn_fs_fs/fs.h @@ -25,7 +25,6 @@ #include <apr_pools.h> #include <apr_hash.h> -#include <apr_thread_mutex.h> #include <apr_network_io.h> #include "svn_fs.h" @@ -34,6 +33,8 @@ #include "private/svn_cache.h" #include "private/svn_fs_private.h" #include "private/svn_sqlite.h" +#include "private/svn_mutex.h" +#include "private/svn_named_atomic.h" #ifdef __cplusplus extern "C" { @@ -60,6 +61,12 @@ extern "C" { #define PATH_LOCKS_DIR "locks" /* Directory of locks */ #define PATH_MIN_UNPACKED_REV "min-unpacked-rev" /* Oldest revision which has not been packed. */ +#define PATH_REVPROP_GENERATION "revprop-generation" + /* Current revprop generation*/ +#define PATH_MANIFEST "manifest" /* Manifest file name */ +#define PATH_PACKED "pack" /* Packed revision data file */ +#define PATH_EXT_PACKED_SHARD ".pack" /* Extension for packed + shards */ /* If you change this, look at tests/svn_test_fs.c(maybe_install_fsfs_conf) */ #define PATH_CONFIG "fsfs.conf" /* Configuration */ @@ -82,11 +89,19 @@ extern "C" { #define CONFIG_OPTION_FAIL_STOP "fail-stop" #define CONFIG_SECTION_REP_SHARING "rep-sharing" #define CONFIG_OPTION_ENABLE_REP_SHARING "enable-rep-sharing" +#define CONFIG_SECTION_DELTIFICATION "deltification" +#define CONFIG_OPTION_ENABLE_DIR_DELTIFICATION "enable-dir-deltification" +#define CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION "enable-props-deltification" +#define CONFIG_OPTION_MAX_DELTIFICATION_WALK "max-deltification-walk" +#define CONFIG_OPTION_MAX_LINEAR_DELTIFICATION "max-linear-deltification" +#define CONFIG_SECTION_PACKED_REVPROPS "packed-revprops" +#define CONFIG_OPTION_REVPROP_PACK_SIZE "revprop-pack-size" +#define CONFIG_OPTION_COMPRESS_PACKED_REVPROPS "compress-packed-revprops" /* The format number of this filesystem. This is independent of the repository format number, and independent of any other FS back ends. */ -#define SVN_FS_FS__FORMAT_NUMBER 4 +#define SVN_FS_FS__FORMAT_NUMBER 6 /* The minimum format number that supports svndiff version 1. */ #define SVN_FS_FS__MIN_SVNDIFF1_FORMAT 2 @@ -118,12 +133,16 @@ extern "C" { /* The minimum format number that stores node kinds in changed-paths lists. */ #define SVN_FS_FS__MIN_KIND_IN_CHANGED_FORMAT 4 +/* 1.8 deltification options should work with any FSFS repo but to avoid + * issues with very old servers, restrict those options to the 1.6+ format*/ +#define SVN_FS_FS__MIN_DELTIFICATION_FORMAT 4 + /* The 1.7-dev format, never released, that packed revprops into SQLite revprops.db . */ #define SVN_FS_FS__PACKED_REVPROP_SQLITE_DEV_FORMAT 5 -/* The minimum format number that supports packed revprop shards. */ -#define SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT SVN_FS_FS__PACKED_REVPROP_SQLITE_DEV_FORMAT +/* The minimum format number that supports packed revprops. */ +#define SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT 6 /* The minimum format number that supports a configuration file (fsfs.conf) */ #define SVN_FS_FS__MIN_CONFIG_FILE 4 @@ -161,6 +180,7 @@ typedef struct fs_fs_shared_txn_data_t per file. On Windows apr implements the locking as per file handle locks, so we don't have to add our own mutex for just in-process synchronization. */ +/* Compare ../libsvn_subr/named_atomic.c:USE_THREAD_MUTEX */ #if APR_HAS_THREADS && !defined(WIN32) #define SVN_FS_FS__USE_LOCK_MUTEX 1 #else @@ -182,26 +202,35 @@ typedef struct fs_fs_shared_data_t Access to this object is synchronised under TXN_LIST_LOCK. */ fs_fs_shared_txn_data_t *free_txn; -#if APR_HAS_THREADS /* A lock for intra-process synchronization when accessing the TXNS list. */ - apr_thread_mutex_t *txn_list_lock; -#endif -#if SVN_FS_FS__USE_LOCK_MUTEX + svn_mutex__t *txn_list_lock; + /* A lock for intra-process synchronization when grabbing the repository write lock. */ - apr_thread_mutex_t *fs_write_lock; + svn_mutex__t *fs_write_lock; /* A lock for intra-process synchronization when locking the txn-current file. */ - apr_thread_mutex_t *txn_current_lock; -#endif + svn_mutex__t *txn_current_lock; /* The common pool, under which this object is allocated, subpools of which are used to allocate the transaction objects. */ apr_pool_t *common_pool; } fs_fs_shared_data_t; -/* Private (non-shared) FSFS-specific data for each svn_fs_t object. */ +/* Data structure for the 1st level DAG node cache. */ +typedef struct fs_fs_dag_cache_t fs_fs_dag_cache_t; + +/* Key type for all caches that use revision + offset / counter as key. */ +typedef struct pair_cache_key_t +{ + svn_revnum_t revision; + + apr_int64_t second; +} pair_cache_key_t; + +/* Private (non-shared) FSFS-specific data for each svn_fs_t object. + Any caches in here may be NULL. */ typedef struct fs_fs_data_t { /* The format number of this FS. */ @@ -210,9 +239,6 @@ typedef struct fs_fs_data_t layouts) or zero (for linear layouts). */ int max_files_per_dir; - /* The uuid of this FS. */ - const char *uuid; - /* The revision that was youngest, last time we checked. */ svn_revnum_t youngest_rev_cache; @@ -227,17 +253,40 @@ typedef struct fs_fs_data_t (svn_fs_id_t *). (Not threadsafe.) */ svn_cache__t *rev_root_id_cache; - /* DAG node cache for immutable nodes */ + /* Caches native dag_node_t* instances and acts as a 1st level cache */ + fs_fs_dag_cache_t *dag_node_cache; + + /* DAG node cache for immutable nodes. Maps (revision, fspath) + to (dag_node_t *). This is the 2nd level cache for DAG nodes. */ svn_cache__t *rev_node_cache; /* A cache of the contents of immutable directories; maps from - unparsed FS ID to ###x. */ + unparsed FS ID to a apr_hash_t * mapping (const char *) dirent + names to (svn_fs_dirent_t *). */ svn_cache__t *dir_cache; /* Fulltext cache; currently only used with memcached. Maps from - rep key to svn_string_t. */ + rep key (revision/offset) to svn_stringbuf_t. */ svn_cache__t *fulltext_cache; + /* Access object to the atomics namespace used by revprop caching. + Will be NULL until the first access. */ + svn_atomic_namespace__t *revprop_namespace; + + /* Access object to the revprop "generation". Will be NULL until + the first access. */ + svn_named_atomic__t *revprop_generation; + + /* Access object to the revprop update timeout. Will be NULL until + the first access. */ + svn_named_atomic__t *revprop_timeout; + + /* Revision property cache. Maps from (rev,generation) to apr_hash_t. */ + svn_cache__t *revprop_cache; + + /* Node properties cache. Maps from rep key to apr_hash_t. */ + svn_cache__t *properties_cache; + /* Pack manifest cache; a cache mapping (svn_revnum_t) shard number to a manifest; and a manifest is a mapping from (svn_revnum_t) revision number offset within a shard to (apr_off_t) byte-offset in the @@ -247,13 +296,33 @@ typedef struct fs_fs_data_t /* Cache for txdelta_window_t objects; the key is (revFilePath, offset) */ svn_cache__t *txdelta_window_cache; + /* Cache for combined windows as svn_stringbuf_t objects; + the key is (revFilePath, offset) */ + svn_cache__t *combined_window_cache; + /* Cache for node_revision_t objects; the key is (revision, id offset) */ svn_cache__t *node_revision_cache; + /* Cache for change lists as APR arrays of change_t * objects; the key + is the revision */ + svn_cache__t *changes_cache; + + /* Cache for svn_mergeinfo_t objects; the key is a combination of + revision, inheritance flags and path. */ + svn_cache__t *mergeinfo_cache; + + /* Cache for presence of svn_mergeinfo_t on a noderev; the key is a + combination of revision, inheritance flags and path; value is "1" + if the node has mergeinfo, "0" if it doesn't. */ + svn_cache__t *mergeinfo_existence_cache; + + /* TRUE while the we hold a lock on the write lock file. */ + svn_boolean_t has_write_lock; + /* If set, there are or have been more than one concurrent transaction */ svn_boolean_t concurrent_transactions; - /* Tempoary cache for changed directories yet to be committed; maps from + /* Temporary cache for changed directories yet to be committed; maps from unparsed FS ID to ###x. NULL outside transactions. */ svn_cache__t *txn_dir_cache; @@ -266,12 +335,37 @@ typedef struct fs_fs_data_t /* Thread-safe boolean */ svn_atomic_t rep_cache_db_opened; - /* The oldest revision not in a pack file. */ + /* The oldest revision not in a pack file. It also applies to revprops + * if revprop packing has been enabled by the FSFS format version. */ svn_revnum_t min_unpacked_rev; /* Whether rep-sharing is supported by the filesystem * and allowed by the configuration. */ svn_boolean_t rep_sharing_allowed; + + /* File size limit in bytes up to which multiple revprops shall be packed + * into a single file. */ + apr_int64_t revprop_pack_size; + + /* Whether packed revprop files shall be compressed. */ + svn_boolean_t compress_packed_revprops; + + /* Whether directory nodes shall be deltified just like file nodes. */ + svn_boolean_t deltify_directories; + + /* Whether nodes properties shall be deltified. */ + svn_boolean_t deltify_properties; + + /* Restart deltification histories after each multiple of this value */ + apr_int64_t max_deltification_walk; + + /* Maximum number of length of the linear part at the top of the + * deltification history after which skip deltas will be used. */ + apr_int64_t max_linear_deltification; + + /* Pointer to svn_fs_open. */ + svn_error_t *(*svn_fs_open_)(svn_fs_t **, const char *, apr_hash_t *, + apr_pool_t *); } fs_fs_data_t; diff --git a/subversion/libsvn_fs_fs/fs_fs.c b/subversion/libsvn_fs_fs/fs_fs.c index 46f3bcd..8fb3a36 100644 --- a/subversion/libsvn_fs_fs/fs_fs.c +++ b/subversion/libsvn_fs_fs/fs_fs.c @@ -49,6 +49,7 @@ #include "svn_mergeinfo.h" #include "svn_config.h" #include "svn_ctype.h" +#include "svn_version.h" #include "fs.h" #include "tree.h" @@ -59,7 +60,10 @@ #include "rep-cache.h" #include "temp_serializer.h" +#include "private/svn_string_private.h" #include "private/svn_fs_util.h" +#include "private/svn_subr_private.h" +#include "private/svn_delta_private.h" #include "../libsvn_fs/fs-loader.h" #include "svn_private_config.h" @@ -71,13 +75,38 @@ /* The default maximum number of files per directory to store in the rev and revprops directory. The number below is somewhat arbitrary, - and can be overriden by defining the macro while compiling; the + and can be overridden by defining the macro while compiling; the figure of 1000 is reasonable for VFAT filesystems, which are by far the worst performers in this area. */ #ifndef SVN_FS_FS_DEFAULT_MAX_FILES_PER_DIR #define SVN_FS_FS_DEFAULT_MAX_FILES_PER_DIR 1000 #endif +/* Begin deltification after a node history exceeded this this limit. + Useful values are 4 to 64 with 16 being a good compromise between + computational overhead and repository size savings. + Should be a power of 2. + Values < 2 will result in standard skip-delta behavior. */ +#define SVN_FS_FS_MAX_LINEAR_DELTIFICATION 16 + +/* Finding a deltification base takes operations proportional to the + number of changes being skipped. To prevent exploding runtime + during commits, limit the deltification range to this value. + Should be a power of 2 minus one. + Values < 1 disable deltification. */ +#define SVN_FS_FS_MAX_DELTIFICATION_WALK 1023 + +/* Give writing processes 10 seconds to replace an existing revprop + file with a new one. After that time, we assume that the writing + process got aborted and that we have re-read revprops. */ +#define REVPROP_CHANGE_TIMEOUT (10 * 1000000) + +/* The following are names of atomics that will be used to communicate + * revprop updates across all processes on this machine. */ +#define ATOMIC_REVPROP_GENERATION "rev-prop-generation" +#define ATOMIC_REVPROP_TIMEOUT "rev-prop-timeout" +#define ATOMIC_REVPROP_NAMESPACE "rev-prop-atomics" + /* Following are defines that specify the textual elements of the native filesystem directories and revision files. */ @@ -147,6 +176,15 @@ read_min_unpacked_rev(svn_revnum_t *min_unpacked_rev, static svn_error_t * update_min_unpacked_rev(svn_fs_t *fs, apr_pool_t *pool); +static svn_error_t * +get_youngest(svn_revnum_t *youngest_p, const char *fs_path, apr_pool_t *pool); + +static svn_error_t * +verify_walker(representation_t *rep, + void *baton, + svn_fs_t *fs, + apr_pool_t *scratch_pool); + /* Pathname helper functions */ /* Return TRUE is REV is packed in FS, FALSE otherwise. */ @@ -162,13 +200,12 @@ is_packed_rev(svn_fs_t *fs, svn_revnum_t rev) static svn_boolean_t is_packed_revprop(svn_fs_t *fs, svn_revnum_t rev) { -#if 0 fs_fs_data_t *ffd = fs->fsap_data; - return (rev < ffd->min_unpacked_revprop); -#else - return FALSE; -#endif + /* rev 0 will not be packed */ + return (rev < ffd->min_unpacked_rev) + && (rev != 0) + && (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT); } static const char * @@ -208,6 +245,12 @@ path_lock(svn_fs_t *fs, apr_pool_t *pool) } static const char * +path_revprop_generation(svn_fs_t *fs, apr_pool_t *pool) +{ + return svn_dirent_join(fs->path, PATH_REVPROP_GENERATION, pool); +} + +static const char * path_rev_packed(svn_fs_t *fs, svn_revnum_t rev, const char *kind, apr_pool_t *pool) { @@ -217,7 +260,8 @@ path_rev_packed(svn_fs_t *fs, svn_revnum_t rev, const char *kind, assert(is_packed_rev(fs, rev)); return svn_dirent_join_many(pool, fs->path, PATH_REVS_DIR, - apr_psprintf(pool, "%ld.pack", + apr_psprintf(pool, + "%ld" PATH_EXT_PACKED_SHARD, rev / ffd->max_files_per_dir), kind, NULL); } @@ -267,7 +311,7 @@ svn_fs_fs__path_rev_absolute(const char **path, } else { - *path = path_rev_packed(fs, rev, "pack", pool); + *path = path_rev_packed(fs, rev, PATH_PACKED, pool); } return SVN_NO_ERROR; @@ -286,6 +330,18 @@ path_revprops_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) } static const char * +path_revprops_pack_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + + assert(ffd->max_files_per_dir); + return svn_dirent_join_many(pool, fs->path, PATH_REVPROPS_DIR, + apr_psprintf(pool, "%ld" PATH_EXT_PACKED_SHARD, + rev / ffd->max_files_per_dir), + NULL); +} + +static const char * path_revprops(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) { fs_fs_data_t *ffd = fs->fsap_data; @@ -311,6 +367,18 @@ path_txn_dir(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool) NULL); } +/* Return the name of the sha1->rep mapping file in transaction TXN_ID + * within FS for the given SHA1 checksum. Use POOL for allocations. + */ +static APR_INLINE const char * +path_txn_sha1(svn_fs_t *fs, const char *txn_id, svn_checksum_t *sha1, + apr_pool_t *pool) +{ + return svn_dirent_join(path_txn_dir(fs, txn_id, pool), + svn_checksum_to_cstring(sha1, pool), + pool); +} + static APR_INLINE const char * path_txn_changes(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool) { @@ -415,7 +483,7 @@ path_and_offset_of(apr_file_t *file, apr_pool_t *pool) } - + /* Functions for working with shared transaction data. */ /* Return the transaction object for transaction TXN_ID from the @@ -509,26 +577,13 @@ with_txnlist_lock(svn_fs_t *fs, const void *baton, apr_pool_t *pool) { - svn_error_t *err; -#if APR_HAS_THREADS fs_fs_data_t *ffd = fs->fsap_data; fs_fs_shared_data_t *ffsd = ffd->shared; - apr_status_t apr_err; - - apr_err = apr_thread_mutex_lock(ffsd->txn_list_lock); - if (apr_err) - return svn_error_wrap_apr(apr_err, _("Can't grab FSFS txn list mutex")); -#endif - - err = body(fs, baton, pool); -#if APR_HAS_THREADS - apr_err = apr_thread_mutex_unlock(ffsd->txn_list_lock); - if (apr_err && !err) - return svn_error_wrap_apr(apr_err, _("Can't ungrab FSFS txn list mutex")); -#endif + SVN_MUTEX__WITH_LOCK(ffsd->txn_list_lock, + body(fs, baton, pool)); - return svn_error_trace(err); + return SVN_NO_ERROR; } @@ -553,60 +608,59 @@ get_lock_on_filesystem(const char *lock_filename, return svn_error_trace(err); } +/* Reset the HAS_WRITE_LOCK member in the FFD given as BATON_VOID. + When registered with the pool holding the lock on the lock file, + this makes sure the flag gets reset just before we release the lock. */ +static apr_status_t +reset_lock_flag(void *baton_void) +{ + fs_fs_data_t *ffd = baton_void; + ffd->has_write_lock = FALSE; + return APR_SUCCESS; +} + /* Obtain a write lock on the file LOCK_FILENAME (protecting with LOCK_MUTEX if APR is threaded) in a subpool of POOL, call BODY with BATON and that subpool, destroy the subpool (releasing the write - lock) and return what BODY returned. */ -static svn_error_t * -with_some_lock(svn_fs_t *fs, - svn_error_t *(*body)(void *baton, - apr_pool_t *pool), - void *baton, - const char *lock_filename, -#if SVN_FS_FS__USE_LOCK_MUTEX - apr_thread_mutex_t *lock_mutex, -#endif - apr_pool_t *pool) + lock) and return what BODY returned. If IS_GLOBAL_LOCK is set, + set the HAS_WRITE_LOCK flag while we keep the write lock. */ +static svn_error_t * +with_some_lock_file(svn_fs_t *fs, + svn_error_t *(*body)(void *baton, + apr_pool_t *pool), + void *baton, + const char *lock_filename, + svn_boolean_t is_global_lock, + apr_pool_t *pool) { apr_pool_t *subpool = svn_pool_create(pool); - svn_error_t *err; - -#if SVN_FS_FS__USE_LOCK_MUTEX - apr_status_t status; - - /* POSIX fcntl locks are per-process, so we need to serialize locks - within the process. */ - status = apr_thread_mutex_lock(lock_mutex); - if (status) - return svn_error_wrap_apr(status, - _("Can't grab FSFS mutex for '%s'"), - lock_filename); -#endif - - err = get_lock_on_filesystem(lock_filename, subpool); + svn_error_t *err = get_lock_on_filesystem(lock_filename, subpool); if (!err) { fs_fs_data_t *ffd = fs->fsap_data; + + if (is_global_lock) + { + /* set the "got the lock" flag and register reset function */ + apr_pool_cleanup_register(subpool, + ffd, + reset_lock_flag, + apr_pool_cleanup_null); + ffd->has_write_lock = TRUE; + } + + /* nobody else will modify the repo state + => read HEAD & pack info once */ if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) SVN_ERR(update_min_unpacked_rev(fs, pool)); -#if 0 /* Might be a good idea? */ SVN_ERR(get_youngest(&ffd->youngest_rev_cache, fs->path, pool)); -#endif err = body(baton, subpool); } svn_pool_destroy(subpool); -#if SVN_FS_FS__USE_LOCK_MUTEX - status = apr_thread_mutex_unlock(lock_mutex); - if (status && !err) - return svn_error_wrap_apr(status, - _("Can't ungrab FSFS mutex for '%s'"), - lock_filename); -#endif - return svn_error_trace(err); } @@ -617,18 +671,16 @@ svn_fs_fs__with_write_lock(svn_fs_t *fs, void *baton, apr_pool_t *pool) { -#if SVN_FS_FS__USE_LOCK_MUTEX fs_fs_data_t *ffd = fs->fsap_data; fs_fs_shared_data_t *ffsd = ffd->shared; - apr_thread_mutex_t *mutex = ffsd->fs_write_lock; -#endif - return with_some_lock(fs, body, baton, - path_lock(fs, pool), -#if SVN_FS_FS__USE_LOCK_MUTEX - mutex, -#endif - pool); + SVN_MUTEX__WITH_LOCK(ffsd->fs_write_lock, + with_some_lock_file(fs, body, baton, + path_lock(fs, pool), + TRUE, + pool)); + + return SVN_NO_ERROR; } /* Run BODY (with BATON and POOL) while the txn-current file @@ -640,18 +692,16 @@ with_txn_current_lock(svn_fs_t *fs, void *baton, apr_pool_t *pool) { -#if SVN_FS_FS__USE_LOCK_MUTEX fs_fs_data_t *ffd = fs->fsap_data; fs_fs_shared_data_t *ffsd = ffd->shared; - apr_thread_mutex_t *mutex = ffsd->txn_current_lock; -#endif - return with_some_lock(fs, body, baton, - path_txn_current_lock(fs, pool), -#if SVN_FS_FS__USE_LOCK_MUTEX - mutex, -#endif - pool); + SVN_MUTEX__WITH_LOCK(ffsd->txn_current_lock, + with_some_lock_file(fs, body, baton, + path_txn_current_lock(fs, pool), + FALSE, + pool)); + + return SVN_NO_ERROR; } /* A structure used by unlock_proto_rev() and unlock_proto_rev_body(), @@ -829,7 +879,7 @@ get_writable_proto_rev_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool) err = svn_error_compose_create( err, unlock_proto_rev_list_locked(fs, txn_id, *lockcookie, pool)); - + *lockcookie = NULL; } @@ -899,25 +949,64 @@ get_file_offset(apr_off_t *offset_p, apr_file_t *file, apr_pool_t *pool) } -/* Check that BUF, a nul-terminated buffer of text from format file PATH, +/* Check that BUF, a nul-terminated buffer of text from file PATH, contains only digits at OFFSET and beyond, raising an error if not. + TITLE contains a user-visible description of the file, usually the + short file name. Uses POOL for temporary allocation. */ static svn_error_t * -check_format_file_buffer_numeric(const char *buf, apr_off_t offset, - const char *path, apr_pool_t *pool) +check_file_buffer_numeric(const char *buf, apr_off_t offset, + const char *path, const char *title, + apr_pool_t *pool) { const char *p; for (p = buf + offset; *p; p++) if (!svn_ctype_isdigit(*p)) return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL, - _("Format file '%s' contains unexpected non-digit '%c' within '%s'"), - svn_dirent_local_style(path, pool), *p, buf); + _("%s file '%s' contains unexpected non-digit '%c' within '%s'"), + title, svn_dirent_local_style(path, pool), *p, buf); return SVN_NO_ERROR; } +/* Check that BUF, a nul-terminated buffer of text from format file PATH, + contains only digits at OFFSET and beyond, raising an error if not. + + Uses POOL for temporary allocation. */ +static svn_error_t * +check_format_file_buffer_numeric(const char *buf, apr_off_t offset, + const char *path, apr_pool_t *pool) +{ + return check_file_buffer_numeric(buf, offset, path, "Format", pool); +} + +/* Return the error SVN_ERR_FS_UNSUPPORTED_FORMAT if FS's format + number is not the same as a format number supported by this + Subversion. */ +static svn_error_t * +check_format(int format) +{ + /* Blacklist. These formats may be either younger or older than + SVN_FS_FS__FORMAT_NUMBER, but we don't support them. */ + if (format == SVN_FS_FS__PACKED_REVPROP_SQLITE_DEV_FORMAT) + return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, + _("Found format '%d', only created by " + "unreleased dev builds; see " + "http://subversion.apache.org" + "/docs/release-notes/1.7#revprop-packing"), + format); + + /* We support all formats from 1-current simultaneously */ + if (1 <= format && format <= SVN_FS_FS__FORMAT_NUMBER) + return SVN_NO_ERROR; + + return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, + _("Expected FS format between '1' and '%d'; found format '%d'"), + SVN_FS_FS__FORMAT_NUMBER, format); +} + /* Read the format number and maximum number of files per directory from PATH and return them in *PFORMAT and *MAX_FILES_PER_DIR respectively. @@ -931,12 +1020,12 @@ read_format(int *pformat, int *max_files_per_dir, const char *path, apr_pool_t *pool) { svn_error_t *err; - apr_file_t *file; - char buf[80]; - apr_size_t len; + svn_stream_t *stream; + svn_stringbuf_t *content; + svn_stringbuf_t *buf; + svn_boolean_t eos = FALSE; - err = svn_io_file_open(&file, path, APR_READ | APR_BUFFERED, - APR_OS_DEFAULT, pool); + err = svn_stringbuf_from_file2(&content, path, pool); if (err && APR_STATUS_IS_ENOENT(err->apr_err)) { /* Treat an absent format file as format 1. Do not try to @@ -954,62 +1043,57 @@ read_format(int *pformat, int *max_files_per_dir, } SVN_ERR(err); - len = sizeof(buf); - err = svn_io_read_length_line(file, buf, &len, pool); - if (err && APR_STATUS_IS_EOF(err->apr_err)) + stream = svn_stream_from_stringbuf(content, pool); + SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, pool)); + if (buf->len == 0 && eos) { /* Return a more useful error message. */ - svn_error_clear(err); return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL, _("Can't read first line of format file '%s'"), svn_dirent_local_style(path, pool)); } - SVN_ERR(err); /* Check that the first line contains only digits. */ - SVN_ERR(check_format_file_buffer_numeric(buf, 0, path, pool)); - SVN_ERR(svn_cstring_atoi(pformat, buf)); + SVN_ERR(check_format_file_buffer_numeric(buf->data, 0, path, pool)); + SVN_ERR(svn_cstring_atoi(pformat, buf->data)); + + /* Check that we support this format at all */ + SVN_ERR(check_format(*pformat)); /* Set the default values for anything that can be set via an option. */ *max_files_per_dir = 0; /* Read any options. */ - while (1) + while (!eos) { - len = sizeof(buf); - err = svn_io_read_length_line(file, buf, &len, pool); - if (err && APR_STATUS_IS_EOF(err->apr_err)) - { - /* No more options; that's okay. */ - svn_error_clear(err); - break; - } - SVN_ERR(err); + SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, pool)); + if (buf->len == 0) + break; if (*pformat >= SVN_FS_FS__MIN_LAYOUT_FORMAT_OPTION_FORMAT && - strncmp(buf, "layout ", 7) == 0) + strncmp(buf->data, "layout ", 7) == 0) { - if (strcmp(buf+7, "linear") == 0) + if (strcmp(buf->data + 7, "linear") == 0) { *max_files_per_dir = 0; continue; } - if (strncmp(buf+7, "sharded ", 8) == 0) + if (strncmp(buf->data + 7, "sharded ", 8) == 0) { /* Check that the argument is numeric. */ - SVN_ERR(check_format_file_buffer_numeric(buf, 15, path, pool)); - SVN_ERR(svn_cstring_atoi(max_files_per_dir, buf + 15)); + SVN_ERR(check_format_file_buffer_numeric(buf->data, 15, path, pool)); + SVN_ERR(svn_cstring_atoi(max_files_per_dir, buf->data + 15)); continue; } } return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL, _("'%s' contains invalid filesystem format option '%s'"), - svn_dirent_local_style(path, pool), buf); + svn_dirent_local_style(path, pool), buf->data); } - return svn_io_file_close(file, pool); + return SVN_NO_ERROR; } /* Write the format number and maximum number of files per directory @@ -1061,31 +1145,6 @@ write_format(const char *path, int format, int max_files_per_dir, return svn_io_set_file_read_only(path, FALSE, pool); } -/* Return the error SVN_ERR_FS_UNSUPPORTED_FORMAT if FS's format - number is not the same as a format number supported by this - Subversion. */ -static svn_error_t * -check_format(int format) -{ - /* Blacklist. These formats may be either younger or older than - SVN_FS_FS__FORMAT_NUMBER, but we don't support them. */ - if (format == SVN_FS_FS__PACKED_REVPROP_SQLITE_DEV_FORMAT) - return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, - _("Found format '%d', only created by " - "unreleased dev builds; see " - "http://subversion.apache.org" - "/docs/release-notes/1.7#revprop-packing"), - format); - - /* We support all formats from 1-current simultaneously */ - if (1 <= format && format <= SVN_FS_FS__FORMAT_NUMBER) - return SVN_NO_ERROR; - - return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, - _("Expected FS format between '1' and '%d'; found format '%d'"), - SVN_FS_FS__FORMAT_NUMBER, format); -} - svn_boolean_t svn_fs_fs__fs_supports_mergeinfo(svn_fs_t *fs) { @@ -1093,15 +1152,17 @@ svn_fs_fs__fs_supports_mergeinfo(svn_fs_t *fs) return ffd->format >= SVN_FS_FS__MIN_MERGEINFO_FORMAT; } +/* Read the configuration information of the file system at FS_PATH + * and set the respective values in FFD. Use POOL for allocations. + */ static svn_error_t * -read_config(svn_fs_t *fs, +read_config(fs_fs_data_t *ffd, + const char *fs_path, apr_pool_t *pool) { - fs_fs_data_t *ffd = fs->fsap_data; - - SVN_ERR(svn_config_read2(&ffd->config, - svn_dirent_join(fs->path, PATH_CONFIG, pool), - FALSE, FALSE, fs->pool)); + SVN_ERR(svn_config_read3(&ffd->config, + svn_dirent_join(fs_path, PATH_CONFIG, pool), + FALSE, FALSE, FALSE, pool)); /* Initialize ffd->rep_sharing_allowed. */ if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) @@ -1111,6 +1172,56 @@ read_config(svn_fs_t *fs, else ffd->rep_sharing_allowed = FALSE; + /* Initialize deltification settings in ffd. */ + if (ffd->format >= SVN_FS_FS__MIN_DELTIFICATION_FORMAT) + { + SVN_ERR(svn_config_get_bool(ffd->config, &ffd->deltify_directories, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_ENABLE_DIR_DELTIFICATION, + FALSE)); + SVN_ERR(svn_config_get_bool(ffd->config, &ffd->deltify_properties, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION, + FALSE)); + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->max_deltification_walk, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_MAX_DELTIFICATION_WALK, + SVN_FS_FS_MAX_DELTIFICATION_WALK)); + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->max_linear_deltification, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_MAX_LINEAR_DELTIFICATION, + SVN_FS_FS_MAX_LINEAR_DELTIFICATION)); + } + else + { + ffd->deltify_directories = FALSE; + ffd->deltify_properties = FALSE; + ffd->max_deltification_walk = SVN_FS_FS_MAX_DELTIFICATION_WALK; + ffd->max_linear_deltification = SVN_FS_FS_MAX_LINEAR_DELTIFICATION; + } + + /* Initialize revprop packing settings in ffd. */ + if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) + { + SVN_ERR(svn_config_get_bool(ffd->config, &ffd->compress_packed_revprops, + CONFIG_SECTION_PACKED_REVPROPS, + CONFIG_OPTION_COMPRESS_PACKED_REVPROPS, + FALSE)); + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->revprop_pack_size, + CONFIG_SECTION_PACKED_REVPROPS, + CONFIG_OPTION_REVPROP_PACK_SIZE, + ffd->compress_packed_revprops + ? 0x100 + : 0x40)); + + ffd->revprop_pack_size *= 1024; + } + else + { + ffd->revprop_pack_size = 0x10000; + ffd->compress_packed_revprops = FALSE; + } + return SVN_NO_ERROR; } @@ -1156,9 +1267,95 @@ write_config(svn_fs_t *fs, "### The following parameter enables rep-sharing in the repository. It can" NL "### be switched on and off at will, but for best space-saving results" NL "### should be enabled consistently over the life of the repository." NL +"### 'svnadmin verify' will check the rep-cache regardless of this setting." NL "### rep-sharing is enabled by default." NL "# " CONFIG_OPTION_ENABLE_REP_SHARING " = true" NL - +"" NL +"[" CONFIG_SECTION_DELTIFICATION "]" NL +"### To conserve space, the filesystem stores data as differences against" NL +"### existing representations. This comes at a slight cost in performance," NL +"### as calculating differences can increase commit times. Reading data" NL +"### will also create higher CPU load and the data will be fragmented." NL +"### Since deltification tends to save significant amounts of disk space," NL +"### the overall I/O load can actually be lower." NL +"###" NL +"### The options in this section allow for tuning the deltification" NL +"### strategy. Their effects on data size and server performance may vary" NL +"### from one repository to another. Versions prior to 1.8 will ignore" NL +"### this section." NL +"###" NL +"### The following parameter enables deltification for directories. It can" NL +"### be switched on and off at will, but for best space-saving results" NL +"### should be enabled consistently over the life of the repository." NL +"### Repositories containing large directories will benefit greatly." NL +"### In rarely read repositories, the I/O overhead may be significant as" NL +"### cache hit rates will most likely be low" NL +"### directory deltification is disabled by default." NL +"# " CONFIG_OPTION_ENABLE_DIR_DELTIFICATION " = false" NL +"###" NL +"### The following parameter enables deltification for properties on files" NL +"### and directories. Overall, this is a minor tuning option but can save" NL +"### some disk space if you merge frequently or frequently change node" NL +"### properties. You should not activate this if rep-sharing has been" NL +"### disabled because this may result in a net increase in repository size." NL +"### property deltification is disabled by default." NL +"# " CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION " = false" NL +"###" NL +"### During commit, the server may need to walk the whole change history of" NL +"### of a given node to find a suitable deltification base. This linear" NL +"### process can impact commit times, svnadmin load and similar operations." NL +"### This setting limits the depth of the deltification history. If the" NL +"### threshold has been reached, the node will be stored as fulltext and a" NL +"### new deltification history begins." NL +"### Note, this is unrelated to svn log." NL +"### Very large values rarely provide significant additional savings but" NL +"### can impact performance greatly - in particular if directory" NL +"### deltification has been activated. Very small values may be useful in" NL +"### repositories that are dominated by large, changing binaries." NL +"### Should be a power of two minus 1. A value of 0 will effectively" NL +"### disable deltification." NL +"### For 1.8, the default value is 1023; earlier versions have no limit." NL +"# " CONFIG_OPTION_MAX_DELTIFICATION_WALK " = 1023" NL +"###" NL +"### The skip-delta scheme used by FSFS tends to repeatably store redundant" NL +"### delta information where a simple delta against the latest version is" NL +"### often smaller. By default, 1.8+ will therefore use skip deltas only" NL +"### after the linear chain of deltas has grown beyond the threshold" NL +"### specified by this setting." NL +"### Values up to 64 can result in some reduction in repository size for" NL +"### the cost of quickly increasing I/O and CPU costs. Similarly, smaller" NL +"### numbers can reduce those costs at the cost of more disk space. For" NL +"### rarely read repositories or those containing larger binaries, this may" NL +"### present a better trade-off." NL +"### Should be a power of two. A value of 1 or smaller will cause the" NL +"### exclusive use of skip-deltas (as in pre-1.8)." NL +"### For 1.8, the default value is 16; earlier versions use 1." NL +"# " CONFIG_OPTION_MAX_LINEAR_DELTIFICATION " = 16" NL +"" NL +"[" CONFIG_SECTION_PACKED_REVPROPS "]" NL +"### This parameter controls the size (in kBytes) of packed revprop files." NL +"### Revprops of consecutive revisions will be concatenated into a single" NL +"### file up to but not exceeding the threshold given here. However, each" NL +"### pack file may be much smaller and revprops of a single revision may be" NL +"### much larger than the limit set here. The threshold will be applied" NL +"### before optional compression takes place." NL +"### Large values will reduce disk space usage at the expense of increased" NL +"### latency and CPU usage reading and changing individual revprops. They" NL +"### become an advantage when revprop caching has been enabled because a" NL +"### lot of data can be read in one go. Values smaller than 4 kByte will" NL +"### not improve latency any further and quickly render revprop packing" NL +"### ineffective." NL +"### revprop-pack-size is 64 kBytes by default for non-compressed revprop" NL +"### pack files and 256 kBytes when compression has been enabled." NL +"# " CONFIG_OPTION_REVPROP_PACK_SIZE " = 64" NL +"###" NL +"### To save disk space, packed revprop files may be compressed. Standard" NL +"### revprops tend to allow for very effective compression. Reading and" NL +"### even more so writing, become significantly more CPU intensive. With" NL +"### revprop caching enabled, the overhead can be offset by reduced I/O" NL +"### unless you often modify revprops after packing." NL +"### Compressing packed revprops is disabled by default." NL +"# " CONFIG_OPTION_COMPRESS_PACKED_REVPROPS " = false" NL ; #undef NL return svn_io_file_create(svn_dirent_join(fs->path, PATH_CONFIG, pool), @@ -1196,9 +1393,6 @@ update_min_unpacked_rev(svn_fs_t *fs, apr_pool_t *pool) pool); } -static svn_error_t * -get_youngest(svn_revnum_t *youngest_p, const char *fs_path, apr_pool_t *pool); - svn_error_t * svn_fs_fs__open(svn_fs_t *fs, const char *path, apr_pool_t *pool) { @@ -1213,7 +1407,6 @@ svn_fs_fs__open(svn_fs_t *fs, const char *path, apr_pool_t *pool) /* Read the FS format number. */ SVN_ERR(read_format(&format, &max_files_per_dir, path_format(fs, pool), pool)); - SVN_ERR(check_format(format)); /* Now we've got a format number no matter what. */ ffd->format = format; @@ -1225,7 +1418,7 @@ svn_fs_fs__open(svn_fs_t *fs, const char *path, apr_pool_t *pool) limit = sizeof(buf); SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit, pool)); - ffd->uuid = apr_pstrdup(fs->pool, buf); + fs->uuid = apr_pstrdup(fs->pool, buf); SVN_ERR(svn_io_file_close(uuid_file, pool)); @@ -1234,7 +1427,7 @@ svn_fs_fs__open(svn_fs_t *fs, const char *path, apr_pool_t *pool) SVN_ERR(update_min_unpacked_rev(fs, pool)); /* Read the configuration file. */ - SVN_ERR(read_config(fs, pool)); + SVN_ERR(read_config(ffd, fs->path, pool)); return get_youngest(&(ffd->youngest_rev_cache), path, pool); } @@ -1254,6 +1447,114 @@ create_file_ignore_eexist(const char *file, return svn_error_trace(err); } +/* forward declarations */ + +static svn_error_t * +pack_revprops_shard(const char *pack_file_dir, + const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + apr_off_t max_pack_size, + int compression_level, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool); + +static svn_error_t * +delete_revprops_shard(const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool); + +/* In the filesystem FS, pack all revprop shards up to min_unpacked_rev. + * + * NOTE: Keep the old non-packed shards around until after the format bump. + * Otherwise, re-running upgrade will drop the packed revprop shard but + * have no unpacked data anymore. Call upgrade_cleanup_pack_revprops after + * the bump. + * + * Use SCRATCH_POOL for temporary allocations. + */ +static svn_error_t * +upgrade_pack_revprops(svn_fs_t *fs, + apr_pool_t *scratch_pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + const char *revprops_shard_path; + const char *revprops_pack_file_dir; + apr_int64_t shard; + apr_int64_t first_unpacked_shard + = ffd->min_unpacked_rev / ffd->max_files_per_dir; + + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, + scratch_pool); + int compression_level = ffd->compress_packed_revprops + ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT + : SVN_DELTA_COMPRESSION_LEVEL_NONE; + + /* first, pack all revprops shards to match the packed revision shards */ + for (shard = 0; shard < first_unpacked_shard; ++shard) + { + revprops_pack_file_dir = svn_dirent_join(revsprops_dir, + apr_psprintf(iterpool, + "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, + shard), + iterpool); + revprops_shard_path = svn_dirent_join(revsprops_dir, + apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), + iterpool); + + SVN_ERR(pack_revprops_shard(revprops_pack_file_dir, revprops_shard_path, + shard, ffd->max_files_per_dir, + (int)(0.9 * ffd->revprop_pack_size), + compression_level, + NULL, NULL, iterpool)); + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* In the filesystem FS, remove all non-packed revprop shards up to + * min_unpacked_rev. Use SCRATCH_POOL for temporary allocations. + * See upgrade_pack_revprops for more info. + */ +static svn_error_t * +upgrade_cleanup_pack_revprops(svn_fs_t *fs, + apr_pool_t *scratch_pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + const char *revprops_shard_path; + apr_int64_t shard; + apr_int64_t first_unpacked_shard + = ffd->min_unpacked_rev / ffd->max_files_per_dir; + + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, + scratch_pool); + + /* delete the non-packed revprops shards afterwards */ + for (shard = 0; shard < first_unpacked_shard; ++shard) + { + revprops_shard_path = svn_dirent_join(revsprops_dir, + apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), + iterpool); + SVN_ERR(delete_revprops_shard(revprops_shard_path, + shard, ffd->max_files_per_dir, + NULL, NULL, iterpool)); + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + static svn_error_t * upgrade_body(void *baton, apr_pool_t *pool) { @@ -1261,10 +1562,10 @@ upgrade_body(void *baton, apr_pool_t *pool) int format, max_files_per_dir; const char *format_path = path_format(fs, pool); svn_node_kind_t kind; + svn_boolean_t needs_revprop_shard_cleanup = FALSE; /* Read the FS format number and max-files-per-dir setting. */ SVN_ERR(read_format(&format, &max_files_per_dir, format_path, pool)); - SVN_ERR(check_format(format)); /* If the config file does not exist, create one. */ SVN_ERR(svn_io_check_path(svn_dirent_join(fs->path, PATH_CONFIG, pool), @@ -1312,9 +1613,28 @@ upgrade_body(void *baton, apr_pool_t *pool) if (format < SVN_FS_FS__MIN_PACKED_FORMAT) SVN_ERR(svn_io_file_create(path_min_unpacked_rev(fs, pool), "0\n", pool)); + /* If the file system supports revision packing but not revprop packing + *and* the FS has been sharded, pack the revprops up to the point that + revision data has been packed. However, keep the non-packed revprop + files around until after the format bump */ + if ( format >= SVN_FS_FS__MIN_PACKED_FORMAT + && format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT + && max_files_per_dir > 0) + { + needs_revprop_shard_cleanup = TRUE; + SVN_ERR(upgrade_pack_revprops(fs, pool)); + } + /* Bump the format file. */ - return write_format(format_path, SVN_FS_FS__FORMAT_NUMBER, max_files_per_dir, - TRUE, pool); + SVN_ERR(write_format(format_path, SVN_FS_FS__FORMAT_NUMBER, + max_files_per_dir, TRUE, pool)); + + /* Now, it is safe to remove the redundant revprop files. */ + if (needs_revprop_shard_cleanup) + SVN_ERR(upgrade_cleanup_pack_revprops(fs, pool)); + + /* Done */ + return SVN_NO_ERROR; } @@ -1325,7 +1645,7 @@ svn_fs_fs__upgrade(svn_fs_t *fs, apr_pool_t *pool) } -/* SVN_ERR-like macros for dealing with recoverable errors on mutable files +/* Functions for dealing with recoverable errors on mutable files * * Revprops, current, and txn-current files are mutable; that is, they * change as part of normal fsfs operation, in constrat to revs files, or @@ -1358,94 +1678,82 @@ svn_fs_fs__upgrade(svn_fs_t *fs, apr_pool_t *pool) * ** Solution * - * Wrap opens and reads of such files with RETRY_RECOVERABLE and - * closes with IGNORE_RECOVERABLE. Call these macros within a loop of - * RECOVERABLE_RETRY_COUNT iterations (though, realistically, the - * second try will succeed). Make sure you put a break statement - * after the close, at the end of your loop. Immediately after your - * loop, return err if err. - * - * You must initialize err to SVN_NO_ERROR and filehandle to NULL, as - * these macros do not. + * Try open and read of such files in try_stringbuf_from_file(). Call + * this function within a loop of RECOVERABLE_RETRY_COUNT iterations + * (though, realistically, the second try will succeed). */ #define RECOVERABLE_RETRY_COUNT 10 +/* Read the file at PATH and return its content in *CONTENT. *CONTENT will + * not be modified unless the whole file was read successfully. + * + * ESTALE, EIO and ENOENT will not cause this function to return an error + * unless LAST_ATTEMPT has been set. If MISSING is not NULL, indicate + * missing files (ENOENT) there. + * + * Use POOL for allocations. + */ +static svn_error_t * +try_stringbuf_from_file(svn_stringbuf_t **content, + svn_boolean_t *missing, + const char *path, + svn_boolean_t last_attempt, + apr_pool_t *pool) +{ + svn_error_t *err = svn_stringbuf_from_file2(content, path, pool); + if (missing) + *missing = FALSE; + + if (err) + { + *content = NULL; + + if (APR_STATUS_IS_ENOENT(err->apr_err)) + { + if (!last_attempt) + { + svn_error_clear(err); + if (missing) + *missing = TRUE; + return SVN_NO_ERROR; + } + } #ifdef ESTALE -/* Do not use do-while due to the embedded 'continue'. */ -#define RETRY_RECOVERABLE(err, filehandle, expr) \ - if (1) { \ - svn_error_clear(err); \ - err = (expr); \ - if (err) \ - { \ - apr_status_t _e = APR_TO_OS_ERROR(err->apr_err); \ - if ((_e == ESTALE) || (_e == EIO) || (_e == ENOENT)) { \ - if (NULL != filehandle) \ - (void)apr_file_close(filehandle); \ - continue; \ - } \ - return svn_error_trace(err); \ - } \ - } else -#define IGNORE_RECOVERABLE(err, expr) \ - if (1) { \ - svn_error_clear(err); \ - err = (expr); \ - if (err) \ - { \ - apr_status_t _e = APR_TO_OS_ERROR(err->apr_err); \ - if ((_e != ESTALE) && (_e != EIO)) \ - return svn_error_trace(err); \ - } \ - } else -#else -#define RETRY_RECOVERABLE(err, filehandle, expr) SVN_ERR(expr) -#define IGNORE_RECOVERABLE(err, expr) SVN_ERR(expr) + else if (APR_TO_OS_ERROR(err->apr_err) == ESTALE + || APR_TO_OS_ERROR(err->apr_err) == EIO) + { + if (!last_attempt) + { + svn_error_clear(err); + return SVN_NO_ERROR; + } + } #endif + } -/* Long enough to hold: "<svn_revnum_t> <node id> <copy id>\0" - * 19 bytes for svn_revnum_t (room for 32 or 64 bit values) - * + 2 spaces - * + 26 bytes for each id (these are actually unbounded, so we just - * have to pick something; 2^64 is 13 bytes in base-36) - * + 1 terminating null - */ -#define CURRENT_BUF_LEN 48 + return svn_error_trace(err); +} /* Read the 'current' file FNAME and store the contents in *BUF. Allocations are performed in POOL. */ static svn_error_t * -read_current(const char *fname, char **buf, apr_pool_t *pool) +read_content(svn_stringbuf_t **content, const char *fname, apr_pool_t *pool) { - apr_file_t *revision_file = NULL; - apr_size_t len; int i; - svn_error_t *err = SVN_NO_ERROR; - apr_pool_t *iterpool; - - *buf = apr_palloc(pool, CURRENT_BUF_LEN); - iterpool = svn_pool_create(pool); - for (i = 0; i < RECOVERABLE_RETRY_COUNT; i++) - { - svn_pool_clear(iterpool); + *content = NULL; - RETRY_RECOVERABLE(err, revision_file, - svn_io_file_open(&revision_file, fname, - APR_READ | APR_BUFFERED, - APR_OS_DEFAULT, iterpool)); - - len = CURRENT_BUF_LEN; - RETRY_RECOVERABLE(err, revision_file, - svn_io_read_length_line(revision_file, - *buf, &len, iterpool)); - IGNORE_RECOVERABLE(err, svn_io_file_close(revision_file, iterpool)); + for (i = 0; !*content && (i < RECOVERABLE_RETRY_COUNT); ++i) + SVN_ERR(try_stringbuf_from_file(content, NULL, + fname, i + 1 < RECOVERABLE_RETRY_COUNT, + pool)); - break; - } - svn_pool_destroy(iterpool); + if (!*content) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Can't read '%s'"), + svn_dirent_local_style(fname, pool)); - return svn_error_trace(err); + return SVN_NO_ERROR; } /* Find the youngest revision in a repository at path FS_PATH and @@ -1456,248 +1764,15 @@ get_youngest(svn_revnum_t *youngest_p, const char *fs_path, apr_pool_t *pool) { - char *buf; - - SVN_ERR(read_current(svn_dirent_join(fs_path, PATH_CURRENT, pool), - &buf, pool)); + svn_stringbuf_t *buf; + SVN_ERR(read_content(&buf, svn_dirent_join(fs_path, PATH_CURRENT, pool), + pool)); - *youngest_p = SVN_STR_TO_REV(buf); + *youngest_p = SVN_STR_TO_REV(buf->data); return SVN_NO_ERROR; } -svn_error_t * -svn_fs_fs__hotcopy(const char *src_path, - const char *dst_path, - apr_pool_t *pool) -{ - const char *src_subdir, *dst_subdir; - svn_revnum_t youngest, rev, min_unpacked_rev; - apr_pool_t *iterpool; - svn_node_kind_t kind; - int format, max_files_per_dir; - - /* Check format to be sure we know how to hotcopy this FS. */ - SVN_ERR(read_format(&format, &max_files_per_dir, - svn_dirent_join(src_path, PATH_FORMAT, pool), - pool)); - SVN_ERR(check_format(format)); - - /* Try to copy the config. - * - * ### We try copying the config file before doing anything else, - * ### because higher layers will abort the hotcopy if we throw - * ### an error from this function, and that renders the hotcopy - * ### unusable anyway. */ - if (format >= SVN_FS_FS__MIN_CONFIG_FILE) - { - svn_error_t *err; - - err = svn_io_dir_file_copy(src_path, dst_path, PATH_CONFIG, pool); - if (err) - { - if (APR_STATUS_IS_ENOENT(err->apr_err)) - { - /* 1.6.0 to 1.6.11 did not copy the configuration file during - * hotcopy. So if we're hotcopying a repository which has been - * created as a hotcopy itself, it's possible that fsfs.conf - * does not exist. Ask the user to re-create it. - * - * ### It would be nice to make this a non-fatal error, - * ### but this function does not get an svn_fs_t object - * ### so we have no way of just printing a warning via - * ### the fs->warning() callback. */ - - const char *msg; - const char *src_abspath; - const char *dst_abspath; - const char *config_relpath; - svn_error_t *err2; - - config_relpath = svn_dirent_join(src_path, PATH_CONFIG, pool); - err2 = svn_dirent_get_absolute(&src_abspath, src_path, pool); - if (err2) - return svn_error_trace(svn_error_compose_create(err, err2)); - err2 = svn_dirent_get_absolute(&dst_abspath, dst_path, pool); - if (err2) - return svn_error_trace(svn_error_compose_create(err, err2)); - - /* ### hack: strip off the 'db/' directory from paths so - * ### they make sense to the user */ - src_abspath = svn_dirent_dirname(src_abspath, pool); - dst_abspath = svn_dirent_dirname(dst_abspath, pool); - - msg = apr_psprintf(pool, - _("Failed to create hotcopy at '%s'. " - "The file '%s' is missing from the source " - "repository. Please create this file, for " - "instance by running 'svnadmin upgrade %s'"), - dst_abspath, config_relpath, src_abspath); - return svn_error_quick_wrap(err, msg); - } - else - return svn_error_trace(err); - } - } - - /* Copy the 'current' file. */ - SVN_ERR(svn_io_dir_file_copy(src_path, dst_path, PATH_CURRENT, pool)); - - /* Copy the uuid. */ - SVN_ERR(svn_io_dir_file_copy(src_path, dst_path, PATH_UUID, pool)); - - /* Copy the rep cache before copying the rev files to make sure all - cached references will be present in the copy. */ - src_subdir = svn_dirent_join(src_path, REP_CACHE_DB_NAME, pool); - dst_subdir = svn_dirent_join(dst_path, REP_CACHE_DB_NAME, pool); - SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); - if (kind == svn_node_file) - SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, pool)); - - /* Copy the min unpacked rev, and read its value. */ - if (format >= SVN_FS_FS__MIN_PACKED_FORMAT) - { - const char *min_unpacked_rev_path; - min_unpacked_rev_path = svn_dirent_join(src_path, PATH_MIN_UNPACKED_REV, - pool); - - SVN_ERR(svn_io_dir_file_copy(src_path, dst_path, PATH_MIN_UNPACKED_REV, - pool)); - SVN_ERR(read_min_unpacked_rev(&min_unpacked_rev, min_unpacked_rev_path, - pool)); - } - else - { - min_unpacked_rev = 0; - } - - /* Find the youngest revision from this 'current' file. */ - SVN_ERR(get_youngest(&youngest, dst_path, pool)); - - /* Copy the necessary rev files. */ - src_subdir = svn_dirent_join(src_path, PATH_REVS_DIR, pool); - dst_subdir = svn_dirent_join(dst_path, PATH_REVS_DIR, pool); - - SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool)); - - iterpool = svn_pool_create(pool); - /* First, copy packed shards. */ - for (rev = 0; rev < min_unpacked_rev; rev += max_files_per_dir) - { - const char *packed_shard = apr_psprintf(iterpool, "%ld.pack", - rev / max_files_per_dir); - const char *src_subdir_packed_shard; - src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, - iterpool); - - SVN_ERR(svn_io_copy_dir_recursively(src_subdir_packed_shard, - dst_subdir, packed_shard, - TRUE /* copy_perms */, - NULL /* cancel_func */, NULL, - iterpool)); - svn_pool_clear(iterpool); - } - - /* Then, copy non-packed shards. */ - SVN_ERR_ASSERT(rev == min_unpacked_rev); - for (; rev <= youngest; rev++) - { - const char *src_subdir_shard = src_subdir, - *dst_subdir_shard = dst_subdir; - - if (max_files_per_dir) - { - const char *shard = apr_psprintf(iterpool, "%ld", - rev / max_files_per_dir); - src_subdir_shard = svn_dirent_join(src_subdir, shard, iterpool); - dst_subdir_shard = svn_dirent_join(dst_subdir, shard, iterpool); - - if (rev % max_files_per_dir == 0) - { - SVN_ERR(svn_io_dir_make(dst_subdir_shard, APR_OS_DEFAULT, - iterpool)); - SVN_ERR(svn_io_copy_perms(dst_subdir, dst_subdir_shard, - iterpool)); - } - } - - SVN_ERR(svn_io_dir_file_copy(src_subdir_shard, dst_subdir_shard, - apr_psprintf(iterpool, "%ld", rev), - iterpool)); - svn_pool_clear(iterpool); - } - - /* Copy the necessary revprop files. */ - src_subdir = svn_dirent_join(src_path, PATH_REVPROPS_DIR, pool); - dst_subdir = svn_dirent_join(dst_path, PATH_REVPROPS_DIR, pool); - - SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool)); - - for (rev = 0; rev <= youngest; rev++) - { - const char *src_subdir_shard = src_subdir, - *dst_subdir_shard = dst_subdir; - - svn_pool_clear(iterpool); - - if (max_files_per_dir) - { - const char *shard = apr_psprintf(iterpool, "%ld", - rev / max_files_per_dir); - src_subdir_shard = svn_dirent_join(src_subdir, shard, iterpool); - dst_subdir_shard = svn_dirent_join(dst_subdir, shard, iterpool); - - if (rev % max_files_per_dir == 0) - { - SVN_ERR(svn_io_dir_make(dst_subdir_shard, APR_OS_DEFAULT, - iterpool)); - SVN_ERR(svn_io_copy_perms(dst_subdir, dst_subdir_shard, - iterpool)); - } - } - - SVN_ERR(svn_io_dir_file_copy(src_subdir_shard, dst_subdir_shard, - apr_psprintf(iterpool, "%ld", rev), - iterpool)); - } - - svn_pool_destroy(iterpool); - - /* Make an empty transactions directory for now. Eventually some - method of copying in progress transactions will need to be - developed.*/ - dst_subdir = svn_dirent_join(dst_path, PATH_TXNS_DIR, pool); - SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool)); - if (format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT) - { - dst_subdir = svn_dirent_join(dst_path, PATH_TXN_PROTOS_DIR, pool); - SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool)); - } - - /* Now copy the locks tree. */ - src_subdir = svn_dirent_join(src_path, PATH_LOCKS_DIR, pool); - SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); - if (kind == svn_node_dir) - SVN_ERR(svn_io_copy_dir_recursively(src_subdir, dst_path, - PATH_LOCKS_DIR, TRUE, NULL, - NULL, pool)); - - /* Now copy the node-origins cache tree. */ - src_subdir = svn_dirent_join(src_path, PATH_NODE_ORIGINS_DIR, pool); - SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); - if (kind == svn_node_dir) - SVN_ERR(svn_io_copy_dir_recursively(src_subdir, dst_path, - PATH_NODE_ORIGINS_DIR, TRUE, NULL, - NULL, pool)); - - /* Copy the txn-current file. */ - if (format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT) - SVN_ERR(svn_io_dir_file_copy(src_path, dst_path, PATH_TXN_CURRENT, pool)); - - /* Hotcopied FS is complete. Stamp it with a format file. */ - return write_format(svn_dirent_join(dst_path, PATH_FORMAT, pool), - format, max_files_per_dir, FALSE, pool); -} svn_error_t * svn_fs_fs__youngest_rev(svn_revnum_t *youngest_p, @@ -1766,7 +1841,7 @@ static svn_error_t * read_header_block(apr_hash_t **headers, /* header_str is safely in our pool, so we can use bits of it as key and value. */ - apr_hash_set(*headers, name, APR_HASH_KEY_STRING, value); + svn_hash_sets(*headers, name, value); } return SVN_NO_ERROR; @@ -1811,6 +1886,16 @@ ensure_revision_exists(svn_fs_t *fs, _("No such revision %ld"), rev); } +svn_error_t * +svn_fs_fs__revision_exists(svn_revnum_t rev, + svn_fs_t *fs, + apr_pool_t *pool) +{ + /* Different order of parameters. */ + SVN_ERR(ensure_revision_exists(fs, rev, pool)); + return SVN_NO_ERROR; +} + /* Open the correct revision file for REV. If the filesystem FS has been packed, *FILE will be set to the packed file; otherwise, set *FILE to the revision file for REV. Return SVN_ERR_FS_NO_SUCH_REVISION if the @@ -1842,22 +1927,30 @@ open_pack_or_rev_file(apr_file_t **file, err = svn_io_file_open(file, path, APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool); - if (err && APR_STATUS_IS_ENOENT(err->apr_err) - && ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) + if (err && APR_STATUS_IS_ENOENT(err->apr_err)) { - /* Could not open the file. This may happen if the - * file once existed but got packed later. */ - svn_error_clear(err); + if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) + { + /* Could not open the file. This may happen if the + * file once existed but got packed later. */ + svn_error_clear(err); - /* if that was our 2nd attempt, leave it at that. */ - if (retry) - return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, - _("No such revision %ld"), rev); + /* if that was our 2nd attempt, leave it at that. */ + if (retry) + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("No such revision %ld"), rev); - /* We failed for the first time. Refresh cache & retry. */ - SVN_ERR(update_min_unpacked_rev(fs, pool)); + /* We failed for the first time. Refresh cache & retry. */ + SVN_ERR(update_min_unpacked_rev(fs, pool)); - retry = TRUE; + retry = TRUE; + } + else + { + svn_error_clear(err); + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("No such revision %ld"), rev); + } } else { @@ -1869,6 +1962,41 @@ open_pack_or_rev_file(apr_file_t **file, return svn_error_trace(err); } +/* Reads a line from STREAM and converts it to a 64 bit integer to be + * returned in *RESULT. If we encounter eof, set *HIT_EOF and leave + * *RESULT unchanged. If HIT_EOF is NULL, EOF causes an "corrupt FS" + * error return. + * SCRATCH_POOL is used for temporary allocations. + */ +static svn_error_t * +read_number_from_stream(apr_int64_t *result, + svn_boolean_t *hit_eof, + svn_stream_t *stream, + apr_pool_t *scratch_pool) +{ + svn_stringbuf_t *sb; + svn_boolean_t eof; + svn_error_t *err; + + SVN_ERR(svn_stream_readline(stream, &sb, "\n", &eof, scratch_pool)); + if (hit_eof) + *hit_eof = eof; + else + if (eof) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Unexpected EOF")); + + if (!eof) + { + err = svn_cstring_atoi64(result, sb->data); + if (err) + return svn_error_createf(SVN_ERR_FS_CORRUPT, err, + _("Number '%s' invalid or too large"), + sb->data); + } + + return SVN_NO_ERROR; +} + /* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file. Use POOL for temporary allocations. */ static svn_error_t * @@ -1902,7 +2030,8 @@ get_packed_offset(apr_off_t *rev_offset, /* Open the manifest file. */ SVN_ERR(svn_stream_open_readonly(&manifest_stream, - path_rev_packed(fs, rev, "manifest", pool), + path_rev_packed(fs, rev, PATH_MANIFEST, + pool), pool, pool)); /* While we're here, let's just read the entire manifest file into an array, @@ -1911,21 +2040,14 @@ get_packed_offset(apr_off_t *rev_offset, manifest = apr_array_make(pool, ffd->max_files_per_dir, sizeof(apr_off_t)); while (1) { - svn_stringbuf_t *sb; svn_boolean_t eof; apr_int64_t val; - svn_error_t *err; svn_pool_clear(iterpool); - SVN_ERR(svn_stream_readline(manifest_stream, &sb, "\n", &eof, iterpool)); + SVN_ERR(read_number_from_stream(&val, &eof, manifest_stream, iterpool)); if (eof) break; - err = svn_cstring_atoi64(&val, sb->data); - if (err) - return svn_error_createf(SVN_ERR_FS_CORRUPT, err, - _("Manifest offset '%s' too large"), - sb->data); APR_ARRAY_PUSH(manifest, apr_off_t) = (apr_off_t)val; } svn_pool_destroy(iterpool); @@ -2026,13 +2148,13 @@ read_rep_offsets_body(representation_t **rep_p, apr_pool_t *pool) { representation_t *rep; - char *str, *last_str; + char *str; apr_int64_t val; rep = apr_pcalloc(pool, sizeof(*rep)); *rep_p = rep; - str = apr_strtok(string, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Malformed text representation offset line in node-rev")); @@ -2046,7 +2168,7 @@ read_rep_offsets_body(representation_t **rep_p, return SVN_NO_ERROR; } - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Malformed text representation offset line in node-rev")); @@ -2054,7 +2176,7 @@ read_rep_offsets_body(representation_t **rep_p, SVN_ERR(svn_cstring_atoi64(&val, str)); rep->offset = (apr_off_t)val; - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Malformed text representation offset line in node-rev")); @@ -2062,7 +2184,7 @@ read_rep_offsets_body(representation_t **rep_p, SVN_ERR(svn_cstring_atoi64(&val, str)); rep->size = (svn_filesize_t)val; - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Malformed text representation offset line in node-rev")); @@ -2071,7 +2193,7 @@ read_rep_offsets_body(representation_t **rep_p, rep->expanded_size = (svn_filesize_t)val; /* Read in the MD5 hash. */ - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if ((str == NULL) || (strlen(str) != (APR_MD5_DIGESTSIZE * 2))) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Malformed text representation offset line in node-rev")); @@ -2080,7 +2202,7 @@ read_rep_offsets_body(representation_t **rep_p, pool)); /* The remaining fields are only used for formats >= 4, so check that. */ - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if (str == NULL) return SVN_NO_ERROR; @@ -2093,7 +2215,7 @@ read_rep_offsets_body(representation_t **rep_p, pool)); /* Read the uniquifier. */ - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Malformed text representation offset line in node-rev")); @@ -2147,16 +2269,6 @@ err_dangling_id(svn_fs_t *fs, const svn_fs_id_t *id) id_str->data, fs->path); } -/* Return a string that uniquely identifies the noderev with the - * given ID, for use as a cache key. - */ -static const char * -get_noderev_cache_key(const svn_fs_id_t *id, apr_pool_t *pool) -{ - const svn_string_t *id_unparsed = svn_fs_fs__id_unparse(id, pool); - return id_unparsed->data; -} - /* Look up the NODEREV_P for ID in FS' node revsion cache. If noderev * caching has been enabled and the data can be found, IS_CACHED will * be set to TRUE. The noderev will be allocated from POOL. @@ -2172,13 +2284,21 @@ get_cached_node_revision_body(node_revision_t **noderev_p, { fs_fs_data_t *ffd = fs->fsap_data; if (! ffd->node_revision_cache || svn_fs_fs__id_txn_id(id)) - *is_cached = FALSE; + { + *is_cached = FALSE; + } else - SVN_ERR(svn_cache__get((void **) noderev_p, - is_cached, - ffd->node_revision_cache, - get_noderev_cache_key(id, pool), - pool)); + { + pair_cache_key_t key = { 0 }; + + key.revision = svn_fs_fs__id_rev(id); + key.second = svn_fs_fs__id_offset(id); + SVN_ERR(svn_cache__get((void **) noderev_p, + is_cached, + ffd->node_revision_cache, + &key, + pool)); + } return SVN_NO_ERROR; } @@ -2197,10 +2317,16 @@ set_cached_node_revision_body(node_revision_t *noderev_p, fs_fs_data_t *ffd = fs->fsap_data; if (ffd->node_revision_cache && !svn_fs_fs__id_txn_id(id)) - return svn_cache__set(ffd->node_revision_cache, - get_noderev_cache_key(id, scratch_pool), - noderev_p, - scratch_pool); + { + pair_cache_key_t key = { 0 }; + + key.revision = svn_fs_fs__id_rev(id); + key.second = svn_fs_fs__id_offset(id); + return svn_cache__set(ffd->node_revision_cache, + &key, + noderev_p, + scratch_pool); + } return SVN_NO_ERROR; } @@ -2274,7 +2400,7 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, noderev = apr_pcalloc(pool, sizeof(*noderev)); /* Read the node-rev id. */ - value = apr_hash_get(headers, HEADER_ID, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_ID); if (value == NULL) /* ### More information: filename/offset coordinates */ return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, @@ -2286,7 +2412,7 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, noderev_id = value; /* for error messages later */ /* Read the type. */ - value = apr_hash_get(headers, HEADER_TYPE, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_TYPE); if ((value == NULL) || (strcmp(value, KIND_FILE) != 0 && strcmp(value, KIND_DIR))) @@ -2299,14 +2425,14 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, : svn_node_dir; /* Read the 'count' field. */ - value = apr_hash_get(headers, HEADER_COUNT, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_COUNT); if (value) SVN_ERR(svn_cstring_atoi(&noderev->predecessor_count, value)); else noderev->predecessor_count = 0; /* Get the properties location. */ - value = apr_hash_get(headers, HEADER_PROPS, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_PROPS); if (value) { SVN_ERR(read_rep_offsets(&noderev->prop_rep, value, @@ -2314,7 +2440,7 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, } /* Get the data location. */ - value = apr_hash_get(headers, HEADER_TEXT, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_TEXT); if (value) { SVN_ERR(read_rep_offsets(&noderev->data_rep, value, @@ -2323,7 +2449,7 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, } /* Get the created path. */ - value = apr_hash_get(headers, HEADER_CPATH, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_CPATH); if (value == NULL) { return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, @@ -2336,13 +2462,13 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, } /* Get the predecessor ID. */ - value = apr_hash_get(headers, HEADER_PRED, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_PRED); if (value) noderev->predecessor_id = svn_fs_fs__id_parse(value, strlen(value), pool); /* Get the copyroot. */ - value = apr_hash_get(headers, HEADER_COPYROOT, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_COPYROOT); if (value == NULL) { noderev->copyroot_path = apr_pstrdup(pool, noderev->created_path); @@ -2350,9 +2476,9 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, } else { - char *str, *last_str; + char *str; - str = apr_strtok(value, " ", &last_str); + str = svn_cstring_tokenize(" ", &value); if (str == NULL) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Malformed copyroot line in node-rev '%s'"), @@ -2360,15 +2486,15 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, noderev->copyroot_rev = SVN_STR_TO_REV(str); - if (last_str == NULL) + if (*value == '\0') return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Malformed copyroot line in node-rev '%s'"), noderev_id); - noderev->copyroot_path = apr_pstrdup(pool, last_str); + noderev->copyroot_path = apr_pstrdup(pool, value); } /* Get the copyfrom. */ - value = apr_hash_get(headers, HEADER_COPYFROM, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_COPYFROM); if (value == NULL) { noderev->copyfrom_path = NULL; @@ -2376,9 +2502,7 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, } else { - char *str, *last_str; - - str = apr_strtok(value, " ", &last_str); + char *str = svn_cstring_tokenize(" ", &value); if (str == NULL) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Malformed copyfrom line in node-rev '%s'"), @@ -2386,26 +2510,26 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, noderev->copyfrom_rev = SVN_STR_TO_REV(str); - if (last_str == NULL) + if (*value == 0) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Malformed copyfrom line in node-rev '%s'"), noderev_id); - noderev->copyfrom_path = apr_pstrdup(pool, last_str); + noderev->copyfrom_path = apr_pstrdup(pool, value); } /* Get whether this is a fresh txn root. */ - value = apr_hash_get(headers, HEADER_FRESHTXNRT, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_FRESHTXNRT); noderev->is_fresh_txn_root = (value != NULL); /* Get the mergeinfo count. */ - value = apr_hash_get(headers, HEADER_MINFO_CNT, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_MINFO_CNT); if (value) SVN_ERR(svn_cstring_atoi64(&noderev->mergeinfo_count, value)); else noderev->mergeinfo_count = 0; /* Get whether *this* node has mergeinfo. */ - value = apr_hash_get(headers, HEADER_MINFO_HERE, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_MINFO_HERE); noderev->has_mergeinfo = (value != NULL); *noderev_p = noderev; @@ -2435,33 +2559,40 @@ svn_fs_fs__get_node_revision(node_revision_t **noderev_p, that represents the location of representation REP. If MUTABLE_REP_TRUNCATED is given, the rep is for props or dir contents, and only a "-1" revision number will be given for a mutable rep. + If MAY_BE_CORRUPT is true, guard for NULL when constructing the string. Perform the allocation from POOL. */ static const char * representation_string(representation_t *rep, int format, svn_boolean_t mutable_rep_truncated, + svn_boolean_t may_be_corrupt, apr_pool_t *pool) { if (rep->txn_id && mutable_rep_truncated) return "-1"; +#define DISPLAY_MAYBE_NULL_CHECKSUM(checksum) \ + ((!may_be_corrupt || (checksum) != NULL) \ + ? svn_checksum_to_cstring_display((checksum), pool) \ + : "(null)") + if (format < SVN_FS_FS__MIN_REP_SHARING_FORMAT || rep->sha1_checksum == NULL) return apr_psprintf(pool, "%ld %" APR_OFF_T_FMT " %" SVN_FILESIZE_T_FMT " %" SVN_FILESIZE_T_FMT " %s", rep->revision, rep->offset, rep->size, rep->expanded_size, - svn_checksum_to_cstring_display(rep->md5_checksum, - pool)); + DISPLAY_MAYBE_NULL_CHECKSUM(rep->md5_checksum)); return apr_psprintf(pool, "%ld %" APR_OFF_T_FMT " %" SVN_FILESIZE_T_FMT " %" SVN_FILESIZE_T_FMT " %s %s %s", rep->revision, rep->offset, rep->size, rep->expanded_size, - svn_checksum_to_cstring_display(rep->md5_checksum, - pool), - svn_checksum_to_cstring_display(rep->sha1_checksum, - pool), + DISPLAY_MAYBE_NULL_CHECKSUM(rep->md5_checksum), + DISPLAY_MAYBE_NULL_CHECKSUM(rep->sha1_checksum), rep->uniquifier); + +#undef DISPLAY_MAYBE_NULL_CHECKSUM + } @@ -2494,12 +2625,13 @@ svn_fs_fs__write_noderev(svn_stream_t *outfile, format, (noderev->kind == svn_node_dir), + FALSE, pool))); if (noderev->prop_rep) SVN_ERR(svn_stream_printf(outfile, pool, HEADER_PROPS ": %s\n", representation_string(noderev->prop_rep, format, - TRUE, pool))); + TRUE, FALSE, pool))); SVN_ERR(svn_stream_printf(outfile, pool, HEADER_CPATH ": %s\n", noderev->created_path)); @@ -2518,7 +2650,7 @@ svn_fs_fs__write_noderev(svn_stream_t *outfile, noderev->copyroot_path)); if (noderev->is_fresh_txn_root) - SVN_ERR(svn_stream_printf(outfile, pool, HEADER_FRESHTXNRT ": y\n")); + SVN_ERR(svn_stream_puts(outfile, HEADER_FRESHTXNRT ": y\n")); if (include_mergeinfo) { @@ -2528,10 +2660,10 @@ svn_fs_fs__write_noderev(svn_stream_t *outfile, noderev->mergeinfo_count)); if (noderev->has_mergeinfo) - SVN_ERR(svn_stream_printf(outfile, pool, HEADER_MINFO_HERE ": y\n")); + SVN_ERR(svn_stream_puts(outfile, HEADER_MINFO_HERE ": y\n")); } - return svn_stream_printf(outfile, pool, "\n"); + return svn_stream_puts(outfile, "\n"); } svn_error_t * @@ -2562,7 +2694,50 @@ svn_fs_fs__put_node_revision(svn_fs_t *fs, svn_fs_fs__fs_supports_mergeinfo(fs), pool)); - return svn_io_file_close(noderev_file, pool); + SVN_ERR(svn_io_file_close(noderev_file, pool)); + + return SVN_NO_ERROR; +} + +/* For the in-transaction NODEREV within FS, write the sha1->rep mapping + * file in the respective transaction, if rep sharing has been enabled etc. + * Use POOL for temporary allocations. + */ +static svn_error_t * +store_sha1_rep_mapping(svn_fs_t *fs, + node_revision_t *noderev, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + + /* if rep sharing has been enabled and the noderev has a data rep and + * its SHA-1 is known, store the rep struct under its SHA1. */ + if ( ffd->rep_sharing_allowed + && noderev->data_rep + && noderev->data_rep->sha1_checksum) + { + apr_file_t *rep_file; + const char *file_name = path_txn_sha1(fs, + svn_fs_fs__id_txn_id(noderev->id), + noderev->data_rep->sha1_checksum, + pool); + const char *rep_string = representation_string(noderev->data_rep, + ffd->format, + (noderev->kind + == svn_node_dir), + FALSE, + pool); + SVN_ERR(svn_io_file_open(&rep_file, file_name, + APR_WRITE | APR_CREATE | APR_TRUNCATE + | APR_BUFFERED, APR_OS_DEFAULT, pool)); + + SVN_ERR(svn_io_file_write_full(rep_file, rep_string, + strlen(rep_string), NULL, pool)); + + SVN_ERR(svn_io_file_close(rep_file, pool)); + } + + return SVN_NO_ERROR; } @@ -2589,7 +2764,7 @@ read_rep_line(struct rep_args **rep_args_p, char buffer[160]; apr_size_t limit; struct rep_args *rep_args; - char *str, *last_str; + char *str, *last_str = buffer; apr_int64_t val; limit = sizeof(buffer); @@ -2617,22 +2792,22 @@ read_rep_line(struct rep_args **rep_args_p, rep_args->is_delta_vs_empty = FALSE; /* We have hopefully a DELTA vs. a non-empty base revision. */ - str = apr_strtok(buffer, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (! str || (strcmp(str, REP_DELTA) != 0)) goto error; - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (! str) goto error; rep_args->base_revision = SVN_STR_TO_REV(str); - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (! str) goto error; SVN_ERR(svn_cstring_atoi64(&val, str)); rep_args->base_offset = (apr_off_t)val; - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (! str) goto error; SVN_ERR(svn_cstring_atoi64(&val, str)); @@ -2671,7 +2846,7 @@ get_fs_id_at_offset(svn_fs_id_t **id_p, /* In error messages, the offset is relative to the pack file, not to the rev file. */ - node_id_str = apr_hash_get(headers, HEADER_ID, APR_HASH_KEY_STRING); + node_id_str = svn_hash_gets(headers, HEADER_ID); if (node_id_str == NULL) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, @@ -2934,109 +3109,1324 @@ svn_fs_fs__rev_get_root(svn_fs_id_t **root_id_p, return SVN_NO_ERROR; } -/* Set the revision property list of revision REV in filesystem FS to - PROPLIST. Use POOL for temporary allocations. */ +/* Revprop caching management. + * + * Mechanism: + * ---------- + * + * Revprop caching needs to be activated and will be deactivated for the + * respective FS instance if the necessary infrastructure could not be + * initialized. In deactivated mode, there is almost no runtime overhead + * associated with revprop caching. As long as no revprops are being read + * or changed, revprop caching imposes no overhead. + * + * When activated, we cache revprops using (revision, generation) pairs + * as keys with the generation being incremented upon every revprop change. + * Since the cache is process-local, the generation needs to be tracked + * for at least as long as the process lives but may be reset afterwards. + * + * To track the revprop generation, we use two-layer approach. On the lower + * level, we use named atomics to have a system-wide consistent value for + * the current revprop generation. However, those named atomics will only + * remain valid for as long as at least one process / thread in the system + * accesses revprops in the respective repository. The underlying shared + * memory gets cleaned up afterwards. + * + * On the second level, we will use a persistent file to track the latest + * revprop generation. It will be written upon each revprop change but + * only be read if we are the first process to initialize the named atomics + * with that value. + * + * The overhead for the second and following accesses to revprops is + * almost zero on most systems. + * + * + * Tech aspects: + * ------------- + * + * A problem is that we need to provide a globally available file name to + * back the SHM implementation on OSes that need it. We can only assume + * write access to some file within the respective repositories. Because + * a given server process may access thousands of repositories during its + * lifetime, keeping the SHM data alive for all of them is also not an + * option. + * + * So, we store the new revprop generation on disk as part of each + * setrevprop call, i.e. this write will be serialized and the write order + * be guaranteed by the repository write lock. + * + * The only racy situation occurs when the data is being read again by two + * processes concurrently but in that situation, the first process to + * finish that procedure is guaranteed to be the only one that initializes + * the SHM data. Since even writers will first go through that + * initialization phase, they will never operate on stale data. + */ + +/* Read revprop generation as stored on disk for repository FS. The result + * is returned in *CURRENT. Default to 2 if no such file is available. + */ static svn_error_t * -set_revision_proplist(svn_fs_t *fs, - svn_revnum_t rev, - apr_hash_t *proplist, - apr_pool_t *pool) +read_revprop_generation_file(apr_int64_t *current, + svn_fs_t *fs, + apr_pool_t *pool) { - SVN_ERR(ensure_revision_exists(fs, rev, pool)); + svn_error_t *err; + apr_file_t *file; + char buf[80]; + apr_size_t len; + const char *path = path_revprop_generation(fs, pool); - if (1) + err = svn_io_file_open(&file, path, + APR_READ | APR_BUFFERED, + APR_OS_DEFAULT, pool); + if (err && APR_STATUS_IS_ENOENT(err->apr_err)) { - const char *final_path = path_revprops(fs, rev, pool); - const char *tmp_path; - const char *perms_reference; - svn_stream_t *stream; + svn_error_clear(err); + *current = 2; - /* ### do we have a directory sitting around already? we really shouldn't - ### have to get the dirname here. */ - SVN_ERR(svn_stream_open_unique(&stream, &tmp_path, - svn_dirent_dirname(final_path, pool), - svn_io_file_del_none, pool, pool)); - SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); - SVN_ERR(svn_stream_close(stream)); + return SVN_NO_ERROR; + } + SVN_ERR(err); - /* We use the rev file of this revision as the perms reference, - because when setting revprops for the first time, the revprop - file won't exist and therefore can't serve as its own reference. - (Whereas the rev file should already exist at this point.) */ - SVN_ERR(svn_fs_fs__path_rev_absolute(&perms_reference, fs, rev, pool)); - SVN_ERR(move_into_place(tmp_path, final_path, perms_reference, pool)); + len = sizeof(buf); + SVN_ERR(svn_io_read_length_line(file, buf, &len, pool)); - return SVN_NO_ERROR; + /* Check that the first line contains only digits. */ + SVN_ERR(check_file_buffer_numeric(buf, 0, path, + "Revprop Generation", pool)); + SVN_ERR(svn_cstring_atoi64(current, buf)); + + return svn_io_file_close(file, pool); +} + +/* Write the CURRENT revprop generation to disk for repository FS. + */ +static svn_error_t * +write_revprop_generation_file(svn_fs_t *fs, + apr_int64_t current, + apr_pool_t *pool) +{ + apr_file_t *file; + const char *tmp_path; + + char buf[SVN_INT64_BUFFER_SIZE]; + apr_size_t len = svn__i64toa(buf, current); + buf[len] = '\n'; + + SVN_ERR(svn_io_open_unique_file3(&file, &tmp_path, fs->path, + svn_io_file_del_none, pool, pool)); + SVN_ERR(svn_io_file_write_full(file, buf, len + 1, NULL, pool)); + SVN_ERR(svn_io_file_close(file, pool)); + + return move_into_place(tmp_path, path_revprop_generation(fs, pool), + tmp_path, pool); +} + +/* Make sure the revprop_namespace member in FS is set. */ +static svn_error_t * +ensure_revprop_namespace(svn_fs_t *fs) +{ + fs_fs_data_t *ffd = fs->fsap_data; + + return ffd->revprop_namespace == NULL + ? svn_atomic_namespace__create(&ffd->revprop_namespace, + svn_dirent_join(fs->path, + ATOMIC_REVPROP_NAMESPACE, + fs->pool), + fs->pool) + : SVN_NO_ERROR; +} + +/* Make sure the revprop_namespace member in FS is set. */ +static svn_error_t * +cleanup_revprop_namespace(svn_fs_t *fs) +{ + const char *name = svn_dirent_join(fs->path, + ATOMIC_REVPROP_NAMESPACE, + fs->pool); + return svn_error_trace(svn_atomic_namespace__cleanup(name, fs->pool)); +} + +/* Make sure the revprop_generation member in FS is set and, if necessary, + * initialized with the latest value stored on disk. + */ +static svn_error_t * +ensure_revprop_generation(svn_fs_t *fs, apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + + SVN_ERR(ensure_revprop_namespace(fs)); + if (ffd->revprop_generation == NULL) + { + apr_int64_t current = 0; + + SVN_ERR(svn_named_atomic__get(&ffd->revprop_generation, + ffd->revprop_namespace, + ATOMIC_REVPROP_GENERATION, + TRUE)); + + /* If the generation is at 0, we just created a new namespace + * (it would be at least 2 otherwise). Read the latest generation + * from disk and if we are the first one to initialize the atomic + * (i.e. is still 0), set it to the value just gotten. + */ + SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation)); + if (current == 0) + { + SVN_ERR(read_revprop_generation_file(¤t, fs, pool)); + SVN_ERR(svn_named_atomic__cmpxchg(NULL, current, 0, + ffd->revprop_generation)); + } } return SVN_NO_ERROR; } +/* Make sure the revprop_timeout member in FS is set. */ static svn_error_t * -revision_proplist(apr_hash_t **proplist_p, +ensure_revprop_timeout(svn_fs_t *fs) +{ + fs_fs_data_t *ffd = fs->fsap_data; + + SVN_ERR(ensure_revprop_namespace(fs)); + return ffd->revprop_timeout == NULL + ? svn_named_atomic__get(&ffd->revprop_timeout, + ffd->revprop_namespace, + ATOMIC_REVPROP_TIMEOUT, + TRUE) + : SVN_NO_ERROR; +} + +/* Create an error object with the given MESSAGE and pass it to the + WARNING member of FS. */ +static void +log_revprop_cache_init_warning(svn_fs_t *fs, + svn_error_t *underlying_err, + const char *message) +{ + svn_error_t *err = svn_error_createf(SVN_ERR_FS_REVPROP_CACHE_INIT_FAILURE, + underlying_err, + message, fs->path); + + if (fs->warning) + (fs->warning)(fs->warning_baton, err); + + svn_error_clear(err); +} + +/* Test whether revprop cache and necessary infrastructure are + available in FS. */ +static svn_boolean_t +has_revprop_cache(svn_fs_t *fs, apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + svn_error_t *error; + + /* is the cache (still) enabled? */ + if (ffd->revprop_cache == NULL) + return FALSE; + + /* is it efficient? */ + if (!svn_named_atomic__is_efficient()) + { + /* access to it would be quite slow + * -> disable the revprop cache for good + */ + ffd->revprop_cache = NULL; + log_revprop_cache_init_warning(fs, NULL, + "Revprop caching for '%s' disabled" + " because it would be inefficient."); + + return FALSE; + } + + /* try to access our SHM-backed infrastructure */ + error = ensure_revprop_generation(fs, pool); + if (error) + { + /* failure -> disable revprop cache for good */ + + ffd->revprop_cache = NULL; + log_revprop_cache_init_warning(fs, error, + "Revprop caching for '%s' disabled " + "because SHM infrastructure for revprop " + "caching failed to initialize."); + + return FALSE; + } + + return TRUE; +} + +/* Baton structure for revprop_generation_fixup. */ +typedef struct revprop_generation_fixup_t +{ + /* revprop generation to read */ + apr_int64_t *generation; + + /* containing the revprop_generation member to query */ + fs_fs_data_t *ffd; +} revprop_generation_upgrade_t; + +/* If the revprop generation has an odd value, it means the original writer + of the revprop got killed. We don't know whether that process as able + to change the revprop data but we assume that it was. Therefore, we + increase the generation in that case to basically invalidate everyones + cache content. + Execute this onlx while holding the write lock to the repo in baton->FFD. + */ +static svn_error_t * +revprop_generation_fixup(void *void_baton, + apr_pool_t *pool) +{ + revprop_generation_upgrade_t *baton = void_baton; + assert(baton->ffd->has_write_lock); + + /* Maybe, either the original revprop writer or some other reader has + already corrected / bumped the revprop generation. Thus, we need + to read it again. */ + SVN_ERR(svn_named_atomic__read(baton->generation, + baton->ffd->revprop_generation)); + + /* Cause everyone to re-read revprops upon their next access, if the + last revprop write did not complete properly. */ + while (*baton->generation % 2) + SVN_ERR(svn_named_atomic__add(baton->generation, + 1, + baton->ffd->revprop_generation)); + + return SVN_NO_ERROR; +} + +/* Read the current revprop generation and return it in *GENERATION. + Also, detect aborted / crashed writers and recover from that. + Use the access object in FS to set the shared mem values. */ +static svn_error_t * +read_revprop_generation(apr_int64_t *generation, + svn_fs_t *fs, + apr_pool_t *pool) +{ + apr_int64_t current = 0; + fs_fs_data_t *ffd = fs->fsap_data; + + /* read the current revprop generation number */ + SVN_ERR(ensure_revprop_generation(fs, pool)); + SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation)); + + /* is an unfinished revprop write under the way? */ + if (current % 2) + { + apr_int64_t timeout = 0; + + /* read timeout for the write operation */ + SVN_ERR(ensure_revprop_timeout(fs)); + SVN_ERR(svn_named_atomic__read(&timeout, ffd->revprop_timeout)); + + /* has the writer process been aborted, + * i.e. has the timeout been reached? + */ + if (apr_time_now() > timeout) + { + revprop_generation_upgrade_t baton; + baton.generation = ¤t; + baton.ffd = ffd; + + /* Ensure that the original writer process no longer exists by + * acquiring the write lock to this repository. Then, fix up + * the revprop generation. + */ + if (ffd->has_write_lock) + SVN_ERR(revprop_generation_fixup(&baton, pool)); + else + SVN_ERR(svn_fs_fs__with_write_lock(fs, revprop_generation_fixup, + &baton, pool)); + } + } + + /* return the value we just got */ + *generation = current; + return SVN_NO_ERROR; +} + +/* Set the revprop generation to the next odd number to indicate that + there is a revprop write process under way. If that times out, + readers shall recover from that state & re-read revprops. + Use the access object in FS to set the shared mem value. */ +static svn_error_t * +begin_revprop_change(svn_fs_t *fs, apr_pool_t *pool) +{ + apr_int64_t current; + fs_fs_data_t *ffd = fs->fsap_data; + + /* set the timeout for the write operation */ + SVN_ERR(ensure_revprop_timeout(fs)); + SVN_ERR(svn_named_atomic__write(NULL, + apr_time_now() + REVPROP_CHANGE_TIMEOUT, + ffd->revprop_timeout)); + + /* set the revprop generation to an odd value to indicate + * that a write is in progress + */ + SVN_ERR(ensure_revprop_generation(fs, pool)); + do + { + SVN_ERR(svn_named_atomic__add(¤t, + 1, + ffd->revprop_generation)); + } + while (current % 2 == 0); + + return SVN_NO_ERROR; +} + +/* Set the revprop generation to the next even number to indicate that + a) readers shall re-read revprops, and + b) the write process has been completed (no recovery required) + Use the access object in FS to set the shared mem value. */ +static svn_error_t * +end_revprop_change(svn_fs_t *fs, apr_pool_t *pool) +{ + apr_int64_t current = 1; + fs_fs_data_t *ffd = fs->fsap_data; + + /* set the revprop generation to an even value to indicate + * that a write has been completed + */ + SVN_ERR(ensure_revprop_generation(fs, pool)); + do + { + SVN_ERR(svn_named_atomic__add(¤t, + 1, + ffd->revprop_generation)); + } + while (current % 2); + + /* Save the latest generation to disk. FS is currently in a "locked" + * state such that we can be sure the be the only ones to write that + * file. + */ + return write_revprop_generation_file(fs, current, pool); +} + +/* Container for all data required to access the packed revprop file + * for a given REVISION. This structure will be filled incrementally + * by read_pack_revprops() its sub-routines. + */ +typedef struct packed_revprops_t +{ + /* revision number to read (not necessarily the first in the pack) */ + svn_revnum_t revision; + + /* current revprop generation. Used when populating the revprop cache */ + apr_int64_t generation; + + /* the actual revision properties */ + apr_hash_t *properties; + + /* their size when serialized to a single string + * (as found in PACKED_REVPROPS) */ + apr_size_t serialized_size; + + + /* name of the pack file (without folder path) */ + const char *filename; + + /* packed shard folder path */ + const char *folder; + + /* sum of values in SIZES */ + apr_size_t total_size; + + /* first revision in the pack (>= MANIFEST_START) */ + svn_revnum_t start_revision; + + /* size of the revprops in PACKED_REVPROPS */ + apr_array_header_t *sizes; + + /* offset of the revprops in PACKED_REVPROPS */ + apr_array_header_t *offsets; + + + /* concatenation of the serialized representation of all revprops + * in the pack, i.e. the pack content without header and compression */ + svn_stringbuf_t *packed_revprops; + + /* First revision covered by MANIFEST. + * Will equal the shard start revision or 1, for the 1st shard. */ + svn_revnum_t manifest_start; + + /* content of the manifest. + * Maps long(rev - MANIFEST_START) to const char* pack file name */ + apr_array_header_t *manifest; +} packed_revprops_t; + +/* Parse the serialized revprops in CONTENT and return them in *PROPERTIES. + * Also, put them into the revprop cache, if activated, for future use. + * Three more parameters are being used to update the revprop cache: FS is + * our file system, the revprops belong to REVISION and the global revprop + * GENERATION is used as well. + * + * The returned hash will be allocated in POOL, SCRATCH_POOL is being used + * for temporary allocations. + */ +static svn_error_t * +parse_revprop(apr_hash_t **properties, + svn_fs_t *fs, + svn_revnum_t revision, + apr_int64_t generation, + svn_string_t *content, + apr_pool_t *pool, + apr_pool_t *scratch_pool) +{ + svn_stream_t *stream = svn_stream_from_string(content, scratch_pool); + *properties = apr_hash_make(pool); + + SVN_ERR(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool)); + if (has_revprop_cache(fs, pool)) + { + fs_fs_data_t *ffd = fs->fsap_data; + pair_cache_key_t key = { 0 }; + + key.revision = revision; + key.second = generation; + SVN_ERR(svn_cache__set(ffd->revprop_cache, &key, *properties, + scratch_pool)); + } + + return SVN_NO_ERROR; +} + +/* Read the non-packed revprops for revision REV in FS, put them into the + * revprop cache if activated and return them in *PROPERTIES. GENERATION + * is the current revprop generation. + * + * If the data could not be read due to an otherwise recoverable error, + * leave *PROPERTIES unchanged. No error will be returned in that case. + * + * Allocations will be done in POOL. + */ +static svn_error_t * +read_non_packed_revprop(apr_hash_t **properties, + svn_fs_t *fs, + svn_revnum_t rev, + apr_int64_t generation, + apr_pool_t *pool) +{ + svn_stringbuf_t *content = NULL; + apr_pool_t *iterpool = svn_pool_create(pool); + svn_boolean_t missing = FALSE; + int i; + + for (i = 0; i < RECOVERABLE_RETRY_COUNT && !missing && !content; ++i) + { + svn_pool_clear(iterpool); + SVN_ERR(try_stringbuf_from_file(&content, + &missing, + path_revprops(fs, rev, iterpool), + i + 1 < RECOVERABLE_RETRY_COUNT, + iterpool)); + } + + if (content) + SVN_ERR(parse_revprop(properties, fs, rev, generation, + svn_stringbuf__morph_into_string(content), + pool, iterpool)); + + svn_pool_clear(iterpool); + + return SVN_NO_ERROR; +} + +/* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST + * members. Use POOL for allocating results and SCRATCH_POOL for temporaries. + */ +static svn_error_t * +get_revprop_packname(svn_fs_t *fs, + packed_revprops_t *revprops, + apr_pool_t *pool, + apr_pool_t *scratch_pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + svn_stringbuf_t *content = NULL; + const char *manifest_file_path; + int idx; + + /* read content of the manifest file */ + revprops->folder = path_revprops_pack_shard(fs, revprops->revision, pool); + manifest_file_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); + + SVN_ERR(read_content(&content, manifest_file_path, pool)); + + /* parse the manifest. Every line is a file name */ + revprops->manifest = apr_array_make(pool, ffd->max_files_per_dir, + sizeof(const char*)); + + /* Read all lines. Since the last line ends with a newline, we will + end up with a valid but empty string after the last entry. */ + while (content->data && *content->data) + { + APR_ARRAY_PUSH(revprops->manifest, const char*) = content->data; + content->data = strchr(content->data, '\n'); + if (content->data) + { + *content->data = 0; + content->data++; + } + } + + /* Index for our revision. Rev 0 is excluded from the first shard. */ + revprops->manifest_start = revprops->revision + - (revprops->revision % ffd->max_files_per_dir); + if (revprops->manifest_start == 0) + ++revprops->manifest_start; + idx = (int)(revprops->revision - revprops->manifest_start); + + if (revprops->manifest->nelts <= idx) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Packed revprop manifest for r%ld too " + "small"), revprops->revision); + + /* Now get the file name */ + revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*); + + return SVN_NO_ERROR; +} + +/* Return TRUE, if revision R1 and R2 refer to the same shard in FS. + */ +static svn_boolean_t +same_shard(svn_fs_t *fs, + svn_revnum_t r1, + svn_revnum_t r2) +{ + fs_fs_data_t *ffd = fs->fsap_data; + return (r1 / ffd->max_files_per_dir) == (r2 / ffd->max_files_per_dir); +} + +/* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS, + * fill the START_REVISION, SIZES, OFFSETS members. Also, make + * PACKED_REVPROPS point to the first serialized revprop. + * + * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as + * well as the SERIALIZED_SIZE member. If revprop caching has been + * enabled, parse all revprops in the pack and cache them. + */ +static svn_error_t * +parse_packed_revprops(svn_fs_t *fs, + packed_revprops_t *revprops, + apr_pool_t *pool, + apr_pool_t *scratch_pool) +{ + svn_stream_t *stream; + apr_int64_t first_rev, count, i; + apr_off_t offset; + const char *header_end; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + + /* decompress (even if the data is only "stored", there is still a + * length header to remove) */ + svn_string_t *compressed + = svn_stringbuf__morph_into_string(revprops->packed_revprops); + svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(pool); + SVN_ERR(svn__decompress(compressed, uncompressed, APR_SIZE_MAX)); + + /* read first revision number and number of revisions in the pack */ + stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); + SVN_ERR(read_number_from_stream(&first_rev, NULL, stream, iterpool)); + SVN_ERR(read_number_from_stream(&count, NULL, stream, iterpool)); + + /* Check revision range for validity. */ + if ( !same_shard(fs, revprops->revision, first_rev) + || !same_shard(fs, revprops->revision, first_rev + count - 1) + || count < 1) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Revprop pack for revision r%ld" + " contains revprops for r%ld .. r%ld"), + revprops->revision, + (svn_revnum_t)first_rev, + (svn_revnum_t)(first_rev + count -1)); + + /* Since start & end are in the same shard, it is enough to just test + * the FIRST_REV for being actually packed. That will also cover the + * special case of rev 0 never being packed. */ + if (!is_packed_revprop(fs, first_rev)) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Revprop pack for revision r%ld" + " starts at non-packed revisions r%ld"), + revprops->revision, (svn_revnum_t)first_rev); + + /* make PACKED_REVPROPS point to the first char after the header. + * This is where the serialized revprops are. */ + header_end = strstr(uncompressed->data, "\n\n"); + if (header_end == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Header end not found")); + + offset = header_end - uncompressed->data + 2; + + revprops->packed_revprops = svn_stringbuf_create_empty(pool); + revprops->packed_revprops->data = uncompressed->data + offset; + revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset); + revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset); + + /* STREAM still points to the first entry in the sizes list. + * Init / construct REVPROPS members. */ + revprops->start_revision = (svn_revnum_t)first_rev; + revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset)); + revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset)); + + /* Now parse, revision by revision, the size and content of each + * revisions' revprops. */ + for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i) + { + apr_int64_t size; + svn_string_t serialized; + apr_hash_t *properties; + svn_revnum_t revision = (svn_revnum_t)(first_rev + i); + + /* read & check the serialized size */ + SVN_ERR(read_number_from_stream(&size, NULL, stream, iterpool)); + if (size + offset > (apr_int64_t)revprops->packed_revprops->len) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Packed revprop size exceeds pack file size")); + + /* Parse this revprops list, if necessary */ + serialized.data = revprops->packed_revprops->data + offset; + serialized.len = (apr_size_t)size; + + if (revision == revprops->revision) + { + SVN_ERR(parse_revprop(&revprops->properties, fs, revision, + revprops->generation, &serialized, + pool, iterpool)); + revprops->serialized_size = serialized.len; + } + else + { + /* If revprop caching is enabled, parse any revprops. + * They will get cached as a side-effect of this. */ + if (has_revprop_cache(fs, pool)) + SVN_ERR(parse_revprop(&properties, fs, revision, + revprops->generation, &serialized, + iterpool, iterpool)); + } + + /* fill REVPROPS data structures */ + APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len; + APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset; + revprops->total_size += serialized.len; + + offset += serialized.len; + + svn_pool_clear(iterpool); + } + + return SVN_NO_ERROR; +} + +/* In filesystem FS, read the packed revprops for revision REV into + * *REVPROPS. Use GENERATION to populate the revprop cache, if enabled. + * Allocate data in POOL. + */ +static svn_error_t * +read_pack_revprop(packed_revprops_t **revprops, svn_fs_t *fs, svn_revnum_t rev, + apr_int64_t generation, apr_pool_t *pool) { - apr_hash_t *proplist; + apr_pool_t *iterpool = svn_pool_create(pool); + svn_boolean_t missing = FALSE; + svn_error_t *err; + packed_revprops_t *result; + int i; + + /* someone insisted that REV is packed. Double-check if necessary */ + if (!is_packed_revprop(fs, rev)) + SVN_ERR(update_min_unpacked_rev(fs, iterpool)); + + if (!is_packed_revprop(fs, rev)) + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("No such packed revision %ld"), rev); + + /* initialize the result data structure */ + result = apr_pcalloc(pool, sizeof(*result)); + result->revision = rev; + result->generation = generation; + + /* try to read the packed revprops. This may require retries if we have + * concurrent writers. */ + for (i = 0; i < RECOVERABLE_RETRY_COUNT && !result->packed_revprops; ++i) + { + const char *file_path; + + /* there might have been concurrent writes. + * Re-read the manifest and the pack file. + */ + SVN_ERR(get_revprop_packname(fs, result, pool, iterpool)); + file_path = svn_dirent_join(result->folder, + result->filename, + iterpool); + SVN_ERR(try_stringbuf_from_file(&result->packed_revprops, + &missing, + file_path, + i + 1 < RECOVERABLE_RETRY_COUNT, + pool)); + + /* If we could not find the file, there was a write. + * So, we should refresh our revprop generation info as well such + * that others may find data we will put into the cache. They would + * consider it outdated, otherwise. + */ + if (missing && has_revprop_cache(fs, pool)) + SVN_ERR(read_revprop_generation(&result->generation, fs, pool)); + + svn_pool_clear(iterpool); + } + + /* the file content should be available now */ + if (!result->packed_revprops) + return svn_error_createf(SVN_ERR_FS_PACKED_REVPROP_READ_FAILURE, NULL, + _("Failed to read revprop pack file for r%ld"), rev); + /* parse it. RESULT will be complete afterwards. */ + err = parse_packed_revprops(fs, result, pool, iterpool); + svn_pool_destroy(iterpool); + if (err) + return svn_error_createf(SVN_ERR_FS_CORRUPT, err, + _("Revprop pack file for r%ld is corrupt"), rev); + + *revprops = result; + + return SVN_NO_ERROR; +} + +/* Read the revprops for revision REV in FS and return them in *PROPERTIES_P. + * + * Allocations will be done in POOL. + */ +static svn_error_t * +get_revision_proplist(apr_hash_t **proplist_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + apr_int64_t generation = 0; + + /* not found, yet */ + *proplist_p = NULL; + + /* should they be available at all? */ SVN_ERR(ensure_revision_exists(fs, rev, pool)); - if (1) + /* Try cache lookup first. */ + if (has_revprop_cache(fs, pool)) { - apr_file_t *revprop_file = NULL; - svn_error_t *err = SVN_NO_ERROR; - int i; - apr_pool_t *iterpool; + svn_boolean_t is_cached; + pair_cache_key_t key = { 0 }; - proplist = apr_hash_make(pool); - iterpool = svn_pool_create(pool); - for (i = 0; i < RECOVERABLE_RETRY_COUNT; i++) + SVN_ERR(read_revprop_generation(&generation, fs, pool)); + + key.revision = rev; + key.second = generation; + SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached, + ffd->revprop_cache, &key, pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + /* if REV had not been packed when we began, try reading it from the + * non-packed shard. If that fails, we will fall through to packed + * shard reads. */ + if (!is_packed_revprop(fs, rev)) + { + svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev, + generation, pool); + if (err) { - svn_pool_clear(iterpool); + if (!APR_STATUS_IS_ENOENT(err->apr_err) + || ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) + return svn_error_trace(err); - /* Clear err here rather than after finding a recoverable error so - * we can return that error on the last iteration of the loop. */ svn_error_clear(err); - err = svn_io_file_open(&revprop_file, path_revprops(fs, rev, - iterpool), - APR_READ | APR_BUFFERED, APR_OS_DEFAULT, - iterpool); - if (err) - { - if (APR_STATUS_IS_ENOENT(err->apr_err)) - { - svn_error_clear(err); - return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, - _("No such revision %ld"), rev); - } -#ifdef ESTALE - else if (APR_TO_OS_ERROR(err->apr_err) == ESTALE - || APR_TO_OS_ERROR(err->apr_err) == EIO - || APR_TO_OS_ERROR(err->apr_err) == ENOENT) - continue; -#endif - return svn_error_trace(err); - } + *proplist_p = NULL; /* in case read_non_packed_revprop changed it */ + } + } - SVN_ERR(svn_hash__clear(proplist, iterpool)); - RETRY_RECOVERABLE(err, revprop_file, - svn_hash_read2(proplist, - svn_stream_from_aprfile2( - revprop_file, TRUE, iterpool), - SVN_HASH_TERMINATOR, pool)); + /* if revprop packing is available and we have not read the revprops, yet, + * try reading them from a packed shard. If that fails, REV is most + * likely invalid (or its revprops highly contested). */ + if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p) + { + packed_revprops_t *packed_revprops; + SVN_ERR(read_pack_revprop(&packed_revprops, fs, rev, generation, pool)); + *proplist_p = packed_revprops->properties; + } - IGNORE_RECOVERABLE(err, svn_io_file_close(revprop_file, iterpool)); + /* The revprops should have been there. Did we get them? */ + if (!*proplist_p) + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("Could not read revprops for revision %ld"), + rev); - break; + return SVN_NO_ERROR; +} + +/* Serialize the revision property list PROPLIST of revision REV in + * filesystem FS to a non-packed file. Return the name of that temporary + * file in *TMP_PATH and the file path that it must be moved to in + * *FINAL_PATH. + * + * Use POOL for allocations. + */ +static svn_error_t * +write_non_packed_revprop(const char **final_path, + const char **tmp_path, + svn_fs_t *fs, + svn_revnum_t rev, + apr_hash_t *proplist, + apr_pool_t *pool) +{ + svn_stream_t *stream; + *final_path = path_revprops(fs, rev, pool); + + /* ### do we have a directory sitting around already? we really shouldn't + ### have to get the dirname here. */ + SVN_ERR(svn_stream_open_unique(&stream, tmp_path, + svn_dirent_dirname(*final_path, pool), + svn_io_file_del_none, pool, pool)); + SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_stream_close(stream)); + + return SVN_NO_ERROR; +} + +/* After writing the new revprop file(s), call this function to move the + * file at TMP_PATH to FINAL_PATH and give it the permissions from + * PERMS_REFERENCE. + * + * If indicated in BUMP_GENERATION, increase FS' revprop generation. + * Finally, delete all the temporary files given in FILES_TO_DELETE. + * The latter may be NULL. + * + * Use POOL for temporary allocations. + */ +static svn_error_t * +switch_to_new_revprop(svn_fs_t *fs, + const char *final_path, + const char *tmp_path, + const char *perms_reference, + apr_array_header_t *files_to_delete, + svn_boolean_t bump_generation, + apr_pool_t *pool) +{ + /* Now, we may actually be replacing revprops. Make sure that all other + threads and processes will know about this. */ + if (bump_generation) + SVN_ERR(begin_revprop_change(fs, pool)); + + SVN_ERR(move_into_place(tmp_path, final_path, perms_reference, pool)); + + /* Indicate that the update (if relevant) has been completed. */ + if (bump_generation) + SVN_ERR(end_revprop_change(fs, pool)); + + /* Clean up temporary files, if necessary. */ + if (files_to_delete) + { + apr_pool_t *iterpool = svn_pool_create(pool); + int i; + + for (i = 0; i < files_to_delete->nelts; ++i) + { + const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*); + SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); + svn_pool_clear(iterpool); } - if (err) - return svn_error_trace(err); svn_pool_destroy(iterpool); } + return SVN_NO_ERROR; +} - *proplist_p = proplist; +/* Write a pack file header to STREAM that starts at revision START_REVISION + * and contains the indexes [START,END) of SIZES. + */ +static svn_error_t * +serialize_revprops_header(svn_stream_t *stream, + svn_revnum_t start_revision, + apr_array_header_t *sizes, + int start, + int end, + apr_pool_t *pool) +{ + apr_pool_t *iterpool = svn_pool_create(pool); + int i; + + SVN_ERR_ASSERT(start < end); + + /* start revision and entry count */ + SVN_ERR(svn_stream_printf(stream, pool, "%ld\n", start_revision)); + SVN_ERR(svn_stream_printf(stream, pool, "%d\n", end - start)); + + /* the sizes array */ + for (i = start; i < end; ++i) + { + apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t); + SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n", + size)); + } + + /* the double newline char indicates the end of the header */ + SVN_ERR(svn_stream_printf(stream, iterpool, "\n")); + + svn_pool_clear(iterpool); + return SVN_NO_ERROR; +} + +/* Writes the a pack file to FILE_STREAM. It copies the serialized data + * from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX. + * + * The data for the latter is taken from NEW_SERIALIZED. Note, that + * CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is + * taken in that case but only a subset of the old data will be copied. + * + * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size. + * POOL is used for temporary allocations. + */ +static svn_error_t * +repack_revprops(svn_fs_t *fs, + packed_revprops_t *revprops, + int start, + int end, + int changed_index, + svn_stringbuf_t *new_serialized, + apr_off_t new_total_size, + svn_stream_t *file_stream, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + svn_stream_t *stream; + int i; + + /* create data empty buffers and the stream object */ + svn_stringbuf_t *uncompressed + = svn_stringbuf_create_ensure((apr_size_t)new_total_size, pool); + svn_stringbuf_t *compressed + = svn_stringbuf_create_empty(pool); + stream = svn_stream_from_stringbuf(uncompressed, pool); + + /* write the header*/ + SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start, + revprops->sizes, start, end, pool)); + + /* append the serialized revprops */ + for (i = start; i < end; ++i) + if (i == changed_index) + { + SVN_ERR(svn_stream_write(stream, + new_serialized->data, + &new_serialized->len)); + } + else + { + apr_size_t size + = (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t); + apr_size_t offset + = (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t); + + SVN_ERR(svn_stream_write(stream, + revprops->packed_revprops->data + offset, + &size)); + } + + /* flush the stream buffer (if any) to our underlying data buffer */ + SVN_ERR(svn_stream_close(stream)); + + /* compress / store the data */ + SVN_ERR(svn__compress(svn_stringbuf__morph_into_string(uncompressed), + compressed, + ffd->compress_packed_revprops + ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT + : SVN_DELTA_COMPRESSION_LEVEL_NONE)); + + /* finally, write the content to the target stream and close it */ + SVN_ERR(svn_stream_write(file_stream, compressed->data, &compressed->len)); + SVN_ERR(svn_stream_close(file_stream)); + + return SVN_NO_ERROR; +} + +/* Allocate a new pack file name for revisions + * [REVPROPS->START_REVISION + START, REVPROPS->START_REVISION + END - 1] + * of REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE, + * auto-create that array if necessary. Return an open file stream to + * the new file in *STREAM allocated in POOL. + */ +static svn_error_t * +repack_stream_open(svn_stream_t **stream, + svn_fs_t *fs, + packed_revprops_t *revprops, + int start, + int end, + apr_array_header_t **files_to_delete, + apr_pool_t *pool) +{ + apr_int64_t tag; + const char *tag_string; + svn_string_t *new_filename; + int i; + apr_file_t *file; + int manifest_offset + = (int)(revprops->start_revision - revprops->manifest_start); + + /* get the old (= current) file name and enlist it for later deletion */ + const char *old_filename = APR_ARRAY_IDX(revprops->manifest, + start + manifest_offset, + const char*); + + if (*files_to_delete == NULL) + *files_to_delete = apr_array_make(pool, 3, sizeof(const char*)); + + APR_ARRAY_PUSH(*files_to_delete, const char*) + = svn_dirent_join(revprops->folder, old_filename, pool); + + /* increase the tag part, i.e. the counter after the dot */ + tag_string = strchr(old_filename, '.'); + if (tag_string == NULL) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Packed file '%s' misses a tag"), + old_filename); + + SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1)); + new_filename = svn_string_createf(pool, "%ld.%" APR_INT64_T_FMT, + revprops->start_revision + start, + ++tag); + + /* update the manifest to point to the new file */ + for (i = start; i < end; ++i) + APR_ARRAY_IDX(revprops->manifest, i + manifest_offset, const char*) + = new_filename->data; + + /* create a file stream for the new file */ + SVN_ERR(svn_io_file_open(&file, svn_dirent_join(revprops->folder, + new_filename->data, + pool), + APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool)); + *stream = svn_stream_from_aprfile2(file, FALSE, pool); + + return SVN_NO_ERROR; +} + +/* For revision REV in filesystem FS, set the revision properties to + * PROPLIST. Return a new file in *TMP_PATH that the caller shall move + * to *FINAL_PATH to make the change visible. Files to be deleted will + * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated. + * Use POOL for allocations. + */ +static svn_error_t * +write_packed_revprop(const char **final_path, + const char **tmp_path, + apr_array_header_t **files_to_delete, + svn_fs_t *fs, + svn_revnum_t rev, + apr_hash_t *proplist, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + packed_revprops_t *revprops; + apr_int64_t generation = 0; + svn_stream_t *stream; + svn_stringbuf_t *serialized; + apr_off_t new_total_size; + int changed_index; + + /* read the current revprop generation. This value will not change + * while we hold the global write lock to this FS. */ + if (has_revprop_cache(fs, pool)) + SVN_ERR(read_revprop_generation(&generation, fs, pool)); + + /* read contents of the current pack file */ + SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, pool)); + + /* serialize the new revprops */ + serialized = svn_stringbuf_create_empty(pool); + stream = svn_stream_from_stringbuf(serialized, pool); + SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_stream_close(stream)); + + /* calculate the size of the new data */ + changed_index = (int)(rev - revprops->start_revision); + new_total_size = revprops->total_size - revprops->serialized_size + + serialized->len + + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE; + + APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len; + + /* can we put the new data into the same pack as the before? */ + if ( new_total_size < ffd->revprop_pack_size + || revprops->sizes->nelts == 1) + { + /* simply replace the old pack file with new content as we do it + * in the non-packed case */ + + *final_path = svn_dirent_join(revprops->folder, revprops->filename, + pool); + SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder, + svn_io_file_del_none, pool, pool)); + SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts, + changed_index, serialized, new_total_size, + stream, pool)); + } + else + { + /* split the pack file into two of roughly equal size */ + int right_count, left_count, i; + + int left = 0; + int right = revprops->sizes->nelts - 1; + apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE; + apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE; + + /* let left and right side grow such that their size difference + * is minimal after each step. */ + while (left <= right) + if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) + < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)) + { + left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) + + SVN_INT64_BUFFER_SIZE; + ++left; + } + else + { + right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t) + + SVN_INT64_BUFFER_SIZE; + --right; + } + + /* since the items need much less than SVN_INT64_BUFFER_SIZE + * bytes to represent their length, the split may not be optimal */ + left_count = left; + right_count = revprops->sizes->nelts - left; + + /* if new_size is large, one side may exceed the pack size limit. + * In that case, split before and after the modified revprop.*/ + if ( left_size > ffd->revprop_pack_size + || right_size > ffd->revprop_pack_size) + { + left_count = changed_index; + right_count = revprops->sizes->nelts - left_count - 1; + } + + /* write the new, split files */ + if (left_count) + { + SVN_ERR(repack_stream_open(&stream, fs, revprops, 0, + left_count, files_to_delete, pool)); + SVN_ERR(repack_revprops(fs, revprops, 0, left_count, + changed_index, serialized, new_total_size, + stream, pool)); + } + + if (left_count + right_count < revprops->sizes->nelts) + { + SVN_ERR(repack_stream_open(&stream, fs, revprops, changed_index, + changed_index + 1, files_to_delete, + pool)); + SVN_ERR(repack_revprops(fs, revprops, changed_index, + changed_index + 1, + changed_index, serialized, new_total_size, + stream, pool)); + } + + if (right_count) + { + SVN_ERR(repack_stream_open(&stream, fs, revprops, + revprops->sizes->nelts - right_count, + revprops->sizes->nelts, + files_to_delete, pool)); + SVN_ERR(repack_revprops(fs, revprops, + revprops->sizes->nelts - right_count, + revprops->sizes->nelts, changed_index, + serialized, new_total_size, stream, + pool)); + } + + /* write the new manifest */ + *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); + SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder, + svn_io_file_del_none, pool, pool)); + + for (i = 0; i < revprops->manifest->nelts; ++i) + { + const char *filename = APR_ARRAY_IDX(revprops->manifest, i, + const char*); + SVN_ERR(svn_stream_printf(stream, pool, "%s\n", filename)); + } + + SVN_ERR(svn_stream_close(stream)); + } + + return SVN_NO_ERROR; +} + +/* Set the revision property list of revision REV in filesystem FS to + PROPLIST. Use POOL for temporary allocations. */ +static svn_error_t * +set_revision_proplist(svn_fs_t *fs, + svn_revnum_t rev, + apr_hash_t *proplist, + apr_pool_t *pool) +{ + svn_boolean_t is_packed; + svn_boolean_t bump_generation = FALSE; + const char *final_path; + const char *tmp_path; + const char *perms_reference; + apr_array_header_t *files_to_delete = NULL; + + SVN_ERR(ensure_revision_exists(fs, rev, pool)); + + /* this info will not change while we hold the global FS write lock */ + is_packed = is_packed_revprop(fs, rev); + + /* Test whether revprops already exist for this revision. + * Only then will we need to bump the revprop generation. */ + if (has_revprop_cache(fs, pool)) + { + if (is_packed) + { + bump_generation = TRUE; + } + else + { + svn_node_kind_t kind; + SVN_ERR(svn_io_check_path(path_revprops(fs, rev, pool), &kind, + pool)); + bump_generation = kind != svn_node_none; + } + } + + /* Serialize the new revprop data */ + if (is_packed) + SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete, + fs, rev, proplist, pool)); + else + SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path, + fs, rev, proplist, pool)); + + /* We use the rev file of this revision as the perms reference, + * because when setting revprops for the first time, the revprop + * file won't exist and therefore can't serve as its own reference. + * (Whereas the rev file should already exist at this point.) + */ + SVN_ERR(svn_fs_fs__path_rev_absolute(&perms_reference, fs, rev, pool)); + + /* Now, switch to the new revprop data. */ + SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference, + files_to_delete, bump_generation, pool)); return SVN_NO_ERROR; } @@ -3047,7 +4437,7 @@ svn_fs_fs__revision_proplist(apr_hash_t **proplist_p, svn_revnum_t rev, apr_pool_t *pool) { - SVN_ERR(revision_proplist(proplist_p, fs, rev, pool)); + SVN_ERR(get_revision_proplist(proplist_p, fs, rev, pool)); return SVN_NO_ERROR; } @@ -3059,6 +4449,8 @@ struct rep_state apr_file_t *file; /* The txdelta window cache to use or NULL. */ svn_cache__t *window_cache; + /* Caches un-deltified windows. May be NULL. */ + svn_cache__t *combined_cache; apr_off_t start; /* The starting offset for the raw svndiff/plaintext data minus header. */ apr_off_t off; /* The current offset into the file. */ @@ -3071,6 +4463,8 @@ struct rep_state static svn_error_t * create_rep_state_body(struct rep_state **rep_state, struct rep_args **rep_args, + apr_file_t **file_hint, + svn_revnum_t *rev_hint, representation_t *rep, svn_fs_t *fs, apr_pool_t *pool) @@ -3080,8 +4474,47 @@ create_rep_state_body(struct rep_state **rep_state, struct rep_args *ra; unsigned char buf[4]; - SVN_ERR(open_and_seek_representation(&rs->file, fs, rep, pool)); + /* If the hint is + * - given, + * - refers to a valid revision, + * - refers to a packed revision, + * - as does the rep we want to read, and + * - refers to the same pack file as the rep + * ... + */ + if ( file_hint && rev_hint && *file_hint + && SVN_IS_VALID_REVNUM(*rev_hint) + && *rev_hint < ffd->min_unpacked_rev + && rep->revision < ffd->min_unpacked_rev + && ( (*rev_hint / ffd->max_files_per_dir) + == (rep->revision / ffd->max_files_per_dir))) + { + /* ... we can re-use the same, already open file object + */ + apr_off_t offset; + SVN_ERR(get_packed_offset(&offset, fs, rep->revision, pool)); + + offset += rep->offset; + SVN_ERR(svn_io_file_seek(*file_hint, APR_SET, &offset, pool)); + + rs->file = *file_hint; + } + else + { + /* otherwise, create a new file object + */ + SVN_ERR(open_and_seek_representation(&rs->file, fs, rep, pool)); + } + + /* remember the current file, if suggested by the caller */ + if (file_hint) + *file_hint = rs->file; + if (rev_hint) + *rev_hint = rep->revision; + + /* continue constructing RS and RA */ rs->window_cache = ffd->txdelta_window_cache; + rs->combined_cache = ffd->combined_window_cache; SVN_ERR(read_rep_line(&ra, rs->file, pool)); SVN_ERR(get_file_offset(&rs->start, rs->file, pool)); @@ -3090,7 +4523,7 @@ create_rep_state_body(struct rep_state **rep_state, *rep_state = rs; *rep_args = ra; - if (ra->is_delta == FALSE) + if (!ra->is_delta) /* This is a plaintext, so just return the current rep_state. */ return SVN_NO_ERROR; @@ -3111,15 +4544,26 @@ create_rep_state_body(struct rep_state **rep_state, /* Read the rep args for REP in filesystem FS and create a rep_state for reading the representation. Return the rep_state in *REP_STATE - and the rep args in *REP_ARGS, both allocated in POOL. */ + and the rep args in *REP_ARGS, both allocated in POOL. + + When reading multiple reps, i.e. a skip delta chain, you may provide + non-NULL FILE_HINT and REV_HINT. (If FILE_HINT is not NULL, in the first + call it should be a pointer to NULL.) The function will use these variables + to store the previous call results and tries to re-use them. This may + result in significant savings in I/O for packed files. + */ static svn_error_t * create_rep_state(struct rep_state **rep_state, struct rep_args **rep_args, + apr_file_t **file_hint, + svn_revnum_t *rev_hint, representation_t *rep, svn_fs_t *fs, apr_pool_t *pool) { - svn_error_t *err = create_rep_state_body(rep_state, rep_args, rep, fs, pool); + svn_error_t *err = create_rep_state_body(rep_state, rep_args, + file_hint, rev_hint, + rep, fs, pool); if (err && err->apr_err == SVN_ERR_FS_CORRUPT) { fs_fs_data_t *ffd = fs->fsap_data; @@ -3133,66 +4577,23 @@ create_rep_state(struct rep_state **rep_state, ### going to jump straight to this comment anyway! */ return svn_error_createf(SVN_ERR_FS_CORRUPT, err, "Corrupt representation '%s'", - rep + rep ? representation_string(rep, ffd->format, TRUE, - pool) + TRUE, pool) : "(null)"); } /* ### Call representation_string() ? */ return svn_error_trace(err); } -/* Build an array of rep_state structures in *LIST giving the delta - reps from first_rep to a plain-text or self-compressed rep. Set - *SRC_STATE to the plain-text rep we find at the end of the chain, - or to NULL if the final delta representation is self-compressed. - The representation to start from is designated by filesystem FS, id - ID, and representation REP. */ -static svn_error_t * -build_rep_list(apr_array_header_t **list, - struct rep_state **src_state, - svn_fs_t *fs, - representation_t *first_rep, - apr_pool_t *pool) -{ - representation_t rep; - struct rep_state *rs; - struct rep_args *rep_args; - - *list = apr_array_make(pool, 1, sizeof(struct rep_state *)); - rep = *first_rep; - - while (1) - { - SVN_ERR(create_rep_state(&rs, &rep_args, &rep, fs, pool)); - if (rep_args->is_delta == FALSE) - { - /* This is a plaintext, so just return the current rep_state. */ - *src_state = rs; - return SVN_NO_ERROR; - } - - /* Push this rep onto the list. If it's self-compressed, we're done. */ - APR_ARRAY_PUSH(*list, struct rep_state *) = rs; - if (rep_args->is_delta_vs_empty) - { - *src_state = NULL; - return SVN_NO_ERROR; - } - - rep.revision = rep_args->base_revision; - rep.offset = rep_args->base_offset; - rep.size = rep_args->base_length; - rep.txn_id = NULL; - } -} - - struct rep_read_baton { /* The FS from which we're reading. */ svn_fs_t *fs; + /* If not NULL, this is the base for the first delta window in rs_list */ + svn_stringbuf_t *base_window; + /* The state of all prior delta representations. */ apr_array_header_t *rs_list; @@ -3225,7 +4626,7 @@ struct rep_read_baton /* The key for the fulltext cache for this rep, if there is a fulltext cache. */ - const char *fulltext_cache_key; + pair_cache_key_t fulltext_cache_key; /* The text we've been reading, if we're going to cache it. */ svn_stringbuf_t *current_fulltext; @@ -3237,51 +4638,9 @@ struct rep_read_baton apr_pool_t *filehandle_pool; }; -/* Create a rep_read_baton structure for node revision NODEREV in - filesystem FS and store it in *RB_P. If FULLTEXT_CACHE_KEY is not - NULL, it is the rep's key in the fulltext cache, and a stringbuf - must be allocated to store the text. Perform all allocations in - POOL. If rep is mutable, it must be for file contents. */ -static svn_error_t * -rep_read_get_baton(struct rep_read_baton **rb_p, - svn_fs_t *fs, - representation_t *rep, - const char *fulltext_cache_key, - apr_pool_t *pool) -{ - struct rep_read_baton *b; - - b = apr_pcalloc(pool, sizeof(*b)); - b->fs = fs; - b->chunk_index = 0; - b->buf = NULL; - b->md5_checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); - b->checksum_finalized = FALSE; - b->md5_checksum = svn_checksum_dup(rep->md5_checksum, pool); - b->len = rep->expanded_size ? rep->expanded_size : rep->size; - b->off = 0; - b->fulltext_cache_key = fulltext_cache_key; - b->pool = svn_pool_create(pool); - b->filehandle_pool = svn_pool_create(pool); - - if (fulltext_cache_key) - b->current_fulltext = svn_stringbuf_create_ensure - ((apr_size_t)b->len, - b->filehandle_pool); - else - b->current_fulltext = NULL; - - SVN_ERR(build_rep_list(&b->rs_list, &b->src_state, fs, rep, - b->filehandle_pool)); - - /* Save our output baton. */ - *rb_p = b; - - return SVN_NO_ERROR; -} - /* Combine the name of the rev file in RS with the given OFFSET to form - * a cache lookup key. Allocations will be made from POOL. */ + * a cache lookup key. Allocations will be made from POOL. May return + * NULL if the key cannot be constructed. */ static const char* get_window_key(struct rep_state *rs, apr_off_t offset, apr_pool_t *pool) { @@ -3295,7 +4654,7 @@ get_window_key(struct rep_state *rs, apr_off_t offset, apr_pool_t *pool) * comparison _will_ find them. */ if (apr_file_name_get(&name, rs->file)) - return ""; + return NULL; /* Handle packed files as well by scanning backwards until we find the * revision or pack number. */ @@ -3383,6 +4742,7 @@ set_cached_window(svn_txdelta_window_t *window, { /* store the window and the first offset _past_ it */ svn_fs_fs__txdelta_cached_window_t cached_window; + cached_window.window = window; cached_window.end_offset = rs->off; @@ -3397,11 +4757,200 @@ set_cached_window(svn_txdelta_window_t *window, return SVN_NO_ERROR; } +/* Read the WINDOW_P for the rep state RS from the current FSFS session's + * cache. This will be a no-op and IS_CACHED will be set to FALSE if no + * cache has been given. If a cache is available IS_CACHED will inform + * the caller about the success of the lookup. Allocations (of the window + * in particualar) will be made from POOL. + */ +static svn_error_t * +get_cached_combined_window(svn_stringbuf_t **window_p, + struct rep_state *rs, + svn_boolean_t *is_cached, + apr_pool_t *pool) +{ + if (! rs->combined_cache) + { + /* txdelta window has not been enabled */ + *is_cached = FALSE; + } + else + { + /* ask the cache for the desired txdelta window */ + return svn_cache__get((void **)window_p, + is_cached, + rs->combined_cache, + get_window_key(rs, rs->start, pool), + pool); + } + + return SVN_NO_ERROR; +} + +/* Store the WINDOW read at OFFSET for the rep state RS in the current + * FSFS session's cache. This will be a no-op if no cache has been given. + * Temporary allocations will be made from SCRATCH_POOL. */ +static svn_error_t * +set_cached_combined_window(svn_stringbuf_t *window, + struct rep_state *rs, + apr_off_t offset, + apr_pool_t *scratch_pool) +{ + if (rs->combined_cache) + { + /* but key it with the start offset because that is the known state + * when we will look it up */ + return svn_cache__set(rs->combined_cache, + get_window_key(rs, offset, scratch_pool), + window, + scratch_pool); + } + + return SVN_NO_ERROR; +} + +/* Build an array of rep_state structures in *LIST giving the delta + reps from first_rep to a plain-text or self-compressed rep. Set + *SRC_STATE to the plain-text rep we find at the end of the chain, + or to NULL if the final delta representation is self-compressed. + The representation to start from is designated by filesystem FS, id + ID, and representation REP. + Also, set *WINDOW_P to the base window content for *LIST, if it + could be found in cache. Otherwise, *LIST will contain the base + representation for the whole delta chain. + Finally, return the expanded size of the representation in + *EXPANDED_SIZE. It will take care of cases where only the on-disk + size is known. */ +static svn_error_t * +build_rep_list(apr_array_header_t **list, + svn_stringbuf_t **window_p, + struct rep_state **src_state, + svn_filesize_t *expanded_size, + svn_fs_t *fs, + representation_t *first_rep, + apr_pool_t *pool) +{ + representation_t rep; + struct rep_state *rs = NULL; + struct rep_args *rep_args; + svn_boolean_t is_cached = FALSE; + apr_file_t *last_file = NULL; + svn_revnum_t last_revision; + + *list = apr_array_make(pool, 1, sizeof(struct rep_state *)); + rep = *first_rep; + + /* The value as stored in the data struct. + 0 is either for unknown length or actually zero length. */ + *expanded_size = first_rep->expanded_size; + + /* for the top-level rep, we need the rep_args */ + SVN_ERR(create_rep_state(&rs, &rep_args, &last_file, + &last_revision, &rep, fs, pool)); + + /* Unknown size or empty representation? + That implies the this being the first iteration. + Usually size equals on-disk size, except for empty, + compressed representations (delta, size = 4). + Please note that for all non-empty deltas have + a 4-byte header _plus_ some data. */ + if (*expanded_size == 0) + if (! rep_args->is_delta || first_rep->size != 4) + *expanded_size = first_rep->size; + + while (1) + { + /* fetch state, if that has not been done already */ + if (!rs) + SVN_ERR(create_rep_state(&rs, &rep_args, &last_file, + &last_revision, &rep, fs, pool)); + + SVN_ERR(get_cached_combined_window(window_p, rs, &is_cached, pool)); + if (is_cached) + { + /* We already have a reconstructed window in our cache. + Write a pseudo rep_state with the full length. */ + rs->off = rs->start; + rs->end = rs->start + (*window_p)->len; + *src_state = rs; + return SVN_NO_ERROR; + } + + if (!rep_args->is_delta) + { + /* This is a plaintext, so just return the current rep_state. */ + *src_state = rs; + return SVN_NO_ERROR; + } + + /* Push this rep onto the list. If it's self-compressed, we're done. */ + APR_ARRAY_PUSH(*list, struct rep_state *) = rs; + if (rep_args->is_delta_vs_empty) + { + *src_state = NULL; + return SVN_NO_ERROR; + } + + rep.revision = rep_args->base_revision; + rep.offset = rep_args->base_offset; + rep.size = rep_args->base_length; + rep.txn_id = NULL; + + rs = NULL; + } +} + + +/* Create a rep_read_baton structure for node revision NODEREV in + filesystem FS and store it in *RB_P. If FULLTEXT_CACHE_KEY is not + NULL, it is the rep's key in the fulltext cache, and a stringbuf + must be allocated to store the text. Perform all allocations in + POOL. If rep is mutable, it must be for file contents. */ +static svn_error_t * +rep_read_get_baton(struct rep_read_baton **rb_p, + svn_fs_t *fs, + representation_t *rep, + pair_cache_key_t fulltext_cache_key, + apr_pool_t *pool) +{ + struct rep_read_baton *b; + + b = apr_pcalloc(pool, sizeof(*b)); + b->fs = fs; + b->base_window = NULL; + b->chunk_index = 0; + b->buf = NULL; + b->md5_checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); + b->checksum_finalized = FALSE; + b->md5_checksum = svn_checksum_dup(rep->md5_checksum, pool); + b->len = rep->expanded_size; + b->off = 0; + b->fulltext_cache_key = fulltext_cache_key; + b->pool = svn_pool_create(pool); + b->filehandle_pool = svn_pool_create(pool); + + SVN_ERR(build_rep_list(&b->rs_list, &b->base_window, + &b->src_state, &b->len, fs, rep, + b->filehandle_pool)); + + if (SVN_IS_VALID_REVNUM(fulltext_cache_key.revision)) + b->current_fulltext = svn_stringbuf_create_ensure + ((apr_size_t)b->len, + b->filehandle_pool); + else + b->current_fulltext = NULL; + + /* Save our output baton. */ + *rb_p = b; + + return SVN_NO_ERROR; +} + /* Skip forwards to THIS_CHUNK in REP_STATE and then read the next delta window into *NWIN. */ static svn_error_t * -read_window(svn_txdelta_window_t **nwin, int this_chunk, struct rep_state *rs, - apr_pool_t *pool) +read_delta_window(svn_txdelta_window_t **nwin, int this_chunk, + struct rep_state *rs, apr_pool_t *pool) { svn_stream_t *stream; svn_boolean_t is_cached; @@ -3409,6 +4958,10 @@ read_window(svn_txdelta_window_t **nwin, int this_chunk, struct rep_state *rs, SVN_ERR_ASSERT(rs->chunk_index <= this_chunk); + /* RS->FILE may be shared between RS instances -> make sure we point + * to the right data. */ + SVN_ERR(svn_io_file_seek(rs->file, APR_SET, &rs->off, pool)); + /* Skip windows to reach the current chunk if we aren't there yet. */ while (rs->chunk_index < this_chunk) { @@ -3444,45 +4997,103 @@ read_window(svn_txdelta_window_t **nwin, int this_chunk, struct rep_state *rs, return set_cached_window(*nwin, rs, old_offset, pool); } -/* Get one delta window that is a result of combining all but the last deltas - from the current desired representation identified in *RB, to its - final base representation. Store the window in *RESULT. */ +/* Read SIZE bytes from the representation RS and return it in *NWIN. */ static svn_error_t * -get_combined_window(svn_txdelta_window_t **result, +read_plain_window(svn_stringbuf_t **nwin, struct rep_state *rs, + apr_size_t size, apr_pool_t *pool) +{ + /* RS->FILE may be shared between RS instances -> make sure we point + * to the right data. */ + SVN_ERR(svn_io_file_seek(rs->file, APR_SET, &rs->off, pool)); + + /* Read the plain data. */ + *nwin = svn_stringbuf_create_ensure(size, pool); + SVN_ERR(svn_io_file_read_full2(rs->file, (*nwin)->data, size, NULL, NULL, + pool)); + (*nwin)->data[size] = 0; + + /* Update RS. */ + rs->off += (apr_off_t)size; + + return SVN_NO_ERROR; +} + +/* Get the undeltified window that is a result of combining all deltas + from the current desired representation identified in *RB with its + base representation. Store the window in *RESULT. */ +static svn_error_t * +get_combined_window(svn_stringbuf_t **result, struct rep_read_baton *rb) { - apr_pool_t *pool, *new_pool; + apr_pool_t *pool, *new_pool, *window_pool; int i; - svn_txdelta_window_t *window, *nwin; + svn_txdelta_window_t *window; + apr_array_header_t *windows; + svn_stringbuf_t *source, *buf = rb->base_window; struct rep_state *rs; - SVN_ERR_ASSERT(rb->rs_list->nelts >= 2); - - pool = svn_pool_create(rb->pool); + /* Read all windows that we need to combine. This is fine because + the size of each window is relatively small (100kB) and skip- + delta limits the number of deltas in a chain to well under 100. + Stop early if one of them does not depend on its predecessors. */ + window_pool = svn_pool_create(rb->pool); + windows = apr_array_make(window_pool, 0, sizeof(svn_txdelta_window_t *)); + for (i = 0; i < rb->rs_list->nelts; ++i) + { + rs = APR_ARRAY_IDX(rb->rs_list, i, struct rep_state *); + SVN_ERR(read_delta_window(&window, rb->chunk_index, rs, window_pool)); - /* Read the next window from the original rep. */ - rs = APR_ARRAY_IDX(rb->rs_list, 0, struct rep_state *); - SVN_ERR(read_window(&window, rb->chunk_index, rs, pool)); + APR_ARRAY_PUSH(windows, svn_txdelta_window_t *) = window; + if (window->src_ops == 0) + { + ++i; + break; + } + } - /* Combine in the windows from the other delta reps, if needed. */ - for (i = 1; i < rb->rs_list->nelts - 1; i++) + /* Combine in the windows from the other delta reps. */ + pool = svn_pool_create(rb->pool); + for (--i; i >= 0; --i) { - if (window->src_ops == 0) - break; rs = APR_ARRAY_IDX(rb->rs_list, i, struct rep_state *); + window = APR_ARRAY_IDX(windows, i, svn_txdelta_window_t *); + + /* Maybe, we've got a PLAIN start representation. If we do, read + as much data from it as the needed for the txdelta window's source + view. + Note that BUF / SOURCE may only be NULL in the first iteration. */ + source = buf; + if (source == NULL && rb->src_state != NULL) + SVN_ERR(read_plain_window(&source, rb->src_state, window->sview_len, + pool)); - SVN_ERR(read_window(&nwin, rb->chunk_index, rs, pool)); - - /* Combine this window with the current one. Cycle pools so that we - only need to hold three windows at a time. */ + /* Combine this window with the current one. */ new_pool = svn_pool_create(rb->pool); - window = svn_txdelta_compose_windows(nwin, window, new_pool); + buf = svn_stringbuf_create_ensure(window->tview_len, new_pool); + buf->len = window->tview_len; + + svn_txdelta_apply_instructions(window, source ? source->data : NULL, + buf->data, &buf->len); + if (buf->len != window->tview_len) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("svndiff window length is " + "corrupt")); + + /* Cache windows only if the whole rep content could be read as a + single chunk. Only then will no other chunk need a deeper RS + list than the cached chunk. */ + if ((rb->chunk_index == 0) && (rs->off == rs->end)) + SVN_ERR(set_cached_combined_window(buf, rs, rs->start, new_pool)); + + /* Cycle pools so that we only need to hold three windows at a time. */ svn_pool_destroy(pool); pool = new_pool; } - *result = window; + svn_pool_destroy(window_pool); + + *result = buf; return SVN_NO_ERROR; } @@ -3515,10 +5126,9 @@ get_contents(struct rep_read_baton *rb, char *buf, apr_size_t *len) { - apr_size_t copy_len, remaining = *len, tlen; - char *sbuf, *tbuf, *cur = buf; + apr_size_t copy_len, remaining = *len; + char *cur = buf; struct rep_state *rs; - svn_txdelta_window_t *cwindow, *lwindow; /* Special case for when there are no delta reps, only a plain text. */ @@ -3526,10 +5136,28 @@ get_contents(struct rep_read_baton *rb, { copy_len = remaining; rs = rb->src_state; - if (((apr_off_t) copy_len) > rs->end - rs->off) - copy_len = (apr_size_t) (rs->end - rs->off); - SVN_ERR(svn_io_file_read_full2(rs->file, cur, copy_len, NULL, - NULL, rb->pool)); + + if (rb->base_window != NULL) + { + /* We got the desired rep directly from the cache. + This is where we need the pseudo rep_state created + by build_rep_list(). */ + apr_size_t offset = (apr_size_t)(rs->off - rs->start); + if (copy_len + offset > rb->base_window->len) + copy_len = offset < rb->base_window->len + ? rb->base_window->len - offset + : 0ul; + + memcpy (cur, rb->base_window->data + offset, copy_len); + } + else + { + if (((apr_off_t) copy_len) > rs->end - rs->off) + copy_len = (apr_size_t) (rs->end - rs->off); + SVN_ERR(svn_io_file_read_full2(rs->file, cur, copy_len, NULL, + NULL, rb->pool)); + } + rs->off += copy_len; *len = copy_len; return SVN_NO_ERROR; @@ -3561,90 +5189,18 @@ get_contents(struct rep_read_baton *rb, } else { + svn_stringbuf_t *sbuf = NULL; rs = APR_ARRAY_IDX(rb->rs_list, 0, struct rep_state *); if (rs->off == rs->end) break; /* Get more buffered data by evaluating a chunk. */ - if (rb->rs_list->nelts > 1) - SVN_ERR(get_combined_window(&cwindow, rb)); - else - cwindow = NULL; - if (!cwindow || cwindow->src_ops > 0) - { - rs = APR_ARRAY_IDX(rb->rs_list, rb->rs_list->nelts - 1, - struct rep_state *); - /* Read window from last representation in list. */ - /* We apply this window directly instead of combining it - with the others. We do this because vdelta used to - be used for deltas against the empty stream, which - will trigger quadratic behaviour in the delta - combiner. It's still likely that we'll find such - deltas in an old repository; it may be worth - considering whether or not this special case is still - needed in the future, though. */ - SVN_ERR(read_window(&lwindow, rb->chunk_index, rs, rb->pool)); - - if (lwindow->src_ops > 0) - { - if (! rb->src_state) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("svndiff data requested " - "non-existent source")); - rs = rb->src_state; - sbuf = apr_palloc(rb->pool, lwindow->sview_len); - if (! ((rs->start + lwindow->sview_offset) < rs->end)) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("svndiff requested position " - "beyond end of stream")); - if ((rs->start + lwindow->sview_offset) != rs->off) - { - rs->off = rs->start + lwindow->sview_offset; - SVN_ERR(svn_io_file_seek(rs->file, APR_SET, &rs->off, - rb->pool)); - } - SVN_ERR(svn_io_file_read_full2(rs->file, sbuf, - lwindow->sview_len, - NULL, NULL, rb->pool)); - rs->off += lwindow->sview_len; - } - else - sbuf = NULL; - - /* Apply lwindow to source. */ - tlen = lwindow->tview_len; - tbuf = apr_palloc(rb->pool, tlen); - svn_txdelta_apply_instructions(lwindow, sbuf, tbuf, - &tlen); - if (tlen != lwindow->tview_len) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("svndiff window length is " - "corrupt")); - sbuf = tbuf; - } - else - sbuf = NULL; + SVN_ERR(get_combined_window(&sbuf, rb)); rb->chunk_index++; - - if (cwindow) - { - rb->buf_len = cwindow->tview_len; - rb->buf = apr_palloc(rb->pool, rb->buf_len); - svn_txdelta_apply_instructions(cwindow, sbuf, rb->buf, - &rb->buf_len); - if (rb->buf_len != cwindow->tview_len) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("svndiff window length is " - "corrupt")); - } - else - { - rb->buf_len = lwindow->tview_len; - rb->buf = sbuf; - } - + rb->buf_len = sbuf->len; + rb->buf = sbuf->data; rb->buf_pos = 0; } } @@ -3697,7 +5253,7 @@ rep_read_contents(void *baton, if (rb->off == rb->len && rb->current_fulltext) { fs_fs_data_t *ffd = rb->fs->fsap_data; - SVN_ERR(svn_cache__set(ffd->fulltext_cache, rb->fulltext_cache_key, + SVN_ERR(svn_cache__set(ffd->fulltext_cache, &rb->fulltext_cache_key, rb->current_fulltext, rb->pool)); rb->current_fulltext = NULL; } @@ -3728,27 +5284,30 @@ read_representation(svn_stream_t **contents_p, else { fs_fs_data_t *ffd = fs->fsap_data; - const char *fulltext_key = NULL; + pair_cache_key_t fulltext_cache_key = { 0 }; svn_filesize_t len = rep->expanded_size ? rep->expanded_size : rep->size; struct rep_read_baton *rb; + fulltext_cache_key.revision = rep->revision; + fulltext_cache_key.second = rep->offset; if (ffd->fulltext_cache && SVN_IS_VALID_REVNUM(rep->revision) && fulltext_size_is_cachable(ffd, len)) { - svn_string_t *fulltext; + svn_stringbuf_t *fulltext; svn_boolean_t is_cached; - fulltext_key = apr_psprintf(pool, "%ld/%" APR_OFF_T_FMT, - rep->revision, rep->offset); SVN_ERR(svn_cache__get((void **) &fulltext, &is_cached, - ffd->fulltext_cache, fulltext_key, pool)); + ffd->fulltext_cache, &fulltext_cache_key, + pool)); if (is_cached) { - *contents_p = svn_stream_from_string(fulltext, pool); + *contents_p = svn_stream_from_stringbuf(fulltext, pool); return SVN_NO_ERROR; } } + else + fulltext_cache_key.revision = SVN_INVALID_REVNUM; - SVN_ERR(rep_read_get_baton(&rb, fs, rep, fulltext_key, pool)); + SVN_ERR(rep_read_get_baton(&rb, fs, rep, fulltext_cache_key, pool)); *contents_p = svn_stream_create(rb, pool); svn_stream_set_read(*contents_p, rep_read_contents); @@ -3787,7 +5346,7 @@ delta_read_next_window(svn_txdelta_window_t **window, void *baton, return SVN_NO_ERROR; } - return read_window(window, drb->rs->chunk_index, drb->rs, pool); + return read_delta_window(window, drb->rs->chunk_index, drb->rs, pool); } /* This implements the svn_txdelta_md5_digest_fn_t interface. */ @@ -3819,13 +5378,15 @@ svn_fs_fs__get_file_delta_stream(svn_txdelta_stream_t **stream_p, struct rep_args *rep_args; /* Read target's base rep if any. */ - SVN_ERR(create_rep_state(&rep_state, &rep_args, target->data_rep, - fs, pool)); - /* If that matches source, then use this delta as is. */ + SVN_ERR(create_rep_state(&rep_state, &rep_args, NULL, NULL, + target->data_rep, fs, pool)); + + /* If that matches source, then use this delta as is. + Note that we want an actual delta here. E.g. a self-delta would + not be good enough. */ if (rep_args->is_delta - && (rep_args->is_delta_vs_empty - || (rep_args->base_revision == source->data_rep->revision - && rep_args->base_offset == source->data_rep->offset))) + && rep_args->base_revision == source->data_rep->revision + && rep_args->base_offset == source->data_rep->offset) { /* Create the delta read baton. */ struct delta_read_baton *drb = apr_pcalloc(pool, sizeof(*drb)); @@ -3846,11 +5407,83 @@ svn_fs_fs__get_file_delta_stream(svn_txdelta_stream_t **stream_p, else source_stream = svn_stream_empty(pool); SVN_ERR(read_representation(&target_stream, fs, target->data_rep, pool)); - svn_txdelta(stream_p, source_stream, target_stream, pool); + + /* Because source and target stream will already verify their content, + * there is no need to do this once more. In particular if the stream + * content is being fetched from cache. */ + svn_txdelta2(stream_p, source_stream, target_stream, FALSE, pool); return SVN_NO_ERROR; } +/* Baton for cache_access_wrapper. Wraps the original parameters of + * svn_fs_fs__try_process_file_content(). + */ +typedef struct cache_access_wrapper_baton_t +{ + svn_fs_process_contents_func_t func; + void* baton; +} cache_access_wrapper_baton_t; + +/* Wrapper to translate between svn_fs_process_contents_func_t and + * svn_cache__partial_getter_func_t. + */ +static svn_error_t * +cache_access_wrapper(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool) +{ + cache_access_wrapper_baton_t *wrapper_baton = baton; + + SVN_ERR(wrapper_baton->func((const unsigned char *)data, + data_len - 1, /* cache adds terminating 0 */ + wrapper_baton->baton, + pool)); + + /* non-NULL value to signal the calling cache that all went well */ + *out = baton; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_fs__try_process_file_contents(svn_boolean_t *success, + svn_fs_t *fs, + node_revision_t *noderev, + svn_fs_process_contents_func_t processor, + void* baton, + apr_pool_t *pool) +{ + representation_t *rep = noderev->data_rep; + if (rep) + { + fs_fs_data_t *ffd = fs->fsap_data; + pair_cache_key_t fulltext_cache_key = { 0 }; + + fulltext_cache_key.revision = rep->revision; + fulltext_cache_key.second = rep->offset; + if (ffd->fulltext_cache && SVN_IS_VALID_REVNUM(rep->revision) + && fulltext_size_is_cachable(ffd, rep->expanded_size)) + { + cache_access_wrapper_baton_t wrapper_baton; + void *dummy = NULL; + + wrapper_baton.func = processor; + wrapper_baton.baton = baton; + return svn_cache__get_partial(&dummy, success, + ffd->fulltext_cache, + &fulltext_cache_key, + cache_access_wrapper, + &wrapper_baton, + pool); + } + } + + *success = FALSE; + return SVN_NO_ERROR; +} /* Fetch the contents of a directory into ENTRIES. Values are stored as filename to string mappings; further conversion is necessary to @@ -3877,10 +5510,27 @@ get_dir_contents(apr_hash_t *entries, } else if (noderev->data_rep) { + /* use a temporary pool for temp objects. + * Also undeltify content before parsing it. Otherwise, we could only + * parse it byte-by-byte. + */ + apr_pool_t *text_pool = svn_pool_create(pool); + apr_size_t len = noderev->data_rep->expanded_size + ? (apr_size_t)noderev->data_rep->expanded_size + : (apr_size_t)noderev->data_rep->size; + svn_stringbuf_t *text = svn_stringbuf_create_ensure(len, text_pool); + text->len = len; + /* The representation is immutable. Read it normally. */ - SVN_ERR(read_representation(&contents, fs, noderev->data_rep, pool)); - SVN_ERR(svn_hash_read2(entries, contents, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(read_representation(&contents, fs, noderev->data_rep, text_pool)); + SVN_ERR(svn_stream_read(contents, text->data, &text->len)); SVN_ERR(svn_stream_close(contents)); + + /* de-serialize hash */ + contents = svn_stream_from_stringbuf(text, text_pool); + SVN_ERR(svn_hash_read2(entries, contents, SVN_HASH_TERMINATOR, pool)); + + svn_pool_destroy(text_pool); } return SVN_NO_ERROR; @@ -3907,7 +5557,14 @@ unparse_dir_entries(apr_hash_t **str_entries_p, { apr_hash_index_t *hi; - *str_entries_p = apr_hash_make(pool); + /* For now, we use a our own hash function to ensure that we get a + * (largely) stable order when serializing the data. It also gives + * us some performance improvement. + * + * ### TODO ### + * Use some sorted or other fixed order data container. + */ + *str_entries_p = svn_hash__make(pool); for (hi = apr_hash_first(pool, entries); hi; hi = apr_hash_next(hi)) { @@ -3947,10 +5604,10 @@ parse_dir_entries(apr_hash_t **entries_p, char *str, *last_str; svn_fs_dirent_t *dirent = apr_pcalloc(pool, sizeof(*dirent)); - str = apr_pstrdup(pool, str_val->data); + last_str = apr_pstrdup(pool, str_val->data); dirent->name = apr_pstrdup(pool, name); - str = apr_strtok(str, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Directory entry corrupt in '%s'"), @@ -3971,7 +5628,7 @@ parse_dir_entries(apr_hash_t **entries_p, unparsed_id); } - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Directory entry corrupt in '%s'"), @@ -3979,7 +5636,7 @@ parse_dir_entries(apr_hash_t **entries_p, dirent->id = svn_fs_fs__id_parse(str, strlen(str), pool); - apr_hash_set(*entries_p, dirent->name, APR_HASH_KEY_STRING, dirent); + svn_hash_sets(*entries_p, dirent->name, dirent); } return SVN_NO_ERROR; @@ -4038,7 +5695,8 @@ svn_fs_fs__rep_contents_dir_entry(svn_fs_dirent_t **dirent, svn_fs_t *fs, node_revision_t *noderev, const char *name, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { svn_boolean_t found = FALSE; @@ -4047,7 +5705,7 @@ svn_fs_fs__rep_contents_dir_entry(svn_fs_dirent_t **dirent, if (cache) { const char *unparsed_id = - svn_fs_fs__id_unparse(noderev->id, pool)->data; + svn_fs_fs__id_unparse(noderev->id, scratch_pool)->data; /* Cache lookup. */ SVN_ERR(svn_cache__get_partial((void **)dirent, @@ -4056,7 +5714,7 @@ svn_fs_fs__rep_contents_dir_entry(svn_fs_dirent_t **dirent, unparsed_id, svn_fs_fs__extract_dir_entry, (void*)name, - pool)); + result_pool)); } /* fetch data from disk if we did not find it in the cache */ @@ -4066,29 +5724,22 @@ svn_fs_fs__rep_contents_dir_entry(svn_fs_dirent_t **dirent, svn_fs_dirent_t *entry; svn_fs_dirent_t *entry_copy = NULL; - /* since we don't need the directory content later on, put it into - some sub-pool that will be reclaimed immedeately after exiting - this function successfully. Opon failure, it will live as long - as pool. - */ - apr_pool_t *sub_pool = svn_pool_create(pool); - /* read the dir from the file system. It will probably be put it into the cache for faster lookup in future calls. */ - SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, noderev, sub_pool)); + SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, noderev, + scratch_pool)); /* find desired entry and return a copy in POOL, if found */ - entry = apr_hash_get(entries, name, APR_HASH_KEY_STRING); + entry = svn_hash_gets(entries, name); if (entry != NULL) { - entry_copy = apr_palloc(pool, sizeof(*entry_copy)); - entry_copy->name = apr_pstrdup(pool, entry->name); - entry_copy->id = svn_fs_fs__id_copy(entry->id, pool); + entry_copy = apr_palloc(result_pool, sizeof(*entry_copy)); + entry_copy->name = apr_pstrdup(result_pool, entry->name); + entry_copy->id = svn_fs_fs__id_copy(entry->id, result_pool); entry_copy->kind = entry->kind; } *dirent = entry_copy; - apr_pool_destroy(sub_pool); } return SVN_NO_ERROR; @@ -4103,11 +5754,10 @@ svn_fs_fs__get_proplist(apr_hash_t **proplist_p, apr_hash_t *proplist; svn_stream_t *stream; - proplist = apr_hash_make(pool); - if (noderev->prop_rep && noderev->prop_rep->txn_id) { const char *filename = path_txn_node_props(fs, noderev->id, pool); + proplist = apr_hash_make(pool); SVN_ERR(svn_stream_open_readonly(&stream, filename, pool, pool)); SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool)); @@ -4115,9 +5765,33 @@ svn_fs_fs__get_proplist(apr_hash_t **proplist_p, } else if (noderev->prop_rep) { + fs_fs_data_t *ffd = fs->fsap_data; + representation_t *rep = noderev->prop_rep; + pair_cache_key_t key = { 0 }; + + key.revision = rep->revision; + key.second = rep->offset; + if (ffd->properties_cache && SVN_IS_VALID_REVNUM(rep->revision)) + { + svn_boolean_t is_cached; + SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached, + ffd->properties_cache, &key, pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + proplist = apr_hash_make(pool); SVN_ERR(read_representation(&stream, fs, noderev->prop_rep, pool)); SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool)); SVN_ERR(svn_stream_close(stream)); + + if (ffd->properties_cache && SVN_IS_VALID_REVNUM(rep->revision)) + SVN_ERR(svn_cache__set(ffd->properties_cache, &key, proplist, pool)); + } + else + { + /* return an empty prop list if the node doesn't have any props */ + proplist = apr_hash_make(pool); } *proplist_p = proplist; @@ -4222,8 +5896,9 @@ fold_change(apr_hash_t *changes, apr_pool_t *pool = apr_hash_pool_get(changes); svn_fs_path_change2_t *old_change, *new_change; const char *path; + apr_size_t path_len = strlen(change->path); - if ((old_change = apr_hash_get(changes, change->path, APR_HASH_KEY_STRING))) + if ((old_change = apr_hash_get(changes, change->path, path_len))) { /* This path already exists in the hash, so we have to merge this change into the already existing one. */ @@ -4361,8 +6036,8 @@ fold_change(apr_hash_t *changes, re-use its value, but there is no way to fetch it. The API makes no guarantees that this (new) key will not be retained. Thus, we (again) copy the key into the target pool to ensure a proper lifetime. */ - path = apr_pstrdup(pool, change->path); - apr_hash_set(changes, path, APR_HASH_KEY_STRING, new_change); + path = apr_pstrmemdup(pool, change->path, path_len); + apr_hash_set(changes, path, path_len, new_change); /* Update the copyfrom cache, if any. */ if (copyfrom_cache) @@ -4380,10 +6055,12 @@ fold_change(apr_hash_t *changes, } /* We need to allocate a copy of the key in the copyfrom_pool if * we're not doing a deletion and if it isn't already there. */ - if (copyfrom_string && ! apr_hash_get(copyfrom_cache, copyfrom_key, - APR_HASH_KEY_STRING)) - copyfrom_key = apr_pstrdup(copyfrom_pool, copyfrom_key); - apr_hash_set(copyfrom_cache, copyfrom_key, APR_HASH_KEY_STRING, + if ( copyfrom_string + && ( ! apr_hash_count(copyfrom_cache) + || ! apr_hash_get(copyfrom_cache, copyfrom_key, path_len))) + copyfrom_key = apr_pstrmemdup(copyfrom_pool, copyfrom_key, path_len); + + apr_hash_set(copyfrom_cache, copyfrom_key, path_len, copyfrom_string); } @@ -4405,7 +6082,7 @@ read_change(change_t **change_p, char buf[MAX_CHANGE_LINE_LEN]; apr_size_t len = sizeof(buf); change_t *change; - char *str, *last_str, *kind_str; + char *str, *last_str = buf, *kind_str; svn_error_t *err; /* Default return value. */ @@ -4429,7 +6106,7 @@ read_change(change_t **change_p, change = apr_pcalloc(pool, sizeof(*change)); /* Get the node-id of the change. */ - str = apr_strtok(buf, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Invalid changes line in rev-file")); @@ -4440,7 +6117,7 @@ read_change(change_t **change_p, _("Invalid changes line in rev-file")); /* Get the change type. */ - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Invalid changes line in rev-file")); @@ -4490,7 +6167,7 @@ read_change(change_t **change_p, } /* Get the text-mod flag. */ - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Invalid changes line in rev-file")); @@ -4510,7 +6187,7 @@ read_change(change_t **change_p, } /* Get the prop-mod flag. */ - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Invalid changes line in rev-file")); @@ -4544,7 +6221,8 @@ read_change(change_t **change_p, } else { - str = apr_strtok(buf, " ", &last_str); + last_str = buf; + str = svn_cstring_tokenize(" ", &last_str); if (! str) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Invalid changes line in rev-file")); @@ -4562,32 +6240,32 @@ read_change(change_t **change_p, return SVN_NO_ERROR; } -/* Fetch all the changed path entries from FILE and store then in +/* Examine all the changed path entries in CHANGES and store them in *CHANGED_PATHS. Folding is done to remove redundant or unnecessary - *data. Store a hash of paths to copyfrom revisions/paths in + *data. Store a hash of paths to copyfrom "REV PATH" strings in COPYFROM_HASH if it is non-NULL. If PREFOLDED is true, assume that the changed-path entries have already been folded (by write_final_changed_path_info) and may be out of order, so we shouldn't remove children of replaced or deleted directories. Do all allocations in POOL. */ static svn_error_t * -fetch_all_changes(apr_hash_t *changed_paths, - apr_hash_t *copyfrom_hash, - apr_file_t *file, - svn_boolean_t prefolded, - apr_pool_t *pool) +process_changes(apr_hash_t *changed_paths, + apr_hash_t *copyfrom_cache, + apr_array_header_t *changes, + svn_boolean_t prefolded, + apr_pool_t *pool) { - change_t *change; apr_pool_t *iterpool = svn_pool_create(pool); + int i; /* Read in the changes one by one, folding them into our local hash as necessary. */ - SVN_ERR(read_change(&change, file, iterpool)); - - while (change) + for (i = 0; i < changes->nelts; ++i) { - SVN_ERR(fold_change(changed_paths, change, copyfrom_hash)); + change_t *change = APR_ARRAY_IDX(changes, i, change_t *); + + SVN_ERR(fold_change(changed_paths, change, copyfrom_cache)); /* Now, if our change was a deletion or replacement, we have to blow away any changes thus far on paths that are (or, were) @@ -4603,28 +6281,43 @@ fetch_all_changes(apr_hash_t *changed_paths, { apr_hash_index_t *hi; + /* a potential child path must contain at least 2 more chars + (the path separator plus at least one char for the name). + Also, we should not assume that all paths have been normalized + i.e. some might have trailing path separators. + */ + apr_ssize_t change_path_len = strlen(change->path); + apr_ssize_t min_child_len = change_path_len == 0 + ? 1 + : change->path[change_path_len-1] == '/' + ? change_path_len + 1 + : change_path_len + 2; + + /* CAUTION: This is the inner loop of an O(n^2) algorithm. + The number of changes to process may be >> 1000. + Therefore, keep the inner loop as tight as possible. + */ for (hi = apr_hash_first(iterpool, changed_paths); hi; hi = apr_hash_next(hi)) { /* KEY is the path. */ - const char *path = svn__apr_hash_index_key(hi); - apr_ssize_t klen = svn__apr_hash_index_klen(hi); - - /* If we come across our own path, ignore it. */ - if (strcmp(change->path, path) == 0) - continue; - - /* If we come across a child of our path, remove it. */ - if (svn_dirent_is_child(change->path, path, iterpool)) + const void *path; + apr_ssize_t klen; + apr_hash_this(hi, &path, &klen, NULL); + + /* If we come across a child of our path, remove it. + Call svn_dirent_is_child only if there is a chance that + this is actually a sub-path. + */ + if ( klen >= min_child_len + && svn_dirent_is_child(change->path, path, iterpool)) apr_hash_set(changed_paths, path, klen, NULL); } } /* Clear the per-iteration subpool. */ svn_pool_clear(iterpool); - - SVN_ERR(read_change(&change, file, iterpool)); } /* Destroy the per-iteration subpool. */ @@ -4633,6 +6326,29 @@ fetch_all_changes(apr_hash_t *changed_paths, return SVN_NO_ERROR; } +/* Fetch all the changes from FILE and store them in *CHANGES. Do all + allocations in POOL. */ +static svn_error_t * +read_all_changes(apr_array_header_t **changes, + apr_file_t *file, + apr_pool_t *pool) +{ + change_t *change; + + /* pre-allocate enough room for most change lists + (will be auto-expanded as necessary) */ + *changes = apr_array_make(pool, 30, sizeof(change_t *)); + + SVN_ERR(read_change(&change, file, pool)); + while (change) + { + APR_ARRAY_PUSH(*changes, change_t*) = change; + SVN_ERR(read_change(&change, file, pool)); + } + + return SVN_NO_ERROR; +} + svn_error_t * svn_fs_fs__txn_changes_fetch(apr_hash_t **changed_paths_p, svn_fs_t *fs, @@ -4641,11 +6357,15 @@ svn_fs_fs__txn_changes_fetch(apr_hash_t **changed_paths_p, { apr_file_t *file; apr_hash_t *changed_paths = apr_hash_make(pool); + apr_array_header_t *changes; + apr_pool_t *scratch_pool = svn_pool_create(pool); SVN_ERR(svn_io_file_open(&file, path_txn_changes(fs, txn_id, pool), APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool)); - SVN_ERR(fetch_all_changes(changed_paths, NULL, file, FALSE, pool)); + SVN_ERR(read_all_changes(&changes, file, scratch_pool)); + SVN_ERR(process_changes(changed_paths, NULL, changes, FALSE, pool)); + svn_pool_destroy(scratch_pool); SVN_ERR(svn_io_file_close(file, pool)); @@ -4654,16 +6374,31 @@ svn_fs_fs__txn_changes_fetch(apr_hash_t **changed_paths_p, return SVN_NO_ERROR; } -svn_error_t * -svn_fs_fs__paths_changed(apr_hash_t **changed_paths_p, - svn_fs_t *fs, - svn_revnum_t rev, - apr_hash_t *copyfrom_cache, - apr_pool_t *pool) +/* Fetch the list of change in revision REV in FS and return it in *CHANGES. + * Allocate the result in POOL. + */ +static svn_error_t * +get_changes(apr_array_header_t **changes, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) { apr_off_t changes_offset; - apr_hash_t *changed_paths; apr_file_t *revision_file; + svn_boolean_t found; + fs_fs_data_t *ffd = fs->fsap_data; + + /* try cache lookup first */ + + if (ffd->changes_cache) + { + SVN_ERR(svn_cache__get((void **) changes, &found, ffd->changes_cache, + &rev, pool)); + if (found) + return SVN_NO_ERROR; + } + + /* read changes from revision file */ SVN_ERR(ensure_revision_exists(fs, rev, pool)); @@ -4673,14 +6408,37 @@ svn_fs_fs__paths_changed(apr_hash_t **changed_paths_p, rev, pool)); SVN_ERR(svn_io_file_seek(revision_file, APR_SET, &changes_offset, pool)); + SVN_ERR(read_all_changes(changes, revision_file, pool)); - changed_paths = apr_hash_make(pool); + SVN_ERR(svn_io_file_close(revision_file, pool)); - SVN_ERR(fetch_all_changes(changed_paths, copyfrom_cache, revision_file, - TRUE, pool)); + /* cache for future reference */ - /* Close the revision file. */ - SVN_ERR(svn_io_file_close(revision_file, pool)); + if (ffd->changes_cache) + SVN_ERR(svn_cache__set(ffd->changes_cache, &rev, *changes, pool)); + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_fs__paths_changed(apr_hash_t **changed_paths_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_hash_t *copyfrom_cache, + apr_pool_t *pool) +{ + apr_hash_t *changed_paths; + apr_array_header_t *changes; + apr_pool_t *scratch_pool = svn_pool_create(pool); + + SVN_ERR(get_changes(&changes, fs, rev, scratch_pool)); + + changed_paths = svn_hash__make(pool); + + SVN_ERR(process_changes(changed_paths, copyfrom_cache, changes, + TRUE, pool)); + svn_pool_destroy(scratch_pool); *changed_paths_p = changed_paths; @@ -4734,40 +6492,17 @@ get_and_increment_txn_key_body(void *baton, apr_pool_t *pool) { struct get_and_increment_txn_key_baton *cb = baton; const char *txn_current_filename = path_txn_current(cb->fs, pool); - apr_file_t *txn_current_file = NULL; const char *tmp_filename; char next_txn_id[MAX_KEY_SIZE+3]; - svn_error_t *err = SVN_NO_ERROR; - apr_pool_t *iterpool; apr_size_t len; - int i; - - cb->txn_id = apr_palloc(cb->pool, MAX_KEY_SIZE); - - iterpool = svn_pool_create(pool); - for (i = 0; i < RECOVERABLE_RETRY_COUNT; ++i) - { - svn_pool_clear(iterpool); - RETRY_RECOVERABLE(err, txn_current_file, - svn_io_file_open(&txn_current_file, - txn_current_filename, - APR_READ | APR_BUFFERED, - APR_OS_DEFAULT, iterpool)); - len = MAX_KEY_SIZE; - RETRY_RECOVERABLE(err, txn_current_file, - svn_io_read_length_line(txn_current_file, - cb->txn_id, - &len, - iterpool)); - IGNORE_RECOVERABLE(err, svn_io_file_close(txn_current_file, - iterpool)); + svn_stringbuf_t *buf; + SVN_ERR(read_content(&buf, txn_current_filename, cb->pool)); - break; - } - SVN_ERR(err); - - svn_pool_destroy(iterpool); + /* remove trailing newlines */ + svn_stringbuf_strip_whitespace(buf); + cb->txn_id = buf->data; + len = buf->len; /* Increment the key and add a trailing \n to the string so the txn-current file has a newline in it. */ @@ -4979,7 +6714,7 @@ svn_fs_fs__change_txn_props(svn_fs_txn_t *txn, { svn_prop_t *prop = &APR_ARRAY_IDX(props, i, svn_prop_t); - apr_hash_set(txn_prop, prop->name, APR_HASH_KEY_STRING, prop->value); + svn_hash_sets(txn_prop, prop->name, prop->value); } /* Create a new version of the file and write out the new props. */ @@ -5068,7 +6803,7 @@ read_next_ids(const char **node_id, apr_file_t *file; char buf[MAX_KEY_SIZE*2+3]; apr_size_t limit; - char *str, *last_str; + char *str, *last_str = buf; SVN_ERR(svn_io_file_open(&file, path_txn_next_ids(fs, txn_id, pool), APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool)); @@ -5080,14 +6815,14 @@ read_next_ids(const char **node_id, /* Parse this into two separate strings. */ - str = apr_strtok(buf, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (! str) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("next-id file corrupt")); *node_id = apr_pstrdup(pool, str); - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (! str) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("next-id file corrupt")); @@ -5207,35 +6942,37 @@ svn_fs_fs__set_entry(svn_fs_t *fs, apr_file_t *file; svn_stream_t *out; fs_fs_data_t *ffd = fs->fsap_data; + apr_pool_t *subpool = svn_pool_create(pool); if (!rep || !rep->txn_id) { const char *unique_suffix; + apr_hash_t *entries; - { - apr_hash_t *entries; - apr_pool_t *subpool = svn_pool_create(pool); - - /* Before we can modify the directory, we need to dump its old - contents into a mutable representation file. */ - SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, parent_noderev, - subpool)); - SVN_ERR(unparse_dir_entries(&entries, entries, subpool)); - SVN_ERR(svn_io_file_open(&file, filename, - APR_WRITE | APR_CREATE | APR_BUFFERED, - APR_OS_DEFAULT, pool)); - out = svn_stream_from_aprfile2(file, TRUE, pool); - SVN_ERR(svn_hash_write2(entries, out, SVN_HASH_TERMINATOR, subpool)); - - svn_pool_destroy(subpool); - } + /* Before we can modify the directory, we need to dump its old + contents into a mutable representation file. */ + SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, parent_noderev, + subpool)); + SVN_ERR(unparse_dir_entries(&entries, entries, subpool)); + SVN_ERR(svn_io_file_open(&file, filename, + APR_WRITE | APR_CREATE | APR_BUFFERED, + APR_OS_DEFAULT, pool)); + out = svn_stream_from_aprfile2(file, TRUE, pool); + SVN_ERR(svn_hash_write2(entries, out, SVN_HASH_TERMINATOR, subpool)); + + svn_pool_clear(subpool); /* Mark the node-rev's data rep as mutable. */ rep = apr_pcalloc(pool, sizeof(*rep)); rep->revision = SVN_INVALID_REVNUM; rep->txn_id = txn_id; - SVN_ERR(get_new_txn_node_id(&unique_suffix, fs, txn_id, pool)); - rep->uniquifier = apr_psprintf(pool, "%s/%s", txn_id, unique_suffix); + + if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) + { + SVN_ERR(get_new_txn_node_id(&unique_suffix, fs, txn_id, pool)); + rep->uniquifier = apr_psprintf(pool, "%s/%s", txn_id, unique_suffix); + } + parent_noderev->data_rep = rep; SVN_ERR(svn_fs_fs__put_node_revision(fs, parent_noderev->id, parent_noderev, FALSE, pool)); @@ -5251,12 +6988,11 @@ svn_fs_fs__set_entry(svn_fs_t *fs, /* if we have a directory cache for this transaction, update it */ if (ffd->txn_dir_cache) { - apr_pool_t *subpool = svn_pool_create(pool); - /* build parameters: (name, new entry) pair */ const char *key = svn_fs_fs__id_unparse(parent_noderev->id, subpool)->data; replace_baton_t baton; + baton.name = name; baton.new_entry = NULL; @@ -5269,28 +7005,31 @@ svn_fs_fs__set_entry(svn_fs_t *fs, } /* actually update the cached directory (if cached) */ - SVN_ERR(svn_cache__set_partial(ffd->txn_dir_cache, key, svn_fs_fs__replace_dir_entry, &baton, subpool)); - - svn_pool_destroy(subpool); + SVN_ERR(svn_cache__set_partial(ffd->txn_dir_cache, key, + svn_fs_fs__replace_dir_entry, &baton, + subpool)); } + svn_pool_clear(subpool); /* Append an incremental hash entry for the entry change. */ if (id) { - const char *val = unparse_dir_entry(kind, id, pool); + const char *val = unparse_dir_entry(kind, id, subpool); - SVN_ERR(svn_stream_printf(out, pool, "K %" APR_SIZE_T_FMT "\n%s\n" + SVN_ERR(svn_stream_printf(out, subpool, "K %" APR_SIZE_T_FMT "\n%s\n" "V %" APR_SIZE_T_FMT "\n%s\n", strlen(name), name, strlen(val), val)); } else { - SVN_ERR(svn_stream_printf(out, pool, "D %" APR_SIZE_T_FMT "\n%s\n", + SVN_ERR(svn_stream_printf(out, subpool, "D %" APR_SIZE_T_FMT "\n%s\n", strlen(name), name)); } - return svn_io_file_close(file, pool); + SVN_ERR(svn_io_file_close(file, subpool)); + svn_pool_destroy(subpool); + return SVN_NO_ERROR; } /* Write a single change entry, path PATH, change CHANGE, and copyfrom @@ -5458,15 +7197,21 @@ rep_write_contents(void *baton, /* Given a node-revision NODEREV in filesystem FS, return the representation in *REP to use as the base for a text representation - delta. Perform temporary allocations in *POOL. */ + delta if PROPS is FALSE. If PROPS has been set, a suitable props + base representation will be returned. Perform temporary allocations + in *POOL. */ static svn_error_t * choose_delta_base(representation_t **rep, svn_fs_t *fs, node_revision_t *noderev, + svn_boolean_t props, apr_pool_t *pool) { int count; + int walk; node_revision_t *base; + fs_fs_data_t *ffd = fs->fsap_data; + svn_boolean_t maybe_shared_rep = FALSE; /* If we have no predecessors, then use the empty stream as a base. */ @@ -5483,20 +7228,141 @@ choose_delta_base(representation_t **rep, count = noderev->predecessor_count; count = count & (count - 1); + /* We use skip delta for limiting the number of delta operations + along very long node histories. Close to HEAD however, we create + a linear history to minimize delta size. */ + walk = noderev->predecessor_count - count; + if (walk < (int)ffd->max_linear_deltification) + count = noderev->predecessor_count - 1; + + /* Finding the delta base over a very long distance can become extremely + expensive for very deep histories, possibly causing client timeouts etc. + OTOH, this is a rare operation and its gains are minimal. Lets simply + start deltification anew close every other 1000 changes or so. */ + if (walk > (int)ffd->max_deltification_walk) + { + *rep = NULL; + return SVN_NO_ERROR; + } + /* Walk back a number of predecessors equal to the difference between count and the original predecessor count. (For example, if noderev has ten predecessors and we want the eighth file rev, walk back two predecessors.) */ base = noderev; while ((count++) < noderev->predecessor_count) - SVN_ERR(svn_fs_fs__get_node_revision(&base, fs, - base->predecessor_id, pool)); + { + SVN_ERR(svn_fs_fs__get_node_revision(&base, fs, + base->predecessor_id, pool)); + + /* If there is a shared rep along the way, we need to limit the + * length of the deltification chain. + * + * Please note that copied nodes - such as branch directories - will + * look the same (false positive) while reps shared within the same + * revision will not be caught (false negative). + */ + if (props) + { + if ( base->prop_rep + && svn_fs_fs__id_rev(base->id) > base->prop_rep->revision) + maybe_shared_rep = TRUE; + } + else + { + if ( base->data_rep + && svn_fs_fs__id_rev(base->id) > base->data_rep->revision) + maybe_shared_rep = TRUE; + } + } + + /* return a suitable base representation */ + *rep = props ? base->prop_rep : base->data_rep; + + /* if we encountered a shared rep, it's parent chain may be different + * from the node-rev parent chain. */ + if (*rep && maybe_shared_rep) + { + /* Check whether the length of the deltification chain is acceptable. + * Otherwise, shared reps may form a non-skipping delta chain in + * extreme cases. */ + apr_pool_t *sub_pool = svn_pool_create(pool); + representation_t base_rep = **rep; + + /* Some reasonable limit, depending on how acceptable longer linear + * chains are in this repo. Also, allow for some minimal chain. */ + int max_chain_length = 2 * (int)ffd->max_linear_deltification + 2; - *rep = base->data_rep; + /* re-use open files between iterations */ + svn_revnum_t rev_hint = SVN_INVALID_REVNUM; + apr_file_t *file_hint = NULL; + /* follow the delta chain towards the end but for at most + * MAX_CHAIN_LENGTH steps. */ + for (; max_chain_length; --max_chain_length) + { + struct rep_state *rep_state; + struct rep_args *rep_args; + + SVN_ERR(create_rep_state_body(&rep_state, + &rep_args, + &file_hint, + &rev_hint, + &base_rep, + fs, + sub_pool)); + if (!rep_args->is_delta || !rep_args->base_revision) + break; + + base_rep.revision = rep_args->base_revision; + base_rep.offset = rep_args->base_offset; + base_rep.size = rep_args->base_length; + base_rep.txn_id = NULL; + } + + /* start new delta chain if the current one has grown too long */ + if (max_chain_length == 0) + *rep = NULL; + + svn_pool_destroy(sub_pool); + } + + /* verify that the reps don't form a degenerated '*/ return SVN_NO_ERROR; } +/* Something went wrong and the pool for the rep write is being + cleared before we've finished writing the rep. So we need + to remove the rep from the protorevfile and we need to unlock + the protorevfile. */ +static apr_status_t +rep_write_cleanup(void *data) +{ + struct rep_write_baton *b = data; + const char *txn_id = svn_fs_fs__id_txn_id(b->noderev->id); + svn_error_t *err; + + /* Truncate and close the protorevfile. */ + err = svn_io_file_trunc(b->file, b->rep_offset, b->pool); + err = svn_error_compose_create(err, svn_io_file_close(b->file, b->pool)); + + /* Remove our lock regardless of any preceeding errors so that the + being_written flag is always removed and stays consistent with the + file lock which will be removed no matter what since the pool is + going away. */ + err = svn_error_compose_create(err, unlock_proto_rev(b->fs, txn_id, + b->lockcookie, b->pool)); + if (err) + { + apr_status_t rc = err->apr_err; + svn_error_clear(err); + return rc; + } + + return APR_SUCCESS; +} + + /* Get a rep_write_baton and store it in *WB_P for the representation indicated by NODEREV in filesystem FS. Perform allocations in POOL. Only appropriate for file contents, not for props or @@ -5539,7 +7405,7 @@ rep_write_get_baton(struct rep_write_baton **wb_p, SVN_ERR(get_file_offset(&b->rep_offset, file, b->pool)); /* Get the base for this delta. */ - SVN_ERR(choose_delta_base(&base_rep, fs, noderev, b->pool)); + SVN_ERR(choose_delta_base(&base_rep, fs, noderev, FALSE, b->pool)); SVN_ERR(read_representation(&source, fs, base_rep, b->pool)); /* Write out the rep header. */ @@ -5560,6 +7426,10 @@ rep_write_get_baton(struct rep_write_baton **wb_p, /* Now determine the offset of the actual svndiff data. */ SVN_ERR(get_file_offset(&b->delta_start, file, b->pool)); + /* Cleanup in case something goes wrong. */ + apr_pool_cleanup_register(b->pool, b, rep_write_cleanup, + apr_pool_cleanup_null); + /* Prepare to write the svndiff data. */ svn_txdelta_to_svndiff3(&wh, &whb, @@ -5575,6 +7445,106 @@ rep_write_get_baton(struct rep_write_baton **wb_p, return SVN_NO_ERROR; } +/* For the hash REP->SHA1, try to find an already existing representation + in FS and return it in *OUT_REP. If no such representation exists or + if rep sharing has been disabled for FS, NULL will be returned. Since + there may be new duplicate representations within the same uncommitted + revision, those can be passed in REPS_HASH (maps a sha1 digest onto + representation_t*), otherwise pass in NULL for REPS_HASH. + POOL will be used for allocations. The lifetime of the returned rep is + limited by both, POOL and REP lifetime. + */ +static svn_error_t * +get_shared_rep(representation_t **old_rep, + svn_fs_t *fs, + representation_t *rep, + apr_hash_t *reps_hash, + apr_pool_t *pool) +{ + svn_error_t *err; + fs_fs_data_t *ffd = fs->fsap_data; + + /* Return NULL, if rep sharing has been disabled. */ + *old_rep = NULL; + if (!ffd->rep_sharing_allowed) + return SVN_NO_ERROR; + + /* Check and see if we already have a representation somewhere that's + identical to the one we just wrote out. Start with the hash lookup + because it is cheepest. */ + if (reps_hash) + *old_rep = apr_hash_get(reps_hash, + rep->sha1_checksum->digest, + APR_SHA1_DIGESTSIZE); + + /* If we haven't found anything yet, try harder and consult our DB. */ + if (*old_rep == NULL) + { + err = svn_fs_fs__get_rep_reference(old_rep, fs, rep->sha1_checksum, + pool); + /* ### Other error codes that we shouldn't mask out? */ + if (err == SVN_NO_ERROR) + { + if (*old_rep) + SVN_ERR(verify_walker(*old_rep, NULL, fs, pool)); + } + else if (err->apr_err == SVN_ERR_FS_CORRUPT + || SVN_ERROR_IN_CATEGORY(err->apr_err, + SVN_ERR_MALFUNC_CATEGORY_START)) + { + /* Fatal error; don't mask it. + + In particular, this block is triggered when the rep-cache refers + to revisions in the future. We signal that as a corruption situation + since, once those revisions are less than youngest (because of more + commits), the rep-cache would be invalid. + */ + SVN_ERR(err); + } + else + { + /* Something's wrong with the rep-sharing index. We can continue + without rep-sharing, but warn. + */ + (fs->warning)(fs->warning_baton, err); + svn_error_clear(err); + *old_rep = NULL; + } + } + + /* look for intra-revision matches (usually data reps but not limited + to them in case props happen to look like some data rep) + */ + if (*old_rep == NULL && rep->txn_id) + { + svn_node_kind_t kind; + const char *file_name + = path_txn_sha1(fs, rep->txn_id, rep->sha1_checksum, pool); + + /* in our txn, is there a rep file named with the wanted SHA1? + If so, read it and use that rep. + */ + SVN_ERR(svn_io_check_path(file_name, &kind, pool)); + if (kind == svn_node_file) + { + svn_stringbuf_t *rep_string; + SVN_ERR(svn_stringbuf_from_file2(&rep_string, file_name, pool)); + SVN_ERR(read_rep_offsets_body(old_rep, rep_string->data, + rep->txn_id, FALSE, pool)); + } + } + + /* Add information that is missing in the cached data. */ + if (*old_rep) + { + /* Use the old rep for this content. */ + (*old_rep)->md5_checksum = rep->md5_checksum; + (*old_rep)->uniquifier = rep->uniquifier; + } + + return SVN_NO_ERROR; +} + /* Close handler for the representation write stream. BATON is a rep_write_baton. Writes out a new node-rev that correctly references the representation we just finished writing. */ @@ -5582,11 +7552,11 @@ static svn_error_t * rep_write_contents_close(void *baton) { struct rep_write_baton *b = baton; - fs_fs_data_t *ffd = b->fs->fsap_data; const char *unique_suffix; representation_t *rep; representation_t *old_rep; apr_off_t offset; + fs_fs_data_t *ffd = b->fs->fsap_data; rep = apr_pcalloc(b->parent_pool, sizeof(*rep)); rep->offset = b->rep_offset; @@ -5603,9 +7573,13 @@ rep_write_contents_close(void *baton) /* Fill in the rest of the representation field. */ rep->expanded_size = b->rep_size; rep->txn_id = svn_fs_fs__id_txn_id(b->noderev->id); - SVN_ERR(get_new_txn_node_id(&unique_suffix, b->fs, rep->txn_id, b->pool)); - rep->uniquifier = apr_psprintf(b->parent_pool, "%s/%s", rep->txn_id, - unique_suffix); + + if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) + { + SVN_ERR(get_new_txn_node_id(&unique_suffix, b->fs, rep->txn_id, b->pool)); + rep->uniquifier = apr_psprintf(b->parent_pool, "%s/%s", rep->txn_id, + unique_suffix); + } rep->revision = SVN_INVALID_REVNUM; /* Finalize the checksum. */ @@ -5616,38 +7590,7 @@ rep_write_contents_close(void *baton) /* Check and see if we already have a representation somewhere that's identical to the one we just wrote out. */ - if (ffd->rep_sharing_allowed) - { - svn_error_t *err; - err = svn_fs_fs__get_rep_reference(&old_rep, b->fs, rep->sha1_checksum, - b->parent_pool); - /* ### Other error codes that we shouldn't mask out? */ - if (err == SVN_NO_ERROR - || err->apr_err == SVN_ERR_FS_CORRUPT - || SVN_ERROR_IN_CATEGORY(err->apr_err, - SVN_ERR_MALFUNC_CATEGORY_START)) - { - /* Fatal error; don't mask it. - - In particular, this block is triggered when the rep-cache refers - to revisions in the future. We signal that as a corruption situation - since, once those revisions are less than youngest (because of more - commits), the rep-cache would be invalid. - */ - SVN_ERR(err); - } - else - { - /* Something's wrong with the rep-sharing index. We can continue - without rep-sharing, but warn. - */ - (b->fs->warning)(b->fs->warning_baton, err); - svn_error_clear(err); - old_rep = NULL; - } - } - else - old_rep = NULL; + SVN_ERR(get_shared_rep(&old_rep, b->fs, rep, NULL, b->parent_pool)); if (old_rep) { @@ -5655,21 +7598,24 @@ rep_write_contents_close(void *baton) SVN_ERR(svn_io_file_trunc(b->file, b->rep_offset, b->pool)); /* Use the old rep for this content. */ - old_rep->md5_checksum = rep->md5_checksum; - old_rep->uniquifier = rep->uniquifier; b->noderev->data_rep = old_rep; } else { /* Write out our cosmetic end marker. */ - SVN_ERR(svn_stream_printf(b->rep_stream, b->pool, "ENDREP\n")); + SVN_ERR(svn_stream_puts(b->rep_stream, "ENDREP\n")); b->noderev->data_rep = rep; } + /* Remove cleanup callback. */ + apr_pool_cleanup_kill(b->pool, b, rep_write_cleanup); + /* Write out the new node-rev information. */ SVN_ERR(svn_fs_fs__put_node_revision(b->fs, b->noderev->id, b->noderev, FALSE, b->pool)); + if (!old_rep) + SVN_ERR(store_sha1_rep_mapping(b->fs, b->noderev, b->pool)); SVN_ERR(svn_io_file_close(b->file, b->pool)); SVN_ERR(unlock_proto_rev(b->fs, rep->txn_id, b->lockcookie, b->pool)); @@ -5790,23 +7736,25 @@ get_next_revision_ids(const char **node_id, apr_pool_t *pool) { char *buf; - char *str, *last_str; + char *str; + svn_stringbuf_t *content; - SVN_ERR(read_current(svn_fs_fs__path_current(fs, pool), &buf, pool)); + SVN_ERR(read_content(&content, svn_fs_fs__path_current(fs, pool), pool)); + buf = content->data; - str = apr_strtok(buf, " ", &last_str); + str = svn_cstring_tokenize(" ", &buf); if (! str) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Corrupt 'current' file")); - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &buf); if (! str) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Corrupt 'current' file")); *node_id = apr_pstrdup(pool, str); - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" \n", &buf); if (! str) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Corrupt 'current' file")); @@ -5823,7 +7771,8 @@ struct write_hash_baton apr_size_t size; - svn_checksum_ctx_t *checksum_ctx; + svn_checksum_ctx_t *md5_ctx; + svn_checksum_ctx_t *sha1_ctx; }; /* The handler for the write_hash_rep stream. BATON is a @@ -5836,7 +7785,8 @@ write_hash_handler(void *baton, { struct write_hash_baton *whb = baton; - SVN_ERR(svn_checksum_update(whb->checksum_ctx, data, *len)); + SVN_ERR(svn_checksum_update(whb->md5_ctx, data, *len)); + SVN_ERR(svn_checksum_update(whb->sha1_ctx, data, *len)); SVN_ERR(svn_stream_write(whb->stream, data, len)); whb->size += *len; @@ -5845,41 +7795,184 @@ write_hash_handler(void *baton, } /* Write out the hash HASH as a text representation to file FILE. In - the process, record the total size of the dump in *SIZE, and the - md5 digest in CHECKSUM. Perform temporary allocations in POOL. */ + the process, record position, the total size of the dump and MD5 as + well as SHA1 in REP. If rep sharing has been enabled and REPS_HASH + is not NULL, it will be used in addition to the on-disk cache to find + earlier reps with the same content. When such existing reps can be + found, we will truncate the one just written from the file and return + the existing rep. Perform temporary allocations in POOL. */ static svn_error_t * -write_hash_rep(svn_filesize_t *size, - svn_checksum_t **checksum, +write_hash_rep(representation_t *rep, apr_file_t *file, apr_hash_t *hash, + svn_fs_t *fs, + apr_hash_t *reps_hash, apr_pool_t *pool) { svn_stream_t *stream; struct write_hash_baton *whb; + representation_t *old_rep; + + SVN_ERR(get_file_offset(&rep->offset, file, pool)); whb = apr_pcalloc(pool, sizeof(*whb)); whb->stream = svn_stream_from_aprfile2(file, TRUE, pool); whb->size = 0; - whb->checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); + whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); + whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool); stream = svn_stream_create(whb, pool); svn_stream_set_write(stream, write_hash_handler); - SVN_ERR(svn_stream_printf(whb->stream, pool, "PLAIN\n")); + SVN_ERR(svn_stream_puts(whb->stream, "PLAIN\n")); SVN_ERR(svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, pool)); /* Store the results. */ - SVN_ERR(svn_checksum_final(checksum, whb->checksum_ctx, pool)); - *size = whb->size; + SVN_ERR(svn_checksum_final(&rep->md5_checksum, whb->md5_ctx, pool)); + SVN_ERR(svn_checksum_final(&rep->sha1_checksum, whb->sha1_ctx, pool)); + + /* Check and see if we already have a representation somewhere that's + identical to the one we just wrote out. */ + SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, pool)); + + if (old_rep) + { + /* We need to erase from the protorev the data we just wrote. */ + SVN_ERR(svn_io_file_trunc(file, rep->offset, pool)); + + /* Use the old rep for this content. */ + memcpy(rep, old_rep, sizeof (*rep)); + } + else + { + /* Write out our cosmetic end marker. */ + SVN_ERR(svn_stream_puts(whb->stream, "ENDREP\n")); + + /* update the representation */ + rep->size = whb->size; + rep->expanded_size = whb->size; + } + + return SVN_NO_ERROR; +} + +/* Write out the hash HASH pertaining to the NODEREV in FS as a deltified + text representation to file FILE. In the process, record the total size + and the md5 digest in REP. If rep sharing has been enabled and REPS_HASH + is not NULL, it will be used in addition to the on-disk cache to find + earlier reps with the same content. When such existing reps can be found, + we will truncate the one just written from the file and return the existing + rep. If PROPS is set, assume that we want to a props representation as + the base for our delta. Perform temporary allocations in POOL. */ +static svn_error_t * +write_hash_delta_rep(representation_t *rep, + apr_file_t *file, + apr_hash_t *hash, + svn_fs_t *fs, + node_revision_t *noderev, + apr_hash_t *reps_hash, + svn_boolean_t props, + apr_pool_t *pool) +{ + svn_txdelta_window_handler_t diff_wh; + void *diff_whb; + + svn_stream_t *file_stream; + svn_stream_t *stream; + representation_t *base_rep; + representation_t *old_rep; + svn_stream_t *source; + const char *header; + + apr_off_t rep_end = 0; + apr_off_t delta_start = 0; + + struct write_hash_baton *whb; + fs_fs_data_t *ffd = fs->fsap_data; + int diff_version = ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT ? 1 : 0; + + /* Get the base for this delta. */ + SVN_ERR(choose_delta_base(&base_rep, fs, noderev, props, pool)); + SVN_ERR(read_representation(&source, fs, base_rep, pool)); + + SVN_ERR(get_file_offset(&rep->offset, file, pool)); + + /* Write out the rep header. */ + if (base_rep) + { + header = apr_psprintf(pool, REP_DELTA " %ld %" APR_OFF_T_FMT " %" + SVN_FILESIZE_T_FMT "\n", + base_rep->revision, base_rep->offset, + base_rep->size); + } + else + { + header = REP_DELTA "\n"; + } + SVN_ERR(svn_io_file_write_full(file, header, strlen(header), NULL, + pool)); + + SVN_ERR(get_file_offset(&delta_start, file, pool)); + file_stream = svn_stream_from_aprfile2(file, TRUE, pool); + + /* Prepare to write the svndiff data. */ + svn_txdelta_to_svndiff3(&diff_wh, + &diff_whb, + file_stream, + diff_version, + SVN_DELTA_COMPRESSION_LEVEL_DEFAULT, + pool); + + whb = apr_pcalloc(pool, sizeof(*whb)); + whb->stream = svn_txdelta_target_push(diff_wh, diff_whb, source, pool); + whb->size = 0; + whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); + whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool); + + /* serialize the hash */ + stream = svn_stream_create(whb, pool); + svn_stream_set_write(stream, write_hash_handler); + + SVN_ERR(svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_stream_close(whb->stream)); + + /* Store the results. */ + SVN_ERR(svn_checksum_final(&rep->md5_checksum, whb->md5_ctx, pool)); + SVN_ERR(svn_checksum_final(&rep->sha1_checksum, whb->sha1_ctx, pool)); + + /* Check and see if we already have a representation somewhere that's + identical to the one we just wrote out. */ + SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, pool)); + + if (old_rep) + { + /* We need to erase from the protorev the data we just wrote. */ + SVN_ERR(svn_io_file_trunc(file, rep->offset, pool)); + + /* Use the old rep for this content. */ + memcpy(rep, old_rep, sizeof (*rep)); + } + else + { + /* Write out our cosmetic end marker. */ + SVN_ERR(get_file_offset(&rep_end, file, pool)); + SVN_ERR(svn_stream_puts(file_stream, "ENDREP\n")); + + /* update the representation */ + rep->expanded_size = whb->size; + rep->size = rep_end - delta_start; + } - return svn_stream_printf(whb->stream, pool, "ENDREP\n"); + return SVN_NO_ERROR; } /* Sanity check ROOT_NODEREV, a candidate for being the root node-revision of (not yet committed) revision REV in FS. Use POOL for temporary allocations. + + If you change this function, consider updating svn_fs_fs__verify() too. */ static svn_error_t * validate_root_noderev(svn_fs_t *fs, @@ -5911,7 +8004,7 @@ validate_root_noderev(svn_fs_t *fs, This kind of corruption was seen on svn.apache.org (both on the root noderev and on other fspaths' noderevs); see - http://mid.gmane.org/20111002202833.GA12373@daniel3.local + issue #4129. Normally (rev == root_noderev->predecessor_count), but here we use a more roundabout check that should only trigger on new instances @@ -5953,6 +8046,10 @@ validate_root_noderev(svn_fs_t *fs, If REPS_TO_CACHE is not NULL, append to it a copy (allocated in REPS_POOL) of each data rep that is new in this revision. + If REPS_HASH is not NULL, append copies (allocated in REPS_POOL) + of the representations of each property rep that is new in this + revision. + AT_ROOT is true if the node revision being written is the root node-revision. It is only controls additional sanity checking logic. @@ -5968,6 +8065,7 @@ write_final_rev(const svn_fs_id_t **new_id_p, const char *start_copy_id, apr_off_t initial_offset, apr_array_header_t *reps_to_cache, + apr_hash_t *reps_hash, apr_pool_t *reps_pool, svn_boolean_t at_root, apr_pool_t *pool) @@ -6013,7 +8111,7 @@ write_final_rev(const svn_fs_id_t **new_id_p, svn_pool_clear(subpool); SVN_ERR(write_final_rev(&new_id, file, rev, fs, dirent->id, start_node_id, start_copy_id, initial_offset, - reps_to_cache, reps_pool, FALSE, + reps_to_cache, reps_hash, reps_pool, FALSE, subpool)); if (new_id && (svn_fs_fs__id_rev(new_id) == rev)) dirent->id = svn_fs_fs__id_copy(new_id, pool); @@ -6027,11 +8125,14 @@ write_final_rev(const svn_fs_id_t **new_id_p, noderev->data_rep->txn_id = NULL; noderev->data_rep->revision = rev; - SVN_ERR(get_file_offset(&noderev->data_rep->offset, file, pool)); - SVN_ERR(write_hash_rep(&noderev->data_rep->size, - &noderev->data_rep->md5_checksum, file, - str_entries, pool)); - noderev->data_rep->expanded_size = noderev->data_rep->size; + + if (ffd->deltify_directories) + SVN_ERR(write_hash_delta_rep(noderev->data_rep, file, + str_entries, fs, noderev, NULL, + FALSE, pool)); + else + SVN_ERR(write_hash_rep(noderev->data_rep, file, str_entries, + fs, NULL, pool)); } } else @@ -6059,15 +8160,18 @@ write_final_rev(const svn_fs_id_t **new_id_p, if (noderev->prop_rep && noderev->prop_rep->txn_id) { apr_hash_t *proplist; - SVN_ERR(svn_fs_fs__get_proplist(&proplist, fs, noderev, pool)); - SVN_ERR(get_file_offset(&noderev->prop_rep->offset, file, pool)); - SVN_ERR(write_hash_rep(&noderev->prop_rep->size, - &noderev->prop_rep->md5_checksum, file, - proplist, pool)); noderev->prop_rep->txn_id = NULL; noderev->prop_rep->revision = rev; + + if (ffd->deltify_properties) + SVN_ERR(write_hash_delta_rep(noderev->prop_rep, file, + proplist, fs, noderev, reps_hash, + TRUE, pool)); + else + SVN_ERR(write_hash_rep(noderev->prop_rep, file, proplist, + fs, reps_hash, pool)); } @@ -6110,24 +8214,53 @@ write_final_rev(const svn_fs_id_t **new_id_p, noderev->id = new_id; + if (ffd->rep_sharing_allowed) + { + /* Save the data representation's hash in the rep cache. */ + if ( noderev->data_rep && noderev->kind == svn_node_file + && noderev->data_rep->revision == rev) + { + SVN_ERR_ASSERT(reps_to_cache && reps_pool); + APR_ARRAY_PUSH(reps_to_cache, representation_t *) + = svn_fs_fs__rep_copy(noderev->data_rep, reps_pool); + } + + if (noderev->prop_rep && noderev->prop_rep->revision == rev) + { + /* Add new property reps to hash and on-disk cache. */ + representation_t *copy + = svn_fs_fs__rep_copy(noderev->prop_rep, reps_pool); + + SVN_ERR_ASSERT(reps_to_cache && reps_pool); + APR_ARRAY_PUSH(reps_to_cache, representation_t *) = copy; + + apr_hash_set(reps_hash, + copy->sha1_checksum->digest, + APR_SHA1_DIGESTSIZE, + copy); + } + } + + /* don't serialize SHA1 for dirs to disk (waste of space) */ + if (noderev->data_rep && noderev->kind == svn_node_dir) + noderev->data_rep->sha1_checksum = NULL; + + /* don't serialize SHA1 for props to disk (waste of space) */ + if (noderev->prop_rep) + noderev->prop_rep->sha1_checksum = NULL; + + /* Workaround issue #4031: is-fresh-txn-root in revision files. */ + noderev->is_fresh_txn_root = FALSE; + /* Write out our new node-revision. */ if (at_root) SVN_ERR(validate_root_noderev(fs, noderev, rev, pool)); + SVN_ERR(svn_fs_fs__write_noderev(svn_stream_from_aprfile2(file, TRUE, pool), noderev, ffd->format, svn_fs_fs__fs_supports_mergeinfo(fs), pool)); - /* Save the data representation's hash in the rep cache. */ - if (ffd->rep_sharing_allowed - && noderev->data_rep && noderev->kind == svn_node_file - && noderev->data_rep->revision == rev) - { - SVN_ERR_ASSERT(reps_to_cache && reps_pool); - APR_ARRAY_PUSH(reps_to_cache, representation_t *) - = svn_fs_fs__rep_copy(noderev->data_rep, reps_pool); - } - /* Return our ID that references the revision file. */ *new_id_p = noderev->id; @@ -6231,6 +8364,51 @@ write_current(svn_fs_t *fs, svn_revnum_t rev, const char *next_node_id, return move_into_place(tmp_name, name, name, pool); } +/* Open a new svn_fs_t handle to FS, set that handle's concept of "current + youngest revision" to NEW_REV, and call svn_fs_fs__verify_root() on + NEW_REV's revision root. + + Intended to be called as the very last step in a commit before 'current' + is bumped. This implies that we are holding the write lock. */ +static svn_error_t * +verify_as_revision_before_current_plus_plus(svn_fs_t *fs, + svn_revnum_t new_rev, + apr_pool_t *pool) +{ +#ifdef SVN_DEBUG + fs_fs_data_t *ffd = fs->fsap_data; + svn_fs_t *ft; /* fs++ == ft */ + svn_fs_root_t *root; + fs_fs_data_t *ft_ffd; + apr_hash_t *fs_config; + + SVN_ERR_ASSERT(ffd->svn_fs_open_); + + /* make sure FT does not simply return data cached by other instances + * but actually retrieves it from disk at least once. + */ + fs_config = apr_hash_make(pool); + svn_hash_sets(fs_config, SVN_FS_CONFIG_FSFS_CACHE_NS, + svn_uuid_generate(pool)); + SVN_ERR(ffd->svn_fs_open_(&ft, fs->path, + fs_config, + pool)); + ft_ffd = ft->fsap_data; + /* Don't let FT consult rep-cache.db, either. */ + ft_ffd->rep_sharing_allowed = FALSE; + + /* Time travel! */ + ft_ffd->youngest_rev_cache = new_rev; + + SVN_ERR(svn_fs_fs__revision_root(&root, ft, new_rev, pool)); + SVN_ERR_ASSERT(root->is_txn_root == FALSE && root->rev == new_rev); + SVN_ERR_ASSERT(ft_ffd->youngest_rev_cache == new_rev); + SVN_ERR(svn_fs_fs__verify_root(root, pool)); +#endif /* SVN_DEBUG */ + + return SVN_NO_ERROR; +} + /* Update the 'current' file to hold the correct next node and copy_ids from transaction TXN_ID in filesystem FS. The current revision is set to REV. Perform temporary allocations in POOL. */ @@ -6305,7 +8483,7 @@ verify_locks(svn_fs_t *fs, continue; /* Fetch the change associated with our path. */ - change = apr_hash_get(changes, path, APR_HASH_KEY_STRING); + change = svn_hash_gets(changes, path); /* What does it mean to succeed at lock verification for a given path? For an existing file or directory getting modified @@ -6341,6 +8519,7 @@ struct commit_baton { svn_fs_t *fs; svn_fs_txn_t *txn; apr_array_header_t *reps_to_cache; + apr_hash_t *reps_hash; apr_pool_t *reps_pool; }; @@ -6398,8 +8577,8 @@ commit_body(void *baton, apr_pool_t *pool) root_id = svn_fs_fs__id_txn_create("0", "0", cb->txn->id, pool); SVN_ERR(write_final_rev(&new_root_id, proto_file, new_rev, cb->fs, root_id, start_node_id, start_copy_id, initial_offset, - cb->reps_to_cache, cb->reps_pool, TRUE, - pool)); + cb->reps_to_cache, cb->reps_hash, cb->reps_pool, + TRUE, pool)); /* Write the changed-path information. */ SVN_ERR(write_final_changed_path_info(&changed_path_offset, proto_file, @@ -6423,14 +8602,13 @@ commit_body(void *baton, apr_pool_t *pool) txnprop_list = apr_array_make(pool, 3, sizeof(svn_prop_t)); prop.value = NULL; - if (apr_hash_get(txnprops, SVN_FS__PROP_TXN_CHECK_OOD, APR_HASH_KEY_STRING)) + if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_OOD)) { prop.name = SVN_FS__PROP_TXN_CHECK_OOD; APR_ARRAY_PUSH(txnprop_list, svn_prop_t) = prop; } - if (apr_hash_get(txnprops, SVN_FS__PROP_TXN_CHECK_LOCKS, - APR_HASH_KEY_STRING)) + if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_LOCKS)) { prop.name = SVN_FS__PROP_TXN_CHECK_LOCKS; APR_ARRAY_PUSH(txnprop_list, svn_prop_t) = prop; @@ -6444,7 +8622,7 @@ commit_body(void *baton, apr_pool_t *pool) fails because the shard already existed for some reason. */ if (ffd->max_files_per_dir && new_rev % ffd->max_files_per_dir == 0) { - if (1) + /* Create the revs shard. */ { const char *new_dir = path_rev_shard(cb->fs, new_rev, pool); svn_error_t *err = svn_io_dir_make(new_dir, APR_OS_DEFAULT, pool); @@ -6501,6 +8679,7 @@ commit_body(void *baton, apr_pool_t *pool) old_rev_filename, pool)); /* Update the 'current' file. */ + SVN_ERR(verify_as_revision_before_current_plus_plus(cb->fs, new_rev, pool)); SVN_ERR(write_final_current(cb->fs, cb->txn->id, new_rev, start_node_id, start_copy_id, pool)); @@ -6540,23 +8719,6 @@ write_reps_to_cache(svn_fs_t *fs, return SVN_NO_ERROR; } -/* Implements svn_sqlite__transaction_callback_t. */ -static svn_error_t * -commit_sqlite_txn_callback(void *baton, svn_sqlite__db_t *db, - apr_pool_t *scratch_pool) -{ - struct commit_baton *cb = baton; - - /* Write new entries to the rep-sharing database. - * - * We use an sqlite transcation to speed things up; - * see <http://www.sqlite.org/faq.html#q19>. - */ - SVN_ERR(write_reps_to_cache(cb->fs, cb->reps_to_cache, scratch_pool)); - - return SVN_NO_ERROR; -} - svn_error_t * svn_fs_fs__commit(svn_revnum_t *new_rev_p, svn_fs_t *fs, @@ -6573,24 +8735,33 @@ svn_fs_fs__commit(svn_revnum_t *new_rev_p, if (ffd->rep_sharing_allowed) { cb.reps_to_cache = apr_array_make(pool, 5, sizeof(representation_t *)); + cb.reps_hash = apr_hash_make(pool); cb.reps_pool = pool; } else { cb.reps_to_cache = NULL; + cb.reps_hash = NULL; cb.reps_pool = NULL; } SVN_ERR(svn_fs_fs__with_write_lock(fs, commit_body, &cb, pool)); + /* At this point, *NEW_REV_P has been set, so errors below won't affect + the success of the commit. (See svn_fs_commit_txn().) */ + if (ffd->rep_sharing_allowed) { - /* At this point, *NEW_REV_P has been set, so errors here won't affect - the success of the commit. (See svn_fs_commit_txn().) */ SVN_ERR(svn_fs_fs__open_rep_cache(fs, pool)); - SVN_ERR(svn_sqlite__with_transaction(ffd->rep_cache_db, - commit_sqlite_txn_callback, - &cb, pool)); + + /* Write new entries to the rep-sharing database. + * + * We use an sqlite transaction to speed things up; + * see <http://www.sqlite.org/faq.html#q19>. + */ + SVN_SQLITE__WITH_TXN( + write_reps_to_cache(fs, cb.reps_to_cache, pool), + ffd->rep_cache_db); } return SVN_NO_ERROR; @@ -6646,7 +8817,7 @@ write_revision_zero(svn_fs_t *fs) date.data = svn_time_to_cstring(apr_time_now(), fs->pool); date.len = strlen(date.data); proplist = apr_hash_make(fs->pool); - apr_hash_set(proplist, SVN_PROP_REVISION_DATE, APR_HASH_KEY_STRING, &date); + svn_hash_sets(proplist, SVN_PROP_REVISION_DATE, &date); return set_revision_proplist(fs, 0, proplist, fs->pool); } @@ -6662,15 +8833,14 @@ svn_fs_fs__create(svn_fs_t *fs, /* See if compatibility with older versions was explicitly requested. */ if (fs->config) { - if (apr_hash_get(fs->config, SVN_FS_CONFIG_PRE_1_4_COMPATIBLE, - APR_HASH_KEY_STRING)) + if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_4_COMPATIBLE)) format = 1; - else if (apr_hash_get(fs->config, SVN_FS_CONFIG_PRE_1_5_COMPATIBLE, - APR_HASH_KEY_STRING)) + else if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_5_COMPATIBLE)) format = 2; - else if (apr_hash_get(fs->config, SVN_FS_CONFIG_PRE_1_6_COMPATIBLE, - APR_HASH_KEY_STRING)) + else if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_6_COMPATIBLE)) format = 3; + else if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_8_COMPATIBLE)) + format = 4; } ffd->format = format; @@ -6713,13 +8883,18 @@ svn_fs_fs__create(svn_fs_t *fs, ? "0\n" : "0 1 1\n"), pool)); SVN_ERR(svn_io_file_create(path_lock(fs, pool), "", pool)); - SVN_ERR(svn_fs_fs__set_uuid(fs, svn_uuid_generate(pool), pool)); + SVN_ERR(svn_fs_fs__set_uuid(fs, NULL, pool)); SVN_ERR(write_revision_zero(fs)); - SVN_ERR(write_config(fs, pool)); + /* Create the fsfs.conf file if supported. Older server versions would + simply ignore the file but that might result in a different behavior + than with the later releases. Also, hotcopy would ignore, i.e. not + copy, a fsfs.conf with old formats. */ + if (ffd->format >= SVN_FS_FS__MIN_CONFIG_FILE) + SVN_ERR(write_config(fs, pool)); - SVN_ERR(read_config(fs, pool)); + SVN_ERR(read_config(ffd, fs->path, pool)); /* Create the min unpacked rev file. */ if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) @@ -6873,13 +9048,13 @@ recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev, pool)); /* Check that this is a directory. It should be. */ - value = apr_hash_get(headers, HEADER_TYPE, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_TYPE); if (value == NULL || strcmp(value, KIND_DIR) != 0) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Recovery encountered a non-directory node")); /* Get the data location. No data location indicates an empty directory. */ - value = apr_hash_get(headers, HEADER_TEXT, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_TEXT); if (!value) return SVN_NO_ERROR; SVN_ERR(read_rep_offsets(&data_rep, value, NULL, FALSE, pool)); @@ -6905,7 +9080,9 @@ recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev, stored in the representation. */ baton.file = rev_file; baton.pool = pool; - baton.remaining = data_rep->expanded_size; + baton.remaining = data_rep->expanded_size + ? data_rep->expanded_size + : data_rep->size; stream = svn_stream_create(&baton, pool); svn_stream_set_read(stream, read_handler_recover); @@ -6920,7 +9097,7 @@ recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev, for (hi = apr_hash_first(pool, entries); hi; hi = apr_hash_next(hi)) { char *str_val; - char *str, *last_str; + char *str; svn_node_kind_t kind; svn_fs_id_t *id; const char *node_id, *copy_id; @@ -6931,7 +9108,7 @@ recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev, str_val = apr_pstrdup(iterpool, path->data); - str = apr_strtok(str_val, " ", &last_str); + str = svn_cstring_tokenize(" ", &str_val); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Directory entry corrupt")); @@ -6946,7 +9123,7 @@ recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev, _("Directory entry corrupt")); } - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &str_val); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Directory entry corrupt")); @@ -6986,6 +9163,77 @@ recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev, return SVN_NO_ERROR; } +/* Return TRUE, if for REVISION in FS, we can find the revprop pack file. + * Use POOL for temporary allocations. + * Set *MISSING, if the reason is a missing manifest or pack file. + */ +static svn_boolean_t +packed_revprop_available(svn_boolean_t *missing, + svn_fs_t *fs, + svn_revnum_t revision, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + svn_stringbuf_t *content = NULL; + + /* try to read the manifest file */ + const char *folder = path_revprops_pack_shard(fs, revision, pool); + const char *manifest_path = svn_dirent_join(folder, PATH_MANIFEST, pool); + + svn_error_t *err = try_stringbuf_from_file(&content, + missing, + manifest_path, + FALSE, + pool); + + /* if the manifest cannot be read, consider the pack files inaccessible + * even if the file itself exists. */ + if (err) + { + svn_error_clear(err); + return FALSE; + } + + if (*missing) + return FALSE; + + /* parse manifest content until we find the entry for REVISION. + * Revision 0 is never packed. */ + revision = revision < ffd->max_files_per_dir + ? revision - 1 + : revision % ffd->max_files_per_dir; + while (content->data) + { + char *next = strchr(content->data, '\n'); + if (next) + { + *next = 0; + ++next; + } + + if (revision-- == 0) + { + /* the respective pack file must exist (and be a file) */ + svn_node_kind_t kind; + err = svn_io_check_path(svn_dirent_join(folder, content->data, + pool), + &kind, pool); + if (err) + { + svn_error_clear(err); + return FALSE; + } + + *missing = kind == svn_node_none; + return kind == svn_node_file; + } + + content->data = next; + } + + return FALSE; +} + /* Baton used for recover_body below. */ struct recover_baton { svn_fs_t *fs; @@ -7008,7 +9256,10 @@ recover_body(void *baton, apr_pool_t *pool) svn_revnum_t youngest_rev; svn_node_kind_t youngest_revprops_kind; - /* First, we need to know the largest revision in the filesystem. */ + /* Lose potentially corrupted data in temp files */ + SVN_ERR(cleanup_revprop_namespace(fs)); + + /* We need to know the largest revision in the filesystem. */ SVN_ERR(recover_get_largest_revision(fs, &max_rev, pool)); /* Get the expected youngest revision */ @@ -7099,13 +9350,24 @@ recover_body(void *baton, apr_pool_t *pool) &youngest_revprops_kind, pool)); if (youngest_revprops_kind == svn_node_none) { - if (1) + svn_boolean_t missing = TRUE; + if (!packed_revprop_available(&missing, fs, max_rev, pool)) { - return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Revision %ld has a revs file but no " - "revprops file"), - max_rev); - } + if (missing) + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Revision %ld has a revs file but no " + "revprops file"), + max_rev); + } + else + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Revision %ld has a revs file but the " + "revprops file is inaccessible"), + max_rev); + } + } } else if (youngest_revprops_kind != svn_node_file) { @@ -7115,9 +9377,17 @@ recover_body(void *baton, apr_pool_t *pool) max_rev); } - /* Prune younger-than-(newfound-youngest) revisions from the rep cache. */ - if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) - SVN_ERR(svn_fs_fs__del_rep_reference(fs, max_rev, pool)); + /* Prune younger-than-(newfound-youngest) revisions from the rep + cache if sharing is enabled taking care not to create the cache + if it does not exist. */ + if (ffd->rep_sharing_allowed) + { + svn_boolean_t rep_cache_exists; + + SVN_ERR(svn_fs_fs__exists_rep_cache(&rep_cache_exists, fs, pool)); + if (rep_cache_exists) + SVN_ERR(svn_fs_fs__del_rep_reference(fs, max_rev, pool)); + } /* Now store the discovered youngest revision, and the next IDs if relevant, in a new 'current' file. */ @@ -7143,17 +9413,6 @@ svn_fs_fs__recover(svn_fs_t *fs, } svn_error_t * -svn_fs_fs__get_uuid(svn_fs_t *fs, - const char **uuid_p, - apr_pool_t *pool) -{ - fs_fs_data_t *ffd = fs->fsap_data; - - *uuid_p = apr_pstrdup(pool, ffd->uuid); - return SVN_NO_ERROR; -} - -svn_error_t * svn_fs_fs__set_uuid(svn_fs_t *fs, const char *uuid, apr_pool_t *pool) @@ -7162,7 +9421,6 @@ svn_fs_fs__set_uuid(svn_fs_t *fs, apr_size_t my_uuid_len; const char *tmp_path; const char *uuid_path = path_uuid(fs, pool); - fs_fs_data_t *ffd = fs->fsap_data; if (! uuid) uuid = svn_uuid_generate(pool); @@ -7183,7 +9441,7 @@ svn_fs_fs__set_uuid(svn_fs_t *fs, /* Remove the newline we added, and stash the UUID. */ my_uuid[my_uuid_len - 1] = '\0'; - ffd->uuid = my_uuid; + fs->uuid = my_uuid; return SVN_NO_ERROR; } @@ -7191,7 +9449,7 @@ svn_fs_fs__set_uuid(svn_fs_t *fs, /** Node origin lazy cache. */ /* If directory PATH does not exist, create it and give it the same - permissions as FS->path.*/ + permissions as FS_path.*/ svn_error_t * svn_fs_fs__ensure_dir_exists(const char *path, const char *fs_path, @@ -7253,7 +9511,7 @@ svn_fs_fs__get_node_origin(const svn_fs_id_t **origin_id, if (node_origins) { svn_string_t *origin_id_str = - apr_hash_get(node_origins, node_id, APR_HASH_KEY_STRING); + svn_hash_gets(node_origins, node_id); if (origin_id_str) *origin_id = svn_fs_fs__id_parse(origin_id_str->data, origin_id_str->len, pool); @@ -7288,7 +9546,7 @@ set_node_origins_for_file(svn_fs_t *fs, if (! origins_hash) origins_hash = apr_hash_make(pool); - old_node_rev_id = apr_hash_get(origins_hash, node_id, APR_HASH_KEY_STRING); + old_node_rev_id = svn_hash_gets(origins_hash, node_id); if (old_node_rev_id && !svn_string_compare(node_rev_id, old_node_rev_id)) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, @@ -7297,7 +9555,7 @@ set_node_origins_for_file(svn_fs_t *fs, "(%s)"), node_id, old_node_rev_id->data, node_rev_id->data); - apr_hash_set(origins_hash, node_id, APR_HASH_KEY_STRING, node_rev_id); + svn_hash_sets(origins_hash, node_id, node_rev_id); /* Sure, there's a race condition here. Two processes could be trying to add different cache elements to the same file at the @@ -7478,7 +9736,7 @@ svn_fs_fs__revision_prop(svn_string_t **value_p, SVN_ERR(svn_fs__check_fs(fs, TRUE)); SVN_ERR(svn_fs_fs__revision_proplist(&table, fs, rev, pool)); - *value_p = apr_hash_get(table, propname, APR_HASH_KEY_STRING); + *value_p = svn_hash_gets(table, propname); return SVN_NO_ERROR; } @@ -7496,7 +9754,6 @@ struct change_rev_prop_baton { /* The work-horse for svn_fs_fs__change_rev_prop, called with the FS write lock. This implements the svn_fs_fs__with_write_lock() 'body' callback type. BATON is a 'struct change_rev_prop_baton *'. */ - static svn_error_t * change_rev_prop_body(void *baton, apr_pool_t *pool) { @@ -7508,8 +9765,7 @@ change_rev_prop_body(void *baton, apr_pool_t *pool) if (cb->old_value_p) { const svn_string_t *wanted_value = *cb->old_value_p; - const svn_string_t *present_value = apr_hash_get(table, cb->name, - APR_HASH_KEY_STRING); + const svn_string_t *present_value = svn_hash_gets(table, cb->name); if ((!wanted_value != !present_value) || (wanted_value && present_value && !svn_string_compare(wanted_value, present_value))) @@ -7522,7 +9778,7 @@ change_rev_prop_body(void *baton, apr_pool_t *pool) } /* Fall through. */ } - apr_hash_set(table, cb->name, APR_HASH_KEY_STRING, cb->value); + svn_hash_sets(table, cb->name, cb->value); return set_revision_proplist(cb->fs, cb->rev, table, pool); } @@ -7581,7 +9837,7 @@ svn_fs_fs__txn_prop(svn_string_t **value_p, SVN_ERR(svn_fs__check_fs(fs, TRUE)); SVN_ERR(svn_fs_fs__txn_proplist(&table, txn, pool)); - *value_p = apr_hash_get(table, propname, APR_HASH_KEY_STRING); + *value_p = svn_hash_gets(table, propname); return SVN_NO_ERROR; } @@ -7659,43 +9915,31 @@ write_revnum_file(const char *fs_path, return SVN_NO_ERROR; } -/* Pack a single shard SHARD in REVS_DIR, using POOL for allocations. - CANCEL_FUNC and CANCEL_BATON are what you think they are. - - If for some reason we detect a partial packing already performed, we - remove the pack file and start again. */ +/* Pack the revision SHARD containing exactly MAX_FILES_PER_DIR revisions + * from SHARD_PATH into the PACK_FILE_DIR, using POOL for allocations. + * CANCEL_FUNC and CANCEL_BATON are what you think they are. + * + * If for some reason we detect a partial packing already performed, we + * remove the pack file and start again. + */ static svn_error_t * -pack_shard(const char *revs_dir, - const char *fs_path, - apr_int64_t shard, - int max_files_per_dir, - svn_fs_pack_notify_t notify_func, - void *notify_baton, - svn_cancel_func_t cancel_func, - void *cancel_baton, - apr_pool_t *pool) +pack_rev_shard(const char *pack_file_dir, + const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) { - const char *pack_file_path, *manifest_file_path, *shard_path; - const char *pack_file_dir; + const char *pack_file_path, *manifest_file_path; svn_stream_t *pack_stream, *manifest_stream; svn_revnum_t start_rev, end_rev, rev; apr_off_t next_offset; apr_pool_t *iterpool; /* Some useful paths. */ - pack_file_dir = svn_dirent_join(revs_dir, - apr_psprintf(pool, "%" APR_INT64_T_FMT ".pack", shard), - pool); - pack_file_path = svn_dirent_join(pack_file_dir, "pack", pool); - manifest_file_path = svn_dirent_join(pack_file_dir, "manifest", pool); - shard_path = svn_dirent_join(revs_dir, - apr_psprintf(pool, "%" APR_INT64_T_FMT, shard), - pool); - - /* Notify caller we're starting to pack this shard. */ - if (notify_func) - SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start, - pool)); + pack_file_path = svn_dirent_join(pack_file_dir, PATH_PACKED, pool); + manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, pool); /* Remove any existing pack file for this shard, since it is incomplete. */ SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton, @@ -7741,21 +9985,338 @@ pack_shard(const char *revs_dir, SVN_ERR(svn_stream_close(manifest_stream)); SVN_ERR(svn_stream_close(pack_stream)); - SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, pool)); - SVN_ERR(svn_io_set_file_read_only(pack_file_path, FALSE, pool)); - SVN_ERR(svn_io_set_file_read_only(manifest_file_path, FALSE, pool)); + SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); + SVN_ERR(svn_io_set_file_read_only(pack_file_path, FALSE, iterpool)); + SVN_ERR(svn_io_set_file_read_only(manifest_file_path, FALSE, iterpool)); + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Copy revprop files for revisions [START_REV, END_REV) from SHARD_PATH + * to the pack file at PACK_FILE_NAME in PACK_FILE_DIR. + * + * The file sizes have already been determined and written to SIZES. + * Please note that this function will be executed while the filesystem + * has been locked and that revprops files will therefore not be modified + * while the pack is in progress. + * + * COMPRESSION_LEVEL defines how well the resulting pack file shall be + * compressed or whether is shall be compressed at all. TOTAL_SIZE is + * a hint on which initial buffer size we should use to hold the pack file + * content. + * + * CANCEL_FUNC and CANCEL_BATON are used as usual. Temporary allocations + * are done in SCRATCH_POOL. + */ +static svn_error_t * +copy_revprops(const char *pack_file_dir, + const char *pack_filename, + const char *shard_path, + svn_revnum_t start_rev, + svn_revnum_t end_rev, + apr_array_header_t *sizes, + apr_size_t total_size, + int compression_level, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + svn_stream_t *pack_stream; + apr_file_t *pack_file; + svn_revnum_t rev; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + svn_stream_t *stream; + + /* create empty data buffer and a write stream on top of it */ + svn_stringbuf_t *uncompressed + = svn_stringbuf_create_ensure(total_size, scratch_pool); + svn_stringbuf_t *compressed + = svn_stringbuf_create_empty(scratch_pool); + pack_stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); + + /* write the pack file header */ + SVN_ERR(serialize_revprops_header(pack_stream, start_rev, sizes, 0, + sizes->nelts, iterpool)); + + /* Some useful paths. */ + SVN_ERR(svn_io_file_open(&pack_file, svn_dirent_join(pack_file_dir, + pack_filename, + scratch_pool), + APR_WRITE | APR_CREATE, APR_OS_DEFAULT, + scratch_pool)); + + /* Iterate over the revisions in this shard, squashing them together. */ + for (rev = start_rev; rev <= end_rev; rev++) + { + const char *path; + + svn_pool_clear(iterpool); + + /* Construct the file name. */ + path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), + iterpool); + + /* Copy all the bits from the non-packed revprop file to the end of + * the pack file. */ + SVN_ERR(svn_stream_open_readonly(&stream, path, iterpool, iterpool)); + SVN_ERR(svn_stream_copy3(stream, pack_stream, + cancel_func, cancel_baton, iterpool)); + } + + /* flush stream buffers to content buffer */ + SVN_ERR(svn_stream_close(pack_stream)); + + /* compress the content (or just store it for COMPRESSION_LEVEL 0) */ + SVN_ERR(svn__compress(svn_stringbuf__morph_into_string(uncompressed), + compressed, compression_level)); + + /* write the pack file content to disk */ + stream = svn_stream_from_aprfile2(pack_file, FALSE, scratch_pool); + SVN_ERR(svn_stream_write(stream, compressed->data, &compressed->len)); + SVN_ERR(svn_stream_close(stream)); + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* For the revprop SHARD at SHARD_PATH with exactly MAX_FILES_PER_DIR + * revprop files in it, create a packed shared at PACK_FILE_DIR. + * + * COMPRESSION_LEVEL defines how well the resulting pack file shall be + * compressed or whether is shall be compressed at all. Individual pack + * file containing more than one revision will be limited to a size of + * MAX_PACK_SIZE bytes before compression. + * + * CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary + * allocations are done in SCRATCH_POOL. + */ +static svn_error_t * +pack_revprops_shard(const char *pack_file_dir, + const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + apr_off_t max_pack_size, + int compression_level, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + const char *manifest_file_path, *pack_filename = NULL; + svn_stream_t *manifest_stream; + svn_revnum_t start_rev, end_rev, rev; + apr_off_t total_size; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + apr_array_header_t *sizes; + + /* Some useful paths. */ + manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, + scratch_pool); + + /* Remove any existing pack file for this shard, since it is incomplete. */ + SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton, + scratch_pool)); + + /* Create the new directory and manifest file stream. */ + SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool)); + SVN_ERR(svn_stream_open_writable(&manifest_stream, manifest_file_path, + scratch_pool, scratch_pool)); + + /* revisions to handle. Special case: revision 0 */ + start_rev = (svn_revnum_t) (shard * max_files_per_dir); + end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1); + if (start_rev == 0) + ++start_rev; + + /* initialize the revprop size info */ + sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_off_t)); + total_size = 2 * SVN_INT64_BUFFER_SIZE; + + /* Iterate over the revisions in this shard, determine their size and + * squashing them together into pack files. */ + for (rev = start_rev; rev <= end_rev; rev++) + { + apr_finfo_t finfo; + const char *path; + + svn_pool_clear(iterpool); + + /* Get the size of the file. */ + path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), + iterpool); + SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool)); + + /* if we already have started a pack file and this revprop cannot be + * appended to it, write the previous pack file. */ + if (sizes->nelts != 0 && + total_size + SVN_INT64_BUFFER_SIZE + finfo.size > max_pack_size) + { + SVN_ERR(copy_revprops(pack_file_dir, pack_filename, shard_path, + start_rev, rev-1, sizes, (apr_size_t)total_size, + compression_level, cancel_func, cancel_baton, + iterpool)); + + /* next pack file starts empty again */ + apr_array_clear(sizes); + total_size = 2 * SVN_INT64_BUFFER_SIZE; + start_rev = rev; + } + + /* Update the manifest. Allocate a file name for the current pack + * file if it is a new one */ + if (sizes->nelts == 0) + pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev); + + SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%s\n", + pack_filename)); + + /* add to list of files to put into the current pack file */ + APR_ARRAY_PUSH(sizes, apr_off_t) = finfo.size; + total_size += SVN_INT64_BUFFER_SIZE + finfo.size; + } + + /* write the last pack file */ + if (sizes->nelts != 0) + SVN_ERR(copy_revprops(pack_file_dir, pack_filename, shard_path, + start_rev, rev-1, sizes, (apr_size_t)total_size, + compression_level, cancel_func, cancel_baton, + iterpool)); + + /* flush the manifest file and update permissions */ + SVN_ERR(svn_stream_close(manifest_stream)); + SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Delete the non-packed revprop SHARD at SHARD_PATH with exactly + * MAX_FILES_PER_DIR revprop files in it. If this is shard 0, keep the + * revprop file for revision 0. + * + * CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary + * allocations are done in SCRATCH_POOL. + */ +static svn_error_t * +delete_revprops_shard(const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + if (shard == 0) + { + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + int i; + + /* delete all files except the one for revision 0 */ + for (i = 1; i < max_files_per_dir; ++i) + { + const char *path = svn_dirent_join(shard_path, + apr_psprintf(iterpool, "%d", i), + iterpool); + if (cancel_func) + SVN_ERR((*cancel_func)(cancel_baton)); + + SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + } + else + SVN_ERR(svn_io_remove_dir2(shard_path, TRUE, + cancel_func, cancel_baton, scratch_pool)); + + return SVN_NO_ERROR; +} + +/* In the file system at FS_PATH, pack the SHARD in REVS_DIR and + * REVPROPS_DIR containing exactly MAX_FILES_PER_DIR revisions, using POOL + * for allocations. REVPROPS_DIR will be NULL if revprop packing is not + * supported. COMPRESSION_LEVEL and MAX_PACK_SIZE will be ignored in that + * case. + * + * CANCEL_FUNC and CANCEL_BATON are what you think they are; similarly + * NOTIFY_FUNC and NOTIFY_BATON. + * + * If for some reason we detect a partial packing already performed, we + * remove the pack file and start again. + */ +static svn_error_t * +pack_shard(const char *revs_dir, + const char *revsprops_dir, + const char *fs_path, + apr_int64_t shard, + int max_files_per_dir, + apr_off_t max_pack_size, + int compression_level, + svn_fs_pack_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + const char *rev_shard_path, *rev_pack_file_dir; + const char *revprops_shard_path, *revprops_pack_file_dir; + + /* Notify caller we're starting to pack this shard. */ + if (notify_func) + SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start, + pool)); + + /* Some useful paths. */ + rev_pack_file_dir = svn_dirent_join(revs_dir, + apr_psprintf(pool, + "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, + shard), + pool); + rev_shard_path = svn_dirent_join(revs_dir, + apr_psprintf(pool, "%" APR_INT64_T_FMT, shard), + pool); + + /* pack the revision content */ + SVN_ERR(pack_rev_shard(rev_pack_file_dir, rev_shard_path, + shard, max_files_per_dir, + cancel_func, cancel_baton, pool)); + + /* if enabled, pack the revprops in an equivalent way */ + if (revsprops_dir) + { + revprops_pack_file_dir = svn_dirent_join(revsprops_dir, + apr_psprintf(pool, + "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, + shard), + pool); + revprops_shard_path = svn_dirent_join(revsprops_dir, + apr_psprintf(pool, "%" APR_INT64_T_FMT, shard), + pool); + + SVN_ERR(pack_revprops_shard(revprops_pack_file_dir, revprops_shard_path, + shard, max_files_per_dir, + (int)(0.9 * max_pack_size), + compression_level, + cancel_func, cancel_baton, pool)); + } /* Update the min-unpacked-rev file to reflect our newly packed shard. * (This doesn't update ffd->min_unpacked_rev. That will be updated by * update_min_unpacked_rev() when necessary.) */ SVN_ERR(write_revnum_file(fs_path, PATH_MIN_UNPACKED_REV, (svn_revnum_t)((shard + 1) * max_files_per_dir), - iterpool)); - svn_pool_destroy(iterpool); + pool)); - /* Finally, remove the existing shard directory. */ - SVN_ERR(svn_io_remove_dir2(shard_path, TRUE, cancel_func, cancel_baton, - pool)); + /* Finally, remove the existing shard directories. */ + SVN_ERR(svn_io_remove_dir2(rev_shard_path, TRUE, + cancel_func, cancel_baton, pool)); + if (revsprops_dir) + SVN_ERR(delete_revprops_shard(revprops_shard_path, + shard, max_files_per_dir, + cancel_func, cancel_baton, pool)); /* Notify caller we're starting to pack this shard. */ if (notify_func) @@ -7786,8 +10347,7 @@ struct pack_baton extension, on not having to use a retry when calling svn_fs_fs__path_rev_absolute() and friends). If you add a call to this function, consider whether you have to call - update_min_unpacked_rev() and update_min_unpacked_revprop() - afterwards. + update_min_unpacked_rev(). See this thread: http://thread.gmane.org/1291206765.3782.3309.camel@edith */ static svn_error_t * @@ -7795,51 +10355,63 @@ pack_body(void *baton, apr_pool_t *pool) { struct pack_baton *pb = baton; - int format, max_files_per_dir; + fs_fs_data_t ffd = {0}; apr_int64_t completed_shards; apr_int64_t i; svn_revnum_t youngest; apr_pool_t *iterpool; - const char *data_path; - svn_revnum_t min_unpacked_rev; + const char *rev_data_path; + const char *revprops_data_path = NULL; - SVN_ERR(read_format(&format, &max_files_per_dir, path_format(pb->fs, pool), - pool)); - SVN_ERR(check_format(format)); + /* read repository settings */ + SVN_ERR(read_format(&ffd.format, &ffd.max_files_per_dir, + path_format(pb->fs, pool), pool)); + SVN_ERR(check_format(ffd.format)); + SVN_ERR(read_config(&ffd, pb->fs->path, pool)); /* If the repository isn't a new enough format, we don't support packing. Return a friendly error to that effect. */ - if (format < SVN_FS_FS__MIN_PACKED_FORMAT) + if (ffd.format < SVN_FS_FS__MIN_PACKED_FORMAT) return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, _("FSFS format (%d) too old to pack; please upgrade the filesystem."), - format); + ffd.format); /* If we aren't using sharding, we can't do any packing, so quit. */ - if (!max_files_per_dir) + if (!ffd.max_files_per_dir) return SVN_NO_ERROR; - SVN_ERR(read_min_unpacked_rev(&min_unpacked_rev, + SVN_ERR(read_min_unpacked_rev(&ffd.min_unpacked_rev, path_min_unpacked_rev(pb->fs, pool), pool)); SVN_ERR(get_youngest(&youngest, pb->fs->path, pool)); - completed_shards = (youngest + 1) / max_files_per_dir; + completed_shards = (youngest + 1) / ffd.max_files_per_dir; /* See if we've already completed all possible shards thus far. */ - if (min_unpacked_rev == (completed_shards * max_files_per_dir)) + if (ffd.min_unpacked_rev == (completed_shards * ffd.max_files_per_dir)) return SVN_NO_ERROR; - data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, pool); + rev_data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, pool); + if (ffd.format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) + revprops_data_path = svn_dirent_join(pb->fs->path, PATH_REVPROPS_DIR, + pool); iterpool = svn_pool_create(pool); - for (i = min_unpacked_rev / max_files_per_dir; i < completed_shards; i++) + for (i = ffd.min_unpacked_rev / ffd.max_files_per_dir; + i < completed_shards; + i++) { svn_pool_clear(iterpool); if (pb->cancel_func) SVN_ERR(pb->cancel_func(pb->cancel_baton)); - SVN_ERR(pack_shard(data_path, pb->fs->path, i, max_files_per_dir, + SVN_ERR(pack_shard(rev_data_path, revprops_data_path, + pb->fs->path, i, ffd.max_files_per_dir, + ffd.revprop_pack_size, + ffd.compress_packed_revprops + ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT + : SVN_DELTA_COMPRESSION_LEVEL_NONE, pb->notify_func, pb->notify_baton, pb->cancel_func, pb->cancel_baton, iterpool)); } @@ -7864,3 +10436,1217 @@ svn_fs_fs__pack(svn_fs_t *fs, pb.cancel_baton = cancel_baton; return svn_fs_fs__with_write_lock(fs, pack_body, &pb, pool); } + + +/** Verifying. **/ + +/* Baton type expected by verify_walker(). The purpose is to reuse open + * rev / pack file handles between calls. Its contents need to be cleaned + * periodically to limit resource usage. + */ +typedef struct verify_walker_baton_t +{ + /* number of calls to verify_walker() since the last clean */ + int iteration_count; + + /* number of files opened since the last clean */ + int file_count; + + /* progress notification callback to invoke periodically (may be NULL) */ + svn_fs_progress_notify_func_t notify_func; + + /* baton to use with NOTIFY_FUNC */ + void *notify_baton; + + /* remember the last revision for which we called notify_func */ + svn_revnum_t last_notified_revision; + + /* current file handle (or NULL) */ + apr_file_t *file_hint; + + /* corresponding revision (or SVN_INVALID_REVNUM) */ + svn_revnum_t rev_hint; + + /* pool to use for the file handles etc. */ + apr_pool_t *pool; +} verify_walker_baton_t; + +/* Used by svn_fs_fs__verify(). + Implements svn_fs_fs__walk_rep_reference().walker. */ +static svn_error_t * +verify_walker(representation_t *rep, + void *baton, + svn_fs_t *fs, + apr_pool_t *scratch_pool) +{ + struct rep_state *rs; + struct rep_args *rep_args; + + if (baton) + { + verify_walker_baton_t *walker_baton = baton; + apr_file_t * previous_file; + + /* notify and free resources periodically */ + if ( walker_baton->iteration_count > 1000 + || walker_baton->file_count > 16) + { + if ( walker_baton->notify_func + && rep->revision != walker_baton->last_notified_revision) + { + walker_baton->notify_func(rep->revision, + walker_baton->notify_baton, + scratch_pool); + walker_baton->last_notified_revision = rep->revision; + } + + svn_pool_clear(walker_baton->pool); + + walker_baton->iteration_count = 0; + walker_baton->file_count = 0; + walker_baton->file_hint = NULL; + walker_baton->rev_hint = SVN_INVALID_REVNUM; + } + + /* access the repo data */ + previous_file = walker_baton->file_hint; + SVN_ERR(create_rep_state(&rs, &rep_args, &walker_baton->file_hint, + &walker_baton->rev_hint, rep, fs, + walker_baton->pool)); + + /* update resource usage counters */ + walker_baton->iteration_count++; + if (previous_file != walker_baton->file_hint) + walker_baton->file_count++; + } + else + { + /* ### Should this be using read_rep_line() directly? */ + SVN_ERR(create_rep_state(&rs, &rep_args, NULL, NULL, rep, fs, + scratch_pool)); + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_fs__verify(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t end, + svn_fs_progress_notify_func_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + svn_boolean_t exists; + svn_revnum_t youngest = ffd->youngest_rev_cache; /* cache is current */ + + if (ffd->format < SVN_FS_FS__MIN_REP_SHARING_FORMAT) + return SVN_NO_ERROR; + + /* Input validation. */ + if (! SVN_IS_VALID_REVNUM(start)) + start = 0; + if (! SVN_IS_VALID_REVNUM(end)) + end = youngest; + SVN_ERR(ensure_revision_exists(fs, start, pool)); + SVN_ERR(ensure_revision_exists(fs, end, pool)); + + /* rep-cache verification. */ + SVN_ERR(svn_fs_fs__exists_rep_cache(&exists, fs, pool)); + if (exists) + { + /* provide a baton to allow the reuse of open file handles between + iterations (saves 2/3 of OS level file operations). */ + verify_walker_baton_t *baton = apr_pcalloc(pool, sizeof(*baton)); + baton->rev_hint = SVN_INVALID_REVNUM; + baton->pool = svn_pool_create(pool); + baton->last_notified_revision = SVN_INVALID_REVNUM; + baton->notify_func = notify_func; + baton->notify_baton = notify_baton; + + /* tell the user that we are now ready to do *something* */ + if (notify_func) + notify_func(SVN_INVALID_REVNUM, notify_baton, baton->pool); + + /* Do not attempt to walk the rep-cache database if its file does + not exist, since doing so would create it --- which may confuse + the administrator. Don't take any lock. */ + SVN_ERR(svn_fs_fs__walk_rep_reference(fs, start, end, + verify_walker, baton, + cancel_func, cancel_baton, + pool)); + + /* walker resource cleanup */ + svn_pool_destroy(baton->pool); + } + + return SVN_NO_ERROR; +} + + +/** Hotcopy. **/ + +/* Like svn_io_dir_file_copy(), but doesn't copy files that exist at + * the destination and do not differ in terms of kind, size, and mtime. */ +static svn_error_t * +hotcopy_io_dir_file_copy(const char *src_path, + const char *dst_path, + const char *file, + apr_pool_t *scratch_pool) +{ + const svn_io_dirent2_t *src_dirent; + const svn_io_dirent2_t *dst_dirent; + const char *src_target; + const char *dst_target; + + /* Does the destination already exist? If not, we must copy it. */ + dst_target = svn_dirent_join(dst_path, file, scratch_pool); + SVN_ERR(svn_io_stat_dirent2(&dst_dirent, dst_target, FALSE, TRUE, + scratch_pool, scratch_pool)); + if (dst_dirent->kind != svn_node_none) + { + /* If the destination's stat information indicates that the file + * is equal to the source, don't bother copying the file again. */ + src_target = svn_dirent_join(src_path, file, scratch_pool); + SVN_ERR(svn_io_stat_dirent2(&src_dirent, src_target, FALSE, FALSE, + scratch_pool, scratch_pool)); + if (src_dirent->kind == dst_dirent->kind && + src_dirent->special == dst_dirent->special && + src_dirent->filesize == dst_dirent->filesize && + src_dirent->mtime <= dst_dirent->mtime) + return SVN_NO_ERROR; + } + + return svn_error_trace(svn_io_dir_file_copy(src_path, dst_path, file, + scratch_pool)); +} + +/* Set *NAME_P to the UTF-8 representation of directory entry NAME. + * NAME is in the internal encoding used by APR; PARENT is in + * UTF-8 and in internal (not local) style. + * + * Use PARENT only for generating an error string if the conversion + * fails because NAME could not be represented in UTF-8. In that + * case, return a two-level error in which the outer error's message + * mentions PARENT, but the inner error's message does not mention + * NAME (except possibly in hex) since NAME may not be printable. + * Such a compound error at least allows the user to go looking in the + * right directory for the problem. + * + * If there is any other error, just return that error directly. + * + * If there is any error, the effect on *NAME_P is undefined. + * + * *NAME_P and NAME may refer to the same storage. + */ +static svn_error_t * +entry_name_to_utf8(const char **name_p, + const char *name, + const char *parent, + apr_pool_t *pool) +{ + svn_error_t *err = svn_path_cstring_to_utf8(name_p, name, pool); + if (err && err->apr_err == APR_EINVAL) + { + return svn_error_createf(err->apr_err, err, + _("Error converting entry " + "in directory '%s' to UTF-8"), + svn_dirent_local_style(parent, pool)); + } + return err; +} + +/* Like svn_io_copy_dir_recursively() but doesn't copy regular files that + * exist in the destination and do not differ from the source in terms of + * kind, size, and mtime. */ +static svn_error_t * +hotcopy_io_copy_dir_recursively(const char *src, + const char *dst_parent, + const char *dst_basename, + svn_boolean_t copy_perms, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + svn_node_kind_t kind; + apr_status_t status; + const char *dst_path; + apr_dir_t *this_dir; + apr_finfo_t this_entry; + apr_int32_t flags = APR_FINFO_TYPE | APR_FINFO_NAME; + + /* Make a subpool for recursion */ + apr_pool_t *subpool = svn_pool_create(pool); + + /* The 'dst_path' is simply dst_parent/dst_basename */ + dst_path = svn_dirent_join(dst_parent, dst_basename, pool); + + /* Sanity checks: SRC and DST_PARENT are directories, and + DST_BASENAME doesn't already exist in DST_PARENT. */ + SVN_ERR(svn_io_check_path(src, &kind, subpool)); + if (kind != svn_node_dir) + return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL, + _("Source '%s' is not a directory"), + svn_dirent_local_style(src, pool)); + + SVN_ERR(svn_io_check_path(dst_parent, &kind, subpool)); + if (kind != svn_node_dir) + return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL, + _("Destination '%s' is not a directory"), + svn_dirent_local_style(dst_parent, pool)); + + SVN_ERR(svn_io_check_path(dst_path, &kind, subpool)); + + /* Create the new directory. */ + /* ### TODO: copy permissions (needs apr_file_attrs_get()) */ + SVN_ERR(svn_io_make_dir_recursively(dst_path, pool)); + + /* Loop over the dirents in SRC. ('.' and '..' are auto-excluded) */ + SVN_ERR(svn_io_dir_open(&this_dir, src, subpool)); + + for (status = apr_dir_read(&this_entry, flags, this_dir); + status == APR_SUCCESS; + status = apr_dir_read(&this_entry, flags, this_dir)) + { + if ((this_entry.name[0] == '.') + && ((this_entry.name[1] == '\0') + || ((this_entry.name[1] == '.') + && (this_entry.name[2] == '\0')))) + { + continue; + } + else + { + const char *entryname_utf8; + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + SVN_ERR(entry_name_to_utf8(&entryname_utf8, this_entry.name, + src, subpool)); + if (this_entry.filetype == APR_REG) /* regular file */ + { + SVN_ERR(hotcopy_io_dir_file_copy(src, dst_path, entryname_utf8, + subpool)); + } + else if (this_entry.filetype == APR_LNK) /* symlink */ + { + const char *src_target = svn_dirent_join(src, entryname_utf8, + subpool); + const char *dst_target = svn_dirent_join(dst_path, + entryname_utf8, + subpool); + SVN_ERR(svn_io_copy_link(src_target, dst_target, + subpool)); + } + else if (this_entry.filetype == APR_DIR) /* recurse */ + { + const char *src_target; + + /* Prevent infinite recursion by filtering off our + newly created destination path. */ + if (strcmp(src, dst_parent) == 0 + && strcmp(entryname_utf8, dst_basename) == 0) + continue; + + src_target = svn_dirent_join(src, entryname_utf8, subpool); + SVN_ERR(hotcopy_io_copy_dir_recursively(src_target, + dst_path, + entryname_utf8, + copy_perms, + cancel_func, + cancel_baton, + subpool)); + } + /* ### support other APR node types someday?? */ + + } + } + + if (! (APR_STATUS_IS_ENOENT(status))) + return svn_error_wrap_apr(status, _("Can't read directory '%s'"), + svn_dirent_local_style(src, pool)); + + status = apr_dir_close(this_dir); + if (status) + return svn_error_wrap_apr(status, _("Error closing directory '%s'"), + svn_dirent_local_style(src, pool)); + + /* Free any memory used by recursion */ + svn_pool_destroy(subpool); + + return SVN_NO_ERROR; +} + +/* Copy an un-packed revision or revprop file for revision REV from SRC_SUBDIR + * to DST_SUBDIR. Assume a sharding layout based on MAX_FILES_PER_DIR. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_copy_shard_file(const char *src_subdir, + const char *dst_subdir, + svn_revnum_t rev, + int max_files_per_dir, + apr_pool_t *scratch_pool) +{ + const char *src_subdir_shard = src_subdir, + *dst_subdir_shard = dst_subdir; + + if (max_files_per_dir) + { + const char *shard = apr_psprintf(scratch_pool, "%ld", + rev / max_files_per_dir); + src_subdir_shard = svn_dirent_join(src_subdir, shard, scratch_pool); + dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); + + if (rev % max_files_per_dir == 0) + { + SVN_ERR(svn_io_make_dir_recursively(dst_subdir_shard, scratch_pool)); + SVN_ERR(svn_io_copy_perms(dst_subdir, dst_subdir_shard, + scratch_pool)); + } + } + + SVN_ERR(hotcopy_io_dir_file_copy(src_subdir_shard, dst_subdir_shard, + apr_psprintf(scratch_pool, "%ld", rev), + scratch_pool)); + return SVN_NO_ERROR; +} + + +/* Copy a packed shard containing revision REV, and which contains + * MAX_FILES_PER_DIR revisions, from SRC_FS to DST_FS. + * Update *DST_MIN_UNPACKED_REV in case the shard is new in DST_FS. + * Do not re-copy data which already exists in DST_FS. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_copy_packed_shard(svn_revnum_t *dst_min_unpacked_rev, + svn_fs_t *src_fs, + svn_fs_t *dst_fs, + svn_revnum_t rev, + int max_files_per_dir, + apr_pool_t *scratch_pool) +{ + const char *src_subdir; + const char *dst_subdir; + const char *packed_shard; + const char *src_subdir_packed_shard; + svn_revnum_t revprop_rev; + apr_pool_t *iterpool; + fs_fs_data_t *src_ffd = src_fs->fsap_data; + + /* Copy the packed shard. */ + src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, scratch_pool); + dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, scratch_pool); + packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD, + rev / max_files_per_dir); + src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, + scratch_pool); + SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard, + dst_subdir, packed_shard, + TRUE /* copy_perms */, + NULL /* cancel_func */, NULL, + scratch_pool)); + + /* Copy revprops belonging to revisions in this pack. */ + src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, scratch_pool); + dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, scratch_pool); + + if ( src_ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT + || src_ffd->min_unpacked_rev < rev + max_files_per_dir) + { + /* copy unpacked revprops rev by rev */ + iterpool = svn_pool_create(scratch_pool); + for (revprop_rev = rev; + revprop_rev < rev + max_files_per_dir; + revprop_rev++) + { + svn_pool_clear(iterpool); + + SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir, + revprop_rev, max_files_per_dir, + iterpool)); + } + svn_pool_destroy(iterpool); + } + else + { + /* revprop for revision 0 will never be packed */ + if (rev == 0) + SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir, + 0, max_files_per_dir, + scratch_pool)); + + /* packed revprops folder */ + packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD, + rev / max_files_per_dir); + src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, + scratch_pool); + SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard, + dst_subdir, packed_shard, + TRUE /* copy_perms */, + NULL /* cancel_func */, NULL, + scratch_pool)); + } + + /* If necessary, update the min-unpacked rev file in the hotcopy. */ + if (*dst_min_unpacked_rev < rev + max_files_per_dir) + { + *dst_min_unpacked_rev = rev + max_files_per_dir; + SVN_ERR(write_revnum_file(dst_fs->path, PATH_MIN_UNPACKED_REV, + *dst_min_unpacked_rev, + scratch_pool)); + } + + return SVN_NO_ERROR; +} + +/* If NEW_YOUNGEST is younger than *DST_YOUNGEST, update the 'current' + * file in DST_FS and set *DST_YOUNGEST to NEW_YOUNGEST. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_update_current(svn_revnum_t *dst_youngest, + svn_fs_t *dst_fs, + svn_revnum_t new_youngest, + apr_pool_t *scratch_pool) +{ + char next_node_id[MAX_KEY_SIZE] = "0"; + char next_copy_id[MAX_KEY_SIZE] = "0"; + fs_fs_data_t *dst_ffd = dst_fs->fsap_data; + + if (*dst_youngest >= new_youngest) + return SVN_NO_ERROR; + + /* If necessary, get new current next_node and next_copy IDs. */ + if (dst_ffd->format < SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT) + { + apr_off_t root_offset; + apr_file_t *rev_file; + char max_node_id[MAX_KEY_SIZE] = "0"; + char max_copy_id[MAX_KEY_SIZE] = "0"; + apr_size_t len; + + if (dst_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) + SVN_ERR(update_min_unpacked_rev(dst_fs, scratch_pool)); + + SVN_ERR(open_pack_or_rev_file(&rev_file, dst_fs, new_youngest, + scratch_pool)); + SVN_ERR(get_root_changes_offset(&root_offset, NULL, rev_file, + dst_fs, new_youngest, scratch_pool)); + SVN_ERR(recover_find_max_ids(dst_fs, new_youngest, rev_file, + root_offset, max_node_id, max_copy_id, + scratch_pool)); + SVN_ERR(svn_io_file_close(rev_file, scratch_pool)); + + /* We store the _next_ ids. */ + len = strlen(max_node_id); + svn_fs_fs__next_key(max_node_id, &len, next_node_id); + len = strlen(max_copy_id); + svn_fs_fs__next_key(max_copy_id, &len, next_copy_id); + } + + /* Update 'current'. */ + SVN_ERR(write_current(dst_fs, new_youngest, next_node_id, next_copy_id, + scratch_pool)); + + *dst_youngest = new_youngest; + + return SVN_NO_ERROR; +} + + +/* Remove revision or revprop files between START_REV (inclusive) and + * END_REV (non-inclusive) from folder DST_SUBDIR in DST_FS. Assume + * sharding as per MAX_FILES_PER_DIR. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_remove_files(svn_fs_t *dst_fs, + const char *dst_subdir, + svn_revnum_t start_rev, + svn_revnum_t end_rev, + int max_files_per_dir, + apr_pool_t *scratch_pool) +{ + const char *shard; + const char *dst_subdir_shard; + svn_revnum_t rev; + apr_pool_t *iterpool; + + /* Pre-compute paths for initial shard. */ + shard = apr_psprintf(scratch_pool, "%ld", start_rev / max_files_per_dir); + dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); + + iterpool = svn_pool_create(scratch_pool); + for (rev = start_rev; rev < end_rev; rev++) + { + const char *path; + svn_pool_clear(iterpool); + + /* If necessary, update paths for shard. */ + if (rev != start_rev && rev % max_files_per_dir == 0) + { + shard = apr_psprintf(iterpool, "%ld", rev / max_files_per_dir); + dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); + } + + /* remove files for REV */ + path = svn_dirent_join(dst_subdir_shard, + apr_psprintf(iterpool, "%ld", rev), + iterpool); + + /* Make the rev file writable and remove it. */ + SVN_ERR(svn_io_set_file_read_write(path, TRUE, iterpool)); + SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Remove revisions between START_REV (inclusive) and END_REV (non-inclusive) + * from DST_FS. Assume sharding as per MAX_FILES_PER_DIR. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_remove_rev_files(svn_fs_t *dst_fs, + svn_revnum_t start_rev, + svn_revnum_t end_rev, + int max_files_per_dir, + apr_pool_t *scratch_pool) +{ + SVN_ERR_ASSERT(start_rev <= end_rev); + SVN_ERR(hotcopy_remove_files(dst_fs, + svn_dirent_join(dst_fs->path, + PATH_REVS_DIR, + scratch_pool), + start_rev, end_rev, + max_files_per_dir, scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Remove revision properties between START_REV (inclusive) and END_REV + * (non-inclusive) from DST_FS. Assume sharding as per MAX_FILES_PER_DIR. + * Use SCRATCH_POOL for temporary allocations. Revision 0 revprops will + * not be deleted. */ +static svn_error_t * +hotcopy_remove_revprop_files(svn_fs_t *dst_fs, + svn_revnum_t start_rev, + svn_revnum_t end_rev, + int max_files_per_dir, + apr_pool_t *scratch_pool) +{ + SVN_ERR_ASSERT(start_rev <= end_rev); + + /* don't delete rev 0 props */ + SVN_ERR(hotcopy_remove_files(dst_fs, + svn_dirent_join(dst_fs->path, + PATH_REVPROPS_DIR, + scratch_pool), + start_rev ? start_rev : 1, end_rev, + max_files_per_dir, scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Verify that DST_FS is a suitable destination for an incremental + * hotcopy from SRC_FS. */ +static svn_error_t * +hotcopy_incremental_check_preconditions(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + apr_pool_t *pool) +{ + fs_fs_data_t *src_ffd = src_fs->fsap_data; + fs_fs_data_t *dst_ffd = dst_fs->fsap_data; + + /* We only support incremental hotcopy between the same format. */ + if (src_ffd->format != dst_ffd->format) + return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("The FSFS format (%d) of the hotcopy source does not match the " + "FSFS format (%d) of the hotcopy destination; please upgrade " + "both repositories to the same format"), + src_ffd->format, dst_ffd->format); + + /* Make sure the UUID of source and destination match up. + * We don't want to copy over a different repository. */ + if (strcmp(src_fs->uuid, dst_fs->uuid) != 0) + return svn_error_create(SVN_ERR_RA_UUID_MISMATCH, NULL, + _("The UUID of the hotcopy source does " + "not match the UUID of the hotcopy " + "destination")); + + /* Also require same shard size. */ + if (src_ffd->max_files_per_dir != dst_ffd->max_files_per_dir) + return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("The sharding layout configuration " + "of the hotcopy source does not match " + "the sharding layout configuration of " + "the hotcopy destination")); + return SVN_NO_ERROR; +} + +/* Remove folder PATH. Ignore errors due to the sub-tree not being empty. + * CANCEL_FUNC and CANCEL_BATON do the usual thing. + * Use POOL for temporary allocations. + */ +static svn_error_t * +remove_folder(const char *path, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + svn_error_t *err = svn_io_remove_dir2(path, TRUE, + cancel_func, cancel_baton, pool); + + if (err && APR_STATUS_IS_ENOTEMPTY(err->apr_err)) + { + svn_error_clear(err); + err = SVN_NO_ERROR; + } + + return svn_error_trace(err); +} + +/* Baton for hotcopy_body(). */ +struct hotcopy_body_baton { + svn_fs_t *src_fs; + svn_fs_t *dst_fs; + svn_boolean_t incremental; + svn_cancel_func_t cancel_func; + void *cancel_baton; +} hotcopy_body_baton; + +/* Perform a hotcopy, either normal or incremental. + * + * Normal hotcopy assumes that the destination exists as an empty + * directory. It behaves like an incremental hotcopy except that + * none of the copied files already exist in the destination. + * + * An incremental hotcopy copies only changed or new files to the destination, + * and removes files from the destination no longer present in the source. + * While the incremental hotcopy is running, readers should still be able + * to access the destintation repository without error and should not see + * revisions currently in progress of being copied. Readers are able to see + * new fully copied revisions even if the entire incremental hotcopy procedure + * has not yet completed. + * + * Writers are blocked out completely during the entire incremental hotcopy + * process to ensure consistency. This function assumes that the repository + * write-lock is held. + */ +static svn_error_t * +hotcopy_body(void *baton, apr_pool_t *pool) +{ + struct hotcopy_body_baton *hbb = baton; + svn_fs_t *src_fs = hbb->src_fs; + fs_fs_data_t *src_ffd = src_fs->fsap_data; + svn_fs_t *dst_fs = hbb->dst_fs; + fs_fs_data_t *dst_ffd = dst_fs->fsap_data; + int max_files_per_dir = src_ffd->max_files_per_dir; + svn_boolean_t incremental = hbb->incremental; + svn_cancel_func_t cancel_func = hbb->cancel_func; + void* cancel_baton = hbb->cancel_baton; + svn_revnum_t src_youngest; + svn_revnum_t dst_youngest; + svn_revnum_t rev; + svn_revnum_t src_min_unpacked_rev; + svn_revnum_t dst_min_unpacked_rev; + const char *src_subdir; + const char *dst_subdir; + const char *revprop_src_subdir; + const char *revprop_dst_subdir; + apr_pool_t *iterpool; + svn_node_kind_t kind; + + /* Try to copy the config. + * + * ### We try copying the config file before doing anything else, + * ### because higher layers will abort the hotcopy if we throw + * ### an error from this function, and that renders the hotcopy + * ### unusable anyway. */ + if (src_ffd->format >= SVN_FS_FS__MIN_CONFIG_FILE) + { + svn_error_t *err; + + err = svn_io_dir_file_copy(src_fs->path, dst_fs->path, PATH_CONFIG, + pool); + if (err) + { + if (APR_STATUS_IS_ENOENT(err->apr_err)) + { + /* 1.6.0 to 1.6.11 did not copy the configuration file during + * hotcopy. So if we're hotcopying a repository which has been + * created as a hotcopy itself, it's possible that fsfs.conf + * does not exist. Ask the user to re-create it. + * + * ### It would be nice to make this a non-fatal error, + * ### but this function does not get an svn_fs_t object + * ### so we have no way of just printing a warning via + * ### the fs->warning() callback. */ + + const char *msg; + const char *src_abspath; + const char *dst_abspath; + const char *config_relpath; + svn_error_t *err2; + + config_relpath = svn_dirent_join(src_fs->path, PATH_CONFIG, pool); + err2 = svn_dirent_get_absolute(&src_abspath, src_fs->path, pool); + if (err2) + return svn_error_trace(svn_error_compose_create(err, err2)); + err2 = svn_dirent_get_absolute(&dst_abspath, dst_fs->path, pool); + if (err2) + return svn_error_trace(svn_error_compose_create(err, err2)); + + /* ### hack: strip off the 'db/' directory from paths so + * ### they make sense to the user */ + src_abspath = svn_dirent_dirname(src_abspath, pool); + dst_abspath = svn_dirent_dirname(dst_abspath, pool); + + msg = apr_psprintf(pool, + _("Failed to create hotcopy at '%s'. " + "The file '%s' is missing from the source " + "repository. Please create this file, for " + "instance by running 'svnadmin upgrade %s'"), + dst_abspath, config_relpath, src_abspath); + return svn_error_quick_wrap(err, msg); + } + else + return svn_error_trace(err); + } + } + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Find the youngest revision in the source and destination. + * We only support hotcopies from sources with an equal or greater amount + * of revisions than the destination. + * This also catches the case where users accidentally swap the + * source and destination arguments. */ + SVN_ERR(get_youngest(&src_youngest, src_fs->path, pool)); + if (incremental) + { + SVN_ERR(get_youngest(&dst_youngest, dst_fs->path, pool)); + if (src_youngest < dst_youngest) + return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("The hotcopy destination already contains more revisions " + "(%lu) than the hotcopy source contains (%lu); are source " + "and destination swapped?"), + dst_youngest, src_youngest); + } + else + dst_youngest = 0; + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Copy the min unpacked rev, and read its value. */ + if (src_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) + { + const char *min_unpacked_rev_path; + + min_unpacked_rev_path = svn_dirent_join(src_fs->path, + PATH_MIN_UNPACKED_REV, + pool); + SVN_ERR(read_min_unpacked_rev(&src_min_unpacked_rev, + min_unpacked_rev_path, + pool)); + + min_unpacked_rev_path = svn_dirent_join(dst_fs->path, + PATH_MIN_UNPACKED_REV, + pool); + SVN_ERR(read_min_unpacked_rev(&dst_min_unpacked_rev, + min_unpacked_rev_path, + pool)); + + /* We only support packs coming from the hotcopy source. + * The destination should not be packed independently from + * the source. This also catches the case where users accidentally + * swap the source and destination arguments. */ + if (src_min_unpacked_rev < dst_min_unpacked_rev) + return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("The hotcopy destination already contains " + "more packed revisions (%lu) than the " + "hotcopy source contains (%lu)"), + dst_min_unpacked_rev - 1, + src_min_unpacked_rev - 1); + + SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path, + PATH_MIN_UNPACKED_REV, pool)); + } + else + { + src_min_unpacked_rev = 0; + dst_min_unpacked_rev = 0; + } + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* + * Copy the necessary rev files. + */ + + src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, pool); + dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, pool); + SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool)); + + iterpool = svn_pool_create(pool); + /* First, copy packed shards. */ + for (rev = 0; rev < src_min_unpacked_rev; rev += max_files_per_dir) + { + svn_pool_clear(iterpool); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Copy the packed shard. */ + SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev, + src_fs, dst_fs, + rev, max_files_per_dir, + iterpool)); + + /* If necessary, update 'current' to the most recent packed rev, + * so readers can see new revisions which arrived in this pack. */ + SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, + rev + max_files_per_dir - 1, + iterpool)); + + /* Remove revision files which are now packed. */ + if (incremental) + { + SVN_ERR(hotcopy_remove_rev_files(dst_fs, rev, + rev + max_files_per_dir, + max_files_per_dir, iterpool)); + if (dst_ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) + SVN_ERR(hotcopy_remove_revprop_files(dst_fs, rev, + rev + max_files_per_dir, + max_files_per_dir, + iterpool)); + } + + /* Now that all revisions have moved into the pack, the original + * rev dir can be removed. */ + SVN_ERR(remove_folder(path_rev_shard(dst_fs, rev, iterpool), + cancel_func, cancel_baton, iterpool)); + if (rev > 0 && dst_ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) + SVN_ERR(remove_folder(path_revprops_shard(dst_fs, rev, iterpool), + cancel_func, cancel_baton, iterpool)); + } + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Now, copy pairs of non-packed revisions and revprop files. + * If necessary, update 'current' after copying all files from a shard. */ + SVN_ERR_ASSERT(rev == src_min_unpacked_rev); + SVN_ERR_ASSERT(src_min_unpacked_rev == dst_min_unpacked_rev); + revprop_src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, pool); + revprop_dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, pool); + SVN_ERR(svn_io_make_dir_recursively(revprop_dst_subdir, pool)); + for (; rev <= src_youngest; rev++) + { + svn_error_t *err; + + svn_pool_clear(iterpool); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Copy the rev file. */ + err = hotcopy_copy_shard_file(src_subdir, dst_subdir, + rev, max_files_per_dir, + iterpool); + if (err) + { + if (APR_STATUS_IS_ENOENT(err->apr_err) && + src_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) + { + svn_error_clear(err); + + /* The source rev file does not exist. This can happen if the + * source repository is being packed concurrently with this + * hotcopy operation. + * + * If the new revision is now packed, and the youngest revision + * we're interested in is not inside this pack, try to copy the + * pack instead. + * + * If the youngest revision ended up being packed, don't try + * to be smart and work around this. Just abort the hotcopy. */ + SVN_ERR(update_min_unpacked_rev(src_fs, pool)); + if (is_packed_rev(src_fs, rev)) + { + if (is_packed_rev(src_fs, src_youngest)) + return svn_error_createf( + SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("The assumed HEAD revision (%lu) of the " + "hotcopy source has been packed while the " + "hotcopy was in progress; please restart " + "the hotcopy operation"), + src_youngest); + + SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev, + src_fs, dst_fs, + rev, max_files_per_dir, + iterpool)); + rev = dst_min_unpacked_rev; + continue; + } + else + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("Revision %lu disappeared from the " + "hotcopy source while hotcopy was " + "in progress"), rev); + } + else + return svn_error_trace(err); + } + + /* Copy the revprop file. */ + SVN_ERR(hotcopy_copy_shard_file(revprop_src_subdir, + revprop_dst_subdir, + rev, max_files_per_dir, + iterpool)); + + /* After completing a full shard, update 'current'. */ + if (max_files_per_dir && rev % max_files_per_dir == 0) + SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, rev, iterpool)); + } + svn_pool_destroy(iterpool); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* We assume that all revisions were copied now, i.e. we didn't exit the + * above loop early. 'rev' was last incremented during exit of the loop. */ + SVN_ERR_ASSERT(rev == src_youngest + 1); + + /* All revisions were copied. Update 'current'. */ + SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, src_youngest, pool)); + + /* Replace the locks tree. + * This is racy in case readers are currently trying to list locks in + * the destination. However, we need to get rid of stale locks. + * This is the simplest way of doing this, so we accept this small race. */ + dst_subdir = svn_dirent_join(dst_fs->path, PATH_LOCKS_DIR, pool); + SVN_ERR(svn_io_remove_dir2(dst_subdir, TRUE, cancel_func, cancel_baton, + pool)); + src_subdir = svn_dirent_join(src_fs->path, PATH_LOCKS_DIR, pool); + SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); + if (kind == svn_node_dir) + SVN_ERR(svn_io_copy_dir_recursively(src_subdir, dst_fs->path, + PATH_LOCKS_DIR, TRUE, + cancel_func, cancel_baton, pool)); + + /* Now copy the node-origins cache tree. */ + src_subdir = svn_dirent_join(src_fs->path, PATH_NODE_ORIGINS_DIR, pool); + SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); + if (kind == svn_node_dir) + SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir, dst_fs->path, + PATH_NODE_ORIGINS_DIR, TRUE, + cancel_func, cancel_baton, pool)); + + /* + * NB: Data copied below is only read by writers, not readers. + * Writers are still locked out at this point. + */ + + if (dst_ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) + { + /* Copy the rep cache and then remove entries for revisions + * younger than the destination's youngest revision. */ + src_subdir = svn_dirent_join(src_fs->path, REP_CACHE_DB_NAME, pool); + dst_subdir = svn_dirent_join(dst_fs->path, REP_CACHE_DB_NAME, pool); + SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); + if (kind == svn_node_file) + { + SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, pool)); + SVN_ERR(svn_fs_fs__del_rep_reference(dst_fs, dst_youngest, pool)); + } + } + + /* Copy the txn-current file. */ + if (dst_ffd->format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT) + SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path, + PATH_TXN_CURRENT, pool)); + + /* If a revprop generation file exists in the source filesystem, + * reset it to zero (since this is on a different path, it will not + * overlap with data already in cache). Also, clean up stale files + * used for the named atomics implementation. */ + SVN_ERR(svn_io_check_path(path_revprop_generation(src_fs, pool), + &kind, pool)); + if (kind == svn_node_file) + SVN_ERR(write_revprop_generation_file(dst_fs, 0, pool)); + + SVN_ERR(cleanup_revprop_namespace(dst_fs)); + + /* Hotcopied FS is complete. Stamp it with a format file. */ + SVN_ERR(write_format(svn_dirent_join(dst_fs->path, PATH_FORMAT, pool), + dst_ffd->format, max_files_per_dir, TRUE, pool)); + + return SVN_NO_ERROR; +} + + +/* Set up shared data between SRC_FS and DST_FS. */ +static void +hotcopy_setup_shared_fs_data(svn_fs_t *src_fs, svn_fs_t *dst_fs) +{ + fs_fs_data_t *src_ffd = src_fs->fsap_data; + fs_fs_data_t *dst_ffd = dst_fs->fsap_data; + + /* The common pool and mutexes are shared between src and dst filesystems. + * During hotcopy we only grab the mutexes for the destination, so there + * is no risk of dead-lock. We don't write to the src filesystem. Shared + * data for the src_fs has already been initialised in fs_hotcopy(). */ + dst_ffd->shared = src_ffd->shared; +} + +/* Create an empty filesystem at DST_FS at DST_PATH with the same + * configuration as SRC_FS (uuid, format, and other parameters). + * After creation DST_FS has no revisions, not even revision zero. */ +static svn_error_t * +hotcopy_create_empty_dest(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + const char *dst_path, + apr_pool_t *pool) +{ + fs_fs_data_t *src_ffd = src_fs->fsap_data; + fs_fs_data_t *dst_ffd = dst_fs->fsap_data; + + dst_fs->path = apr_pstrdup(pool, dst_path); + + dst_ffd->max_files_per_dir = src_ffd->max_files_per_dir; + dst_ffd->config = src_ffd->config; + dst_ffd->format = src_ffd->format; + + /* Create the revision data directories. */ + if (dst_ffd->max_files_per_dir) + SVN_ERR(svn_io_make_dir_recursively(path_rev_shard(dst_fs, 0, pool), + pool)); + else + SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, + PATH_REVS_DIR, pool), + pool)); + + /* Create the revprops directory. */ + if (src_ffd->max_files_per_dir) + SVN_ERR(svn_io_make_dir_recursively(path_revprops_shard(dst_fs, 0, pool), + pool)); + else + SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, + PATH_REVPROPS_DIR, + pool), + pool)); + + /* Create the transaction directory. */ + SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, PATH_TXNS_DIR, + pool), + pool)); + + /* Create the protorevs directory. */ + if (dst_ffd->format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT) + SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, + PATH_TXN_PROTOS_DIR, + pool), + pool)); + + /* Create the 'current' file. */ + SVN_ERR(svn_io_file_create(svn_fs_fs__path_current(dst_fs, pool), + (dst_ffd->format >= + SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT + ? "0\n" : "0 1 1\n"), + pool)); + + /* Create lock file and UUID. */ + SVN_ERR(svn_io_file_create(path_lock(dst_fs, pool), "", pool)); + SVN_ERR(svn_fs_fs__set_uuid(dst_fs, src_fs->uuid, pool)); + + /* Create the min unpacked rev file. */ + if (dst_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) + SVN_ERR(svn_io_file_create(path_min_unpacked_rev(dst_fs, pool), + "0\n", pool)); + /* Create the txn-current file if the repository supports + the transaction sequence file. */ + if (dst_ffd->format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT) + { + SVN_ERR(svn_io_file_create(path_txn_current(dst_fs, pool), + "0\n", pool)); + SVN_ERR(svn_io_file_create(path_txn_current_lock(dst_fs, pool), + "", pool)); + } + + dst_ffd->youngest_rev_cache = 0; + + hotcopy_setup_shared_fs_data(src_fs, dst_fs); + SVN_ERR(svn_fs_fs__initialize_caches(dst_fs, pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_fs__hotcopy(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + const char *src_path, + const char *dst_path, + svn_boolean_t incremental, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + struct hotcopy_body_baton hbb; + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + SVN_ERR(svn_fs_fs__open(src_fs, src_path, pool)); + + if (incremental) + { + const char *dst_format_abspath; + svn_node_kind_t dst_format_kind; + + /* Check destination format to be sure we know how to incrementally + * hotcopy to the destination FS. */ + dst_format_abspath = svn_dirent_join(dst_path, PATH_FORMAT, pool); + SVN_ERR(svn_io_check_path(dst_format_abspath, &dst_format_kind, pool)); + if (dst_format_kind == svn_node_none) + { + /* Destination doesn't exist yet. Perform a normal hotcopy to a + * empty destination using the same configuration as the source. */ + SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path, pool)); + } + else + { + /* Check the existing repository. */ + SVN_ERR(svn_fs_fs__open(dst_fs, dst_path, pool)); + SVN_ERR(hotcopy_incremental_check_preconditions(src_fs, dst_fs, + pool)); + hotcopy_setup_shared_fs_data(src_fs, dst_fs); + SVN_ERR(svn_fs_fs__initialize_caches(dst_fs, pool)); + } + } + else + { + /* Start out with an empty destination using the same configuration + * as the source. */ + SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path, pool)); + } + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + hbb.src_fs = src_fs; + hbb.dst_fs = dst_fs; + hbb.incremental = incremental; + hbb.cancel_func = cancel_func; + hbb.cancel_baton = cancel_baton; + SVN_ERR(svn_fs_fs__with_write_lock(dst_fs, hotcopy_body, &hbb, pool)); + + return SVN_NO_ERROR; +} diff --git a/subversion/libsvn_fs_fs/fs_fs.h b/subversion/libsvn_fs_fs/fs_fs.h index c32ef55..c09f861 100644 --- a/subversion/libsvn_fs_fs/fs_fs.h +++ b/subversion/libsvn_fs_fs/fs_fs.h @@ -38,11 +38,31 @@ svn_error_t *svn_fs_fs__open(svn_fs_t *fs, svn_error_t *svn_fs_fs__upgrade(svn_fs_t *fs, apr_pool_t *pool); -/* Copy the fsfs filesystem at SRC_PATH into a new copy at DST_PATH. - Use POOL for temporary allocations. */ -svn_error_t *svn_fs_fs__hotcopy(const char *src_path, - const char *dst_path, - apr_pool_t *pool); +/* Verify metadata in fsfs filesystem FS. Limit the checks to revisions + * START to END where possible. Indicate progress via the optional + * NOTIFY_FUNC callback using NOTIFY_BATON. The optional CANCEL_FUNC + * will periodically be called with CANCEL_BATON to allow for preemption. + * Use POOL for temporary allocations. */ +svn_error_t *svn_fs_fs__verify(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t end, + svn_fs_progress_notify_func_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool); + +/* Copy the fsfs filesystem SRC_FS at SRC_PATH into a new copy DST_FS at + * DST_PATH. If INCREMENTAL is TRUE, do not re-copy data which already + * exists in DST_FS. Use POOL for temporary allocations. */ +svn_error_t * svn_fs_fs__hotcopy(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + const char *src_path, + const char *dst_path, + svn_boolean_t incremental, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool); /* Recover the fsfs associated with filesystem FS. Use optional CANCEL_FUNC/CANCEL_BATON for cancellation support. @@ -71,6 +91,7 @@ svn_error_t *svn_fs_fs__put_node_revision(svn_fs_t *fs, /* Write the node-revision NODEREV into the stream OUTFILE, compatible with filesystem format FORMAT. Only write mergeinfo-related metadata if INCLUDE_MERGEINFO is true. Temporary allocations are from POOL. */ +/* ### Currently used only by fs_fs.c */ svn_error_t * svn_fs_fs__write_noderev(svn_stream_t *outfile, node_revision_t *noderev, @@ -93,6 +114,12 @@ svn_error_t *svn_fs_fs__youngest_rev(svn_revnum_t *youngest, svn_fs_t *fs, apr_pool_t *pool); +/* Return an error iff REV does not exist in FS. */ +svn_error_t * +svn_fs_fs__revision_exists(svn_revnum_t rev, + svn_fs_t *fs, + apr_pool_t *pool); + /* Set *ROOT_ID to the node-id for the root of revision REV in filesystem FS. Do any allocations in POOL. */ svn_error_t *svn_fs_fs__rev_get_root(svn_fs_id_t **root_id, @@ -110,14 +137,16 @@ svn_error_t *svn_fs_fs__rep_contents_dir(apr_hash_t **entries, apr_pool_t *pool); /* Set *DIRENT to the entry identified by NAME in the directory given - by NODEREV in filesystem FS. The returned object is allocated in POOL, - which is also used for temporary allocations. */ + by NODEREV in filesystem FS. If no such entry exits, *DIRENT will + be NULL. The returned object is allocated in RESULT_POOL; SCRATCH_POOL + used for temporary allocations. */ svn_error_t * svn_fs_fs__rep_contents_dir_entry(svn_fs_dirent_t **dirent, svn_fs_t *fs, node_revision_t *noderev, const char *name, - apr_pool_t *pool); + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); /* Set *CONTENTS to be a readable svn_stream_t that receives the text representation of node-revision NODEREV as seen in filesystem FS. @@ -127,6 +156,20 @@ svn_error_t *svn_fs_fs__get_contents(svn_stream_t **contents, node_revision_t *noderev, apr_pool_t *pool); +/* Attempt to fetch the text representation of node-revision NODEREV as + seen in filesystem FS and pass it along with the BATON to the PROCESSOR. + Set *SUCCESS only of the data could be provided and the processing + had been called. + Use POOL for all allocations. + */ +svn_error_t * +svn_fs_fs__try_process_file_contents(svn_boolean_t *success, + svn_fs_t *fs, + node_revision_t *noderev, + svn_fs_process_contents_func_t processor, + void* baton, + apr_pool_t *pool); + /* Set *STREAM_P to a delta stream turning the contents of the file SOURCE into the contents of the file TARGET, allocated in POOL. If SOURCE is null, the empty string will be used. */ @@ -333,12 +376,6 @@ svn_error_t *svn_fs_fs__create(svn_fs_t *fs, const char *path, apr_pool_t *pool); -/* Store the uuid of the repository FS in *UUID. Allocate space in - POOL. */ -svn_error_t *svn_fs_fs__get_uuid(svn_fs_t *fs, - const char **uuid, - apr_pool_t *pool); - /* Set the uuid of repository FS to UUID, if UUID is not NULL; otherwise, set the uuid of FS to a newly generated UUID. Perform temporary allocations in POOL. */ diff --git a/subversion/libsvn_fs_fs/id.c b/subversion/libsvn_fs_fs/id.c index 8575a82..1317829 100644 --- a/subversion/libsvn_fs_fs/id.c +++ b/subversion/libsvn_fs_fs/id.c @@ -26,6 +26,7 @@ #include "id.h" #include "../libsvn_fs/fs-loader.h" #include "private/svn_temp_serializer.h" +#include "private/svn_string_private.h" typedef struct id_private_t { @@ -88,22 +89,25 @@ svn_string_t * svn_fs_fs__id_unparse(const svn_fs_id_t *id, apr_pool_t *pool) { - const char *txn_rev_id; id_private_t *pvt = id->fsap_data; if ((! pvt->txn_id)) { - txn_rev_id = apr_psprintf(pool, "%ld/%" - APR_OFF_T_FMT, pvt->rev, pvt->offset); + char rev_string[SVN_INT64_BUFFER_SIZE]; + char offset_string[SVN_INT64_BUFFER_SIZE]; + + svn__i64toa(rev_string, pvt->rev); + svn__i64toa(offset_string, pvt->offset); + return svn_string_createf(pool, "%s.%s.r%s/%s", + pvt->node_id, pvt->copy_id, + rev_string, offset_string); } else { - txn_rev_id = pvt->txn_id; + return svn_string_createf(pool, "%s.%s.t%s", + pvt->node_id, pvt->copy_id, + pvt->txn_id); } - return svn_string_createf(pool, "%s.%s.%c%s", - pvt->node_id, pvt->copy_id, - (pvt->txn_id ? 't' : 'r'), - txn_rev_id); } @@ -242,7 +246,7 @@ svn_fs_fs__id_parse(const char *data, { svn_fs_id_t *id; id_private_t *pvt; - char *data_copy, *str, *last_str; + char *data_copy, *str; /* Dup the ID data into POOL. Our returned ID will have references into this memory. */ @@ -255,24 +259,25 @@ svn_fs_fs__id_parse(const char *data, id->fsap_data = pvt; /* Now, we basically just need to "split" this data on `.' - characters. We will use apr_strtok, which will put terminators - where each of the '.'s used to be. Then our new id field will - reference string locations inside our duplicate string.*/ + characters. We will use svn_cstring_tokenize, which will put + terminators where each of the '.'s used to be. Then our new + id field will reference string locations inside our duplicate + string.*/ /* Node Id */ - str = apr_strtok(data_copy, ".", &last_str); + str = svn_cstring_tokenize(".", &data_copy); if (str == NULL) return NULL; pvt->node_id = str; /* Copy Id */ - str = apr_strtok(NULL, ".", &last_str); + str = svn_cstring_tokenize(".", &data_copy); if (str == NULL) return NULL; pvt->copy_id = str; /* Txn/Rev Id */ - str = apr_strtok(NULL, ".", &last_str); + str = svn_cstring_tokenize(".", &data_copy); if (str == NULL) return NULL; @@ -284,12 +289,13 @@ svn_fs_fs__id_parse(const char *data, /* This is a revision type ID */ pvt->txn_id = NULL; - str = apr_strtok(str + 1, "/", &last_str); + data_copy = str + 1; + str = svn_cstring_tokenize("/", &data_copy); if (str == NULL) return NULL; pvt->rev = SVN_STR_TO_REV(str); - str = apr_strtok(NULL, "/", &last_str); + str = svn_cstring_tokenize("/", &data_copy); if (str == NULL) return NULL; err = svn_cstring_atoi64(&val, str); diff --git a/subversion/libsvn_fs_fs/key-gen.c b/subversion/libsvn_fs_fs/key-gen.c index f0f6cce..a65c59d 100644 --- a/subversion/libsvn_fs_fs/key-gen.c +++ b/subversion/libsvn_fs_fs/key-gen.c @@ -41,8 +41,8 @@ void svn_fs_fs__add_keys(const char *key1, const char *key2, char *result) { - int i1 = strlen(key1) - 1; - int i2 = strlen(key2) - 1; + apr_ssize_t i1 = strlen(key1) - 1; + apr_ssize_t i2 = strlen(key2) - 1; int i3 = 0; int val; int carry = 0; @@ -60,7 +60,7 @@ svn_fs_fs__add_keys(const char *key1, const char *key2, char *result) carry = val / 36; val = val % 36; - buf[i3++] = (val <= 9) ? (val + '0') : (val - 10 + 'a'); + buf[i3++] = (char)((val <= 9) ? (val + '0') : (val - 10 + 'a')); if (i1>=0) i1--; @@ -79,7 +79,7 @@ svn_fs_fs__add_keys(const char *key1, const char *key2, char *result) void svn_fs_fs__next_key(const char *this, apr_size_t *len, char *next) { - int i; + apr_ssize_t i; apr_size_t olen = *len; /* remember the first length */ char c; /* current char */ svn_boolean_t carry = TRUE; /* boolean: do we have a carry or not? @@ -115,7 +115,7 @@ svn_fs_fs__next_key(const char *this, apr_size_t *len, char *next) if (c == '9') next[i] = 'a'; else - next[i] = c + 1; + next[i] = ++c; } } else @@ -146,8 +146,8 @@ svn_fs_fs__next_key(const char *this, apr_size_t *len, char *next) int svn_fs_fs__key_compare(const char *a, const char *b) { - int a_len = strlen(a); - int b_len = strlen(b); + apr_size_t a_len = strlen(a); + apr_size_t b_len = strlen(b); int cmp; if (a_len > b_len) diff --git a/subversion/libsvn_fs_fs/lock.c b/subversion/libsvn_fs_fs/lock.c index 0ce5802..95bd943 100644 --- a/subversion/libsvn_fs_fs/lock.c +++ b/subversion/libsvn_fs_fs/lock.c @@ -105,7 +105,7 @@ hash_fetch(apr_hash_t *hash, const char *key, apr_pool_t *pool) { - svn_string_t *str = apr_hash_get(hash, key, APR_HASH_KEY_STRING); + svn_string_t *str = svn_hash_gets(hash, key); return str ? str->data : NULL; } @@ -205,7 +205,7 @@ write_digest_file(apr_hash_t *children, } if (apr_hash_count(children)) { - svn_stringbuf_t *children_list = svn_stringbuf_create("", pool); + svn_stringbuf_t *children_list = svn_stringbuf_create_empty(pool); for (hi = apr_hash_first(pool, children); hi; hi = apr_hash_next(hi)) { svn_stringbuf_appendbytes(children_list, @@ -324,8 +324,8 @@ read_digest_file(apr_hash_t **children_p, for (i = 0; i < kiddos->nelts; i++) { - apr_hash_set(*children_p, APR_ARRAY_IDX(kiddos, i, const char *), - APR_HASH_KEY_STRING, (void *)1); + svn_hash_sets(*children_p, APR_ARRAY_IDX(kiddos, i, const char *), + (void *)1); } } return SVN_NO_ERROR; @@ -387,11 +387,9 @@ set_lock(const char *fs_path, else { /* If we already have an entry for this path, we're done. */ - if (apr_hash_get(this_children, lock_digest_path, - APR_HASH_KEY_STRING)) + if (svn_hash_gets(this_children, lock_digest_path)) break; - apr_hash_set(this_children, lock_digest_path, - APR_HASH_KEY_STRING, (void *)1); + svn_hash_sets(this_children, lock_digest_path, (void *)1); } SVN_ERR(write_digest_file(this_children, this_lock, fs_path, digest_path, perms_reference, subpool)); @@ -448,7 +446,7 @@ delete_lock(svn_fs_t *fs, } if (child_to_kill) - apr_hash_set(this_children, child_to_kill, APR_HASH_KEY_STRING, NULL); + svn_hash_sets(this_children, child_to_kill, NULL); if (! (this_lock || apr_hash_count(this_children) != 0)) { @@ -477,23 +475,31 @@ delete_lock(svn_fs_t *fs, /* Set *LOCK_P to the lock for PATH in FS. HAVE_WRITE_LOCK should be TRUE if the caller (or one of its callers) has taken out the - repository-wide write lock, FALSE otherwise. Use POOL for - allocations. */ + repository-wide write lock, FALSE otherwise. If MUST_EXIST is + not set, the function will simply return NULL in *LOCK_P instead + of creating an SVN_FS__ERR_NO_SUCH_LOCK error in case the lock + was not found (much faster). Use POOL for allocations. */ static svn_error_t * get_lock(svn_lock_t **lock_p, svn_fs_t *fs, const char *path, svn_boolean_t have_write_lock, + svn_boolean_t must_exist, apr_pool_t *pool) { - svn_lock_t *lock; + svn_lock_t *lock = NULL; const char *digest_path; + svn_node_kind_t kind; SVN_ERR(digest_path_from_path(&digest_path, fs->path, path, pool)); + SVN_ERR(svn_io_check_path(digest_path, &kind, pool)); + + *lock_p = NULL; + if (kind != svn_node_none) + SVN_ERR(read_digest_file(NULL, &lock, fs->path, digest_path, pool)); - SVN_ERR(read_digest_file(NULL, &lock, fs->path, digest_path, pool)); if (! lock) - return SVN_FS__ERR_NO_SUCH_LOCK(fs, path, pool); + return must_exist ? SVN_FS__ERR_NO_SUCH_LOCK(fs, path) : SVN_NO_ERROR; /* Don't return an expired lock. */ if (lock->expiration_date && (apr_time_now() > lock->expiration_date)) @@ -502,8 +508,7 @@ get_lock(svn_lock_t **lock_p, Read operations shouldn't change the filesystem. */ if (have_write_lock) SVN_ERR(delete_lock(fs, lock, pool)); - *lock_p = NULL; - return SVN_FS__ERR_LOCK_EXPIRED(fs, lock->token, pool); + return SVN_FS__ERR_LOCK_EXPIRED(fs, lock->token); } *lock_p = lock; @@ -525,7 +530,7 @@ get_lock_helper(svn_fs_t *fs, svn_lock_t *lock; svn_error_t *err; - err = get_lock(&lock, fs, path, have_write_lock, pool); + err = get_lock(&lock, fs, path, have_write_lock, FALSE, pool); /* We've deliberately decided that this function doesn't tell the caller *why* the lock is unavailable. */ @@ -683,8 +688,7 @@ verify_lock(svn_fs_t *fs, _("User '%s' does not own lock on path '%s' (currently locked by '%s')"), fs->access_ctx->username, lock->path, lock->owner); - else if (apr_hash_get(fs->access_ctx->lock_tokens, lock->token, - APR_HASH_KEY_STRING) == NULL) + else if (svn_hash_gets(fs->access_ctx->lock_tokens, lock->token) == NULL) return svn_error_createf (SVN_ERR_FS_BAD_LOCK_TOKEN, NULL, _("Cannot verify lock on path '%s'; no matching lock-token available"), @@ -770,7 +774,7 @@ lock_body(void *baton, apr_pool_t *pool) SVN_ERR(lb->fs->vtable->revision_root(&root, lb->fs, youngest, pool)); SVN_ERR(svn_fs_fs__check_path(&kind, root, lb->path, pool)); if (kind == svn_node_dir) - return SVN_FS__ERR_NOT_FILE(lb->fs, lb->path, pool); + return SVN_FS__ERR_NOT_FILE(lb->fs, lb->path); /* While our locking implementation easily supports the locking of nonexistent paths, we deliberately choose not to allow such madness. */ @@ -790,7 +794,7 @@ lock_body(void *baton, apr_pool_t *pool) /* We need to have a username attached to the fs. */ if (!lb->fs->access_ctx || !lb->fs->access_ctx->username) - return SVN_FS__ERR_NO_USER(lb->fs, pool); + return SVN_FS__ERR_NO_USER(lb->fs); /* Is the caller attempting to lock an out-of-date working file? */ if (SVN_IS_VALID_REVNUM(lb->current_rev)) @@ -836,7 +840,7 @@ lock_body(void *baton, apr_pool_t *pool) if (! lb->steal_lock) { /* Sorry, the path is already locked. */ - return SVN_FS__ERR_PATH_ALREADY_LOCKED(lb->fs, existing_lock, pool); + return SVN_FS__ERR_PATH_ALREADY_LOCKED(lb->fs, existing_lock); } else { @@ -885,23 +889,23 @@ unlock_body(void *baton, apr_pool_t *pool) svn_lock_t *lock; /* This could return SVN_ERR_FS_BAD_LOCK_TOKEN or SVN_ERR_FS_LOCK_EXPIRED. */ - SVN_ERR(get_lock(&lock, ub->fs, ub->path, TRUE, pool)); + SVN_ERR(get_lock(&lock, ub->fs, ub->path, TRUE, TRUE, pool)); /* Unless breaking the lock, we do some checks. */ if (! ub->break_lock) { /* Sanity check: the incoming token should match lock->token. */ if (strcmp(ub->token, lock->token) != 0) - return SVN_FS__ERR_NO_SUCH_LOCK(ub->fs, lock->path, pool); + return SVN_FS__ERR_NO_SUCH_LOCK(ub->fs, lock->path); /* There better be a username attached to the fs. */ if (! (ub->fs->access_ctx && ub->fs->access_ctx->username)) - return SVN_FS__ERR_NO_USER(ub->fs, pool); + return SVN_FS__ERR_NO_USER(ub->fs); /* And that username better be the same as the lock's owner. */ if (strcmp(ub->fs->access_ctx->username, lock->owner) != 0) return SVN_FS__ERR_LOCK_OWNER_MISMATCH( - ub->fs, ub->fs->access_ctx->username, lock->owner, pool); + ub->fs, ub->fs->access_ctx->username, lock->owner); } /* Remove lock and lock token files. */ @@ -1040,7 +1044,7 @@ get_locks_filter_func(void *baton, else if ((b->requested_depth == svn_depth_files) || (b->requested_depth == svn_depth_immediates)) { - const char *rel_uri = svn_fspath__is_child(b->path, lock->path, pool); + const char *rel_uri = svn_fspath__skip_ancestor(b->path, lock->path); if (rel_uri && (svn_path_component_count(rel_uri) == 1)) SVN_ERR(b->get_locks_func(b->get_locks_baton, lock, pool)); } diff --git a/subversion/libsvn_fs_fs/rep-cache-db.h b/subversion/libsvn_fs_fs/rep-cache-db.h index f829e37..255ea07 100644 --- a/subversion/libsvn_fs_fs/rep-cache-db.h +++ b/subversion/libsvn_fs_fs/rep-cache-db.h @@ -1,9 +1,9 @@ -/* This file is automatically generated from rep-cache-db.sql. +/* This file is automatically generated from rep-cache-db.sql and .dist_sandbox/subversion-1.8.13/subversion/libsvn_fs_fs/token-map.h. * Do not edit this file -- edit the source and rerun gen-make.py */ #define STMT_CREATE_SCHEMA 0 +#define STMT_0_INFO {"STMT_CREATE_SCHEMA", NULL} #define STMT_0 \ - "PRAGMA AUTO_VACUUM = 1; " \ "CREATE TABLE rep_cache ( " \ " hash TEXT NOT NULL PRIMARY KEY, " \ " revision INTEGER NOT NULL, " \ @@ -15,6 +15,7 @@ "" #define STMT_GET_REP 1 +#define STMT_1_INFO {"STMT_GET_REP", NULL} #define STMT_1 \ "SELECT revision, offset, size, expanded_size " \ "FROM rep_cache " \ @@ -22,22 +23,61 @@ "" #define STMT_SET_REP 2 +#define STMT_2_INFO {"STMT_SET_REP", NULL} #define STMT_2 \ "INSERT OR FAIL INTO rep_cache (hash, revision, offset, size, expanded_size) " \ "VALUES (?1, ?2, ?3, ?4, ?5) " \ "" -#define STMT_DEL_REPS_YOUNGER_THAN_REV 3 +#define STMT_GET_REPS_FOR_RANGE 3 +#define STMT_3_INFO {"STMT_GET_REPS_FOR_RANGE", NULL} #define STMT_3 \ + "SELECT hash, revision, offset, size, expanded_size " \ + "FROM rep_cache " \ + "WHERE revision >= ?1 AND revision <= ?2 " \ + "" + +#define STMT_GET_MAX_REV 4 +#define STMT_4_INFO {"STMT_GET_MAX_REV", NULL} +#define STMT_4 \ + "SELECT MAX(revision) " \ + "FROM rep_cache " \ + "" + +#define STMT_DEL_REPS_YOUNGER_THAN_REV 5 +#define STMT_5_INFO {"STMT_DEL_REPS_YOUNGER_THAN_REV", NULL} +#define STMT_5 \ "DELETE FROM rep_cache " \ "WHERE revision > ?1 " \ "" +#define STMT_LOCK_REP 6 +#define STMT_6_INFO {"STMT_LOCK_REP", NULL} +#define STMT_6 \ + "BEGIN TRANSACTION; " \ + "INSERT INTO rep_cache VALUES ('dummy', 0, 0, 0, 0) " \ + "" + #define REP_CACHE_DB_SQL_DECLARE_STATEMENTS(varname) \ static const char * const varname[] = { \ STMT_0, \ STMT_1, \ STMT_2, \ STMT_3, \ + STMT_4, \ + STMT_5, \ + STMT_6, \ NULL \ } + +#define REP_CACHE_DB_SQL_DECLARE_STATEMENT_INFO(varname) \ + static const char * const varname[][2] = { \ + STMT_0_INFO, \ + STMT_1_INFO, \ + STMT_2_INFO, \ + STMT_3_INFO, \ + STMT_4_INFO, \ + STMT_5_INFO, \ + STMT_6_INFO, \ + {NULL, NULL} \ + } diff --git a/subversion/libsvn_fs_fs/rep-cache-db.sql b/subversion/libsvn_fs_fs/rep-cache-db.sql index 7703bf3..b88c3e0 100644 --- a/subversion/libsvn_fs_fs/rep-cache-db.sql +++ b/subversion/libsvn_fs_fs/rep-cache-db.sql @@ -22,8 +22,6 @@ */ -- STMT_CREATE_SCHEMA -PRAGMA AUTO_VACUUM = 1; - /* A table mapping representation hashes to locations in a rev file. */ CREATE TABLE rep_cache ( hash TEXT NOT NULL PRIMARY KEY, @@ -41,11 +39,27 @@ SELECT revision, offset, size, expanded_size FROM rep_cache WHERE hash = ?1 - -- STMT_SET_REP INSERT OR FAIL INTO rep_cache (hash, revision, offset, size, expanded_size) VALUES (?1, ?2, ?3, ?4, ?5) +-- STMT_GET_REPS_FOR_RANGE +SELECT hash, revision, offset, size, expanded_size +FROM rep_cache +WHERE revision >= ?1 AND revision <= ?2 + +-- STMT_GET_MAX_REV +SELECT MAX(revision) +FROM rep_cache + -- STMT_DEL_REPS_YOUNGER_THAN_REV DELETE FROM rep_cache WHERE revision > ?1 + +/* An INSERT takes an SQLite reserved lock that prevents other writes + but doesn't block reads. The incomplete transaction means that no + permanent change is made to the database and the transaction is + removed when the database is closed. */ +-- STMT_LOCK_REP +BEGIN TRANSACTION; +INSERT INTO rep_cache VALUES ('dummy', 0, 0, 0, 0) diff --git a/subversion/libsvn_fs_fs/rep-cache.c b/subversion/libsvn_fs_fs/rep-cache.c index b3867d9..0082266 100644 --- a/subversion/libsvn_fs_fs/rep-cache.c +++ b/subversion/libsvn_fs_fs/rep-cache.c @@ -20,6 +20,8 @@ * ==================================================================== */ +#include "svn_pools.h" + #include "svn_private_config.h" #include "fs_fs.h" @@ -39,32 +41,90 @@ REP_CACHE_DB_SQL_DECLARE_STATEMENTS(statements); + +/** Helper functions. **/ +static APR_INLINE const char * +path_rep_cache_db(const char *fs_path, + apr_pool_t *result_pool) +{ + return svn_dirent_join(fs_path, REP_CACHE_DB_NAME, result_pool); +} + +/* Check that REP refers to a revision that exists in FS. */ +static svn_error_t * +rep_has_been_born(representation_t *rep, + svn_fs_t *fs, + apr_pool_t *pool) +{ + SVN_ERR_ASSERT(rep); + + SVN_ERR(svn_fs_fs__revision_exists(rep->revision, fs, pool)); + + return SVN_NO_ERROR; +} + + + +/** Library-private API's. **/ + +/* Body of svn_fs_fs__open_rep_cache(). + Implements svn_atomic__init_once().init_func. + */ static svn_error_t * open_rep_cache(void *baton, apr_pool_t *pool) { svn_fs_t *fs = baton; fs_fs_data_t *ffd = fs->fsap_data; + svn_sqlite__db_t *sdb; const char *db_path; int version; /* Open (or create) the sqlite database. It will be automatically - closed when fs->pool is destoyed. */ - db_path = svn_dirent_join(fs->path, REP_CACHE_DB_NAME, pool); - SVN_ERR(svn_sqlite__open(&ffd->rep_cache_db, db_path, + closed when fs->pool is destoyed. */ + db_path = path_rep_cache_db(fs->path, pool); +#ifndef WIN32 + { + /* We want to extend the permissions that apply to the repository + as a whole when creating a new rep cache and not simply default + to umask. */ + svn_boolean_t exists; + + SVN_ERR(svn_fs_fs__exists_rep_cache(&exists, fs, pool)); + if (!exists) + { + const char *current = svn_fs_fs__path_current(fs, pool); + svn_error_t *err = svn_io_file_create(db_path, "", pool); + + if (err && !APR_STATUS_IS_EEXIST(err->apr_err)) + /* A real error. */ + return svn_error_trace(err); + else if (err) + /* Some other thread/process created the file. */ + svn_error_clear(err); + else + /* We created the file. */ + SVN_ERR(svn_io_copy_perms(current, db_path, pool)); + } + } +#endif + SVN_ERR(svn_sqlite__open(&sdb, db_path, svn_sqlite__mode_rwcreate, statements, 0, NULL, fs->pool, pool)); - SVN_ERR(svn_sqlite__read_schema_version(&version, ffd->rep_cache_db, pool)); + SVN_ERR(svn_sqlite__read_schema_version(&version, sdb, pool)); if (version < REP_CACHE_SCHEMA_FORMAT) { /* Must be 0 -- an uninitialized (no schema) database. Create the schema. Results in schema version of 1. */ - SVN_ERR(svn_sqlite__exec_statements(ffd->rep_cache_db, - STMT_CREATE_SCHEMA)); + SVN_ERR(svn_sqlite__exec_statements(sdb, STMT_CREATE_SCHEMA)); } + /* This is used as a flag that the database is available so don't + set it earlier. */ + ffd->rep_cache_db = sdb; + return SVN_NO_ERROR; } @@ -78,6 +138,112 @@ svn_fs_fs__open_rep_cache(svn_fs_t *fs, return svn_error_quick_wrap(err, _("Couldn't open rep-cache database")); } +svn_error_t * +svn_fs_fs__exists_rep_cache(svn_boolean_t *exists, + svn_fs_t *fs, apr_pool_t *pool) +{ + svn_node_kind_t kind; + + SVN_ERR(svn_io_check_path(path_rep_cache_db(fs->path, pool), + &kind, pool)); + + *exists = (kind != svn_node_none); + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_fs__walk_rep_reference(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t end, + svn_error_t *(*walker)(representation_t *, + void *, + svn_fs_t *, + apr_pool_t *), + void *walker_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + svn_sqlite__stmt_t *stmt; + svn_boolean_t have_row; + int iterations = 0; + + apr_pool_t *iterpool = svn_pool_create(pool); + + /* Don't check ffd->rep_sharing_allowed. */ + SVN_ERR_ASSERT(ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT); + + if (! ffd->rep_cache_db) + SVN_ERR(svn_fs_fs__open_rep_cache(fs, pool)); + + /* Check global invariants. */ + if (start == 0) + { + svn_revnum_t max; + + SVN_ERR(svn_sqlite__get_statement(&stmt, ffd->rep_cache_db, + STMT_GET_MAX_REV)); + SVN_ERR(svn_sqlite__step(&have_row, stmt)); + max = svn_sqlite__column_revnum(stmt, 0); + SVN_ERR(svn_sqlite__reset(stmt)); + if (SVN_IS_VALID_REVNUM(max)) /* The rep-cache could be empty. */ + SVN_ERR(svn_fs_fs__revision_exists(max, fs, iterpool)); + } + + SVN_ERR(svn_sqlite__get_statement(&stmt, ffd->rep_cache_db, + STMT_GET_REPS_FOR_RANGE)); + SVN_ERR(svn_sqlite__bindf(stmt, "rr", + start, end)); + + /* Walk the cache entries. */ + SVN_ERR(svn_sqlite__step(&have_row, stmt)); + while (have_row) + { + representation_t *rep; + const char *sha1_digest; + svn_error_t *err; + + /* Clear ITERPOOL occasionally. */ + if (iterations++ % 16 == 0) + svn_pool_clear(iterpool); + + /* Check for cancellation. */ + if (cancel_func) + { + err = cancel_func(cancel_baton); + if (err) + return svn_error_compose_create(err, svn_sqlite__reset(stmt)); + } + + /* Construct a representation_t. */ + rep = apr_pcalloc(iterpool, sizeof(*rep)); + sha1_digest = svn_sqlite__column_text(stmt, 0, iterpool); + err = svn_checksum_parse_hex(&rep->sha1_checksum, + svn_checksum_sha1, sha1_digest, + iterpool); + if (err) + return svn_error_compose_create(err, svn_sqlite__reset(stmt)); + rep->revision = svn_sqlite__column_revnum(stmt, 1); + rep->offset = svn_sqlite__column_int64(stmt, 2); + rep->size = svn_sqlite__column_int64(stmt, 3); + rep->expanded_size = svn_sqlite__column_int64(stmt, 4); + + /* Walk. */ + err = walker(rep, walker_baton, fs, iterpool); + if (err) + return svn_error_compose_create(err, svn_sqlite__reset(stmt)); + + SVN_ERR(svn_sqlite__step(&have_row, stmt)); + } + + SVN_ERR(svn_sqlite__reset(stmt)); + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + + /* This function's caller ignores most errors it returns. If you extend this function, check the callsite to see if you have to make it not-ignore additional error codes. */ @@ -118,27 +284,12 @@ svn_fs_fs__get_rep_reference(representation_t **rep, else *rep = NULL; - /* Sanity check. */ - if (*rep) - { - svn_revnum_t youngest; + SVN_ERR(svn_sqlite__reset(stmt)); - youngest = ffd->youngest_rev_cache; - if (youngest < (*rep)->revision) - { - /* Stale cache. */ - SVN_ERR(svn_fs_fs__youngest_rev(&youngest, fs, pool)); - - /* Fresh cache. */ - if (youngest < (*rep)->revision) - return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Youngest revision is r%ld, but " - "rep-cache contains r%ld"), - youngest, (*rep)->revision); - } - } + if (*rep) + SVN_ERR(rep_has_been_born(*rep, fs, pool)); - return svn_sqlite__reset(stmt); + return SVN_NO_ERROR; } svn_error_t * @@ -239,3 +390,17 @@ svn_fs_fs__del_rep_reference(svn_fs_t *fs, return SVN_NO_ERROR; } + +svn_error_t * +svn_fs_fs__lock_rep_cache(svn_fs_t *fs, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + + if (! ffd->rep_cache_db) + SVN_ERR(svn_fs_fs__open_rep_cache(fs, pool)); + + SVN_ERR(svn_sqlite__exec_statements(ffd->rep_cache_db, STMT_LOCK_REP)); + + return SVN_NO_ERROR; +} diff --git a/subversion/libsvn_fs_fs/rep-cache.h b/subversion/libsvn_fs_fs/rep-cache.h index 1ff1a70..3ccb056 100644 --- a/subversion/libsvn_fs_fs/rep-cache.h +++ b/subversion/libsvn_fs_fs/rep-cache.h @@ -40,6 +40,25 @@ svn_error_t * svn_fs_fs__open_rep_cache(svn_fs_t *fs, apr_pool_t *pool); +/* Set *EXISTS to TRUE iff the rep-cache DB file exists. */ +svn_error_t * +svn_fs_fs__exists_rep_cache(svn_boolean_t *exists, + svn_fs_t *fs, apr_pool_t *pool); + +/* Iterate all representations currently in FS's cache. */ +svn_error_t * +svn_fs_fs__walk_rep_reference(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t end, + svn_error_t *(*walker)(representation_t *rep, + void *walker_baton, + svn_fs_t *fs, + apr_pool_t *scratch_pool), + void *walker_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool); + /* Return the representation REP in FS which has fulltext CHECKSUM. REP is allocated in POOL. If the rep cache database has not been opened, just set *REP to NULL. */ @@ -69,6 +88,12 @@ svn_fs_fs__del_rep_reference(svn_fs_t *fs, svn_revnum_t youngest, apr_pool_t *pool); +/* Start a transaction to take an SQLite reserved lock that prevents + other writes. */ +svn_error_t * +svn_fs_fs__lock_rep_cache(svn_fs_t *fs, + apr_pool_t *pool); + #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/subversion/libsvn_fs_fs/structure b/subversion/libsvn_fs_fs/structure index 5472a18..41caf1d 100644 --- a/subversion/libsvn_fs_fs/structure +++ b/subversion/libsvn_fs_fs/structure @@ -40,6 +40,9 @@ repository) is: revprops/ Subdirectory containing rev-props <shard>/ Shard directory, if sharding is in use (see below) <revnum> File containing rev-props for <revnum> + <shard>.pack/ Pack directory, if the repo has been packed (see below) + <rev>.<count> Pack file, if the repository has been packed (see below) + manifest Pack manifest file, if a pack file exists (see below) revprops.db SQLite database of the packed revision properties transactions/ Subdirectory containing transactions <txnid>.txn/ Directory containing transaction <txnid> @@ -134,6 +137,7 @@ The formats are: Format 3, understood by Subversion 1.5+ Format 4, understood by Subversion 1.6+ Format 5, understood by Subversion 1.7-dev, never released + Format 6, understood by Subversion 1.8 The differences between the formats are: @@ -173,6 +177,12 @@ Revision changed paths list: Format 1-3: Does not contain the node's kind. Format 4+: Contains the node's kind. +Shard packing: + Format 4: Applied to revision data only. + Format 5: Revprops would be packed independently of revision data. + Format 6+: Applied equally to revision data and revprop data + (i.e. same min packed revision) + # Incomplete list. See SVN_FS_FS__MIN_*_FORMAT @@ -232,6 +242,80 @@ See r1143829 of this file: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/structure?view=markup&pathrev=1143829 +Packing revision properties (format 6+) +--------------------------- + +Similarly to the revision data, packing will concatenate multiple +revprops into a single file. Since they are mutable data, we put an +upper limit to the size of these files: We will concatenate the data +up to the limit and then use a new file for the following revisions. + +The limit can be set and changed at will in the configuration file. +It is 64kB by default. Because a pack file must contain at least one +complete property list, files containing just one revision may exceed +that limit. + +Furthermore, pack files can be compressed which saves about 75% of +disk space. A configuration file flag enables the compression; it is +off by default and may be switched on and off at will. The pack size +limit is always applied to the uncompressed data. For this reason, +the default is 256kB while compression has been enabled. + +Files are named after their start revision as "<rev>.<counter>" where +counter will be increased whenever we rewrite a pack file due to a +revprop change. The manifest file contains the list of pack file +names, one line for each revision. + +Many tools track repository global data in revision properties at +revision 0. To minimize I/O overhead for those applications, we +will never pack that revision, i.e. its data is always being kept +in revprops/0/0. + +Pack file format + + Top level: <packed container> + + We always apply data compression to the pack file - using the + SVN_DELTA_COMPRESSION_LEVEL_NONE level if compression is disabled. + (Note that compression at SVN_DELTA_COMPRESSION_LEVEL_NONE is not + a no-op stream transformation although most of the data will remain + human readable.) + + container := header '\n' (revprops)+ + header := start_rev '\n' rev_count '\n' (size '\n')+ + + All numbers in the header are given as ASCII decimals. rev_count + is the number of revisions packed into this container. There must + be exactly as many "size" and serialized "revprops". The "size" + values in the list are the length in bytes of the serialized + revprops of the respective revision. + +Writing to packed revprops + + The old pack file is being read and the new revprops serialized. + If they fit into the same pack file, a temp file with the new + content gets written and moved into place just like an non-packed + revprop file would. No name change or manifest update required. + + If they don't fit into the same pack file, i.e. exceed the pack + size limit, the pack will be split into 2 or 3 new packs just + before and / or after the modified revision. + + In the current implementation, they will never be merged again. + To minimize fragmentation, the initial packing process will only + use about 90% of the limit, i.e. leave some room for growth. + + When a pack file gets split, its counter is being increased + creating a new file and leaving the old content in place and + available for concurrent readers. Only after the new manifest + file got moved into place, will the old pack files be deleted. + + Write access to revprops is being serialized by the global + filesystem write lock. We only need to build a few retries into + the reader code to gracefully handle manifest changes and pack + file deletions. + + Node-revision IDs ----------------- @@ -266,7 +350,7 @@ Within a revision: "r<rev>/<offset>" txn-id fields. In Format 3 and above, this uniqueness is done by changing a temporary - id of "_<base36>" to "<rev>-<base36>". Note that this means that the + id of "_<base36>" to "<base36>-<rev>". Note that this means that the originating revision of a line of history or a copy can be determined by looking at the node ID. @@ -379,7 +463,8 @@ defined: props "<rev> <offset> <length> <size> <digest>" for props rep <rev> and <offset> give location of rep <length> gives length of rep, sans header and trailer - <size> gives size of expanded rep + <size> gives size of expanded rep; may be 0 if equal + to the length <digest> gives hex MD5 digest of expanded rep ### in formats >=4, also present: <sha1-digest> gives hex SHA1 digest of expanded rep @@ -441,6 +526,7 @@ A transaction directory has the following layout: node.<nid>.<cid> New node-rev data for node node.<nid>.<cid>.props Props for new node-rev, if changed node.<nid>.<cid>.children Directory contents for node-rev + <sha1> Text representation of that sha1 In FS formats 1 and 2, it also contains: @@ -458,6 +544,11 @@ The two kinds of props files are all in hash dump format. The "props" file will always be present. The "node.<nid>.<cid>.props" file will only be present if the node-rev properties have been changed. +The <sha1> files have been introduced in FS format 6. Their content +is that of text rep references: "<rev> <offset> <length> <size> <digest>" +They will be written for text reps in the current transaction and be +used to eliminate duplicate reps within that transaction. + The "next-ids" file contains a single line "<next-temp-node-id> <next-temp-copy-id>\n" giving the next temporary node-ID and copy-ID assignments (without the leading underscores). The next node-ID is diff --git a/subversion/libsvn_fs_fs/temp_serializer.c b/subversion/libsvn_fs_fs/temp_serializer.c index cb0bb6b..0178143 100644 --- a/subversion/libsvn_fs_fs/temp_serializer.c +++ b/subversion/libsvn_fs_fs/temp_serializer.c @@ -30,6 +30,7 @@ #include "private/svn_fs_util.h" #include "private/svn_temp_serializer.h" +#include "private/svn_subr_private.h" #include "temp_serializer.h" @@ -47,16 +48,16 @@ encode_number(apr_int64_t number, char *key_buffer) if (number < 0) { number = -number; - *key_buffer = (number & 63) + ' ' + 65; + *key_buffer = (char)((number & 63) + ' ' + 65); } else - *key_buffer = (number & 63) + ' ' + 1; + *key_buffer = (char)((number & 63) + ' ' + 1); number /= 64; /* write 7 bits / byte until no significant bits are left */ while (number) { - *++key_buffer = (number & 127) + ' ' + 1; + *++key_buffer = (char)((number & 127) + ' ' + 1); number /= 128; } @@ -64,10 +65,6 @@ encode_number(apr_int64_t number, char *key_buffer) return key_buffer; } -/* Prepend the NUMBER to the STRING in a space efficient way that no other - * (number,string) combination can produce the same result. - * Allocate temporaries as well as the result from POOL. - */ const char* svn_fs_fs__combine_number_and_string(apr_int64_t number, const char *string, @@ -91,31 +88,6 @@ svn_fs_fs__combine_number_and_string(apr_int64_t number, return key; } -/* Combine the numbers A and B a space efficient way that no other - * combination of numbers can produce the same result. - * Allocate temporaries as well as the result from POOL. - */ -const char* -svn_fs_fs__combine_two_numbers(apr_int64_t a, - apr_int64_t b, - apr_pool_t *pool) -{ - /* encode numbers as 2x 10x7 bits + 1 space + 1 terminating \0*/ - char *key_buffer = apr_palloc(pool, 22); - const char *key = key_buffer; - - /* combine the numbers. Since the separator is disjoint from any part - * of the encoded numbers, there is no other combination that can yield - * the same result */ - key_buffer = encode_number(a, key_buffer); - *++key_buffer = ' '; - key_buffer = encode_number(b, ++key_buffer); - *++key_buffer = '\0'; - - /* return the start of the key */ - return key; -} - /* Utility function to serialize string S in the given serialization CONTEXT. */ static void @@ -136,7 +108,7 @@ serialize_svn_string(svn_temp_serializer__context_t *context, * Thus, we cannot use svn_temp_serializer__add_string. */ svn_temp_serializer__push(context, (const void * const *)&string->data, - string->len); + string->len + 1); /* back to the caller's nesting level */ svn_temp_serializer__pop(context); @@ -184,7 +156,7 @@ serialize_checksum(svn_temp_serializer__context_t *context, /* Utility function to deserialize the checksum CS inside the BUFFER. */ static void -deserialize_checksum(void *buffer, svn_checksum_t * const *cs) +deserialize_checksum(void *buffer, svn_checksum_t **cs) { svn_temp_deserializer__resolve(buffer, (void **)cs); if (*cs == NULL) @@ -367,7 +339,7 @@ serialize_dir(apr_hash_t *entries, apr_pool_t *pool) static apr_hash_t * deserialize_dir(void *buffer, hash_data_t *hash_data, apr_pool_t *pool) { - apr_hash_t *result = apr_hash_make(pool); + apr_hash_t *result = svn_hash__make(pool); apr_size_t i; apr_size_t count; svn_fs_dirent_t *entry; @@ -388,7 +360,7 @@ deserialize_dir(void *buffer, hash_data_t *hash_data, apr_pool_t *pool) svn_fs_fs__id_deserialize(entry, (svn_fs_id_t **)&entry->id); /* add the entry to the hash */ - apr_hash_set(result, entry->name, APR_HASH_KEY_STRING, entry); + svn_hash_sets(result, entry->name, entry); } /* return the now complete hash */ @@ -489,7 +461,7 @@ serialize_txdeltawindow(svn_temp_serializer__context_t *context, } svn_error_t * -svn_fs_fs__serialize_txdelta_window(char **buffer, +svn_fs_fs__serialize_txdelta_window(void **buffer, apr_size_t *buffer_size, void *item, apr_pool_t *pool) @@ -522,7 +494,7 @@ svn_fs_fs__serialize_txdelta_window(char **buffer, svn_error_t * svn_fs_fs__deserialize_txdelta_window(void **item, - char *buffer, + void *buffer, apr_size_t buffer_size, apr_pool_t *pool) { @@ -548,7 +520,7 @@ svn_fs_fs__deserialize_txdelta_window(void **item, } svn_error_t * -svn_fs_fs__serialize_manifest(char **data, +svn_fs_fs__serialize_manifest(void **data, apr_size_t *data_len, void *in, apr_pool_t *pool) @@ -564,7 +536,7 @@ svn_fs_fs__serialize_manifest(char **data, svn_error_t * svn_fs_fs__deserialize_manifest(void **out, - char *data, + void *data, apr_size_t data_len, apr_pool_t *pool) { @@ -579,8 +551,144 @@ svn_fs_fs__deserialize_manifest(void **out, return SVN_NO_ERROR; } +/* Auxilliary structure representing the content of a properties hash. + This structure is much easier to (de-)serialize than an apr_hash. + */ +typedef struct properties_data_t +{ + /* number of entries in the hash */ + apr_size_t count; + + /* reference to the keys */ + const char **keys; + + /* reference to the values */ + const svn_string_t **values; +} properties_data_t; + +/* Serialize COUNT C-style strings from *STRINGS into CONTEXT. */ +static void +serialize_cstring_array(svn_temp_serializer__context_t *context, + const char ***strings, + apr_size_t count) +{ + apr_size_t i; + const char **entries = *strings; + + /* serialize COUNT entries pointers (the array) */ + svn_temp_serializer__push(context, + (const void * const *)strings, + count * sizeof(const char*)); + + /* serialize array elements */ + for (i = 0; i < count; ++i) + svn_temp_serializer__add_string(context, &entries[i]); + + svn_temp_serializer__pop(context); +} + +/* Serialize COUNT svn_string_t* items from *STRINGS into CONTEXT. */ +static void +serialize_svn_string_array(svn_temp_serializer__context_t *context, + const svn_string_t ***strings, + apr_size_t count) +{ + apr_size_t i; + const svn_string_t **entries = *strings; + + /* serialize COUNT entries pointers (the array) */ + svn_temp_serializer__push(context, + (const void * const *)strings, + count * sizeof(const char*)); + + /* serialize array elements */ + for (i = 0; i < count; ++i) + serialize_svn_string(context, &entries[i]); + + svn_temp_serializer__pop(context); +} + +svn_error_t * +svn_fs_fs__serialize_properties(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + apr_hash_t *hash = in; + properties_data_t properties; + svn_temp_serializer__context_t *context; + apr_hash_index_t *hi; + svn_stringbuf_t *serialized; + apr_size_t i; + + /* create our auxilliary data structure */ + properties.count = apr_hash_count(hash); + properties.keys = apr_palloc(pool, sizeof(const char*) * (properties.count + 1)); + properties.values = apr_palloc(pool, sizeof(const char*) * properties.count); + + /* populate it with the hash entries */ + for (hi = apr_hash_first(pool, hash), i=0; hi; hi = apr_hash_next(hi), ++i) + { + properties.keys[i] = svn__apr_hash_index_key(hi); + properties.values[i] = svn__apr_hash_index_val(hi); + } + + /* serialize it */ + context = svn_temp_serializer__init(&properties, + sizeof(properties), + properties.count * 100, + pool); + + properties.keys[i] = ""; + serialize_cstring_array(context, &properties.keys, properties.count + 1); + serialize_svn_string_array(context, &properties.values, properties.count); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + svn_error_t * -svn_fs_fs__serialize_id(char **data, +svn_fs_fs__deserialize_properties(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + apr_hash_t *hash = svn_hash__make(pool); + properties_data_t *properties = (properties_data_t *)data; + size_t i; + + /* de-serialize our auxilliary data structure */ + svn_temp_deserializer__resolve(properties, (void**)&properties->keys); + svn_temp_deserializer__resolve(properties, (void**)&properties->values); + + /* de-serialize each entry and put it into the hash */ + for (i = 0; i < properties->count; ++i) + { + apr_size_t len = properties->keys[i+1] - properties->keys[i] - 1; + svn_temp_deserializer__resolve((void*)properties->keys, + (void**)&properties->keys[i]); + + deserialize_svn_string((void*)properties->values, + (svn_string_t **)&properties->values[i]); + + apr_hash_set(hash, + properties->keys[i], len, + properties->values[i]); + } + + /* done */ + *out = hash; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_fs__serialize_id(void **data, apr_size_t *data_len, void *in, apr_pool_t *pool) @@ -605,7 +713,7 @@ svn_fs_fs__serialize_id(char **data, svn_error_t * svn_fs_fs__deserialize_id(void **out, - char *data, + void *data, apr_size_t data_len, apr_pool_t *pool) { @@ -623,17 +731,20 @@ svn_fs_fs__deserialize_id(void **out, /** Caching node_revision_t objects. **/ svn_error_t * -svn_fs_fs__serialize_node_revision(char **buffer, - apr_size_t *buffer_size, - void *item, - apr_pool_t *pool) +svn_fs_fs__serialize_node_revision(void **buffer, + apr_size_t *buffer_size, + void *item, + apr_pool_t *pool) { svn_stringbuf_t *serialized; node_revision_t *noderev = item; - /* create an (empty) serialization context with plenty of buffer space */ + /* create an (empty) serialization context with plenty of (initial) + * buffer space. */ svn_temp_serializer__context_t *context = - svn_temp_serializer__init(NULL, 0, 503, pool); + svn_temp_serializer__init(NULL, 0, + 1024 - SVN_TEMP_SERIALIZER__OVERHEAD, + pool); /* serialize the noderev */ svn_fs_fs__noderev_serialize(context, &noderev); @@ -648,7 +759,7 @@ svn_fs_fs__serialize_node_revision(char **buffer, svn_error_t * svn_fs_fs__deserialize_node_revision(void **item, - char *buffer, + void *buffer, apr_size_t buffer_size, apr_pool_t *pool) { @@ -667,7 +778,7 @@ svn_fs_fs__deserialize_node_revision(void **item, * to DATA and DATA_LEN. */ static svn_error_t * return_serialized_dir_context(svn_temp_serializer__context_t *context, - char **data, + void **data, apr_size_t *data_len) { svn_stringbuf_t *serialized = svn_temp_serializer__get(context); @@ -680,7 +791,7 @@ return_serialized_dir_context(svn_temp_serializer__context_t *context, } svn_error_t * -svn_fs_fs__serialize_dir_entries(char **data, +svn_fs_fs__serialize_dir_entries(void **data, apr_size_t *data_len, void *in, apr_pool_t *pool) @@ -696,7 +807,7 @@ svn_fs_fs__serialize_dir_entries(char **data, svn_error_t * svn_fs_fs__deserialize_dir_entries(void **out, - char *data, + void *data, apr_size_t data_len, apr_pool_t *pool) { @@ -711,12 +822,12 @@ svn_fs_fs__deserialize_dir_entries(void **out, svn_error_t * svn_fs_fs__get_sharded_offset(void **out, - const char *data, + const void *data, apr_size_t data_len, void *baton, apr_pool_t *pool) { - apr_off_t *manifest = (apr_off_t *)data; + const apr_off_t *manifest = data; apr_int64_t shard_pos = *(apr_int64_t *)baton; *(apr_off_t *)out = manifest[shard_pos]; @@ -743,9 +854,9 @@ find_entry(svn_fs_dirent_t **entries, for (middle = upper / 2; lower < upper; middle = (upper + lower) / 2) { const svn_fs_dirent_t *entry = - svn_temp_deserializer__ptr(entries, (const void **)&entries[middle]); + svn_temp_deserializer__ptr(entries, (const void *const *)&entries[middle]); const char* entry_name = - svn_temp_deserializer__ptr(entry, (const void **)&entry->name); + svn_temp_deserializer__ptr(entry, (const void *const *)&entry->name); int diff = strcmp(entry_name, name); if (diff < 0) @@ -759,9 +870,9 @@ find_entry(svn_fs_dirent_t **entries, if (lower < count) { const svn_fs_dirent_t *entry = - svn_temp_deserializer__ptr(entries, (const void **)&entries[lower]); + svn_temp_deserializer__ptr(entries, (const void *const *)&entries[lower]); const char* entry_name = - svn_temp_deserializer__ptr(entry, (const void **)&entry->name); + svn_temp_deserializer__ptr(entry, (const void *const *)&entry->name); if (strcmp(entry_name, name) == 0) *found = TRUE; @@ -772,22 +883,22 @@ find_entry(svn_fs_dirent_t **entries, svn_error_t * svn_fs_fs__extract_dir_entry(void **out, - const char *data, + const void *data, apr_size_t data_len, void *baton, apr_pool_t *pool) { - hash_data_t *hash_data = (hash_data_t *)data; + const hash_data_t *hash_data = data; const char* name = baton; svn_boolean_t found; /* resolve the reference to the entries array */ const svn_fs_dirent_t * const *entries = - svn_temp_deserializer__ptr(data, (const void **)&hash_data->entries); + svn_temp_deserializer__ptr(data, (const void *const *)&hash_data->entries); /* resolve the reference to the lengths array */ const apr_uint32_t *lengths = - svn_temp_deserializer__ptr(data, (const void **)&hash_data->lengths); + svn_temp_deserializer__ptr(data, (const void *const *)&hash_data->lengths); /* binary search for the desired entry by name */ apr_size_t pos = find_entry((svn_fs_dirent_t **)entries, @@ -800,10 +911,10 @@ svn_fs_fs__extract_dir_entry(void **out, if (found) { const svn_fs_dirent_t *source = - svn_temp_deserializer__ptr(entries, (const void **)&entries[pos]); + svn_temp_deserializer__ptr(entries, (const void *const *)&entries[pos]); /* Entries have been serialized one-by-one, each time including all - * nestes structures and strings. Therefore, they occupy a single + * nested structures and strings. Therefore, they occupy a single * block of memory whose end-offset is either the beginning of the * next entry or the end of the buffer */ @@ -825,7 +936,7 @@ svn_fs_fs__extract_dir_entry(void **out, * modification as a simply deserialize / modify / serialize sequence. */ static svn_error_t * -slowly_replace_dir_entry(char **data, +slowly_replace_dir_entry(void **data, apr_size_t *data_len, void *baton, apr_pool_t *pool) @@ -838,16 +949,13 @@ slowly_replace_dir_entry(char **data, *data, hash_data->len, pool)); - apr_hash_set(dir, - replace_baton->name, - APR_HASH_KEY_STRING, - replace_baton->new_entry); + svn_hash_sets(dir, replace_baton->name, replace_baton->new_entry); return svn_fs_fs__serialize_dir_entries(data, data_len, dir, pool); } svn_error_t * -svn_fs_fs__replace_dir_entry(char **data, +svn_fs_fs__replace_dir_entry(void **data, apr_size_t *data_len, void *baton, apr_pool_t *pool) @@ -871,12 +979,12 @@ svn_fs_fs__replace_dir_entry(char **data, /* resolve the reference to the entries array */ entries = (svn_fs_dirent_t **) svn_temp_deserializer__ptr((const char *)hash_data, - (const void **)&hash_data->entries); + (const void *const *)&hash_data->entries); /* resolve the reference to the lengths array */ lengths = (apr_uint32_t *) svn_temp_deserializer__ptr((const char *)hash_data, - (const void **)&hash_data->lengths); + (const void *const *)&hash_data->lengths); /* binary search for the desired entry by name */ pos = find_entry(entries, replace_baton->name, hash_data->count, &found); @@ -945,8 +1053,289 @@ svn_fs_fs__replace_dir_entry(char **data, hash_data = (hash_data_t *)*data; lengths = (apr_uint32_t *) svn_temp_deserializer__ptr((const char *)hash_data, - (const void **)&hash_data->lengths); + (const void *const *)&hash_data->lengths); lengths[pos] = length; return SVN_NO_ERROR; } + +/* Utility function to serialize change CHANGE_P in the given serialization + * CONTEXT. + */ +static void +serialize_change(svn_temp_serializer__context_t *context, + change_t * const *change_p) +{ + const change_t * change = *change_p; + if (change == NULL) + return; + + /* serialize the change struct itself */ + svn_temp_serializer__push(context, + (const void * const *)change_p, + sizeof(*change)); + + /* serialize sub-structures */ + svn_fs_fs__id_serialize(context, &change->noderev_id); + + svn_temp_serializer__add_string(context, &change->path); + svn_temp_serializer__add_string(context, &change->copyfrom_path); + + /* return to the caller's nesting level */ + svn_temp_serializer__pop(context); +} + +/* Utility function to serialize the CHANGE_P within the given + * serialization CONTEXT. + */ +static void +deserialize_change(void *buffer, change_t **change_p) +{ + change_t * change; + + /* fix-up of the pointer to the struct in question */ + svn_temp_deserializer__resolve(buffer, (void **)change_p); + + change = *change_p; + if (change == NULL) + return; + + /* fix-up of sub-structures */ + svn_fs_fs__id_deserialize(change, (svn_fs_id_t **)&change->noderev_id); + + svn_temp_deserializer__resolve(change, (void **)&change->path); + svn_temp_deserializer__resolve(change, (void **)&change->copyfrom_path); +} + +/* Auxiliary structure representing the content of a change_t array. + This structure is much easier to (de-)serialize than an APR array. + */ +typedef struct changes_data_t +{ + /* number of entries in the array */ + int count; + + /* reference to the changes */ + change_t **changes; +} changes_data_t; + +svn_error_t * +svn_fs_fs__serialize_changes(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + apr_array_header_t *array = in; + changes_data_t changes; + svn_temp_serializer__context_t *context; + svn_stringbuf_t *serialized; + int i; + + /* initialize our auxiliary data structure */ + changes.count = array->nelts; + changes.changes = apr_palloc(pool, sizeof(change_t*) * changes.count); + + /* populate it with the array elements */ + for (i = 0; i < changes.count; ++i) + changes.changes[i] = APR_ARRAY_IDX(array, i, change_t*); + + /* serialize it and all its elements */ + context = svn_temp_serializer__init(&changes, + sizeof(changes), + changes.count * 100, + pool); + + svn_temp_serializer__push(context, + (const void * const *)&changes.changes, + changes.count * sizeof(change_t*)); + + for (i = 0; i < changes.count; ++i) + serialize_change(context, &changes.changes[i]); + + svn_temp_serializer__pop(context); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_fs__deserialize_changes(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + int i; + changes_data_t *changes = (changes_data_t *)data; + apr_array_header_t *array = apr_array_make(pool, changes->count, + sizeof(change_t *)); + + /* de-serialize our auxiliary data structure */ + svn_temp_deserializer__resolve(changes, (void**)&changes->changes); + + /* de-serialize each entry and add it to the array */ + for (i = 0; i < changes->count; ++i) + { + deserialize_change((void*)changes->changes, + (change_t **)&changes->changes[i]); + APR_ARRAY_PUSH(array, change_t *) = changes->changes[i]; + } + + /* done */ + *out = array; + + return SVN_NO_ERROR; +} + +/* Auxiliary structure representing the content of a svn_mergeinfo_t hash. + This structure is much easier to (de-)serialize than an APR array. + */ +typedef struct mergeinfo_data_t +{ + /* number of paths in the hash */ + unsigned count; + + /* COUNT keys (paths) */ + const char **keys; + + /* COUNT keys lengths (strlen of path) */ + apr_ssize_t *key_lengths; + + /* COUNT entries, each giving the number of ranges for the key */ + int *range_counts; + + /* all ranges in a single, concatenated buffer */ + svn_merge_range_t *ranges; +} mergeinfo_data_t; + +svn_error_t * +svn_fs_fs__serialize_mergeinfo(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool) +{ + svn_mergeinfo_t mergeinfo = in; + mergeinfo_data_t merges; + svn_temp_serializer__context_t *context; + svn_stringbuf_t *serialized; + apr_hash_index_t *hi; + unsigned i; + int k; + apr_size_t range_count; + + /* initialize our auxiliary data structure */ + merges.count = apr_hash_count(mergeinfo); + merges.keys = apr_palloc(pool, sizeof(*merges.keys) * merges.count); + merges.key_lengths = apr_palloc(pool, sizeof(*merges.key_lengths) * + merges.count); + merges.range_counts = apr_palloc(pool, sizeof(*merges.range_counts) * + merges.count); + + i = 0; + range_count = 0; + for (hi = apr_hash_first(pool, mergeinfo); hi; hi = apr_hash_next(hi), ++i) + { + svn_rangelist_t *ranges; + apr_hash_this(hi, (const void**)&merges.keys[i], + &merges.key_lengths[i], + (void **)&ranges); + merges.range_counts[i] = ranges->nelts; + range_count += ranges->nelts; + } + + merges.ranges = apr_palloc(pool, sizeof(*merges.ranges) * range_count); + + i = 0; + for (hi = apr_hash_first(pool, mergeinfo); hi; hi = apr_hash_next(hi)) + { + svn_rangelist_t *ranges = svn__apr_hash_index_val(hi); + for (k = 0; k < ranges->nelts; ++k, ++i) + merges.ranges[i] = *APR_ARRAY_IDX(ranges, k, svn_merge_range_t*); + } + + /* serialize it and all its elements */ + context = svn_temp_serializer__init(&merges, + sizeof(merges), + range_count * 30, + pool); + + /* keys array */ + svn_temp_serializer__push(context, + (const void * const *)&merges.keys, + merges.count * sizeof(*merges.keys)); + + for (i = 0; i < merges.count; ++i) + svn_temp_serializer__add_string(context, &merges.keys[i]); + + svn_temp_serializer__pop(context); + + /* key lengths array */ + svn_temp_serializer__push(context, + (const void * const *)&merges.key_lengths, + merges.count * sizeof(*merges.key_lengths)); + svn_temp_serializer__pop(context); + + /* range counts array */ + svn_temp_serializer__push(context, + (const void * const *)&merges.range_counts, + merges.count * sizeof(*merges.range_counts)); + svn_temp_serializer__pop(context); + + /* ranges */ + svn_temp_serializer__push(context, + (const void * const *)&merges.ranges, + range_count * sizeof(*merges.ranges)); + svn_temp_serializer__pop(context); + + /* return the serialized result */ + serialized = svn_temp_serializer__get(context); + + *data = serialized->data; + *data_len = serialized->len; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_fs__deserialize_mergeinfo(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool) +{ + unsigned i; + int k, n; + mergeinfo_data_t *merges = (mergeinfo_data_t *)data; + svn_mergeinfo_t mergeinfo; + + /* de-serialize our auxiliary data structure */ + svn_temp_deserializer__resolve(merges, (void**)&merges->keys); + svn_temp_deserializer__resolve(merges, (void**)&merges->key_lengths); + svn_temp_deserializer__resolve(merges, (void**)&merges->range_counts); + svn_temp_deserializer__resolve(merges, (void**)&merges->ranges); + + /* de-serialize keys and add entries to the result */ + n = 0; + mergeinfo = svn_hash__make(pool); + for (i = 0; i < merges->count; ++i) + { + svn_rangelist_t *ranges = apr_array_make(pool, + merges->range_counts[i], + sizeof(svn_merge_range_t*)); + for (k = 0; k < merges->range_counts[i]; ++k, ++n) + APR_ARRAY_PUSH(ranges, svn_merge_range_t*) = &merges->ranges[n]; + + svn_temp_deserializer__resolve((void*)merges->keys, + (void**)&merges->keys[i]); + apr_hash_set(mergeinfo, merges->keys[i], merges->key_lengths[i], ranges); + } + + /* done */ + *out = mergeinfo; + + return SVN_NO_ERROR; +} + diff --git a/subversion/libsvn_fs_fs/temp_serializer.h b/subversion/libsvn_fs_fs/temp_serializer.h index 1e1763e..1009d63 100644 --- a/subversion/libsvn_fs_fs/temp_serializer.h +++ b/subversion/libsvn_fs_fs/temp_serializer.h @@ -26,7 +26,7 @@ #include "fs.h" /** - * Prepend the @a number to the @a string in a space efficient way that + * Prepend the @a number to the @a string in a space efficient way such that * no other (number,string) combination can produce the same result. * Allocate temporaries as well as the result from @a pool. */ @@ -36,16 +36,6 @@ svn_fs_fs__combine_number_and_string(apr_int64_t number, apr_pool_t *pool); /** - * Combine the numbers @a a and @a b a space efficient way that no other - * combination of numbers can produce the same result. - * Allocate temporaries as well as the result from @a pool. - */ -const char* -svn_fs_fs__combine_two_numbers(apr_int64_t a, - apr_int64_t b, - apr_pool_t *pool); - -/** * Serialize a @a noderev_p within the serialization @a context. */ void @@ -77,7 +67,7 @@ typedef struct * #svn_fs_fs__txdelta_cached_window_t. */ svn_error_t * -svn_fs_fs__serialize_txdelta_window(char **buffer, +svn_fs_fs__serialize_txdelta_window(void **buffer, apr_size_t *buffer_size, void *item, apr_pool_t *pool); @@ -88,7 +78,7 @@ svn_fs_fs__serialize_txdelta_window(char **buffer, */ svn_error_t * svn_fs_fs__deserialize_txdelta_window(void **item, - char *buffer, + void *buffer, apr_size_t buffer_size, apr_pool_t *pool); @@ -97,7 +87,7 @@ svn_fs_fs__deserialize_txdelta_window(void **item, * (@a in is an #apr_array_header_t of apr_off_t elements). */ svn_error_t * -svn_fs_fs__serialize_manifest(char **data, +svn_fs_fs__serialize_manifest(void **data, apr_size_t *data_len, void *in, apr_pool_t *pool); @@ -108,15 +98,35 @@ svn_fs_fs__serialize_manifest(char **data, */ svn_error_t * svn_fs_fs__deserialize_manifest(void **out, - char *data, + void *data, apr_size_t data_len, apr_pool_t *pool); /** + * Implements #svn_cache__serialize_func_t for a properties hash + * (@a in is an #apr_hash_t of svn_string_t elements, keyed by const char*). + */ +svn_error_t * +svn_fs_fs__serialize_properties(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/** + * Implements #svn_cache__deserialize_func_t for a properties hash + * (@a *out is an #apr_hash_t of svn_string_t elements, keyed by const char*). + */ +svn_error_t * +svn_fs_fs__deserialize_properties(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/** * Implements #svn_cache__serialize_func_t for #svn_fs_id_t */ svn_error_t * -svn_fs_fs__serialize_id(char **data, +svn_fs_fs__serialize_id(void **data, apr_size_t *data_len, void *in, apr_pool_t *pool); @@ -126,7 +136,7 @@ svn_fs_fs__serialize_id(char **data, */ svn_error_t * svn_fs_fs__deserialize_id(void **out, - char *data, + void *data, apr_size_t data_len, apr_pool_t *pool); @@ -134,7 +144,7 @@ svn_fs_fs__deserialize_id(void **out, * Implements #svn_cache__serialize_func_t for #node_revision_t */ svn_error_t * -svn_fs_fs__serialize_node_revision(char **buffer, +svn_fs_fs__serialize_node_revision(void **buffer, apr_size_t *buffer_size, void *item, apr_pool_t *pool); @@ -144,7 +154,7 @@ svn_fs_fs__serialize_node_revision(char **buffer, */ svn_error_t * svn_fs_fs__deserialize_node_revision(void **item, - char *buffer, + void *buffer, apr_size_t buffer_size, apr_pool_t *pool); @@ -152,7 +162,7 @@ svn_fs_fs__deserialize_node_revision(void **item, * Implements #svn_cache__serialize_func_t for a directory contents hash */ svn_error_t * -svn_fs_fs__serialize_dir_entries(char **data, +svn_fs_fs__serialize_dir_entries(void **data, apr_size_t *data_len, void *in, apr_pool_t *pool); @@ -162,7 +172,7 @@ svn_fs_fs__serialize_dir_entries(char **data, */ svn_error_t * svn_fs_fs__deserialize_dir_entries(void **out, - char *data, + void *data, apr_size_t data_len, apr_pool_t *pool); @@ -172,7 +182,7 @@ svn_fs_fs__deserialize_dir_entries(void **out, * serialized manifest array @a data and @a data_len. */ svn_error_t * svn_fs_fs__get_sharded_offset(void **out, - const char *data, + const void *data, apr_size_t data_len, void *baton, apr_pool_t *pool); @@ -184,7 +194,7 @@ svn_fs_fs__get_sharded_offset(void **out, */ svn_error_t * svn_fs_fs__extract_dir_entry(void **out, - const char *data, + const void *data, apr_size_t data_len, void *baton, apr_pool_t *pool); @@ -210,9 +220,47 @@ typedef struct replace_baton_t * identified by its name in the #replace_baton_t in @a baton. */ svn_error_t * -svn_fs_fs__replace_dir_entry(char **data, +svn_fs_fs__replace_dir_entry(void **data, apr_size_t *data_len, void *baton, apr_pool_t *pool); +/** + * Implements #svn_cache__serialize_func_t for an #apr_array_header_t of + * #change_t *. + */ +svn_error_t * +svn_fs_fs__serialize_changes(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/** + * Implements #svn_cache__deserialize_func_t for an #apr_array_header_t of + * #change_t *. + */ +svn_error_t * +svn_fs_fs__deserialize_changes(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + +/** + * Implements #svn_cache__serialize_func_t for #svn_mergeinfo_t objects. + */ +svn_error_t * +svn_fs_fs__serialize_mergeinfo(void **data, + apr_size_t *data_len, + void *in, + apr_pool_t *pool); + +/** + * Implements #svn_cache__deserialize_func_t for #svn_mergeinfo_t objects. + */ +svn_error_t * +svn_fs_fs__deserialize_mergeinfo(void **out, + void *data, + apr_size_t data_len, + apr_pool_t *pool); + #endif diff --git a/subversion/libsvn_fs_fs/tree.c b/subversion/libsvn_fs_fs/tree.c index b3943cc..acd1eb4 100644 --- a/subversion/libsvn_fs_fs/tree.c +++ b/subversion/libsvn_fs_fs/tree.c @@ -41,10 +41,10 @@ #include <apr_pools.h> #include <apr_hash.h> +#include "svn_hash.h" #include "svn_private_config.h" #include "svn_pools.h" #include "svn_error.h" -#include "svn_dirent_uri.h" #include "svn_path.h" #include "svn_mergeinfo.h" #include "svn_fs.h" @@ -57,8 +57,10 @@ #include "tree.h" #include "fs_fs.h" #include "id.h" +#include "temp_serializer.h" #include "private/svn_mergeinfo_private.h" +#include "private/svn_subr_private.h" #include "private/svn_fs_util.h" #include "private/svn_fspath.h" #include "../libsvn_fs/fs-loader.h" @@ -114,14 +116,18 @@ typedef struct fs_rev_root_data_t typedef struct fs_txn_root_data_t { + const char *txn_id; + /* Cache of txn DAG nodes (without their nested noderevs, because - * it's mutable). */ + * it's mutable). Same keys/values as ffd->rev_node_cache. */ svn_cache__t *txn_node_cache; } fs_txn_root_data_t; /* Declared here to resolve the circular dependencies. */ -static svn_error_t * get_dag(dag_node_t **dag_node_p, svn_fs_root_t *root, - const char *path, apr_pool_t *pool); +static svn_error_t * get_dag(dag_node_t **dag_node_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *pool); static svn_fs_root_t *make_revision_root(svn_fs_t *fs, svn_revnum_t rev, dag_node_t *root_dir, @@ -135,6 +141,166 @@ static svn_error_t *make_txn_root(svn_fs_root_t **root_p, /*** Node Caching ***/ +/* 1st level cache */ + +/* An entry in the first-level cache. REVISION and PATH form the key that + will ultimately be matched. + */ +typedef struct cache_entry_t +{ + /* hash value derived from PATH, REVISION. + Used to short-circuit failed lookups. */ + apr_uint32_t hash_value; + + /* revision to which the NODE belongs */ + svn_revnum_t revision; + + /* path of the NODE */ + char *path; + + /* cached value of strlen(PATH). */ + apr_size_t path_len; + + /* the node allocated in the cache's pool. NULL for empty entries. */ + dag_node_t *node; +} cache_entry_t; + +/* Number of entries in the cache. Keep this low to keep pressure on the + CPU caches low as well. A binary value is most efficient. If we walk + a directory tree, we want enough entries to store nodes for all files + without overwriting the nodes for the parent folder. That way, there + will be no unnecessary misses (except for a few random ones caused by + hash collision). + + The actual number of instances may be higher but entries that got + overwritten are no longer visible. + */ +enum { BUCKET_COUNT = 256 }; + +/* The actual cache structure. All nodes will be allocated in POOL. + When the number of INSERTIONS (i.e. objects created form that pool) + exceeds a certain threshold, the pool will be cleared and the cache + with it. + */ +struct fs_fs_dag_cache_t +{ + /* fixed number of (possibly empty) cache entries */ + cache_entry_t buckets[BUCKET_COUNT]; + + /* pool used for all node allocation */ + apr_pool_t *pool; + + /* number of entries created from POOL since the last cleanup */ + apr_size_t insertions; + + /* Property lookups etc. have a very high locality (75% re-hit). + Thus, remember the last hit location for optimistic lookup. */ + apr_size_t last_hit; +}; + +fs_fs_dag_cache_t* +svn_fs_fs__create_dag_cache(apr_pool_t *pool) +{ + fs_fs_dag_cache_t *result = apr_pcalloc(pool, sizeof(*result)); + result->pool = svn_pool_create(pool); + + return result; +} + +/* Clears the CACHE at regular intervals (destroying all cached nodes) + */ +static void +auto_clear_dag_cache(fs_fs_dag_cache_t* cache) +{ + if (cache->insertions > BUCKET_COUNT) + { + svn_pool_clear(cache->pool); + + memset(cache->buckets, 0, sizeof(cache->buckets)); + cache->insertions = 0; + } +} + +/* For the given REVISION and PATH, return the respective entry in CACHE. + If the entry is empty, its NODE member will be NULL and the caller + may then set it to the corresponding DAG node allocated in CACHE->POOL. + */ +static cache_entry_t * +cache_lookup( fs_fs_dag_cache_t *cache + , svn_revnum_t revision + , const char *path) +{ + apr_size_t i, bucket_index; + apr_size_t path_len = strlen(path); + apr_uint32_t hash_value = (apr_uint32_t)revision; + +#if SVN_UNALIGNED_ACCESS_IS_OK + /* "randomizing" / distributing factor used in our hash function */ + const apr_uint32_t factor = 0xd1f3da69; +#endif + + /* optimistic lookup: hit the same bucket again? */ + cache_entry_t *result = &cache->buckets[cache->last_hit]; + if ( (result->revision == revision) + && (result->path_len == path_len) + && !memcmp(result->path, path, path_len)) + { + return result; + } + + /* need to do a full lookup. Calculate the hash value + (HASH_VALUE has been initialized to REVISION). */ + i = 0; +#if SVN_UNALIGNED_ACCESS_IS_OK + /* We relax the dependency chain between iterations by processing + two chunks from the input per hash_value self-multiplication. + The HASH_VALUE update latency is now 1 MUL latency + 1 ADD latency + per 2 chunks instead of 1 chunk. + */ + for (; i + 8 <= path_len; i += 8) + hash_value = hash_value * factor * factor + + ( *(const apr_uint32_t*)(path + i) * factor + + *(const apr_uint32_t*)(path + i + 4)); +#endif + + for (; i < path_len; ++i) + /* Help GCC to minimize the HASH_VALUE update latency by splitting the + MUL 33 of the naive implementation: h = h * 33 + path[i]. This + shortens the dependency chain from 1 shift + 2 ADDs to 1 shift + 1 ADD. + */ + hash_value = hash_value * 32 + (hash_value + (unsigned char)path[i]); + + bucket_index = hash_value + (hash_value >> 16); + bucket_index = (bucket_index + (bucket_index >> 8)) % BUCKET_COUNT; + + /* access the corresponding bucket and remember its location */ + result = &cache->buckets[bucket_index]; + cache->last_hit = bucket_index; + + /* if it is *NOT* a match, clear the bucket, expect the caller to fill + in the node and count it as an insertion */ + if ( (result->hash_value != hash_value) + || (result->revision != revision) + || (result->path_len != path_len) + || memcmp(result->path, path, path_len)) + { + result->hash_value = hash_value; + result->revision = revision; + if (result->path_len < path_len) + result->path = apr_palloc(cache->pool, path_len + 1); + result->path_len = path_len; + memcpy(result->path, path, path_len + 1); + + result->node = NULL; + + cache->insertions++; + } + + return result; +} + +/* 2nd level cache */ + /* Find and return the DAG node cache for ROOT and the key that should be used for PATH. */ static void @@ -154,13 +320,13 @@ locate_cache(svn_cache__t **cache, { fs_fs_data_t *ffd = root->fs->fsap_data; if (cache) *cache = ffd->rev_node_cache; - if (key && path) *key = apr_psprintf(pool, "%ld%s", - root->rev, path); + if (key && path) *key + = svn_fs_fs__combine_number_and_string(root->rev, path, pool); } } -/* Return NODE for PATH from ROOT's node cache, or NULL if the node - isn't cached; the node is copied into POOL. */ +/* Return NODE_P for PATH from ROOT's node cache, or NULL if the node + isn't cached; read it from the FS. *NODE_P is allocated in POOL. */ static svn_error_t * dag_node_cache_get(dag_node_t **node_p, svn_fs_root_t *root, @@ -168,24 +334,59 @@ dag_node_cache_get(dag_node_t **node_p, apr_pool_t *pool) { svn_boolean_t found; - dag_node_t *node; + dag_node_t *node = NULL; svn_cache__t *cache; const char *key; SVN_ERR_ASSERT(*path == '/'); - locate_cache(&cache, &key, root, path, pool); - - SVN_ERR(svn_cache__get((void **) &node, &found, cache, key, pool)); - if (found && node) + if (!root->is_txn_root) { - /* Patch up the FS, since this might have come from an old FS - * object. */ - svn_fs_fs__dag_set_fs(node, root->fs); - *node_p = node; + /* immutable DAG node. use the global caches for it */ + + fs_fs_data_t *ffd = root->fs->fsap_data; + cache_entry_t *bucket; + + auto_clear_dag_cache(ffd->dag_node_cache); + bucket = cache_lookup(ffd->dag_node_cache, root->rev, path); + if (bucket->node == NULL) + { + locate_cache(&cache, &key, root, path, pool); + SVN_ERR(svn_cache__get((void **)&node, &found, cache, key, pool)); + if (found && node) + { + /* Patch up the FS, since this might have come from an old FS + * object. */ + svn_fs_fs__dag_set_fs(node, root->fs); + + /* Retain the DAG node in L1 cache. */ + bucket->node = svn_fs_fs__dag_dup(node, + ffd->dag_node_cache->pool); + } + } + else + { + /* Copy the node from L1 cache into the passed-in POOL. */ + node = svn_fs_fs__dag_dup(bucket->node, pool); + } } else - *node_p = NULL; + { + /* DAG is mutable / may become invalid. Use the TXN-local cache */ + + locate_cache(&cache, &key, root, path, pool); + + SVN_ERR(svn_cache__get((void **) &node, &found, cache, key, pool)); + if (found && node) + { + /* Patch up the FS, since this might have come from an old FS + * object. */ + svn_fs_fs__dag_set_fs(node, root->fs); + } + } + + *node_p = node; + return SVN_NO_ERROR; } @@ -202,6 +403,9 @@ dag_node_cache_set(svn_fs_root_t *root, SVN_ERR_ASSERT(*path == '/'); + /* Do *not* attempt to dup and put the node into L1. + * dup() is twice as expensive as an L2 lookup (which will set also L1). + */ locate_cache(&cache, &key, root, path, pool); return svn_cache__set(cache, key, node, pool); @@ -228,7 +432,7 @@ find_descendents_in_cache(void *baton, struct fdic_baton *b = baton; const char *item_path = key; - if (svn_dirent_is_ancestor(b->path, item_path)) + if (svn_fspath__skip_ancestor(b->path, item_path)) APR_ARRAY_PUSH(b->list, const char *) = apr_pstrdup(b->pool, item_path); return SVN_NO_ERROR; @@ -287,12 +491,10 @@ svn_fs_fs__txn_root(svn_fs_root_t **root_p, SVN_ERR(svn_fs_fs__txn_proplist(&txnprops, txn, pool)); if (txnprops) { - if (apr_hash_get(txnprops, SVN_FS__PROP_TXN_CHECK_OOD, - APR_HASH_KEY_STRING)) + if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_OOD)) flags |= SVN_FS_TXN_CHECK_OOD; - if (apr_hash_get(txnprops, SVN_FS__PROP_TXN_CHECK_LOCKS, - APR_HASH_KEY_STRING)) + if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_LOCKS)) flags |= SVN_FS_TXN_CHECK_LOCKS; } @@ -358,7 +560,7 @@ mutable_root_node(dag_node_t **node_p, return svn_fs_fs__dag_clone_root(node_p, root->fs, root->txn, pool); else /* If it's not a transaction root, we can't change its contents. */ - return SVN_FS__ERR_NOT_MUTABLE(root->fs, root->rev, error_path, pool); + return SVN_FS__ERR_NOT_MUTABLE(root->fs, root->rev, error_path); } @@ -502,7 +704,7 @@ get_copy_inheritance(copy_id_inherit_t *inherit_p, or if it is a branch point that we are accessing via its original copy destination path. */ SVN_ERR(svn_fs_fs__dag_get_copyroot(©root_rev, ©root_path, - child->node,pool)); + child->node)); SVN_ERR(svn_fs_fs__revision_root(©root_root, fs, copyroot_rev, pool)); SVN_ERR(get_dag(©root_node, copyroot_root, copyroot_path, pool)); copyroot_id = svn_fs_fs__dag_get_id(copyroot_node); @@ -552,20 +754,28 @@ typedef enum open_path_flags_t { directories must exist, as usual.) If the last component doesn't exist, simply leave the `node' member of the bottom parent_path component zero. */ - open_path_last_optional = 1 + open_path_last_optional = 1, + + /* When this flag is set, don't bother to lookup the DAG node in + our caches because we already tried this. Ignoring this flag + has no functional impact. */ + open_path_uncached = 2, + /* The caller does not care about the parent node chain but only + the final DAG node. */ + open_path_node_only = 4 } open_path_flags_t; /* Open the node identified by PATH in ROOT, allocating in POOL. Set *PARENT_PATH_P to a path from the node up to ROOT. The resulting **PARENT_PATH_P value is guaranteed to contain at least one - *element, for the root directory. + *element, for the root directory. PATH must be in canonical form. If resulting *PARENT_PATH_P will eventually be made mutable and modified, or if copy ID inheritance information is otherwise needed, TXN_ID should be the ID of the mutability transaction. If - TXN_ID is NULL, no copy ID in heritance information will be + TXN_ID is NULL, no copy ID inheritance information will be calculated for the *PARENT_PATH_P chain. If FLAGS & open_path_last_optional is zero, return the error @@ -576,6 +786,11 @@ typedef enum open_path_flags_t { callers that create new nodes --- we find the parent directory for them, and tell them whether the entry exists already. + The remaining bits in FLAGS are hints that allow this function + to take shortcuts based on knowledge that the caller provides, + such as the caller is not actually being interested in PARENT_PATH_P, + but only in (*PARENT_PATH_P)->NODE. + NOTE: Public interfaces which only *read* from the filesystem should not call this function directly, but should instead use get_dag(). @@ -589,20 +804,44 @@ open_path(parent_path_t **parent_path_p, apr_pool_t *pool) { svn_fs_t *fs = root->fs; - dag_node_t *here; /* The directory we're currently looking at. */ - parent_path_t *parent_path; /* The path from HERE up to the root. */ + dag_node_t *here = NULL; /* The directory we're currently looking at. */ + parent_path_t *parent_path; /* The path from HERE up to the root. */ const char *rest; /* The portion of PATH we haven't traversed yet. */ - const char *canon_path = svn_fs__canonicalize_abspath(path, pool); + + /* ensure a canonical path representation */ const char *path_so_far = "/"; + apr_pool_t *iterpool = svn_pool_create(pool); + + /* callers often traverse the tree in some path-based order. That means + a sibling of PATH has been presently accessed. Try to start the lookup + directly at the parent node, if the caller did not requested the full + parent chain. */ + const char *directory; + assert(svn_fs__is_canonical_abspath(path)); + if (flags & open_path_node_only) + { + directory = svn_dirent_dirname(path, pool); + if (directory[1] != 0) /* root nodes are covered anyway */ + SVN_ERR(dag_node_cache_get(&here, root, directory, pool)); + } + + /* did the shortcut work? */ + if (here) + { + path_so_far = directory; + rest = path + strlen(directory) + 1; + } + else + { + /* Make a parent_path item for the root node, using its own current + copy id. */ + SVN_ERR(root_node(&here, root, pool)); + rest = path + 1; /* skip the leading '/', it saves in iteration */ + } - /* Make a parent_path item for the root node, using its own current - copy id. */ - SVN_ERR(root_node(&here, root, pool)); parent_path = make_parent_path(here, 0, 0, pool); parent_path->copy_inherit = copy_id_inherit_self; - rest = canon_path + 1; /* skip the leading '/', it saves in iteration */ - /* Whenever we are at the top of this loop: - HERE is our current directory, - ID is the node revision ID of HERE, @@ -614,6 +853,8 @@ open_path(parent_path_t **parent_path_p, char *entry; dag_node_t *child; + svn_pool_clear(iterpool); + /* Parse out the next entry from the path. */ entry = svn_fs__next_entry_name(&next, rest, pool); @@ -633,16 +874,20 @@ open_path(parent_path_t **parent_path_p, copy_id_inherit_t inherit; const char *copy_path = NULL; svn_error_t *err = SVN_NO_ERROR; - dag_node_t *cached_node; + dag_node_t *cached_node = NULL; /* If we found a directory entry, follow it. First, we check our node cache, and, failing that, we hit the DAG - layer. */ - SVN_ERR(dag_node_cache_get(&cached_node, root, path_so_far, pool)); + layer. Don't bother to contact the cache for the last + element if we already know the lookup to fail for the + complete path. */ + if (next || !(flags & open_path_uncached)) + SVN_ERR(dag_node_cache_get(&cached_node, root, path_so_far, pool)); + if (cached_node) child = cached_node; else - err = svn_fs_fs__dag_open(&child, here, entry, pool); + err = svn_fs_fs__dag_open(&child, here, entry, pool, iterpool); /* "file not found" requires special handling. */ if (err && err->apr_err == SVN_ERR_FS_NOT_FOUND) @@ -671,19 +916,27 @@ open_path(parent_path_t **parent_path_p, /* Other errors we return normally. */ SVN_ERR(err); - /* Now, make a parent_path item for CHILD. */ - parent_path = make_parent_path(child, entry, parent_path, pool); - if (txn_id) + if (flags & open_path_node_only) { - SVN_ERR(get_copy_inheritance(&inherit, ©_path, - fs, parent_path, txn_id, pool)); - parent_path->copy_inherit = inherit; - parent_path->copy_src_path = apr_pstrdup(pool, copy_path); + /* Shortcut: the caller only wan'ts the final DAG node. */ + parent_path->node = child; + } + else + { + /* Now, make a parent_path item for CHILD. */ + parent_path = make_parent_path(child, entry, parent_path, pool); + if (txn_id) + { + SVN_ERR(get_copy_inheritance(&inherit, ©_path, fs, + parent_path, txn_id, iterpool)); + parent_path->copy_inherit = inherit; + parent_path->copy_src_path = apr_pstrdup(pool, copy_path); + } } /* Cache the node we found (if it wasn't already cached). */ if (! cached_node) - SVN_ERR(dag_node_cache_set(root, path_so_far, child, pool)); + SVN_ERR(dag_node_cache_set(root, path_so_far, child, iterpool)); } /* Are we finished traversing the path? */ @@ -692,13 +945,14 @@ open_path(parent_path_t **parent_path_p, /* The path isn't finished yet; we'd better be in a directory. */ if (svn_fs_fs__dag_node_kind(child) != svn_node_dir) - SVN_ERR_W(SVN_FS__ERR_NOT_DIRECTORY(fs, path_so_far, pool), - apr_psprintf(pool, _("Failure opening '%s'"), path)); + SVN_ERR_W(SVN_FS__ERR_NOT_DIRECTORY(fs, path_so_far), + apr_psprintf(iterpool, _("Failure opening '%s'"), path)); rest = next; here = child; } + svn_pool_destroy(iterpool); *parent_path_p = parent_path; return SVN_NO_ERROR; } @@ -763,7 +1017,7 @@ make_path_mutable(svn_fs_root_t *root, /* Determine what copyroot our new child node should use. */ SVN_ERR(svn_fs_fs__dag_get_copyroot(©root_rev, ©root_path, - parent_path->node, pool)); + parent_path->node)); SVN_ERR(svn_fs_fs__revision_root(©root_root, root->fs, copyroot_rev, pool)); SVN_ERR(get_dag(©root_node, copyroot_root, copyroot_path, pool)); @@ -811,21 +1065,35 @@ get_dag(dag_node_t **dag_node_p, apr_pool_t *pool) { parent_path_t *parent_path; - dag_node_t *node; + dag_node_t *node = NULL; - /* Canonicalize the input PATH. */ - path = svn_fs__canonicalize_abspath(path, pool); + /* First we look for the DAG in our cache + (if the path may be canonical). */ + if (*path == '/') + SVN_ERR(dag_node_cache_get(&node, root, path, pool)); - /* First we look for the DAG in our cache. */ - SVN_ERR(dag_node_cache_get(&node, root, path, pool)); if (! node) { - /* Call open_path with no flags, as we want this to return an error - if the node for which we are searching doesn't exist. */ - SVN_ERR(open_path(&parent_path, root, path, 0, NULL, pool)); - node = parent_path->node; + /* Canonicalize the input PATH. */ + if (! svn_fs__is_canonical_abspath(path)) + { + path = svn_fs__canonicalize_abspath(path, pool); + + /* Try again with the corrected path. */ + SVN_ERR(dag_node_cache_get(&node, root, path, pool)); + } - /* No need to cache our find -- open_path() will do that for us. */ + if (! node) + { + /* Call open_path with no flags, as we want this to return an + * error if the node for which we are searching doesn't exist. */ + SVN_ERR(open_path(&parent_path, root, path, + open_path_uncached | open_path_node_only, + NULL, pool)); + node = parent_path->node; + + /* No need to cache our find -- open_path() will do that for us. */ + } } *dag_node_p = node; @@ -988,7 +1256,7 @@ fs_node_prop(svn_string_t **value_p, SVN_ERR(svn_fs_fs__dag_get_proplist(&proplist, node, pool)); *value_p = NULL; if (proplist) - *value_p = apr_hash_get(proplist, propname, APR_HASH_KEY_STRING); + *value_p = svn_hash_gets(proplist, propname); return SVN_NO_ERROR; } @@ -1048,6 +1316,7 @@ fs_change_node_prop(svn_fs_root_t *root, return SVN_FS__NOT_TXN(root); txn_id = root->txn; + path = svn_fs__canonicalize_abspath(path, pool); SVN_ERR(open_path(&parent_path, root, path, 0, txn_id, pool)); /* Check (non-recursively) to see if path is locked; if so, check @@ -1072,8 +1341,7 @@ fs_change_node_prop(svn_fs_root_t *root, { apr_int64_t increment = 0; svn_boolean_t had_mergeinfo; - SVN_ERR(svn_fs_fs__dag_has_mergeinfo(&had_mergeinfo, parent_path->node, - pool)); + SVN_ERR(svn_fs_fs__dag_has_mergeinfo(&had_mergeinfo, parent_path->node)); if (value && !had_mergeinfo) increment = 1; @@ -1089,7 +1357,7 @@ fs_change_node_prop(svn_fs_root_t *root, } /* Set the property. */ - apr_hash_set(proplist, name, APR_HASH_KEY_STRING, value); + svn_hash_sets(proplist, name, value); /* Overwrite the node's proplist. */ SVN_ERR(svn_fs_fs__dag_set_proplist(parent_path->node, proplist, @@ -1127,7 +1395,7 @@ fs_props_changed(svn_boolean_t *changed_p, SVN_ERR(get_dag(&node1, root1, path1, pool)); SVN_ERR(get_dag(&node2, root2, path2, pool)); return svn_fs_fs__dag_things_different(changed_p, NULL, - node1, node2, pool); + node1, node2); } @@ -1138,7 +1406,7 @@ fs_props_changed(svn_boolean_t *changed_p, static svn_error_t * get_root(dag_node_t **node, svn_fs_root_t *root, apr_pool_t *pool) { - return get_dag(node, root, "", pool); + return get_dag(node, root, "/", pool); } @@ -1194,6 +1462,7 @@ merge(svn_stringbuf_t *conflict_p, svn_fs_t *fs; apr_pool_t *iterpool; apr_int64_t mergeinfo_increment = 0; + svn_boolean_t fs_supports_mergeinfo; /* Make sure everyone comes from the same filesystem. */ fs = svn_fs_fs__dag_get_fs(ancestor); @@ -1337,9 +1606,11 @@ merge(svn_stringbuf_t *conflict_p, /* ### todo: it would be more efficient to simply check for a NULL entries hash where necessary below than to allocate an empty hash here, but another day, another day... */ - SVN_ERR(svn_fs_fs__dag_dir_entries(&s_entries, source, pool, pool)); - SVN_ERR(svn_fs_fs__dag_dir_entries(&t_entries, target, pool, pool)); - SVN_ERR(svn_fs_fs__dag_dir_entries(&a_entries, ancestor, pool, pool)); + SVN_ERR(svn_fs_fs__dag_dir_entries(&s_entries, source, pool)); + SVN_ERR(svn_fs_fs__dag_dir_entries(&t_entries, target, pool)); + SVN_ERR(svn_fs_fs__dag_dir_entries(&a_entries, ancestor, pool)); + + fs_supports_mergeinfo = svn_fs_fs__fs_supports_mergeinfo(fs); /* for each entry E in a_entries... */ iterpool = svn_pool_create(pool); @@ -1372,12 +1643,11 @@ merge(svn_stringbuf_t *conflict_p, dag_node_t *t_ent_node; SVN_ERR(svn_fs_fs__dag_get_node(&t_ent_node, fs, t_entry->id, iterpool)); - if (svn_fs_fs__fs_supports_mergeinfo(fs)) + if (fs_supports_mergeinfo) { apr_int64_t mergeinfo_start; SVN_ERR(svn_fs_fs__dag_get_mergeinfo_count(&mergeinfo_start, - t_ent_node, - iterpool)); + t_ent_node)); mergeinfo_increment -= mergeinfo_start; } @@ -1387,12 +1657,11 @@ merge(svn_stringbuf_t *conflict_p, SVN_ERR(svn_fs_fs__dag_get_node(&s_ent_node, fs, s_entry->id, iterpool)); - if (svn_fs_fs__fs_supports_mergeinfo(fs)) + if (fs_supports_mergeinfo) { apr_int64_t mergeinfo_end; SVN_ERR(svn_fs_fs__dag_get_mergeinfo_count(&mergeinfo_end, - s_ent_node, - iterpool)); + s_ent_node)); mergeinfo_increment += mergeinfo_end; } @@ -1465,7 +1734,7 @@ merge(svn_stringbuf_t *conflict_p, txn_id, &sub_mergeinfo_increment, iterpool)); - if (svn_fs_fs__fs_supports_mergeinfo(fs)) + if (fs_supports_mergeinfo) mergeinfo_increment += sub_mergeinfo_increment; } @@ -1501,12 +1770,11 @@ merge(svn_stringbuf_t *conflict_p, SVN_ERR(svn_fs_fs__dag_get_node(&s_ent_node, fs, s_entry->id, iterpool)); - if (svn_fs_fs__fs_supports_mergeinfo(fs)) + if (fs_supports_mergeinfo) { apr_int64_t mergeinfo_s; SVN_ERR(svn_fs_fs__dag_get_mergeinfo_count(&mergeinfo_s, - s_ent_node, - iterpool)); + s_ent_node)); mergeinfo_increment += mergeinfo_s; } @@ -1518,7 +1786,7 @@ merge(svn_stringbuf_t *conflict_p, SVN_ERR(svn_fs_fs__dag_update_ancestry(target, source, pool)); - if (svn_fs_fs__fs_supports_mergeinfo(fs)) + if (fs_supports_mergeinfo) SVN_ERR(svn_fs_fs__dag_increment_mergeinfo_count(target, mergeinfo_increment, pool)); @@ -1621,7 +1889,7 @@ svn_fs_fs__commit_txn(const char **conflict_p, */ svn_error_t *err = SVN_NO_ERROR; - svn_stringbuf_t *conflict = svn_stringbuf_create("", pool); + svn_stringbuf_t *conflict = svn_stringbuf_create_empty(pool); svn_fs_t *fs = txn->fs; /* Limit memory usage when the repository has a high commit rate and @@ -1729,7 +1997,7 @@ fs_merge(const char **conflict_p, dag_node_t *source, *ancestor; svn_fs_txn_t *txn; svn_error_t *err; - svn_stringbuf_t *conflict = svn_stringbuf_create("", pool); + svn_stringbuf_t *conflict = svn_stringbuf_create_empty(pool); if (! target_root->is_txn_root) return SVN_FS__NOT_TXN(target_root); @@ -1803,9 +2071,23 @@ fs_dir_entries(apr_hash_t **table_p, /* Get the entries for this path in the caller's pool. */ SVN_ERR(get_dag(&node, root, path, pool)); - return svn_fs_fs__dag_dir_entries(table_p, node, pool, pool); + return svn_fs_fs__dag_dir_entries(table_p, node, pool); } +/* Raise an error if PATH contains a newline because FSFS cannot handle + * such paths. See issue #4340. */ +static svn_error_t * +check_newline(const char *path, apr_pool_t *pool) +{ + char *c = strchr(path, '\n'); + + if (c) + return svn_error_createf(SVN_ERR_FS_PATH_SYNTAX, NULL, + _("Invalid control character '0x%02x' in path '%s'"), + (unsigned char)*c, svn_path_illegal_path_escape(path, pool)); + + return SVN_NO_ERROR; +} /* Create a new directory named PATH in ROOT. The new directory has no entries, and no properties. ROOT must be the root of a @@ -1820,6 +2102,9 @@ fs_make_dir(svn_fs_root_t *root, dag_node_t *sub_dir; const char *txn_id = root->txn; + SVN_ERR(check_newline(path, pool)); + + path = svn_fs__canonicalize_abspath(path, pool); SVN_ERR(open_path(&parent_path, root, path, open_path_last_optional, txn_id, pool)); @@ -1833,7 +2118,7 @@ fs_make_dir(svn_fs_root_t *root, /* If there's already a sub-directory by that name, complain. This also catches the case of trying to make a subdirectory named `/'. */ if (parent_path->node) - return SVN_FS__ALREADY_EXISTS(root, path, pool); + return SVN_FS__ALREADY_EXISTS(root, path); /* Create the subdirectory. */ SVN_ERR(make_path_mutable(root, parent_path->parent, path, pool)); @@ -1865,12 +2150,13 @@ fs_delete_node(svn_fs_root_t *root, { parent_path_t *parent_path; const char *txn_id = root->txn; - apr_int64_t mergeinfo_count; + apr_int64_t mergeinfo_count = 0; svn_node_kind_t kind; if (! root->is_txn_root) return SVN_FS__NOT_TXN(root); + path = svn_fs__canonicalize_abspath(path, pool); SVN_ERR(open_path(&parent_path, root, path, 0, txn_id, pool)); kind = svn_fs_fs__dag_node_kind(parent_path->node); @@ -1889,8 +2175,7 @@ fs_delete_node(svn_fs_root_t *root, SVN_ERR(make_path_mutable(root, parent_path->parent, path, pool)); if (svn_fs_fs__fs_supports_mergeinfo(root->fs)) SVN_ERR(svn_fs_fs__dag_get_mergeinfo_count(&mergeinfo_count, - parent_path->node, - pool)); + parent_path->node)); SVN_ERR(svn_fs_fs__dag_delete(parent_path->parent->node, parent_path->entry, txn_id, pool)); @@ -1900,7 +2185,7 @@ fs_delete_node(svn_fs_root_t *root, pool)); /* Update mergeinfo counts for parents */ - if (svn_fs_fs__fs_supports_mergeinfo(root->fs) && mergeinfo_count > 0) + if (mergeinfo_count > 0) SVN_ERR(increment_mergeinfo_up_tree(parent_path->parent, -mergeinfo_count, pool)); @@ -1922,17 +2207,7 @@ fs_same_p(svn_boolean_t *same_p, svn_fs_t *fs2, apr_pool_t *pool) { - const char *uuid1; - const char *uuid2; - - /* Random thought: if fetching UUIDs to compare filesystems is too - expensive, one solution would be to cache the UUID in each fs - object (copying the UUID into fs->pool, of course). */ - - SVN_ERR(fs1->vtable->get_uuid(fs1, &uuid1, pool)); - SVN_ERR(fs2->vtable->get_uuid(fs2, &uuid2, pool)); - - *same_p = ! strcmp(uuid1, uuid2); + *same_p = ! strcmp(fs1->uuid, fs2->uuid); return SVN_NO_ERROR; } @@ -2005,8 +2280,7 @@ copy_helper(svn_fs_root_t *from_root, kind = svn_fs_path_change_replace; if (svn_fs_fs__fs_supports_mergeinfo(to_root->fs)) SVN_ERR(svn_fs_fs__dag_get_mergeinfo_count(&mergeinfo_start, - to_parent_path->node, - pool)); + to_parent_path->node)); } else { @@ -2016,7 +2290,7 @@ copy_helper(svn_fs_root_t *from_root, if (svn_fs_fs__fs_supports_mergeinfo(to_root->fs)) SVN_ERR(svn_fs_fs__dag_get_mergeinfo_count(&mergeinfo_end, - from_node, pool)); + from_node)); /* Make sure the target node's parents are mutable. */ SVN_ERR(make_path_mutable(to_root, to_parent_path->parent, @@ -2082,7 +2356,14 @@ fs_copy(svn_fs_root_t *from_root, const char *to_path, apr_pool_t *pool) { - return svn_error_trace(copy_helper(from_root, from_path, to_root, to_path, + SVN_ERR(check_newline(to_path, pool)); + + return svn_error_trace(copy_helper(from_root, + svn_fs__canonicalize_abspath(from_path, + pool), + to_root, + svn_fs__canonicalize_abspath(to_path, + pool), TRUE, pool)); } @@ -2099,6 +2380,7 @@ fs_revision_link(svn_fs_root_t *from_root, if (! to_root->is_txn_root) return SVN_FS__NOT_TXN(to_root); + path = svn_fs__canonicalize_abspath(path, pool); return svn_error_trace(copy_helper(from_root, path, to_root, path, FALSE, pool)); } @@ -2117,13 +2399,13 @@ fs_copied_from(svn_revnum_t *rev_p, dag_node_t *node; const char *copyfrom_path, *copyfrom_str = NULL; svn_revnum_t copyfrom_rev; - char *str, *last_str, *buf; + char *str, *buf; /* Check to see if there is a cached version of this copyfrom entry. */ if (! root->is_txn_root) { fs_rev_root_data_t *frd = root->fsap_data; - copyfrom_str = apr_hash_get(frd->copyfrom_cache, path, APR_HASH_KEY_STRING); + copyfrom_str = svn_hash_gets(frd->copyfrom_cache, path); } if (copyfrom_str) @@ -2139,9 +2421,9 @@ fs_copied_from(svn_revnum_t *rev_p, { /* Parse the copyfrom string for our cached entry. */ buf = apr_pstrdup(pool, copyfrom_str); - str = apr_strtok(buf, " ", &last_str); + str = svn_cstring_tokenize(" ", &buf); copyfrom_rev = SVN_STR_TO_REV(str); - copyfrom_path = last_str; + copyfrom_path = buf; } } else @@ -2149,8 +2431,8 @@ fs_copied_from(svn_revnum_t *rev_p, /* There is no cached entry, look it up the old-fashioned way. */ SVN_ERR(get_dag(&node, root, path, pool)); - SVN_ERR(svn_fs_fs__dag_get_copyfrom_rev(©from_rev, node, pool)); - SVN_ERR(svn_fs_fs__dag_get_copyfrom_path(©from_path, node, pool)); + SVN_ERR(svn_fs_fs__dag_get_copyfrom_rev(©from_rev, node)); + SVN_ERR(svn_fs_fs__dag_get_copyfrom_path(©from_path, node)); } *rev_p = copyfrom_rev; @@ -2174,13 +2456,16 @@ fs_make_file(svn_fs_root_t *root, dag_node_t *child; const char *txn_id = root->txn; + SVN_ERR(check_newline(path, pool)); + + path = svn_fs__canonicalize_abspath(path, pool); SVN_ERR(open_path(&parent_path, root, path, open_path_last_optional, - txn_id, pool)); + txn_id, pool)); /* If there's already a file by that name, complain. This also catches the case of trying to make a file named `/'. */ if (parent_path->node) - return SVN_FS__ALREADY_EXISTS(root, path, pool); + return SVN_FS__ALREADY_EXISTS(root, path); /* Check (non-recursively) to see if path is locked; if so, check that we can use it. */ @@ -2270,6 +2555,25 @@ fs_file_contents(svn_stream_t **contents, /* --- End machinery for svn_fs_file_contents() --- */ +/* --- Machinery for svn_fs_try_process_file_contents() --- */ + +static svn_error_t * +fs_try_process_file_contents(svn_boolean_t *success, + svn_fs_root_t *root, + const char *path, + svn_fs_process_contents_func_t processor, + void* baton, + apr_pool_t *pool) +{ + dag_node_t *node; + SVN_ERR(get_dag(&node, root, path, pool)); + + return svn_fs_fs__dag_try_process_file_contents(success, node, + processor, baton, pool); +} + +/* --- End machinery for svn_fs_try_process_file_contents() --- */ + /* --- Machinery for svn_fs_apply_textdelta() --- */ @@ -2361,7 +2665,7 @@ window_consumer(svn_txdelta_window_t *window, void *baton) SVN_ERR(svn_stream_write(tb->target_stream, tb->target_string->data, &len)); - svn_stringbuf_set(tb->target_string, ""); + svn_stringbuf_setempty(tb->target_string); } /* Is the window NULL? If so, we're done. */ @@ -2428,7 +2732,7 @@ apply_textdelta(void *baton, apr_pool_t *pool) /* Make a writable "string" stream which writes data to tb->target_string. */ - tb->target_string = svn_stringbuf_create("", tb->pool); + tb->target_string = svn_stringbuf_create_empty(tb->pool); tb->string_stream = svn_stream_create(tb, tb->pool); svn_stream_set_write(tb->string_stream, write_to_string); @@ -2464,19 +2768,10 @@ fs_apply_textdelta(svn_txdelta_window_handler_t *contents_p, txdelta_baton_t *tb = apr_pcalloc(pool, sizeof(*tb)); tb->root = root; - tb->path = path; + tb->path = svn_fs__canonicalize_abspath(path, pool); tb->pool = pool; - - if (base_checksum) - tb->base_checksum = svn_checksum_dup(base_checksum, pool); - else - tb->base_checksum = NULL; - - if (result_checksum) - tb->result_checksum = svn_checksum_dup(result_checksum, pool); - else - tb->result_checksum = NULL; - + tb->base_checksum = svn_checksum_dup(base_checksum, pool); + tb->result_checksum = svn_checksum_dup(result_checksum, pool); SVN_ERR(apply_textdelta(tb, pool)); @@ -2606,13 +2901,9 @@ fs_apply_text(svn_stream_t **contents_p, struct text_baton_t *tb = apr_pcalloc(pool, sizeof(*tb)); tb->root = root; - tb->path = path; + tb->path = svn_fs__canonicalize_abspath(path, pool); tb->pool = pool; - - if (result_checksum) - tb->result_checksum = svn_checksum_dup(result_checksum, pool); - else - tb->result_checksum = NULL; + tb->result_checksum = svn_checksum_dup(result_checksum, pool); SVN_ERR(apply_text(tb, pool)); @@ -2660,7 +2951,7 @@ fs_contents_changed(svn_boolean_t *changed_p, SVN_ERR(get_dag(&node1, root1, path1, pool)); SVN_ERR(get_dag(&node2, root2, path2, pool)); return svn_fs_fs__dag_things_different(NULL, changed_p, - node1, node2, pool); + node1, node2); } @@ -2780,8 +3071,10 @@ find_youngest_copyroot(svn_revnum_t *rev_p, parent_path_t *parent_path, apr_pool_t *pool) { - svn_revnum_t rev_mine, rev_parent = -1; - const char *path_mine, *path_parent; + svn_revnum_t rev_mine; + svn_revnum_t rev_parent = SVN_INVALID_REVNUM; + const char *path_mine; + const char *path_parent = NULL; /* First find our parent's youngest copyroot. */ if (parent_path->parent) @@ -2790,7 +3083,7 @@ find_youngest_copyroot(svn_revnum_t *rev_p, /* Find our copyroot. */ SVN_ERR(svn_fs_fs__dag_get_copyroot(&rev_mine, &path_mine, - parent_path->node, pool)); + parent_path->node)); /* If a parent and child were copied to in the same revision, prefer the child copy target, since it is the copy relevant to the @@ -2828,6 +3121,7 @@ static svn_error_t *fs_closest_copy(svn_fs_root_t **root_p, *root_p = NULL; *path_p = NULL; + path = svn_fs__canonicalize_abspath(path, pool); SVN_ERR(open_path(&parent_path, root, path, 0, NULL, pool)); /* Find the youngest copyroot in the path of this node-rev, which @@ -2846,7 +3140,7 @@ static svn_error_t *fs_closest_copy(svn_fs_root_t **root_p, if (kind == svn_node_none) return SVN_NO_ERROR; SVN_ERR(open_path(©_dst_parent_path, copy_dst_root, path, - 0, NULL, pool)); + open_path_node_only, NULL, pool)); copy_dst_node = copy_dst_parent_path->node; if (! svn_fs_fs__id_check_related(svn_fs_fs__dag_get_id(copy_dst_node), svn_fs_fs__dag_get_id(parent_path->node))) @@ -2870,7 +3164,7 @@ static svn_error_t *fs_closest_copy(svn_fs_root_t **root_p, if (created_rev == copy_dst_rev) { const svn_fs_id_t *pred; - SVN_ERR(svn_fs_fs__dag_get_predecessor_id(&pred, copy_dst_node, pool)); + SVN_ERR(svn_fs_fs__dag_get_predecessor_id(&pred, copy_dst_node)); if (! pred) return SVN_NO_ERROR; } @@ -2898,7 +3192,7 @@ prev_location(const char **prev_path, const char *path, apr_pool_t *pool) { - const char *copy_path, *copy_src_path, *remainder = ""; + const char *copy_path, *copy_src_path, *remainder_path; svn_fs_root_t *copy_root; svn_revnum_t copy_src_rev; @@ -2927,9 +3221,8 @@ prev_location(const char **prev_path, */ SVN_ERR(fs_copied_from(©_src_rev, ©_src_path, copy_root, copy_path, pool)); - if (strcmp(copy_path, path) != 0) - remainder = svn_relpath__is_child(copy_path, path, pool); - *prev_path = svn_fspath__join(copy_src_path, remainder, pool); + remainder_path = svn_fspath__skip_ancestor(copy_path, path); + *prev_path = svn_fspath__join(copy_src_path, remainder_path, pool); *prev_rev = copy_src_rev; return SVN_NO_ERROR; } @@ -2967,6 +3260,14 @@ fs_node_origin_rev(svn_revnum_t *revision, return SVN_NO_ERROR; } + /* The root node always has ID 0, created in revision 0 and will never + use the new-style ID format. */ + if (strcmp(node_id, "0") == 0) + { + *revision = 0; + return SVN_NO_ERROR; + } + /* OK, it's an old-style ID? Maybe it's cached. */ SVN_ERR(svn_fs_fs__get_node_origin(&cached_origin_id, fs, @@ -3021,7 +3322,7 @@ fs_node_origin_rev(svn_revnum_t *revision, /* Walk the predecessor links back to origin. */ SVN_ERR(svn_fs_fs__node_id(&pred_id, curroot, lastpath->data, predidpool)); - while (pred_id) + do { svn_pool_clear(subpool); SVN_ERR(svn_fs_fs__dag_get_node(&node, fs, pred_id, subpool)); @@ -3032,9 +3333,10 @@ fs_node_origin_rev(svn_revnum_t *revision, value cached in the node (which is allocated in SUBPOOL... maybe). */ svn_pool_clear(predidpool); - SVN_ERR(svn_fs_fs__dag_get_predecessor_id(&pred_id, node, subpool)); + SVN_ERR(svn_fs_fs__dag_get_predecessor_id(&pred_id, node)); pred_id = pred_id ? svn_fs_fs__id_copy(pred_id, predidpool) : NULL; } + while (pred_id); /* When we get here, NODE should be the first node-revision in our chain. */ @@ -3133,7 +3435,7 @@ history_prev(void *baton, apr_pool_t *pool) no predecessor, in which case we're all done!). */ const svn_fs_id_t *pred_id; - SVN_ERR(svn_fs_fs__dag_get_predecessor_id(&pred_id, node, pool)); + SVN_ERR(svn_fs_fs__dag_get_predecessor_id(&pred_id, node)); if (! pred_id) return SVN_NO_ERROR; @@ -3157,7 +3459,7 @@ history_prev(void *baton, apr_pool_t *pool) if (copyroot_rev > commit_rev) { - const char *remainder; + const char *remainder_path; const char *copy_dst, *copy_src; svn_fs_root_t *copyroot_root; @@ -3174,21 +3476,18 @@ history_prev(void *baton, apr_pool_t *pool) the copy source. Finally, if our current path doesn't meet one of these other criteria ... ### for now just fallback to the old copy hunt algorithm. */ - if (strcmp(path, copy_dst) == 0) - remainder = ""; - else - remainder = svn_relpath__is_child(copy_dst, path, pool); + remainder_path = svn_fspath__skip_ancestor(copy_dst, path); - if (remainder) + if (remainder_path) { /* If we get here, then our current path is the destination of, or the child of the destination of, a copy. Fill in the return values and get outta here. */ - SVN_ERR(svn_fs_fs__dag_get_copyfrom_rev(&src_rev, node, pool)); - SVN_ERR(svn_fs_fs__dag_get_copyfrom_path(©_src, node, pool)); + SVN_ERR(svn_fs_fs__dag_get_copyfrom_rev(&src_rev, node)); + SVN_ERR(svn_fs_fs__dag_get_copyfrom_path(©_src, node)); dst_rev = copyroot_rev; - src_path = svn_fspath__join(copy_src, remainder, pool); + src_path = svn_fspath__join(copy_src, remainder_path, pool); } } @@ -3309,7 +3608,7 @@ assemble_history(svn_fs_t *fs, { svn_fs_history_t *history = apr_pcalloc(pool, sizeof(*history)); fs_history_data_t *fhd = apr_pcalloc(pool, sizeof(*fhd)); - fhd->path = path; + fhd->path = svn_fs__canonicalize_abspath(path, pool); fhd->revision = revision; fhd->is_interesting = is_interesting; fhd->path_hint = path_hint; @@ -3349,7 +3648,7 @@ crawl_directory_dag_for_mergeinfo(svn_fs_root_t *root, apr_pool_t *iterpool = svn_pool_create(scratch_pool); SVN_ERR(svn_fs_fs__dag_dir_entries(&entries, dir_dag, - scratch_pool, scratch_pool)); + scratch_pool)); for (hi = apr_hash_first(scratch_pool, entries); hi; @@ -3365,9 +3664,8 @@ crawl_directory_dag_for_mergeinfo(svn_fs_root_t *root, kid_path = svn_fspath__join(this_path, dirent->name, iterpool); SVN_ERR(get_dag(&kid_dag, root, kid_path, iterpool)); - SVN_ERR(svn_fs_fs__dag_has_mergeinfo(&has_mergeinfo, kid_dag, iterpool)); - SVN_ERR(svn_fs_fs__dag_has_descendants_with_mergeinfo(&go_down, kid_dag, - iterpool)); + SVN_ERR(svn_fs_fs__dag_has_mergeinfo(&has_mergeinfo, kid_dag)); + SVN_ERR(svn_fs_fs__dag_has_descendants_with_mergeinfo(&go_down, kid_dag)); if (has_mergeinfo) { @@ -3378,8 +3676,7 @@ crawl_directory_dag_for_mergeinfo(svn_fs_root_t *root, svn_error_t *err; SVN_ERR(svn_fs_fs__dag_get_proplist(&proplist, kid_dag, iterpool)); - mergeinfo_string = apr_hash_get(proplist, SVN_PROP_MERGEINFO, - APR_HASH_KEY_STRING); + mergeinfo_string = svn_hash_gets(proplist, SVN_PROP_MERGEINFO); if (!mergeinfo_string) { svn_string_t *idstr = svn_fs_fs__id_unparse(dirent->id, iterpool); @@ -3404,10 +3701,8 @@ crawl_directory_dag_for_mergeinfo(svn_fs_root_t *root, } else { - apr_hash_set(result_catalog, - apr_pstrdup(result_pool, kid_path), - APR_HASH_KEY_STRING, - kid_mergeinfo); + svn_hash_sets(result_catalog, apr_pstrdup(result_pool, kid_path), + kid_mergeinfo); } } @@ -3424,6 +3719,24 @@ crawl_directory_dag_for_mergeinfo(svn_fs_root_t *root, return SVN_NO_ERROR; } +/* Return the cache key as a combination of REV_ROOT->REV, the inheritance + flags INHERIT and ADJUST_INHERITED_MERGEINFO, and the PATH. The result + will be allocated in POOL.. + */ +static const char * +mergeinfo_cache_key(const char *path, + svn_fs_root_t *rev_root, + svn_mergeinfo_inheritance_t inherit, + svn_boolean_t adjust_inherited_mergeinfo, + apr_pool_t *pool) +{ + apr_int64_t number = rev_root->rev; + number = number * 4 + + (inherit == svn_mergeinfo_nearest_ancestor ? 2 : 0) + + (adjust_inherited_mergeinfo ? 1 : 0); + + return svn_fs_fs__combine_number_and_string(number, path, pool); +} /* Calculates the mergeinfo for PATH under REV_ROOT using inheritance type INHERIT. Returns it in *MERGEINFO, or NULL if there is none. @@ -3431,19 +3744,17 @@ crawl_directory_dag_for_mergeinfo(svn_fs_root_t *root, used for temporary allocations. */ static svn_error_t * -get_mergeinfo_for_path(svn_mergeinfo_t *mergeinfo, - svn_fs_root_t *rev_root, - const char *path, - svn_mergeinfo_inheritance_t inherit, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool) +get_mergeinfo_for_path_internal(svn_mergeinfo_t *mergeinfo, + svn_fs_root_t *rev_root, + const char *path, + svn_mergeinfo_inheritance_t inherit, + svn_boolean_t adjust_inherited_mergeinfo, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { parent_path_t *parent_path, *nearest_ancestor; apr_hash_t *proplist; svn_string_t *mergeinfo_string; - apr_pool_t *iterpool = svn_pool_create(scratch_pool); - - *mergeinfo = NULL; path = svn_fs__canonicalize_abspath(path, scratch_pool); @@ -3461,17 +3772,14 @@ get_mergeinfo_for_path(svn_mergeinfo_t *mergeinfo, { svn_boolean_t has_mergeinfo; - svn_pool_clear(iterpool); - SVN_ERR(svn_fs_fs__dag_has_mergeinfo(&has_mergeinfo, - nearest_ancestor->node, iterpool)); + nearest_ancestor->node)); if (has_mergeinfo) break; /* No need to loop if we're looking for explicit mergeinfo. */ if (inherit == svn_mergeinfo_explicit) { - svn_pool_destroy(iterpool); return SVN_NO_ERROR; } @@ -3480,17 +3788,13 @@ get_mergeinfo_for_path(svn_mergeinfo_t *mergeinfo, /* Run out? There's no mergeinfo. */ if (!nearest_ancestor) { - svn_pool_destroy(iterpool); return SVN_NO_ERROR; } } - svn_pool_destroy(iterpool); - SVN_ERR(svn_fs_fs__dag_get_proplist(&proplist, nearest_ancestor->node, scratch_pool)); - mergeinfo_string = apr_hash_get(proplist, SVN_PROP_MERGEINFO, - APR_HASH_KEY_STRING); + mergeinfo_string = svn_hash_gets(proplist, SVN_PROP_MERGEINFO); if (!mergeinfo_string) return svn_error_createf (SVN_ERR_FS_CORRUPT, NULL, @@ -3521,7 +3825,7 @@ get_mergeinfo_for_path(svn_mergeinfo_t *mergeinfo, can return the mergeinfo results directly. Otherwise, we're inheriting the mergeinfo, so we need to a) remove non-inheritable ranges and b) telescope the merged-from paths. */ - if (nearest_ancestor != parent_path) + if (adjust_inherited_mergeinfo && (nearest_ancestor != parent_path)) { svn_mergeinfo_t tmp_mergeinfo; @@ -3539,6 +3843,58 @@ get_mergeinfo_for_path(svn_mergeinfo_t *mergeinfo, return SVN_NO_ERROR; } +/* Caching wrapper around get_mergeinfo_for_path_internal(). + */ +static svn_error_t * +get_mergeinfo_for_path(svn_mergeinfo_t *mergeinfo, + svn_fs_root_t *rev_root, + const char *path, + svn_mergeinfo_inheritance_t inherit, + svn_boolean_t adjust_inherited_mergeinfo, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + fs_fs_data_t *ffd = rev_root->fs->fsap_data; + const char *cache_key; + svn_boolean_t found = FALSE; + svn_stringbuf_t *mergeinfo_exists; + + *mergeinfo = NULL; + + cache_key = mergeinfo_cache_key(path, rev_root, inherit, + adjust_inherited_mergeinfo, scratch_pool); + if (ffd->mergeinfo_existence_cache) + { + SVN_ERR(svn_cache__get((void **)&mergeinfo_exists, &found, + ffd->mergeinfo_existence_cache, + cache_key, result_pool)); + if (found && mergeinfo_exists->data[0] == '1') + SVN_ERR(svn_cache__get((void **)mergeinfo, &found, + ffd->mergeinfo_cache, + cache_key, result_pool)); + } + + if (! found) + { + SVN_ERR(get_mergeinfo_for_path_internal(mergeinfo, rev_root, path, + inherit, + adjust_inherited_mergeinfo, + result_pool, scratch_pool)); + if (ffd->mergeinfo_existence_cache) + { + mergeinfo_exists = svn_stringbuf_create(*mergeinfo ? "1" : "0", + scratch_pool); + SVN_ERR(svn_cache__set(ffd->mergeinfo_existence_cache, + cache_key, mergeinfo_exists, scratch_pool)); + if (*mergeinfo) + SVN_ERR(svn_cache__set(ffd->mergeinfo_cache, + cache_key, *mergeinfo, scratch_pool)); + } + } + + return SVN_NO_ERROR; +} + /* Adds mergeinfo for each descendant of PATH (but not PATH itself) under ROOT to RESULT_CATALOG. Returned values are allocated in RESULT_POOL; temporary values in POOL. */ @@ -3554,8 +3910,7 @@ add_descendant_mergeinfo(svn_mergeinfo_catalog_t result_catalog, SVN_ERR(get_dag(&this_dag, root, path, scratch_pool)); SVN_ERR(svn_fs_fs__dag_has_descendants_with_mergeinfo(&go_down, - this_dag, - scratch_pool)); + this_dag)); if (go_down) SVN_ERR(crawl_directory_dag_for_mergeinfo(root, path, @@ -3576,10 +3931,12 @@ get_mergeinfos_for_paths(svn_fs_root_t *root, const apr_array_header_t *paths, svn_mergeinfo_inheritance_t inherit, svn_boolean_t include_descendants, - apr_pool_t *pool) + svn_boolean_t adjust_inherited_mergeinfo, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { - svn_mergeinfo_catalog_t result_catalog = apr_hash_make(pool); - apr_pool_t *iterpool = svn_pool_create(pool); + svn_mergeinfo_catalog_t result_catalog = svn_hash__make(result_pool); + apr_pool_t *iterpool = svn_pool_create(scratch_pool); int i; for (i = 0; i < paths->nelts; i++) @@ -3591,7 +3948,8 @@ get_mergeinfos_for_paths(svn_fs_root_t *root, svn_pool_clear(iterpool); err = get_mergeinfo_for_path(&path_mergeinfo, root, path, - inherit, pool, iterpool); + inherit, adjust_inherited_mergeinfo, + result_pool, iterpool); if (err) { if (err->apr_err == SVN_ERR_MERGEINFO_PARSE_ERROR) @@ -3607,11 +3965,10 @@ get_mergeinfos_for_paths(svn_fs_root_t *root, } if (path_mergeinfo) - apr_hash_set(result_catalog, path, APR_HASH_KEY_STRING, - path_mergeinfo); + svn_hash_sets(result_catalog, path, path_mergeinfo); if (include_descendants) - SVN_ERR(add_descendant_mergeinfo(result_catalog, root, path, pool, - iterpool)); + SVN_ERR(add_descendant_mergeinfo(result_catalog, root, path, + result_pool, scratch_pool)); } svn_pool_destroy(iterpool); @@ -3627,7 +3984,9 @@ fs_get_mergeinfo(svn_mergeinfo_catalog_t *catalog, const apr_array_header_t *paths, svn_mergeinfo_inheritance_t inherit, svn_boolean_t include_descendants, - apr_pool_t *pool) + svn_boolean_t adjust_inherited_mergeinfo, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { fs_fs_data_t *ffd = root->fs->fsap_data; @@ -3646,7 +4005,9 @@ fs_get_mergeinfo(svn_mergeinfo_catalog_t *catalog, /* Retrieve a path -> mergeinfo hash mapping. */ return get_mergeinfos_for_paths(root, catalog, paths, inherit, - include_descendants, pool); + include_descendants, + adjust_inherited_mergeinfo, + result_pool, scratch_pool); } @@ -3673,6 +4034,7 @@ static root_vtable_t root_vtable = { fs_file_length, fs_file_checksum, fs_file_contents, + fs_try_process_file_contents, fs_make_file, fs_apply_textdelta, fs_apply_text, @@ -3712,7 +4074,7 @@ make_revision_root(svn_fs_t *fs, root->rev = rev; frd->root_dir = root_dir; - frd->copyfrom_cache = apr_hash_make(root->pool); + frd->copyfrom_cache = svn_hash__make(root->pool); root->fsap_data = frd; @@ -3739,6 +4101,8 @@ make_txn_root(svn_fs_root_t **root_p, root->txn_flags = flags; root->rev = base_rev; + frd->txn_id = txn; + /* Because this cache actually tries to invalidate elements, keep the number of elements per page down. @@ -3749,7 +4113,8 @@ make_txn_root(svn_fs_root_t **root_p, svn_fs_fs__dag_deserialize, APR_HASH_KEY_STRING, 32, 20, FALSE, - apr_pstrcat(pool, txn, ":TXN", (char *)NULL), + apr_pstrcat(pool, txn, ":TXN", + (char *)NULL), root->pool)); /* Initialize transaction-local caches in FS. @@ -3763,3 +4128,197 @@ make_txn_root(svn_fs_root_t **root_p, *root_p = root; return SVN_NO_ERROR; } + + + +/* Verify. */ +static APR_INLINE const char * +stringify_node(dag_node_t *node, + apr_pool_t *pool) +{ + /* ### TODO: print some PATH@REV to it, too. */ + return svn_fs_fs__id_unparse(svn_fs_fs__dag_get_id(node), pool)->data; +} + +/* Check metadata sanity on NODE, and on its children. Manually verify + information for DAG nodes in revision REV, and trust the metadata + accuracy for nodes belonging to older revisions. */ +static svn_error_t * +verify_node(dag_node_t *node, + svn_revnum_t rev, + apr_pool_t *pool) +{ + svn_boolean_t has_mergeinfo; + apr_int64_t mergeinfo_count; + const svn_fs_id_t *pred_id; + svn_fs_t *fs = svn_fs_fs__dag_get_fs(node); + int pred_count; + svn_node_kind_t kind; + apr_pool_t *iterpool = svn_pool_create(pool); + + /* Fetch some data. */ + SVN_ERR(svn_fs_fs__dag_has_mergeinfo(&has_mergeinfo, node)); + SVN_ERR(svn_fs_fs__dag_get_mergeinfo_count(&mergeinfo_count, node)); + SVN_ERR(svn_fs_fs__dag_get_predecessor_id(&pred_id, node)); + SVN_ERR(svn_fs_fs__dag_get_predecessor_count(&pred_count, node)); + kind = svn_fs_fs__dag_node_kind(node); + + /* Sanity check. */ + if (mergeinfo_count < 0) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Negative mergeinfo-count %" APR_INT64_T_FMT + " on node '%s'", + mergeinfo_count, stringify_node(node, iterpool)); + + /* Issue #4129. (This check will explicitly catch non-root instances too.) */ + if (pred_id) + { + dag_node_t *pred; + int pred_pred_count; + SVN_ERR(svn_fs_fs__dag_get_node(&pred, fs, pred_id, iterpool)); + SVN_ERR(svn_fs_fs__dag_get_predecessor_count(&pred_pred_count, pred)); + if (pred_pred_count+1 != pred_count) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Predecessor count mismatch: " + "%s has %d, but %s has %d", + stringify_node(node, iterpool), pred_count, + stringify_node(pred, iterpool), + pred_pred_count); + } + + /* Kind-dependent verifications. */ + if (kind == svn_node_none) + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Node '%s' has kind 'none'", + stringify_node(node, iterpool)); + } + if (kind == svn_node_file) + { + if (has_mergeinfo != mergeinfo_count) /* comparing int to bool */ + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "File node '%s' has inconsistent mergeinfo: " + "has_mergeinfo=%d, " + "mergeinfo_count=%" APR_INT64_T_FMT, + stringify_node(node, iterpool), + has_mergeinfo, mergeinfo_count); + } + if (kind == svn_node_dir) + { + apr_hash_t *entries; + apr_hash_index_t *hi; + apr_int64_t children_mergeinfo = 0; + + SVN_ERR(svn_fs_fs__dag_dir_entries(&entries, node, pool)); + + /* Compute CHILDREN_MERGEINFO. */ + for (hi = apr_hash_first(pool, entries); + hi; + hi = apr_hash_next(hi)) + { + svn_fs_dirent_t *dirent = svn__apr_hash_index_val(hi); + dag_node_t *child; + svn_revnum_t child_rev; + apr_int64_t child_mergeinfo; + + svn_pool_clear(iterpool); + + /* Compute CHILD_REV. */ + SVN_ERR(svn_fs_fs__dag_get_node(&child, fs, dirent->id, iterpool)); + SVN_ERR(svn_fs_fs__dag_get_revision(&child_rev, child, iterpool)); + + if (child_rev == rev) + SVN_ERR(verify_node(child, rev, iterpool)); + + SVN_ERR(svn_fs_fs__dag_get_mergeinfo_count(&child_mergeinfo, child)); + children_mergeinfo += child_mergeinfo; + } + + /* Side-effect of issue #4129. */ + if (children_mergeinfo+has_mergeinfo != mergeinfo_count) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Mergeinfo-count discrepancy on '%s': " + "expected %" APR_INT64_T_FMT "+%d, " + "counted %" APR_INT64_T_FMT, + stringify_node(node, iterpool), + mergeinfo_count, has_mergeinfo, + children_mergeinfo); + } + + svn_pool_destroy(iterpool); + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_fs__verify_root(svn_fs_root_t *root, + apr_pool_t *pool) +{ + svn_fs_t *fs = root->fs; + dag_node_t *root_dir; + + /* Issue #4129: bogus pred-counts and minfo-cnt's on the root node-rev + (and elsewhere). This code makes more thorough checks than the + commit-time checks in validate_root_noderev(). */ + + /* Callers should disable caches by setting SVN_FS_CONFIG_FSFS_CACHE_NS; + see r1462436. + + When this code is called in the library, we want to ensure we + use the on-disk data --- rather than some data that was read + in the possibly-distance past and cached since. */ + + if (root->is_txn_root) + { + fs_txn_root_data_t *frd = root->fsap_data; + SVN_ERR(svn_fs_fs__dag_txn_root(&root_dir, fs, frd->txn_id, pool)); + } + else + { + fs_rev_root_data_t *frd = root->fsap_data; + root_dir = frd->root_dir; + } + + /* Recursively verify ROOT_DIR. */ + SVN_ERR(verify_node(root_dir, root->rev, pool)); + + /* Verify explicitly the predecessor of the root. */ + { + const svn_fs_id_t *pred_id; + + /* Only r0 should have no predecessor. */ + SVN_ERR(svn_fs_fs__dag_get_predecessor_id(&pred_id, root_dir)); + if (! root->is_txn_root && !!pred_id != !!root->rev) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "r%ld's root node's predecessor is " + "unexpectedly '%s'", + root->rev, + (pred_id + ? svn_fs_fs__id_unparse(pred_id, pool)->data + : "(null)")); + if (root->is_txn_root && !pred_id) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Transaction '%s''s root node's predecessor is " + "unexpectedly NULL", + root->txn); + + /* Check the predecessor's revision. */ + if (pred_id) + { + svn_revnum_t pred_rev = svn_fs_fs__id_rev(pred_id); + if (! root->is_txn_root && pred_rev+1 != root->rev) + /* Issue #4129. */ + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "r%ld's root node's predecessor is r%ld" + " but should be r%ld", + root->rev, pred_rev, root->rev - 1); + if (root->is_txn_root && pred_rev != root->rev) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Transaction '%s''s root node's predecessor" + " is r%ld" + " but should be r%ld", + root->txn, pred_rev, root->rev); + } + } + + return SVN_NO_ERROR; +} diff --git a/subversion/libsvn_fs_fs/tree.h b/subversion/libsvn_fs_fs/tree.h index 0f445ed..34fa0a2 100644 --- a/subversion/libsvn_fs_fs/tree.h +++ b/subversion/libsvn_fs_fs/tree.h @@ -23,12 +23,19 @@ #ifndef SVN_LIBSVN_FS_TREE_H #define SVN_LIBSVN_FS_TREE_H +#include "fs.h" + #ifdef __cplusplus extern "C" { #endif /* __cplusplus */ +/* In POOL, create an instance of a DAG node 1st level cache. + The POOL will be cleared at regular intervals. */ +fs_fs_dag_cache_t* +svn_fs_fs__create_dag_cache(apr_pool_t *pool); + /* Set *ROOT_P to the root directory of revision REV in filesystem FS. Allocate the structure in POOL. */ svn_error_t *svn_fs_fs__revision_root(svn_fs_root_t **root_p, svn_fs_t *fs, @@ -78,6 +85,12 @@ svn_fs_fs__node_created_rev(svn_revnum_t *revision, const char *path, apr_pool_t *pool); +/* Verify metadata for ROOT. + ### Currently only implemented for revision roots. */ +svn_error_t * +svn_fs_fs__verify_root(svn_fs_root_t *root, + apr_pool_t *pool); + #ifdef __cplusplus } #endif /* __cplusplus */ |