diff options
Diffstat (limited to 'subversion/libsvn_fs_fs/fs_fs.c')
| -rw-r--r-- | subversion/libsvn_fs_fs/fs_fs.c | 6174 |
1 files changed, 4980 insertions, 1194 deletions
diff --git a/subversion/libsvn_fs_fs/fs_fs.c b/subversion/libsvn_fs_fs/fs_fs.c index 46f3bcd..8fb3a36 100644 --- a/subversion/libsvn_fs_fs/fs_fs.c +++ b/subversion/libsvn_fs_fs/fs_fs.c @@ -49,6 +49,7 @@ #include "svn_mergeinfo.h" #include "svn_config.h" #include "svn_ctype.h" +#include "svn_version.h" #include "fs.h" #include "tree.h" @@ -59,7 +60,10 @@ #include "rep-cache.h" #include "temp_serializer.h" +#include "private/svn_string_private.h" #include "private/svn_fs_util.h" +#include "private/svn_subr_private.h" +#include "private/svn_delta_private.h" #include "../libsvn_fs/fs-loader.h" #include "svn_private_config.h" @@ -71,13 +75,38 @@ /* The default maximum number of files per directory to store in the rev and revprops directory. The number below is somewhat arbitrary, - and can be overriden by defining the macro while compiling; the + and can be overridden by defining the macro while compiling; the figure of 1000 is reasonable for VFAT filesystems, which are by far the worst performers in this area. */ #ifndef SVN_FS_FS_DEFAULT_MAX_FILES_PER_DIR #define SVN_FS_FS_DEFAULT_MAX_FILES_PER_DIR 1000 #endif +/* Begin deltification after a node history exceeded this this limit. + Useful values are 4 to 64 with 16 being a good compromise between + computational overhead and repository size savings. + Should be a power of 2. + Values < 2 will result in standard skip-delta behavior. */ +#define SVN_FS_FS_MAX_LINEAR_DELTIFICATION 16 + +/* Finding a deltification base takes operations proportional to the + number of changes being skipped. To prevent exploding runtime + during commits, limit the deltification range to this value. + Should be a power of 2 minus one. + Values < 1 disable deltification. */ +#define SVN_FS_FS_MAX_DELTIFICATION_WALK 1023 + +/* Give writing processes 10 seconds to replace an existing revprop + file with a new one. After that time, we assume that the writing + process got aborted and that we have re-read revprops. */ +#define REVPROP_CHANGE_TIMEOUT (10 * 1000000) + +/* The following are names of atomics that will be used to communicate + * revprop updates across all processes on this machine. */ +#define ATOMIC_REVPROP_GENERATION "rev-prop-generation" +#define ATOMIC_REVPROP_TIMEOUT "rev-prop-timeout" +#define ATOMIC_REVPROP_NAMESPACE "rev-prop-atomics" + /* Following are defines that specify the textual elements of the native filesystem directories and revision files. */ @@ -147,6 +176,15 @@ read_min_unpacked_rev(svn_revnum_t *min_unpacked_rev, static svn_error_t * update_min_unpacked_rev(svn_fs_t *fs, apr_pool_t *pool); +static svn_error_t * +get_youngest(svn_revnum_t *youngest_p, const char *fs_path, apr_pool_t *pool); + +static svn_error_t * +verify_walker(representation_t *rep, + void *baton, + svn_fs_t *fs, + apr_pool_t *scratch_pool); + /* Pathname helper functions */ /* Return TRUE is REV is packed in FS, FALSE otherwise. */ @@ -162,13 +200,12 @@ is_packed_rev(svn_fs_t *fs, svn_revnum_t rev) static svn_boolean_t is_packed_revprop(svn_fs_t *fs, svn_revnum_t rev) { -#if 0 fs_fs_data_t *ffd = fs->fsap_data; - return (rev < ffd->min_unpacked_revprop); -#else - return FALSE; -#endif + /* rev 0 will not be packed */ + return (rev < ffd->min_unpacked_rev) + && (rev != 0) + && (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT); } static const char * @@ -208,6 +245,12 @@ path_lock(svn_fs_t *fs, apr_pool_t *pool) } static const char * +path_revprop_generation(svn_fs_t *fs, apr_pool_t *pool) +{ + return svn_dirent_join(fs->path, PATH_REVPROP_GENERATION, pool); +} + +static const char * path_rev_packed(svn_fs_t *fs, svn_revnum_t rev, const char *kind, apr_pool_t *pool) { @@ -217,7 +260,8 @@ path_rev_packed(svn_fs_t *fs, svn_revnum_t rev, const char *kind, assert(is_packed_rev(fs, rev)); return svn_dirent_join_many(pool, fs->path, PATH_REVS_DIR, - apr_psprintf(pool, "%ld.pack", + apr_psprintf(pool, + "%ld" PATH_EXT_PACKED_SHARD, rev / ffd->max_files_per_dir), kind, NULL); } @@ -267,7 +311,7 @@ svn_fs_fs__path_rev_absolute(const char **path, } else { - *path = path_rev_packed(fs, rev, "pack", pool); + *path = path_rev_packed(fs, rev, PATH_PACKED, pool); } return SVN_NO_ERROR; @@ -286,6 +330,18 @@ path_revprops_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) } static const char * +path_revprops_pack_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + + assert(ffd->max_files_per_dir); + return svn_dirent_join_many(pool, fs->path, PATH_REVPROPS_DIR, + apr_psprintf(pool, "%ld" PATH_EXT_PACKED_SHARD, + rev / ffd->max_files_per_dir), + NULL); +} + +static const char * path_revprops(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) { fs_fs_data_t *ffd = fs->fsap_data; @@ -311,6 +367,18 @@ path_txn_dir(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool) NULL); } +/* Return the name of the sha1->rep mapping file in transaction TXN_ID + * within FS for the given SHA1 checksum. Use POOL for allocations. + */ +static APR_INLINE const char * +path_txn_sha1(svn_fs_t *fs, const char *txn_id, svn_checksum_t *sha1, + apr_pool_t *pool) +{ + return svn_dirent_join(path_txn_dir(fs, txn_id, pool), + svn_checksum_to_cstring(sha1, pool), + pool); +} + static APR_INLINE const char * path_txn_changes(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool) { @@ -415,7 +483,7 @@ path_and_offset_of(apr_file_t *file, apr_pool_t *pool) } - + /* Functions for working with shared transaction data. */ /* Return the transaction object for transaction TXN_ID from the @@ -509,26 +577,13 @@ with_txnlist_lock(svn_fs_t *fs, const void *baton, apr_pool_t *pool) { - svn_error_t *err; -#if APR_HAS_THREADS fs_fs_data_t *ffd = fs->fsap_data; fs_fs_shared_data_t *ffsd = ffd->shared; - apr_status_t apr_err; - - apr_err = apr_thread_mutex_lock(ffsd->txn_list_lock); - if (apr_err) - return svn_error_wrap_apr(apr_err, _("Can't grab FSFS txn list mutex")); -#endif - - err = body(fs, baton, pool); -#if APR_HAS_THREADS - apr_err = apr_thread_mutex_unlock(ffsd->txn_list_lock); - if (apr_err && !err) - return svn_error_wrap_apr(apr_err, _("Can't ungrab FSFS txn list mutex")); -#endif + SVN_MUTEX__WITH_LOCK(ffsd->txn_list_lock, + body(fs, baton, pool)); - return svn_error_trace(err); + return SVN_NO_ERROR; } @@ -553,60 +608,59 @@ get_lock_on_filesystem(const char *lock_filename, return svn_error_trace(err); } +/* Reset the HAS_WRITE_LOCK member in the FFD given as BATON_VOID. + When registered with the pool holding the lock on the lock file, + this makes sure the flag gets reset just before we release the lock. */ +static apr_status_t +reset_lock_flag(void *baton_void) +{ + fs_fs_data_t *ffd = baton_void; + ffd->has_write_lock = FALSE; + return APR_SUCCESS; +} + /* Obtain a write lock on the file LOCK_FILENAME (protecting with LOCK_MUTEX if APR is threaded) in a subpool of POOL, call BODY with BATON and that subpool, destroy the subpool (releasing the write - lock) and return what BODY returned. */ -static svn_error_t * -with_some_lock(svn_fs_t *fs, - svn_error_t *(*body)(void *baton, - apr_pool_t *pool), - void *baton, - const char *lock_filename, -#if SVN_FS_FS__USE_LOCK_MUTEX - apr_thread_mutex_t *lock_mutex, -#endif - apr_pool_t *pool) + lock) and return what BODY returned. If IS_GLOBAL_LOCK is set, + set the HAS_WRITE_LOCK flag while we keep the write lock. */ +static svn_error_t * +with_some_lock_file(svn_fs_t *fs, + svn_error_t *(*body)(void *baton, + apr_pool_t *pool), + void *baton, + const char *lock_filename, + svn_boolean_t is_global_lock, + apr_pool_t *pool) { apr_pool_t *subpool = svn_pool_create(pool); - svn_error_t *err; - -#if SVN_FS_FS__USE_LOCK_MUTEX - apr_status_t status; - - /* POSIX fcntl locks are per-process, so we need to serialize locks - within the process. */ - status = apr_thread_mutex_lock(lock_mutex); - if (status) - return svn_error_wrap_apr(status, - _("Can't grab FSFS mutex for '%s'"), - lock_filename); -#endif - - err = get_lock_on_filesystem(lock_filename, subpool); + svn_error_t *err = get_lock_on_filesystem(lock_filename, subpool); if (!err) { fs_fs_data_t *ffd = fs->fsap_data; + + if (is_global_lock) + { + /* set the "got the lock" flag and register reset function */ + apr_pool_cleanup_register(subpool, + ffd, + reset_lock_flag, + apr_pool_cleanup_null); + ffd->has_write_lock = TRUE; + } + + /* nobody else will modify the repo state + => read HEAD & pack info once */ if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) SVN_ERR(update_min_unpacked_rev(fs, pool)); -#if 0 /* Might be a good idea? */ SVN_ERR(get_youngest(&ffd->youngest_rev_cache, fs->path, pool)); -#endif err = body(baton, subpool); } svn_pool_destroy(subpool); -#if SVN_FS_FS__USE_LOCK_MUTEX - status = apr_thread_mutex_unlock(lock_mutex); - if (status && !err) - return svn_error_wrap_apr(status, - _("Can't ungrab FSFS mutex for '%s'"), - lock_filename); -#endif - return svn_error_trace(err); } @@ -617,18 +671,16 @@ svn_fs_fs__with_write_lock(svn_fs_t *fs, void *baton, apr_pool_t *pool) { -#if SVN_FS_FS__USE_LOCK_MUTEX fs_fs_data_t *ffd = fs->fsap_data; fs_fs_shared_data_t *ffsd = ffd->shared; - apr_thread_mutex_t *mutex = ffsd->fs_write_lock; -#endif - return with_some_lock(fs, body, baton, - path_lock(fs, pool), -#if SVN_FS_FS__USE_LOCK_MUTEX - mutex, -#endif - pool); + SVN_MUTEX__WITH_LOCK(ffsd->fs_write_lock, + with_some_lock_file(fs, body, baton, + path_lock(fs, pool), + TRUE, + pool)); + + return SVN_NO_ERROR; } /* Run BODY (with BATON and POOL) while the txn-current file @@ -640,18 +692,16 @@ with_txn_current_lock(svn_fs_t *fs, void *baton, apr_pool_t *pool) { -#if SVN_FS_FS__USE_LOCK_MUTEX fs_fs_data_t *ffd = fs->fsap_data; fs_fs_shared_data_t *ffsd = ffd->shared; - apr_thread_mutex_t *mutex = ffsd->txn_current_lock; -#endif - return with_some_lock(fs, body, baton, - path_txn_current_lock(fs, pool), -#if SVN_FS_FS__USE_LOCK_MUTEX - mutex, -#endif - pool); + SVN_MUTEX__WITH_LOCK(ffsd->txn_current_lock, + with_some_lock_file(fs, body, baton, + path_txn_current_lock(fs, pool), + FALSE, + pool)); + + return SVN_NO_ERROR; } /* A structure used by unlock_proto_rev() and unlock_proto_rev_body(), @@ -829,7 +879,7 @@ get_writable_proto_rev_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool) err = svn_error_compose_create( err, unlock_proto_rev_list_locked(fs, txn_id, *lockcookie, pool)); - + *lockcookie = NULL; } @@ -899,25 +949,64 @@ get_file_offset(apr_off_t *offset_p, apr_file_t *file, apr_pool_t *pool) } -/* Check that BUF, a nul-terminated buffer of text from format file PATH, +/* Check that BUF, a nul-terminated buffer of text from file PATH, contains only digits at OFFSET and beyond, raising an error if not. + TITLE contains a user-visible description of the file, usually the + short file name. Uses POOL for temporary allocation. */ static svn_error_t * -check_format_file_buffer_numeric(const char *buf, apr_off_t offset, - const char *path, apr_pool_t *pool) +check_file_buffer_numeric(const char *buf, apr_off_t offset, + const char *path, const char *title, + apr_pool_t *pool) { const char *p; for (p = buf + offset; *p; p++) if (!svn_ctype_isdigit(*p)) return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL, - _("Format file '%s' contains unexpected non-digit '%c' within '%s'"), - svn_dirent_local_style(path, pool), *p, buf); + _("%s file '%s' contains unexpected non-digit '%c' within '%s'"), + title, svn_dirent_local_style(path, pool), *p, buf); return SVN_NO_ERROR; } +/* Check that BUF, a nul-terminated buffer of text from format file PATH, + contains only digits at OFFSET and beyond, raising an error if not. + + Uses POOL for temporary allocation. */ +static svn_error_t * +check_format_file_buffer_numeric(const char *buf, apr_off_t offset, + const char *path, apr_pool_t *pool) +{ + return check_file_buffer_numeric(buf, offset, path, "Format", pool); +} + +/* Return the error SVN_ERR_FS_UNSUPPORTED_FORMAT if FS's format + number is not the same as a format number supported by this + Subversion. */ +static svn_error_t * +check_format(int format) +{ + /* Blacklist. These formats may be either younger or older than + SVN_FS_FS__FORMAT_NUMBER, but we don't support them. */ + if (format == SVN_FS_FS__PACKED_REVPROP_SQLITE_DEV_FORMAT) + return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, + _("Found format '%d', only created by " + "unreleased dev builds; see " + "http://subversion.apache.org" + "/docs/release-notes/1.7#revprop-packing"), + format); + + /* We support all formats from 1-current simultaneously */ + if (1 <= format && format <= SVN_FS_FS__FORMAT_NUMBER) + return SVN_NO_ERROR; + + return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, + _("Expected FS format between '1' and '%d'; found format '%d'"), + SVN_FS_FS__FORMAT_NUMBER, format); +} + /* Read the format number and maximum number of files per directory from PATH and return them in *PFORMAT and *MAX_FILES_PER_DIR respectively. @@ -931,12 +1020,12 @@ read_format(int *pformat, int *max_files_per_dir, const char *path, apr_pool_t *pool) { svn_error_t *err; - apr_file_t *file; - char buf[80]; - apr_size_t len; + svn_stream_t *stream; + svn_stringbuf_t *content; + svn_stringbuf_t *buf; + svn_boolean_t eos = FALSE; - err = svn_io_file_open(&file, path, APR_READ | APR_BUFFERED, - APR_OS_DEFAULT, pool); + err = svn_stringbuf_from_file2(&content, path, pool); if (err && APR_STATUS_IS_ENOENT(err->apr_err)) { /* Treat an absent format file as format 1. Do not try to @@ -954,62 +1043,57 @@ read_format(int *pformat, int *max_files_per_dir, } SVN_ERR(err); - len = sizeof(buf); - err = svn_io_read_length_line(file, buf, &len, pool); - if (err && APR_STATUS_IS_EOF(err->apr_err)) + stream = svn_stream_from_stringbuf(content, pool); + SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, pool)); + if (buf->len == 0 && eos) { /* Return a more useful error message. */ - svn_error_clear(err); return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL, _("Can't read first line of format file '%s'"), svn_dirent_local_style(path, pool)); } - SVN_ERR(err); /* Check that the first line contains only digits. */ - SVN_ERR(check_format_file_buffer_numeric(buf, 0, path, pool)); - SVN_ERR(svn_cstring_atoi(pformat, buf)); + SVN_ERR(check_format_file_buffer_numeric(buf->data, 0, path, pool)); + SVN_ERR(svn_cstring_atoi(pformat, buf->data)); + + /* Check that we support this format at all */ + SVN_ERR(check_format(*pformat)); /* Set the default values for anything that can be set via an option. */ *max_files_per_dir = 0; /* Read any options. */ - while (1) + while (!eos) { - len = sizeof(buf); - err = svn_io_read_length_line(file, buf, &len, pool); - if (err && APR_STATUS_IS_EOF(err->apr_err)) - { - /* No more options; that's okay. */ - svn_error_clear(err); - break; - } - SVN_ERR(err); + SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, pool)); + if (buf->len == 0) + break; if (*pformat >= SVN_FS_FS__MIN_LAYOUT_FORMAT_OPTION_FORMAT && - strncmp(buf, "layout ", 7) == 0) + strncmp(buf->data, "layout ", 7) == 0) { - if (strcmp(buf+7, "linear") == 0) + if (strcmp(buf->data + 7, "linear") == 0) { *max_files_per_dir = 0; continue; } - if (strncmp(buf+7, "sharded ", 8) == 0) + if (strncmp(buf->data + 7, "sharded ", 8) == 0) { /* Check that the argument is numeric. */ - SVN_ERR(check_format_file_buffer_numeric(buf, 15, path, pool)); - SVN_ERR(svn_cstring_atoi(max_files_per_dir, buf + 15)); + SVN_ERR(check_format_file_buffer_numeric(buf->data, 15, path, pool)); + SVN_ERR(svn_cstring_atoi(max_files_per_dir, buf->data + 15)); continue; } } return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL, _("'%s' contains invalid filesystem format option '%s'"), - svn_dirent_local_style(path, pool), buf); + svn_dirent_local_style(path, pool), buf->data); } - return svn_io_file_close(file, pool); + return SVN_NO_ERROR; } /* Write the format number and maximum number of files per directory @@ -1061,31 +1145,6 @@ write_format(const char *path, int format, int max_files_per_dir, return svn_io_set_file_read_only(path, FALSE, pool); } -/* Return the error SVN_ERR_FS_UNSUPPORTED_FORMAT if FS's format - number is not the same as a format number supported by this - Subversion. */ -static svn_error_t * -check_format(int format) -{ - /* Blacklist. These formats may be either younger or older than - SVN_FS_FS__FORMAT_NUMBER, but we don't support them. */ - if (format == SVN_FS_FS__PACKED_REVPROP_SQLITE_DEV_FORMAT) - return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, - _("Found format '%d', only created by " - "unreleased dev builds; see " - "http://subversion.apache.org" - "/docs/release-notes/1.7#revprop-packing"), - format); - - /* We support all formats from 1-current simultaneously */ - if (1 <= format && format <= SVN_FS_FS__FORMAT_NUMBER) - return SVN_NO_ERROR; - - return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, - _("Expected FS format between '1' and '%d'; found format '%d'"), - SVN_FS_FS__FORMAT_NUMBER, format); -} - svn_boolean_t svn_fs_fs__fs_supports_mergeinfo(svn_fs_t *fs) { @@ -1093,15 +1152,17 @@ svn_fs_fs__fs_supports_mergeinfo(svn_fs_t *fs) return ffd->format >= SVN_FS_FS__MIN_MERGEINFO_FORMAT; } +/* Read the configuration information of the file system at FS_PATH + * and set the respective values in FFD. Use POOL for allocations. + */ static svn_error_t * -read_config(svn_fs_t *fs, +read_config(fs_fs_data_t *ffd, + const char *fs_path, apr_pool_t *pool) { - fs_fs_data_t *ffd = fs->fsap_data; - - SVN_ERR(svn_config_read2(&ffd->config, - svn_dirent_join(fs->path, PATH_CONFIG, pool), - FALSE, FALSE, fs->pool)); + SVN_ERR(svn_config_read3(&ffd->config, + svn_dirent_join(fs_path, PATH_CONFIG, pool), + FALSE, FALSE, FALSE, pool)); /* Initialize ffd->rep_sharing_allowed. */ if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) @@ -1111,6 +1172,56 @@ read_config(svn_fs_t *fs, else ffd->rep_sharing_allowed = FALSE; + /* Initialize deltification settings in ffd. */ + if (ffd->format >= SVN_FS_FS__MIN_DELTIFICATION_FORMAT) + { + SVN_ERR(svn_config_get_bool(ffd->config, &ffd->deltify_directories, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_ENABLE_DIR_DELTIFICATION, + FALSE)); + SVN_ERR(svn_config_get_bool(ffd->config, &ffd->deltify_properties, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION, + FALSE)); + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->max_deltification_walk, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_MAX_DELTIFICATION_WALK, + SVN_FS_FS_MAX_DELTIFICATION_WALK)); + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->max_linear_deltification, + CONFIG_SECTION_DELTIFICATION, + CONFIG_OPTION_MAX_LINEAR_DELTIFICATION, + SVN_FS_FS_MAX_LINEAR_DELTIFICATION)); + } + else + { + ffd->deltify_directories = FALSE; + ffd->deltify_properties = FALSE; + ffd->max_deltification_walk = SVN_FS_FS_MAX_DELTIFICATION_WALK; + ffd->max_linear_deltification = SVN_FS_FS_MAX_LINEAR_DELTIFICATION; + } + + /* Initialize revprop packing settings in ffd. */ + if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) + { + SVN_ERR(svn_config_get_bool(ffd->config, &ffd->compress_packed_revprops, + CONFIG_SECTION_PACKED_REVPROPS, + CONFIG_OPTION_COMPRESS_PACKED_REVPROPS, + FALSE)); + SVN_ERR(svn_config_get_int64(ffd->config, &ffd->revprop_pack_size, + CONFIG_SECTION_PACKED_REVPROPS, + CONFIG_OPTION_REVPROP_PACK_SIZE, + ffd->compress_packed_revprops + ? 0x100 + : 0x40)); + + ffd->revprop_pack_size *= 1024; + } + else + { + ffd->revprop_pack_size = 0x10000; + ffd->compress_packed_revprops = FALSE; + } + return SVN_NO_ERROR; } @@ -1156,9 +1267,95 @@ write_config(svn_fs_t *fs, "### The following parameter enables rep-sharing in the repository. It can" NL "### be switched on and off at will, but for best space-saving results" NL "### should be enabled consistently over the life of the repository." NL +"### 'svnadmin verify' will check the rep-cache regardless of this setting." NL "### rep-sharing is enabled by default." NL "# " CONFIG_OPTION_ENABLE_REP_SHARING " = true" NL - +"" NL +"[" CONFIG_SECTION_DELTIFICATION "]" NL +"### To conserve space, the filesystem stores data as differences against" NL +"### existing representations. This comes at a slight cost in performance," NL +"### as calculating differences can increase commit times. Reading data" NL +"### will also create higher CPU load and the data will be fragmented." NL +"### Since deltification tends to save significant amounts of disk space," NL +"### the overall I/O load can actually be lower." NL +"###" NL +"### The options in this section allow for tuning the deltification" NL +"### strategy. Their effects on data size and server performance may vary" NL +"### from one repository to another. Versions prior to 1.8 will ignore" NL +"### this section." NL +"###" NL +"### The following parameter enables deltification for directories. It can" NL +"### be switched on and off at will, but for best space-saving results" NL +"### should be enabled consistently over the life of the repository." NL +"### Repositories containing large directories will benefit greatly." NL +"### In rarely read repositories, the I/O overhead may be significant as" NL +"### cache hit rates will most likely be low" NL +"### directory deltification is disabled by default." NL +"# " CONFIG_OPTION_ENABLE_DIR_DELTIFICATION " = false" NL +"###" NL +"### The following parameter enables deltification for properties on files" NL +"### and directories. Overall, this is a minor tuning option but can save" NL +"### some disk space if you merge frequently or frequently change node" NL +"### properties. You should not activate this if rep-sharing has been" NL +"### disabled because this may result in a net increase in repository size." NL +"### property deltification is disabled by default." NL +"# " CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION " = false" NL +"###" NL +"### During commit, the server may need to walk the whole change history of" NL +"### of a given node to find a suitable deltification base. This linear" NL +"### process can impact commit times, svnadmin load and similar operations." NL +"### This setting limits the depth of the deltification history. If the" NL +"### threshold has been reached, the node will be stored as fulltext and a" NL +"### new deltification history begins." NL +"### Note, this is unrelated to svn log." NL +"### Very large values rarely provide significant additional savings but" NL +"### can impact performance greatly - in particular if directory" NL +"### deltification has been activated. Very small values may be useful in" NL +"### repositories that are dominated by large, changing binaries." NL +"### Should be a power of two minus 1. A value of 0 will effectively" NL +"### disable deltification." NL +"### For 1.8, the default value is 1023; earlier versions have no limit." NL +"# " CONFIG_OPTION_MAX_DELTIFICATION_WALK " = 1023" NL +"###" NL +"### The skip-delta scheme used by FSFS tends to repeatably store redundant" NL +"### delta information where a simple delta against the latest version is" NL +"### often smaller. By default, 1.8+ will therefore use skip deltas only" NL +"### after the linear chain of deltas has grown beyond the threshold" NL +"### specified by this setting." NL +"### Values up to 64 can result in some reduction in repository size for" NL +"### the cost of quickly increasing I/O and CPU costs. Similarly, smaller" NL +"### numbers can reduce those costs at the cost of more disk space. For" NL +"### rarely read repositories or those containing larger binaries, this may" NL +"### present a better trade-off." NL +"### Should be a power of two. A value of 1 or smaller will cause the" NL +"### exclusive use of skip-deltas (as in pre-1.8)." NL +"### For 1.8, the default value is 16; earlier versions use 1." NL +"# " CONFIG_OPTION_MAX_LINEAR_DELTIFICATION " = 16" NL +"" NL +"[" CONFIG_SECTION_PACKED_REVPROPS "]" NL +"### This parameter controls the size (in kBytes) of packed revprop files." NL +"### Revprops of consecutive revisions will be concatenated into a single" NL +"### file up to but not exceeding the threshold given here. However, each" NL +"### pack file may be much smaller and revprops of a single revision may be" NL +"### much larger than the limit set here. The threshold will be applied" NL +"### before optional compression takes place." NL +"### Large values will reduce disk space usage at the expense of increased" NL +"### latency and CPU usage reading and changing individual revprops. They" NL +"### become an advantage when revprop caching has been enabled because a" NL +"### lot of data can be read in one go. Values smaller than 4 kByte will" NL +"### not improve latency any further and quickly render revprop packing" NL +"### ineffective." NL +"### revprop-pack-size is 64 kBytes by default for non-compressed revprop" NL +"### pack files and 256 kBytes when compression has been enabled." NL +"# " CONFIG_OPTION_REVPROP_PACK_SIZE " = 64" NL +"###" NL +"### To save disk space, packed revprop files may be compressed. Standard" NL +"### revprops tend to allow for very effective compression. Reading and" NL +"### even more so writing, become significantly more CPU intensive. With" NL +"### revprop caching enabled, the overhead can be offset by reduced I/O" NL +"### unless you often modify revprops after packing." NL +"### Compressing packed revprops is disabled by default." NL +"# " CONFIG_OPTION_COMPRESS_PACKED_REVPROPS " = false" NL ; #undef NL return svn_io_file_create(svn_dirent_join(fs->path, PATH_CONFIG, pool), @@ -1196,9 +1393,6 @@ update_min_unpacked_rev(svn_fs_t *fs, apr_pool_t *pool) pool); } -static svn_error_t * -get_youngest(svn_revnum_t *youngest_p, const char *fs_path, apr_pool_t *pool); - svn_error_t * svn_fs_fs__open(svn_fs_t *fs, const char *path, apr_pool_t *pool) { @@ -1213,7 +1407,6 @@ svn_fs_fs__open(svn_fs_t *fs, const char *path, apr_pool_t *pool) /* Read the FS format number. */ SVN_ERR(read_format(&format, &max_files_per_dir, path_format(fs, pool), pool)); - SVN_ERR(check_format(format)); /* Now we've got a format number no matter what. */ ffd->format = format; @@ -1225,7 +1418,7 @@ svn_fs_fs__open(svn_fs_t *fs, const char *path, apr_pool_t *pool) limit = sizeof(buf); SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit, pool)); - ffd->uuid = apr_pstrdup(fs->pool, buf); + fs->uuid = apr_pstrdup(fs->pool, buf); SVN_ERR(svn_io_file_close(uuid_file, pool)); @@ -1234,7 +1427,7 @@ svn_fs_fs__open(svn_fs_t *fs, const char *path, apr_pool_t *pool) SVN_ERR(update_min_unpacked_rev(fs, pool)); /* Read the configuration file. */ - SVN_ERR(read_config(fs, pool)); + SVN_ERR(read_config(ffd, fs->path, pool)); return get_youngest(&(ffd->youngest_rev_cache), path, pool); } @@ -1254,6 +1447,114 @@ create_file_ignore_eexist(const char *file, return svn_error_trace(err); } +/* forward declarations */ + +static svn_error_t * +pack_revprops_shard(const char *pack_file_dir, + const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + apr_off_t max_pack_size, + int compression_level, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool); + +static svn_error_t * +delete_revprops_shard(const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool); + +/* In the filesystem FS, pack all revprop shards up to min_unpacked_rev. + * + * NOTE: Keep the old non-packed shards around until after the format bump. + * Otherwise, re-running upgrade will drop the packed revprop shard but + * have no unpacked data anymore. Call upgrade_cleanup_pack_revprops after + * the bump. + * + * Use SCRATCH_POOL for temporary allocations. + */ +static svn_error_t * +upgrade_pack_revprops(svn_fs_t *fs, + apr_pool_t *scratch_pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + const char *revprops_shard_path; + const char *revprops_pack_file_dir; + apr_int64_t shard; + apr_int64_t first_unpacked_shard + = ffd->min_unpacked_rev / ffd->max_files_per_dir; + + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, + scratch_pool); + int compression_level = ffd->compress_packed_revprops + ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT + : SVN_DELTA_COMPRESSION_LEVEL_NONE; + + /* first, pack all revprops shards to match the packed revision shards */ + for (shard = 0; shard < first_unpacked_shard; ++shard) + { + revprops_pack_file_dir = svn_dirent_join(revsprops_dir, + apr_psprintf(iterpool, + "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, + shard), + iterpool); + revprops_shard_path = svn_dirent_join(revsprops_dir, + apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), + iterpool); + + SVN_ERR(pack_revprops_shard(revprops_pack_file_dir, revprops_shard_path, + shard, ffd->max_files_per_dir, + (int)(0.9 * ffd->revprop_pack_size), + compression_level, + NULL, NULL, iterpool)); + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* In the filesystem FS, remove all non-packed revprop shards up to + * min_unpacked_rev. Use SCRATCH_POOL for temporary allocations. + * See upgrade_pack_revprops for more info. + */ +static svn_error_t * +upgrade_cleanup_pack_revprops(svn_fs_t *fs, + apr_pool_t *scratch_pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + const char *revprops_shard_path; + apr_int64_t shard; + apr_int64_t first_unpacked_shard + = ffd->min_unpacked_rev / ffd->max_files_per_dir; + + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, + scratch_pool); + + /* delete the non-packed revprops shards afterwards */ + for (shard = 0; shard < first_unpacked_shard; ++shard) + { + revprops_shard_path = svn_dirent_join(revsprops_dir, + apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), + iterpool); + SVN_ERR(delete_revprops_shard(revprops_shard_path, + shard, ffd->max_files_per_dir, + NULL, NULL, iterpool)); + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + static svn_error_t * upgrade_body(void *baton, apr_pool_t *pool) { @@ -1261,10 +1562,10 @@ upgrade_body(void *baton, apr_pool_t *pool) int format, max_files_per_dir; const char *format_path = path_format(fs, pool); svn_node_kind_t kind; + svn_boolean_t needs_revprop_shard_cleanup = FALSE; /* Read the FS format number and max-files-per-dir setting. */ SVN_ERR(read_format(&format, &max_files_per_dir, format_path, pool)); - SVN_ERR(check_format(format)); /* If the config file does not exist, create one. */ SVN_ERR(svn_io_check_path(svn_dirent_join(fs->path, PATH_CONFIG, pool), @@ -1312,9 +1613,28 @@ upgrade_body(void *baton, apr_pool_t *pool) if (format < SVN_FS_FS__MIN_PACKED_FORMAT) SVN_ERR(svn_io_file_create(path_min_unpacked_rev(fs, pool), "0\n", pool)); + /* If the file system supports revision packing but not revprop packing + *and* the FS has been sharded, pack the revprops up to the point that + revision data has been packed. However, keep the non-packed revprop + files around until after the format bump */ + if ( format >= SVN_FS_FS__MIN_PACKED_FORMAT + && format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT + && max_files_per_dir > 0) + { + needs_revprop_shard_cleanup = TRUE; + SVN_ERR(upgrade_pack_revprops(fs, pool)); + } + /* Bump the format file. */ - return write_format(format_path, SVN_FS_FS__FORMAT_NUMBER, max_files_per_dir, - TRUE, pool); + SVN_ERR(write_format(format_path, SVN_FS_FS__FORMAT_NUMBER, + max_files_per_dir, TRUE, pool)); + + /* Now, it is safe to remove the redundant revprop files. */ + if (needs_revprop_shard_cleanup) + SVN_ERR(upgrade_cleanup_pack_revprops(fs, pool)); + + /* Done */ + return SVN_NO_ERROR; } @@ -1325,7 +1645,7 @@ svn_fs_fs__upgrade(svn_fs_t *fs, apr_pool_t *pool) } -/* SVN_ERR-like macros for dealing with recoverable errors on mutable files +/* Functions for dealing with recoverable errors on mutable files * * Revprops, current, and txn-current files are mutable; that is, they * change as part of normal fsfs operation, in constrat to revs files, or @@ -1358,94 +1678,82 @@ svn_fs_fs__upgrade(svn_fs_t *fs, apr_pool_t *pool) * ** Solution * - * Wrap opens and reads of such files with RETRY_RECOVERABLE and - * closes with IGNORE_RECOVERABLE. Call these macros within a loop of - * RECOVERABLE_RETRY_COUNT iterations (though, realistically, the - * second try will succeed). Make sure you put a break statement - * after the close, at the end of your loop. Immediately after your - * loop, return err if err. - * - * You must initialize err to SVN_NO_ERROR and filehandle to NULL, as - * these macros do not. + * Try open and read of such files in try_stringbuf_from_file(). Call + * this function within a loop of RECOVERABLE_RETRY_COUNT iterations + * (though, realistically, the second try will succeed). */ #define RECOVERABLE_RETRY_COUNT 10 +/* Read the file at PATH and return its content in *CONTENT. *CONTENT will + * not be modified unless the whole file was read successfully. + * + * ESTALE, EIO and ENOENT will not cause this function to return an error + * unless LAST_ATTEMPT has been set. If MISSING is not NULL, indicate + * missing files (ENOENT) there. + * + * Use POOL for allocations. + */ +static svn_error_t * +try_stringbuf_from_file(svn_stringbuf_t **content, + svn_boolean_t *missing, + const char *path, + svn_boolean_t last_attempt, + apr_pool_t *pool) +{ + svn_error_t *err = svn_stringbuf_from_file2(content, path, pool); + if (missing) + *missing = FALSE; + + if (err) + { + *content = NULL; + + if (APR_STATUS_IS_ENOENT(err->apr_err)) + { + if (!last_attempt) + { + svn_error_clear(err); + if (missing) + *missing = TRUE; + return SVN_NO_ERROR; + } + } #ifdef ESTALE -/* Do not use do-while due to the embedded 'continue'. */ -#define RETRY_RECOVERABLE(err, filehandle, expr) \ - if (1) { \ - svn_error_clear(err); \ - err = (expr); \ - if (err) \ - { \ - apr_status_t _e = APR_TO_OS_ERROR(err->apr_err); \ - if ((_e == ESTALE) || (_e == EIO) || (_e == ENOENT)) { \ - if (NULL != filehandle) \ - (void)apr_file_close(filehandle); \ - continue; \ - } \ - return svn_error_trace(err); \ - } \ - } else -#define IGNORE_RECOVERABLE(err, expr) \ - if (1) { \ - svn_error_clear(err); \ - err = (expr); \ - if (err) \ - { \ - apr_status_t _e = APR_TO_OS_ERROR(err->apr_err); \ - if ((_e != ESTALE) && (_e != EIO)) \ - return svn_error_trace(err); \ - } \ - } else -#else -#define RETRY_RECOVERABLE(err, filehandle, expr) SVN_ERR(expr) -#define IGNORE_RECOVERABLE(err, expr) SVN_ERR(expr) + else if (APR_TO_OS_ERROR(err->apr_err) == ESTALE + || APR_TO_OS_ERROR(err->apr_err) == EIO) + { + if (!last_attempt) + { + svn_error_clear(err); + return SVN_NO_ERROR; + } + } #endif + } -/* Long enough to hold: "<svn_revnum_t> <node id> <copy id>\0" - * 19 bytes for svn_revnum_t (room for 32 or 64 bit values) - * + 2 spaces - * + 26 bytes for each id (these are actually unbounded, so we just - * have to pick something; 2^64 is 13 bytes in base-36) - * + 1 terminating null - */ -#define CURRENT_BUF_LEN 48 + return svn_error_trace(err); +} /* Read the 'current' file FNAME and store the contents in *BUF. Allocations are performed in POOL. */ static svn_error_t * -read_current(const char *fname, char **buf, apr_pool_t *pool) +read_content(svn_stringbuf_t **content, const char *fname, apr_pool_t *pool) { - apr_file_t *revision_file = NULL; - apr_size_t len; int i; - svn_error_t *err = SVN_NO_ERROR; - apr_pool_t *iterpool; - - *buf = apr_palloc(pool, CURRENT_BUF_LEN); - iterpool = svn_pool_create(pool); - for (i = 0; i < RECOVERABLE_RETRY_COUNT; i++) - { - svn_pool_clear(iterpool); + *content = NULL; - RETRY_RECOVERABLE(err, revision_file, - svn_io_file_open(&revision_file, fname, - APR_READ | APR_BUFFERED, - APR_OS_DEFAULT, iterpool)); - - len = CURRENT_BUF_LEN; - RETRY_RECOVERABLE(err, revision_file, - svn_io_read_length_line(revision_file, - *buf, &len, iterpool)); - IGNORE_RECOVERABLE(err, svn_io_file_close(revision_file, iterpool)); + for (i = 0; !*content && (i < RECOVERABLE_RETRY_COUNT); ++i) + SVN_ERR(try_stringbuf_from_file(content, NULL, + fname, i + 1 < RECOVERABLE_RETRY_COUNT, + pool)); - break; - } - svn_pool_destroy(iterpool); + if (!*content) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Can't read '%s'"), + svn_dirent_local_style(fname, pool)); - return svn_error_trace(err); + return SVN_NO_ERROR; } /* Find the youngest revision in a repository at path FS_PATH and @@ -1456,248 +1764,15 @@ get_youngest(svn_revnum_t *youngest_p, const char *fs_path, apr_pool_t *pool) { - char *buf; - - SVN_ERR(read_current(svn_dirent_join(fs_path, PATH_CURRENT, pool), - &buf, pool)); + svn_stringbuf_t *buf; + SVN_ERR(read_content(&buf, svn_dirent_join(fs_path, PATH_CURRENT, pool), + pool)); - *youngest_p = SVN_STR_TO_REV(buf); + *youngest_p = SVN_STR_TO_REV(buf->data); return SVN_NO_ERROR; } -svn_error_t * -svn_fs_fs__hotcopy(const char *src_path, - const char *dst_path, - apr_pool_t *pool) -{ - const char *src_subdir, *dst_subdir; - svn_revnum_t youngest, rev, min_unpacked_rev; - apr_pool_t *iterpool; - svn_node_kind_t kind; - int format, max_files_per_dir; - - /* Check format to be sure we know how to hotcopy this FS. */ - SVN_ERR(read_format(&format, &max_files_per_dir, - svn_dirent_join(src_path, PATH_FORMAT, pool), - pool)); - SVN_ERR(check_format(format)); - - /* Try to copy the config. - * - * ### We try copying the config file before doing anything else, - * ### because higher layers will abort the hotcopy if we throw - * ### an error from this function, and that renders the hotcopy - * ### unusable anyway. */ - if (format >= SVN_FS_FS__MIN_CONFIG_FILE) - { - svn_error_t *err; - - err = svn_io_dir_file_copy(src_path, dst_path, PATH_CONFIG, pool); - if (err) - { - if (APR_STATUS_IS_ENOENT(err->apr_err)) - { - /* 1.6.0 to 1.6.11 did not copy the configuration file during - * hotcopy. So if we're hotcopying a repository which has been - * created as a hotcopy itself, it's possible that fsfs.conf - * does not exist. Ask the user to re-create it. - * - * ### It would be nice to make this a non-fatal error, - * ### but this function does not get an svn_fs_t object - * ### so we have no way of just printing a warning via - * ### the fs->warning() callback. */ - - const char *msg; - const char *src_abspath; - const char *dst_abspath; - const char *config_relpath; - svn_error_t *err2; - - config_relpath = svn_dirent_join(src_path, PATH_CONFIG, pool); - err2 = svn_dirent_get_absolute(&src_abspath, src_path, pool); - if (err2) - return svn_error_trace(svn_error_compose_create(err, err2)); - err2 = svn_dirent_get_absolute(&dst_abspath, dst_path, pool); - if (err2) - return svn_error_trace(svn_error_compose_create(err, err2)); - - /* ### hack: strip off the 'db/' directory from paths so - * ### they make sense to the user */ - src_abspath = svn_dirent_dirname(src_abspath, pool); - dst_abspath = svn_dirent_dirname(dst_abspath, pool); - - msg = apr_psprintf(pool, - _("Failed to create hotcopy at '%s'. " - "The file '%s' is missing from the source " - "repository. Please create this file, for " - "instance by running 'svnadmin upgrade %s'"), - dst_abspath, config_relpath, src_abspath); - return svn_error_quick_wrap(err, msg); - } - else - return svn_error_trace(err); - } - } - - /* Copy the 'current' file. */ - SVN_ERR(svn_io_dir_file_copy(src_path, dst_path, PATH_CURRENT, pool)); - - /* Copy the uuid. */ - SVN_ERR(svn_io_dir_file_copy(src_path, dst_path, PATH_UUID, pool)); - - /* Copy the rep cache before copying the rev files to make sure all - cached references will be present in the copy. */ - src_subdir = svn_dirent_join(src_path, REP_CACHE_DB_NAME, pool); - dst_subdir = svn_dirent_join(dst_path, REP_CACHE_DB_NAME, pool); - SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); - if (kind == svn_node_file) - SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, pool)); - - /* Copy the min unpacked rev, and read its value. */ - if (format >= SVN_FS_FS__MIN_PACKED_FORMAT) - { - const char *min_unpacked_rev_path; - min_unpacked_rev_path = svn_dirent_join(src_path, PATH_MIN_UNPACKED_REV, - pool); - - SVN_ERR(svn_io_dir_file_copy(src_path, dst_path, PATH_MIN_UNPACKED_REV, - pool)); - SVN_ERR(read_min_unpacked_rev(&min_unpacked_rev, min_unpacked_rev_path, - pool)); - } - else - { - min_unpacked_rev = 0; - } - - /* Find the youngest revision from this 'current' file. */ - SVN_ERR(get_youngest(&youngest, dst_path, pool)); - - /* Copy the necessary rev files. */ - src_subdir = svn_dirent_join(src_path, PATH_REVS_DIR, pool); - dst_subdir = svn_dirent_join(dst_path, PATH_REVS_DIR, pool); - - SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool)); - - iterpool = svn_pool_create(pool); - /* First, copy packed shards. */ - for (rev = 0; rev < min_unpacked_rev; rev += max_files_per_dir) - { - const char *packed_shard = apr_psprintf(iterpool, "%ld.pack", - rev / max_files_per_dir); - const char *src_subdir_packed_shard; - src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, - iterpool); - - SVN_ERR(svn_io_copy_dir_recursively(src_subdir_packed_shard, - dst_subdir, packed_shard, - TRUE /* copy_perms */, - NULL /* cancel_func */, NULL, - iterpool)); - svn_pool_clear(iterpool); - } - - /* Then, copy non-packed shards. */ - SVN_ERR_ASSERT(rev == min_unpacked_rev); - for (; rev <= youngest; rev++) - { - const char *src_subdir_shard = src_subdir, - *dst_subdir_shard = dst_subdir; - - if (max_files_per_dir) - { - const char *shard = apr_psprintf(iterpool, "%ld", - rev / max_files_per_dir); - src_subdir_shard = svn_dirent_join(src_subdir, shard, iterpool); - dst_subdir_shard = svn_dirent_join(dst_subdir, shard, iterpool); - - if (rev % max_files_per_dir == 0) - { - SVN_ERR(svn_io_dir_make(dst_subdir_shard, APR_OS_DEFAULT, - iterpool)); - SVN_ERR(svn_io_copy_perms(dst_subdir, dst_subdir_shard, - iterpool)); - } - } - - SVN_ERR(svn_io_dir_file_copy(src_subdir_shard, dst_subdir_shard, - apr_psprintf(iterpool, "%ld", rev), - iterpool)); - svn_pool_clear(iterpool); - } - - /* Copy the necessary revprop files. */ - src_subdir = svn_dirent_join(src_path, PATH_REVPROPS_DIR, pool); - dst_subdir = svn_dirent_join(dst_path, PATH_REVPROPS_DIR, pool); - - SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool)); - - for (rev = 0; rev <= youngest; rev++) - { - const char *src_subdir_shard = src_subdir, - *dst_subdir_shard = dst_subdir; - - svn_pool_clear(iterpool); - - if (max_files_per_dir) - { - const char *shard = apr_psprintf(iterpool, "%ld", - rev / max_files_per_dir); - src_subdir_shard = svn_dirent_join(src_subdir, shard, iterpool); - dst_subdir_shard = svn_dirent_join(dst_subdir, shard, iterpool); - - if (rev % max_files_per_dir == 0) - { - SVN_ERR(svn_io_dir_make(dst_subdir_shard, APR_OS_DEFAULT, - iterpool)); - SVN_ERR(svn_io_copy_perms(dst_subdir, dst_subdir_shard, - iterpool)); - } - } - - SVN_ERR(svn_io_dir_file_copy(src_subdir_shard, dst_subdir_shard, - apr_psprintf(iterpool, "%ld", rev), - iterpool)); - } - - svn_pool_destroy(iterpool); - - /* Make an empty transactions directory for now. Eventually some - method of copying in progress transactions will need to be - developed.*/ - dst_subdir = svn_dirent_join(dst_path, PATH_TXNS_DIR, pool); - SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool)); - if (format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT) - { - dst_subdir = svn_dirent_join(dst_path, PATH_TXN_PROTOS_DIR, pool); - SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool)); - } - - /* Now copy the locks tree. */ - src_subdir = svn_dirent_join(src_path, PATH_LOCKS_DIR, pool); - SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); - if (kind == svn_node_dir) - SVN_ERR(svn_io_copy_dir_recursively(src_subdir, dst_path, - PATH_LOCKS_DIR, TRUE, NULL, - NULL, pool)); - - /* Now copy the node-origins cache tree. */ - src_subdir = svn_dirent_join(src_path, PATH_NODE_ORIGINS_DIR, pool); - SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); - if (kind == svn_node_dir) - SVN_ERR(svn_io_copy_dir_recursively(src_subdir, dst_path, - PATH_NODE_ORIGINS_DIR, TRUE, NULL, - NULL, pool)); - - /* Copy the txn-current file. */ - if (format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT) - SVN_ERR(svn_io_dir_file_copy(src_path, dst_path, PATH_TXN_CURRENT, pool)); - - /* Hotcopied FS is complete. Stamp it with a format file. */ - return write_format(svn_dirent_join(dst_path, PATH_FORMAT, pool), - format, max_files_per_dir, FALSE, pool); -} svn_error_t * svn_fs_fs__youngest_rev(svn_revnum_t *youngest_p, @@ -1766,7 +1841,7 @@ static svn_error_t * read_header_block(apr_hash_t **headers, /* header_str is safely in our pool, so we can use bits of it as key and value. */ - apr_hash_set(*headers, name, APR_HASH_KEY_STRING, value); + svn_hash_sets(*headers, name, value); } return SVN_NO_ERROR; @@ -1811,6 +1886,16 @@ ensure_revision_exists(svn_fs_t *fs, _("No such revision %ld"), rev); } +svn_error_t * +svn_fs_fs__revision_exists(svn_revnum_t rev, + svn_fs_t *fs, + apr_pool_t *pool) +{ + /* Different order of parameters. */ + SVN_ERR(ensure_revision_exists(fs, rev, pool)); + return SVN_NO_ERROR; +} + /* Open the correct revision file for REV. If the filesystem FS has been packed, *FILE will be set to the packed file; otherwise, set *FILE to the revision file for REV. Return SVN_ERR_FS_NO_SUCH_REVISION if the @@ -1842,22 +1927,30 @@ open_pack_or_rev_file(apr_file_t **file, err = svn_io_file_open(file, path, APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool); - if (err && APR_STATUS_IS_ENOENT(err->apr_err) - && ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) + if (err && APR_STATUS_IS_ENOENT(err->apr_err)) { - /* Could not open the file. This may happen if the - * file once existed but got packed later. */ - svn_error_clear(err); + if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) + { + /* Could not open the file. This may happen if the + * file once existed but got packed later. */ + svn_error_clear(err); - /* if that was our 2nd attempt, leave it at that. */ - if (retry) - return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, - _("No such revision %ld"), rev); + /* if that was our 2nd attempt, leave it at that. */ + if (retry) + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("No such revision %ld"), rev); - /* We failed for the first time. Refresh cache & retry. */ - SVN_ERR(update_min_unpacked_rev(fs, pool)); + /* We failed for the first time. Refresh cache & retry. */ + SVN_ERR(update_min_unpacked_rev(fs, pool)); - retry = TRUE; + retry = TRUE; + } + else + { + svn_error_clear(err); + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("No such revision %ld"), rev); + } } else { @@ -1869,6 +1962,41 @@ open_pack_or_rev_file(apr_file_t **file, return svn_error_trace(err); } +/* Reads a line from STREAM and converts it to a 64 bit integer to be + * returned in *RESULT. If we encounter eof, set *HIT_EOF and leave + * *RESULT unchanged. If HIT_EOF is NULL, EOF causes an "corrupt FS" + * error return. + * SCRATCH_POOL is used for temporary allocations. + */ +static svn_error_t * +read_number_from_stream(apr_int64_t *result, + svn_boolean_t *hit_eof, + svn_stream_t *stream, + apr_pool_t *scratch_pool) +{ + svn_stringbuf_t *sb; + svn_boolean_t eof; + svn_error_t *err; + + SVN_ERR(svn_stream_readline(stream, &sb, "\n", &eof, scratch_pool)); + if (hit_eof) + *hit_eof = eof; + else + if (eof) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Unexpected EOF")); + + if (!eof) + { + err = svn_cstring_atoi64(result, sb->data); + if (err) + return svn_error_createf(SVN_ERR_FS_CORRUPT, err, + _("Number '%s' invalid or too large"), + sb->data); + } + + return SVN_NO_ERROR; +} + /* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file. Use POOL for temporary allocations. */ static svn_error_t * @@ -1902,7 +2030,8 @@ get_packed_offset(apr_off_t *rev_offset, /* Open the manifest file. */ SVN_ERR(svn_stream_open_readonly(&manifest_stream, - path_rev_packed(fs, rev, "manifest", pool), + path_rev_packed(fs, rev, PATH_MANIFEST, + pool), pool, pool)); /* While we're here, let's just read the entire manifest file into an array, @@ -1911,21 +2040,14 @@ get_packed_offset(apr_off_t *rev_offset, manifest = apr_array_make(pool, ffd->max_files_per_dir, sizeof(apr_off_t)); while (1) { - svn_stringbuf_t *sb; svn_boolean_t eof; apr_int64_t val; - svn_error_t *err; svn_pool_clear(iterpool); - SVN_ERR(svn_stream_readline(manifest_stream, &sb, "\n", &eof, iterpool)); + SVN_ERR(read_number_from_stream(&val, &eof, manifest_stream, iterpool)); if (eof) break; - err = svn_cstring_atoi64(&val, sb->data); - if (err) - return svn_error_createf(SVN_ERR_FS_CORRUPT, err, - _("Manifest offset '%s' too large"), - sb->data); APR_ARRAY_PUSH(manifest, apr_off_t) = (apr_off_t)val; } svn_pool_destroy(iterpool); @@ -2026,13 +2148,13 @@ read_rep_offsets_body(representation_t **rep_p, apr_pool_t *pool) { representation_t *rep; - char *str, *last_str; + char *str; apr_int64_t val; rep = apr_pcalloc(pool, sizeof(*rep)); *rep_p = rep; - str = apr_strtok(string, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Malformed text representation offset line in node-rev")); @@ -2046,7 +2168,7 @@ read_rep_offsets_body(representation_t **rep_p, return SVN_NO_ERROR; } - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Malformed text representation offset line in node-rev")); @@ -2054,7 +2176,7 @@ read_rep_offsets_body(representation_t **rep_p, SVN_ERR(svn_cstring_atoi64(&val, str)); rep->offset = (apr_off_t)val; - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Malformed text representation offset line in node-rev")); @@ -2062,7 +2184,7 @@ read_rep_offsets_body(representation_t **rep_p, SVN_ERR(svn_cstring_atoi64(&val, str)); rep->size = (svn_filesize_t)val; - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Malformed text representation offset line in node-rev")); @@ -2071,7 +2193,7 @@ read_rep_offsets_body(representation_t **rep_p, rep->expanded_size = (svn_filesize_t)val; /* Read in the MD5 hash. */ - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if ((str == NULL) || (strlen(str) != (APR_MD5_DIGESTSIZE * 2))) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Malformed text representation offset line in node-rev")); @@ -2080,7 +2202,7 @@ read_rep_offsets_body(representation_t **rep_p, pool)); /* The remaining fields are only used for formats >= 4, so check that. */ - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if (str == NULL) return SVN_NO_ERROR; @@ -2093,7 +2215,7 @@ read_rep_offsets_body(representation_t **rep_p, pool)); /* Read the uniquifier. */ - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &string); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Malformed text representation offset line in node-rev")); @@ -2147,16 +2269,6 @@ err_dangling_id(svn_fs_t *fs, const svn_fs_id_t *id) id_str->data, fs->path); } -/* Return a string that uniquely identifies the noderev with the - * given ID, for use as a cache key. - */ -static const char * -get_noderev_cache_key(const svn_fs_id_t *id, apr_pool_t *pool) -{ - const svn_string_t *id_unparsed = svn_fs_fs__id_unparse(id, pool); - return id_unparsed->data; -} - /* Look up the NODEREV_P for ID in FS' node revsion cache. If noderev * caching has been enabled and the data can be found, IS_CACHED will * be set to TRUE. The noderev will be allocated from POOL. @@ -2172,13 +2284,21 @@ get_cached_node_revision_body(node_revision_t **noderev_p, { fs_fs_data_t *ffd = fs->fsap_data; if (! ffd->node_revision_cache || svn_fs_fs__id_txn_id(id)) - *is_cached = FALSE; + { + *is_cached = FALSE; + } else - SVN_ERR(svn_cache__get((void **) noderev_p, - is_cached, - ffd->node_revision_cache, - get_noderev_cache_key(id, pool), - pool)); + { + pair_cache_key_t key = { 0 }; + + key.revision = svn_fs_fs__id_rev(id); + key.second = svn_fs_fs__id_offset(id); + SVN_ERR(svn_cache__get((void **) noderev_p, + is_cached, + ffd->node_revision_cache, + &key, + pool)); + } return SVN_NO_ERROR; } @@ -2197,10 +2317,16 @@ set_cached_node_revision_body(node_revision_t *noderev_p, fs_fs_data_t *ffd = fs->fsap_data; if (ffd->node_revision_cache && !svn_fs_fs__id_txn_id(id)) - return svn_cache__set(ffd->node_revision_cache, - get_noderev_cache_key(id, scratch_pool), - noderev_p, - scratch_pool); + { + pair_cache_key_t key = { 0 }; + + key.revision = svn_fs_fs__id_rev(id); + key.second = svn_fs_fs__id_offset(id); + return svn_cache__set(ffd->node_revision_cache, + &key, + noderev_p, + scratch_pool); + } return SVN_NO_ERROR; } @@ -2274,7 +2400,7 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, noderev = apr_pcalloc(pool, sizeof(*noderev)); /* Read the node-rev id. */ - value = apr_hash_get(headers, HEADER_ID, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_ID); if (value == NULL) /* ### More information: filename/offset coordinates */ return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, @@ -2286,7 +2412,7 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, noderev_id = value; /* for error messages later */ /* Read the type. */ - value = apr_hash_get(headers, HEADER_TYPE, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_TYPE); if ((value == NULL) || (strcmp(value, KIND_FILE) != 0 && strcmp(value, KIND_DIR))) @@ -2299,14 +2425,14 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, : svn_node_dir; /* Read the 'count' field. */ - value = apr_hash_get(headers, HEADER_COUNT, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_COUNT); if (value) SVN_ERR(svn_cstring_atoi(&noderev->predecessor_count, value)); else noderev->predecessor_count = 0; /* Get the properties location. */ - value = apr_hash_get(headers, HEADER_PROPS, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_PROPS); if (value) { SVN_ERR(read_rep_offsets(&noderev->prop_rep, value, @@ -2314,7 +2440,7 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, } /* Get the data location. */ - value = apr_hash_get(headers, HEADER_TEXT, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_TEXT); if (value) { SVN_ERR(read_rep_offsets(&noderev->data_rep, value, @@ -2323,7 +2449,7 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, } /* Get the created path. */ - value = apr_hash_get(headers, HEADER_CPATH, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_CPATH); if (value == NULL) { return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, @@ -2336,13 +2462,13 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, } /* Get the predecessor ID. */ - value = apr_hash_get(headers, HEADER_PRED, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_PRED); if (value) noderev->predecessor_id = svn_fs_fs__id_parse(value, strlen(value), pool); /* Get the copyroot. */ - value = apr_hash_get(headers, HEADER_COPYROOT, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_COPYROOT); if (value == NULL) { noderev->copyroot_path = apr_pstrdup(pool, noderev->created_path); @@ -2350,9 +2476,9 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, } else { - char *str, *last_str; + char *str; - str = apr_strtok(value, " ", &last_str); + str = svn_cstring_tokenize(" ", &value); if (str == NULL) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Malformed copyroot line in node-rev '%s'"), @@ -2360,15 +2486,15 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, noderev->copyroot_rev = SVN_STR_TO_REV(str); - if (last_str == NULL) + if (*value == '\0') return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Malformed copyroot line in node-rev '%s'"), noderev_id); - noderev->copyroot_path = apr_pstrdup(pool, last_str); + noderev->copyroot_path = apr_pstrdup(pool, value); } /* Get the copyfrom. */ - value = apr_hash_get(headers, HEADER_COPYFROM, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_COPYFROM); if (value == NULL) { noderev->copyfrom_path = NULL; @@ -2376,9 +2502,7 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, } else { - char *str, *last_str; - - str = apr_strtok(value, " ", &last_str); + char *str = svn_cstring_tokenize(" ", &value); if (str == NULL) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Malformed copyfrom line in node-rev '%s'"), @@ -2386,26 +2510,26 @@ svn_fs_fs__read_noderev(node_revision_t **noderev_p, noderev->copyfrom_rev = SVN_STR_TO_REV(str); - if (last_str == NULL) + if (*value == 0) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Malformed copyfrom line in node-rev '%s'"), noderev_id); - noderev->copyfrom_path = apr_pstrdup(pool, last_str); + noderev->copyfrom_path = apr_pstrdup(pool, value); } /* Get whether this is a fresh txn root. */ - value = apr_hash_get(headers, HEADER_FRESHTXNRT, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_FRESHTXNRT); noderev->is_fresh_txn_root = (value != NULL); /* Get the mergeinfo count. */ - value = apr_hash_get(headers, HEADER_MINFO_CNT, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_MINFO_CNT); if (value) SVN_ERR(svn_cstring_atoi64(&noderev->mergeinfo_count, value)); else noderev->mergeinfo_count = 0; /* Get whether *this* node has mergeinfo. */ - value = apr_hash_get(headers, HEADER_MINFO_HERE, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_MINFO_HERE); noderev->has_mergeinfo = (value != NULL); *noderev_p = noderev; @@ -2435,33 +2559,40 @@ svn_fs_fs__get_node_revision(node_revision_t **noderev_p, that represents the location of representation REP. If MUTABLE_REP_TRUNCATED is given, the rep is for props or dir contents, and only a "-1" revision number will be given for a mutable rep. + If MAY_BE_CORRUPT is true, guard for NULL when constructing the string. Perform the allocation from POOL. */ static const char * representation_string(representation_t *rep, int format, svn_boolean_t mutable_rep_truncated, + svn_boolean_t may_be_corrupt, apr_pool_t *pool) { if (rep->txn_id && mutable_rep_truncated) return "-1"; +#define DISPLAY_MAYBE_NULL_CHECKSUM(checksum) \ + ((!may_be_corrupt || (checksum) != NULL) \ + ? svn_checksum_to_cstring_display((checksum), pool) \ + : "(null)") + if (format < SVN_FS_FS__MIN_REP_SHARING_FORMAT || rep->sha1_checksum == NULL) return apr_psprintf(pool, "%ld %" APR_OFF_T_FMT " %" SVN_FILESIZE_T_FMT " %" SVN_FILESIZE_T_FMT " %s", rep->revision, rep->offset, rep->size, rep->expanded_size, - svn_checksum_to_cstring_display(rep->md5_checksum, - pool)); + DISPLAY_MAYBE_NULL_CHECKSUM(rep->md5_checksum)); return apr_psprintf(pool, "%ld %" APR_OFF_T_FMT " %" SVN_FILESIZE_T_FMT " %" SVN_FILESIZE_T_FMT " %s %s %s", rep->revision, rep->offset, rep->size, rep->expanded_size, - svn_checksum_to_cstring_display(rep->md5_checksum, - pool), - svn_checksum_to_cstring_display(rep->sha1_checksum, - pool), + DISPLAY_MAYBE_NULL_CHECKSUM(rep->md5_checksum), + DISPLAY_MAYBE_NULL_CHECKSUM(rep->sha1_checksum), rep->uniquifier); + +#undef DISPLAY_MAYBE_NULL_CHECKSUM + } @@ -2494,12 +2625,13 @@ svn_fs_fs__write_noderev(svn_stream_t *outfile, format, (noderev->kind == svn_node_dir), + FALSE, pool))); if (noderev->prop_rep) SVN_ERR(svn_stream_printf(outfile, pool, HEADER_PROPS ": %s\n", representation_string(noderev->prop_rep, format, - TRUE, pool))); + TRUE, FALSE, pool))); SVN_ERR(svn_stream_printf(outfile, pool, HEADER_CPATH ": %s\n", noderev->created_path)); @@ -2518,7 +2650,7 @@ svn_fs_fs__write_noderev(svn_stream_t *outfile, noderev->copyroot_path)); if (noderev->is_fresh_txn_root) - SVN_ERR(svn_stream_printf(outfile, pool, HEADER_FRESHTXNRT ": y\n")); + SVN_ERR(svn_stream_puts(outfile, HEADER_FRESHTXNRT ": y\n")); if (include_mergeinfo) { @@ -2528,10 +2660,10 @@ svn_fs_fs__write_noderev(svn_stream_t *outfile, noderev->mergeinfo_count)); if (noderev->has_mergeinfo) - SVN_ERR(svn_stream_printf(outfile, pool, HEADER_MINFO_HERE ": y\n")); + SVN_ERR(svn_stream_puts(outfile, HEADER_MINFO_HERE ": y\n")); } - return svn_stream_printf(outfile, pool, "\n"); + return svn_stream_puts(outfile, "\n"); } svn_error_t * @@ -2562,7 +2694,50 @@ svn_fs_fs__put_node_revision(svn_fs_t *fs, svn_fs_fs__fs_supports_mergeinfo(fs), pool)); - return svn_io_file_close(noderev_file, pool); + SVN_ERR(svn_io_file_close(noderev_file, pool)); + + return SVN_NO_ERROR; +} + +/* For the in-transaction NODEREV within FS, write the sha1->rep mapping + * file in the respective transaction, if rep sharing has been enabled etc. + * Use POOL for temporary allocations. + */ +static svn_error_t * +store_sha1_rep_mapping(svn_fs_t *fs, + node_revision_t *noderev, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + + /* if rep sharing has been enabled and the noderev has a data rep and + * its SHA-1 is known, store the rep struct under its SHA1. */ + if ( ffd->rep_sharing_allowed + && noderev->data_rep + && noderev->data_rep->sha1_checksum) + { + apr_file_t *rep_file; + const char *file_name = path_txn_sha1(fs, + svn_fs_fs__id_txn_id(noderev->id), + noderev->data_rep->sha1_checksum, + pool); + const char *rep_string = representation_string(noderev->data_rep, + ffd->format, + (noderev->kind + == svn_node_dir), + FALSE, + pool); + SVN_ERR(svn_io_file_open(&rep_file, file_name, + APR_WRITE | APR_CREATE | APR_TRUNCATE + | APR_BUFFERED, APR_OS_DEFAULT, pool)); + + SVN_ERR(svn_io_file_write_full(rep_file, rep_string, + strlen(rep_string), NULL, pool)); + + SVN_ERR(svn_io_file_close(rep_file, pool)); + } + + return SVN_NO_ERROR; } @@ -2589,7 +2764,7 @@ read_rep_line(struct rep_args **rep_args_p, char buffer[160]; apr_size_t limit; struct rep_args *rep_args; - char *str, *last_str; + char *str, *last_str = buffer; apr_int64_t val; limit = sizeof(buffer); @@ -2617,22 +2792,22 @@ read_rep_line(struct rep_args **rep_args_p, rep_args->is_delta_vs_empty = FALSE; /* We have hopefully a DELTA vs. a non-empty base revision. */ - str = apr_strtok(buffer, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (! str || (strcmp(str, REP_DELTA) != 0)) goto error; - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (! str) goto error; rep_args->base_revision = SVN_STR_TO_REV(str); - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (! str) goto error; SVN_ERR(svn_cstring_atoi64(&val, str)); rep_args->base_offset = (apr_off_t)val; - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (! str) goto error; SVN_ERR(svn_cstring_atoi64(&val, str)); @@ -2671,7 +2846,7 @@ get_fs_id_at_offset(svn_fs_id_t **id_p, /* In error messages, the offset is relative to the pack file, not to the rev file. */ - node_id_str = apr_hash_get(headers, HEADER_ID, APR_HASH_KEY_STRING); + node_id_str = svn_hash_gets(headers, HEADER_ID); if (node_id_str == NULL) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, @@ -2934,109 +3109,1324 @@ svn_fs_fs__rev_get_root(svn_fs_id_t **root_id_p, return SVN_NO_ERROR; } -/* Set the revision property list of revision REV in filesystem FS to - PROPLIST. Use POOL for temporary allocations. */ +/* Revprop caching management. + * + * Mechanism: + * ---------- + * + * Revprop caching needs to be activated and will be deactivated for the + * respective FS instance if the necessary infrastructure could not be + * initialized. In deactivated mode, there is almost no runtime overhead + * associated with revprop caching. As long as no revprops are being read + * or changed, revprop caching imposes no overhead. + * + * When activated, we cache revprops using (revision, generation) pairs + * as keys with the generation being incremented upon every revprop change. + * Since the cache is process-local, the generation needs to be tracked + * for at least as long as the process lives but may be reset afterwards. + * + * To track the revprop generation, we use two-layer approach. On the lower + * level, we use named atomics to have a system-wide consistent value for + * the current revprop generation. However, those named atomics will only + * remain valid for as long as at least one process / thread in the system + * accesses revprops in the respective repository. The underlying shared + * memory gets cleaned up afterwards. + * + * On the second level, we will use a persistent file to track the latest + * revprop generation. It will be written upon each revprop change but + * only be read if we are the first process to initialize the named atomics + * with that value. + * + * The overhead for the second and following accesses to revprops is + * almost zero on most systems. + * + * + * Tech aspects: + * ------------- + * + * A problem is that we need to provide a globally available file name to + * back the SHM implementation on OSes that need it. We can only assume + * write access to some file within the respective repositories. Because + * a given server process may access thousands of repositories during its + * lifetime, keeping the SHM data alive for all of them is also not an + * option. + * + * So, we store the new revprop generation on disk as part of each + * setrevprop call, i.e. this write will be serialized and the write order + * be guaranteed by the repository write lock. + * + * The only racy situation occurs when the data is being read again by two + * processes concurrently but in that situation, the first process to + * finish that procedure is guaranteed to be the only one that initializes + * the SHM data. Since even writers will first go through that + * initialization phase, they will never operate on stale data. + */ + +/* Read revprop generation as stored on disk for repository FS. The result + * is returned in *CURRENT. Default to 2 if no such file is available. + */ static svn_error_t * -set_revision_proplist(svn_fs_t *fs, - svn_revnum_t rev, - apr_hash_t *proplist, - apr_pool_t *pool) +read_revprop_generation_file(apr_int64_t *current, + svn_fs_t *fs, + apr_pool_t *pool) { - SVN_ERR(ensure_revision_exists(fs, rev, pool)); + svn_error_t *err; + apr_file_t *file; + char buf[80]; + apr_size_t len; + const char *path = path_revprop_generation(fs, pool); - if (1) + err = svn_io_file_open(&file, path, + APR_READ | APR_BUFFERED, + APR_OS_DEFAULT, pool); + if (err && APR_STATUS_IS_ENOENT(err->apr_err)) { - const char *final_path = path_revprops(fs, rev, pool); - const char *tmp_path; - const char *perms_reference; - svn_stream_t *stream; + svn_error_clear(err); + *current = 2; - /* ### do we have a directory sitting around already? we really shouldn't - ### have to get the dirname here. */ - SVN_ERR(svn_stream_open_unique(&stream, &tmp_path, - svn_dirent_dirname(final_path, pool), - svn_io_file_del_none, pool, pool)); - SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); - SVN_ERR(svn_stream_close(stream)); + return SVN_NO_ERROR; + } + SVN_ERR(err); - /* We use the rev file of this revision as the perms reference, - because when setting revprops for the first time, the revprop - file won't exist and therefore can't serve as its own reference. - (Whereas the rev file should already exist at this point.) */ - SVN_ERR(svn_fs_fs__path_rev_absolute(&perms_reference, fs, rev, pool)); - SVN_ERR(move_into_place(tmp_path, final_path, perms_reference, pool)); + len = sizeof(buf); + SVN_ERR(svn_io_read_length_line(file, buf, &len, pool)); - return SVN_NO_ERROR; + /* Check that the first line contains only digits. */ + SVN_ERR(check_file_buffer_numeric(buf, 0, path, + "Revprop Generation", pool)); + SVN_ERR(svn_cstring_atoi64(current, buf)); + + return svn_io_file_close(file, pool); +} + +/* Write the CURRENT revprop generation to disk for repository FS. + */ +static svn_error_t * +write_revprop_generation_file(svn_fs_t *fs, + apr_int64_t current, + apr_pool_t *pool) +{ + apr_file_t *file; + const char *tmp_path; + + char buf[SVN_INT64_BUFFER_SIZE]; + apr_size_t len = svn__i64toa(buf, current); + buf[len] = '\n'; + + SVN_ERR(svn_io_open_unique_file3(&file, &tmp_path, fs->path, + svn_io_file_del_none, pool, pool)); + SVN_ERR(svn_io_file_write_full(file, buf, len + 1, NULL, pool)); + SVN_ERR(svn_io_file_close(file, pool)); + + return move_into_place(tmp_path, path_revprop_generation(fs, pool), + tmp_path, pool); +} + +/* Make sure the revprop_namespace member in FS is set. */ +static svn_error_t * +ensure_revprop_namespace(svn_fs_t *fs) +{ + fs_fs_data_t *ffd = fs->fsap_data; + + return ffd->revprop_namespace == NULL + ? svn_atomic_namespace__create(&ffd->revprop_namespace, + svn_dirent_join(fs->path, + ATOMIC_REVPROP_NAMESPACE, + fs->pool), + fs->pool) + : SVN_NO_ERROR; +} + +/* Make sure the revprop_namespace member in FS is set. */ +static svn_error_t * +cleanup_revprop_namespace(svn_fs_t *fs) +{ + const char *name = svn_dirent_join(fs->path, + ATOMIC_REVPROP_NAMESPACE, + fs->pool); + return svn_error_trace(svn_atomic_namespace__cleanup(name, fs->pool)); +} + +/* Make sure the revprop_generation member in FS is set and, if necessary, + * initialized with the latest value stored on disk. + */ +static svn_error_t * +ensure_revprop_generation(svn_fs_t *fs, apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + + SVN_ERR(ensure_revprop_namespace(fs)); + if (ffd->revprop_generation == NULL) + { + apr_int64_t current = 0; + + SVN_ERR(svn_named_atomic__get(&ffd->revprop_generation, + ffd->revprop_namespace, + ATOMIC_REVPROP_GENERATION, + TRUE)); + + /* If the generation is at 0, we just created a new namespace + * (it would be at least 2 otherwise). Read the latest generation + * from disk and if we are the first one to initialize the atomic + * (i.e. is still 0), set it to the value just gotten. + */ + SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation)); + if (current == 0) + { + SVN_ERR(read_revprop_generation_file(¤t, fs, pool)); + SVN_ERR(svn_named_atomic__cmpxchg(NULL, current, 0, + ffd->revprop_generation)); + } } return SVN_NO_ERROR; } +/* Make sure the revprop_timeout member in FS is set. */ static svn_error_t * -revision_proplist(apr_hash_t **proplist_p, +ensure_revprop_timeout(svn_fs_t *fs) +{ + fs_fs_data_t *ffd = fs->fsap_data; + + SVN_ERR(ensure_revprop_namespace(fs)); + return ffd->revprop_timeout == NULL + ? svn_named_atomic__get(&ffd->revprop_timeout, + ffd->revprop_namespace, + ATOMIC_REVPROP_TIMEOUT, + TRUE) + : SVN_NO_ERROR; +} + +/* Create an error object with the given MESSAGE and pass it to the + WARNING member of FS. */ +static void +log_revprop_cache_init_warning(svn_fs_t *fs, + svn_error_t *underlying_err, + const char *message) +{ + svn_error_t *err = svn_error_createf(SVN_ERR_FS_REVPROP_CACHE_INIT_FAILURE, + underlying_err, + message, fs->path); + + if (fs->warning) + (fs->warning)(fs->warning_baton, err); + + svn_error_clear(err); +} + +/* Test whether revprop cache and necessary infrastructure are + available in FS. */ +static svn_boolean_t +has_revprop_cache(svn_fs_t *fs, apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + svn_error_t *error; + + /* is the cache (still) enabled? */ + if (ffd->revprop_cache == NULL) + return FALSE; + + /* is it efficient? */ + if (!svn_named_atomic__is_efficient()) + { + /* access to it would be quite slow + * -> disable the revprop cache for good + */ + ffd->revprop_cache = NULL; + log_revprop_cache_init_warning(fs, NULL, + "Revprop caching for '%s' disabled" + " because it would be inefficient."); + + return FALSE; + } + + /* try to access our SHM-backed infrastructure */ + error = ensure_revprop_generation(fs, pool); + if (error) + { + /* failure -> disable revprop cache for good */ + + ffd->revprop_cache = NULL; + log_revprop_cache_init_warning(fs, error, + "Revprop caching for '%s' disabled " + "because SHM infrastructure for revprop " + "caching failed to initialize."); + + return FALSE; + } + + return TRUE; +} + +/* Baton structure for revprop_generation_fixup. */ +typedef struct revprop_generation_fixup_t +{ + /* revprop generation to read */ + apr_int64_t *generation; + + /* containing the revprop_generation member to query */ + fs_fs_data_t *ffd; +} revprop_generation_upgrade_t; + +/* If the revprop generation has an odd value, it means the original writer + of the revprop got killed. We don't know whether that process as able + to change the revprop data but we assume that it was. Therefore, we + increase the generation in that case to basically invalidate everyones + cache content. + Execute this onlx while holding the write lock to the repo in baton->FFD. + */ +static svn_error_t * +revprop_generation_fixup(void *void_baton, + apr_pool_t *pool) +{ + revprop_generation_upgrade_t *baton = void_baton; + assert(baton->ffd->has_write_lock); + + /* Maybe, either the original revprop writer or some other reader has + already corrected / bumped the revprop generation. Thus, we need + to read it again. */ + SVN_ERR(svn_named_atomic__read(baton->generation, + baton->ffd->revprop_generation)); + + /* Cause everyone to re-read revprops upon their next access, if the + last revprop write did not complete properly. */ + while (*baton->generation % 2) + SVN_ERR(svn_named_atomic__add(baton->generation, + 1, + baton->ffd->revprop_generation)); + + return SVN_NO_ERROR; +} + +/* Read the current revprop generation and return it in *GENERATION. + Also, detect aborted / crashed writers and recover from that. + Use the access object in FS to set the shared mem values. */ +static svn_error_t * +read_revprop_generation(apr_int64_t *generation, + svn_fs_t *fs, + apr_pool_t *pool) +{ + apr_int64_t current = 0; + fs_fs_data_t *ffd = fs->fsap_data; + + /* read the current revprop generation number */ + SVN_ERR(ensure_revprop_generation(fs, pool)); + SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation)); + + /* is an unfinished revprop write under the way? */ + if (current % 2) + { + apr_int64_t timeout = 0; + + /* read timeout for the write operation */ + SVN_ERR(ensure_revprop_timeout(fs)); + SVN_ERR(svn_named_atomic__read(&timeout, ffd->revprop_timeout)); + + /* has the writer process been aborted, + * i.e. has the timeout been reached? + */ + if (apr_time_now() > timeout) + { + revprop_generation_upgrade_t baton; + baton.generation = ¤t; + baton.ffd = ffd; + + /* Ensure that the original writer process no longer exists by + * acquiring the write lock to this repository. Then, fix up + * the revprop generation. + */ + if (ffd->has_write_lock) + SVN_ERR(revprop_generation_fixup(&baton, pool)); + else + SVN_ERR(svn_fs_fs__with_write_lock(fs, revprop_generation_fixup, + &baton, pool)); + } + } + + /* return the value we just got */ + *generation = current; + return SVN_NO_ERROR; +} + +/* Set the revprop generation to the next odd number to indicate that + there is a revprop write process under way. If that times out, + readers shall recover from that state & re-read revprops. + Use the access object in FS to set the shared mem value. */ +static svn_error_t * +begin_revprop_change(svn_fs_t *fs, apr_pool_t *pool) +{ + apr_int64_t current; + fs_fs_data_t *ffd = fs->fsap_data; + + /* set the timeout for the write operation */ + SVN_ERR(ensure_revprop_timeout(fs)); + SVN_ERR(svn_named_atomic__write(NULL, + apr_time_now() + REVPROP_CHANGE_TIMEOUT, + ffd->revprop_timeout)); + + /* set the revprop generation to an odd value to indicate + * that a write is in progress + */ + SVN_ERR(ensure_revprop_generation(fs, pool)); + do + { + SVN_ERR(svn_named_atomic__add(¤t, + 1, + ffd->revprop_generation)); + } + while (current % 2 == 0); + + return SVN_NO_ERROR; +} + +/* Set the revprop generation to the next even number to indicate that + a) readers shall re-read revprops, and + b) the write process has been completed (no recovery required) + Use the access object in FS to set the shared mem value. */ +static svn_error_t * +end_revprop_change(svn_fs_t *fs, apr_pool_t *pool) +{ + apr_int64_t current = 1; + fs_fs_data_t *ffd = fs->fsap_data; + + /* set the revprop generation to an even value to indicate + * that a write has been completed + */ + SVN_ERR(ensure_revprop_generation(fs, pool)); + do + { + SVN_ERR(svn_named_atomic__add(¤t, + 1, + ffd->revprop_generation)); + } + while (current % 2); + + /* Save the latest generation to disk. FS is currently in a "locked" + * state such that we can be sure the be the only ones to write that + * file. + */ + return write_revprop_generation_file(fs, current, pool); +} + +/* Container for all data required to access the packed revprop file + * for a given REVISION. This structure will be filled incrementally + * by read_pack_revprops() its sub-routines. + */ +typedef struct packed_revprops_t +{ + /* revision number to read (not necessarily the first in the pack) */ + svn_revnum_t revision; + + /* current revprop generation. Used when populating the revprop cache */ + apr_int64_t generation; + + /* the actual revision properties */ + apr_hash_t *properties; + + /* their size when serialized to a single string + * (as found in PACKED_REVPROPS) */ + apr_size_t serialized_size; + + + /* name of the pack file (without folder path) */ + const char *filename; + + /* packed shard folder path */ + const char *folder; + + /* sum of values in SIZES */ + apr_size_t total_size; + + /* first revision in the pack (>= MANIFEST_START) */ + svn_revnum_t start_revision; + + /* size of the revprops in PACKED_REVPROPS */ + apr_array_header_t *sizes; + + /* offset of the revprops in PACKED_REVPROPS */ + apr_array_header_t *offsets; + + + /* concatenation of the serialized representation of all revprops + * in the pack, i.e. the pack content without header and compression */ + svn_stringbuf_t *packed_revprops; + + /* First revision covered by MANIFEST. + * Will equal the shard start revision or 1, for the 1st shard. */ + svn_revnum_t manifest_start; + + /* content of the manifest. + * Maps long(rev - MANIFEST_START) to const char* pack file name */ + apr_array_header_t *manifest; +} packed_revprops_t; + +/* Parse the serialized revprops in CONTENT and return them in *PROPERTIES. + * Also, put them into the revprop cache, if activated, for future use. + * Three more parameters are being used to update the revprop cache: FS is + * our file system, the revprops belong to REVISION and the global revprop + * GENERATION is used as well. + * + * The returned hash will be allocated in POOL, SCRATCH_POOL is being used + * for temporary allocations. + */ +static svn_error_t * +parse_revprop(apr_hash_t **properties, + svn_fs_t *fs, + svn_revnum_t revision, + apr_int64_t generation, + svn_string_t *content, + apr_pool_t *pool, + apr_pool_t *scratch_pool) +{ + svn_stream_t *stream = svn_stream_from_string(content, scratch_pool); + *properties = apr_hash_make(pool); + + SVN_ERR(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool)); + if (has_revprop_cache(fs, pool)) + { + fs_fs_data_t *ffd = fs->fsap_data; + pair_cache_key_t key = { 0 }; + + key.revision = revision; + key.second = generation; + SVN_ERR(svn_cache__set(ffd->revprop_cache, &key, *properties, + scratch_pool)); + } + + return SVN_NO_ERROR; +} + +/* Read the non-packed revprops for revision REV in FS, put them into the + * revprop cache if activated and return them in *PROPERTIES. GENERATION + * is the current revprop generation. + * + * If the data could not be read due to an otherwise recoverable error, + * leave *PROPERTIES unchanged. No error will be returned in that case. + * + * Allocations will be done in POOL. + */ +static svn_error_t * +read_non_packed_revprop(apr_hash_t **properties, + svn_fs_t *fs, + svn_revnum_t rev, + apr_int64_t generation, + apr_pool_t *pool) +{ + svn_stringbuf_t *content = NULL; + apr_pool_t *iterpool = svn_pool_create(pool); + svn_boolean_t missing = FALSE; + int i; + + for (i = 0; i < RECOVERABLE_RETRY_COUNT && !missing && !content; ++i) + { + svn_pool_clear(iterpool); + SVN_ERR(try_stringbuf_from_file(&content, + &missing, + path_revprops(fs, rev, iterpool), + i + 1 < RECOVERABLE_RETRY_COUNT, + iterpool)); + } + + if (content) + SVN_ERR(parse_revprop(properties, fs, rev, generation, + svn_stringbuf__morph_into_string(content), + pool, iterpool)); + + svn_pool_clear(iterpool); + + return SVN_NO_ERROR; +} + +/* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST + * members. Use POOL for allocating results and SCRATCH_POOL for temporaries. + */ +static svn_error_t * +get_revprop_packname(svn_fs_t *fs, + packed_revprops_t *revprops, + apr_pool_t *pool, + apr_pool_t *scratch_pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + svn_stringbuf_t *content = NULL; + const char *manifest_file_path; + int idx; + + /* read content of the manifest file */ + revprops->folder = path_revprops_pack_shard(fs, revprops->revision, pool); + manifest_file_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); + + SVN_ERR(read_content(&content, manifest_file_path, pool)); + + /* parse the manifest. Every line is a file name */ + revprops->manifest = apr_array_make(pool, ffd->max_files_per_dir, + sizeof(const char*)); + + /* Read all lines. Since the last line ends with a newline, we will + end up with a valid but empty string after the last entry. */ + while (content->data && *content->data) + { + APR_ARRAY_PUSH(revprops->manifest, const char*) = content->data; + content->data = strchr(content->data, '\n'); + if (content->data) + { + *content->data = 0; + content->data++; + } + } + + /* Index for our revision. Rev 0 is excluded from the first shard. */ + revprops->manifest_start = revprops->revision + - (revprops->revision % ffd->max_files_per_dir); + if (revprops->manifest_start == 0) + ++revprops->manifest_start; + idx = (int)(revprops->revision - revprops->manifest_start); + + if (revprops->manifest->nelts <= idx) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Packed revprop manifest for r%ld too " + "small"), revprops->revision); + + /* Now get the file name */ + revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*); + + return SVN_NO_ERROR; +} + +/* Return TRUE, if revision R1 and R2 refer to the same shard in FS. + */ +static svn_boolean_t +same_shard(svn_fs_t *fs, + svn_revnum_t r1, + svn_revnum_t r2) +{ + fs_fs_data_t *ffd = fs->fsap_data; + return (r1 / ffd->max_files_per_dir) == (r2 / ffd->max_files_per_dir); +} + +/* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS, + * fill the START_REVISION, SIZES, OFFSETS members. Also, make + * PACKED_REVPROPS point to the first serialized revprop. + * + * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as + * well as the SERIALIZED_SIZE member. If revprop caching has been + * enabled, parse all revprops in the pack and cache them. + */ +static svn_error_t * +parse_packed_revprops(svn_fs_t *fs, + packed_revprops_t *revprops, + apr_pool_t *pool, + apr_pool_t *scratch_pool) +{ + svn_stream_t *stream; + apr_int64_t first_rev, count, i; + apr_off_t offset; + const char *header_end; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + + /* decompress (even if the data is only "stored", there is still a + * length header to remove) */ + svn_string_t *compressed + = svn_stringbuf__morph_into_string(revprops->packed_revprops); + svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(pool); + SVN_ERR(svn__decompress(compressed, uncompressed, APR_SIZE_MAX)); + + /* read first revision number and number of revisions in the pack */ + stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); + SVN_ERR(read_number_from_stream(&first_rev, NULL, stream, iterpool)); + SVN_ERR(read_number_from_stream(&count, NULL, stream, iterpool)); + + /* Check revision range for validity. */ + if ( !same_shard(fs, revprops->revision, first_rev) + || !same_shard(fs, revprops->revision, first_rev + count - 1) + || count < 1) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Revprop pack for revision r%ld" + " contains revprops for r%ld .. r%ld"), + revprops->revision, + (svn_revnum_t)first_rev, + (svn_revnum_t)(first_rev + count -1)); + + /* Since start & end are in the same shard, it is enough to just test + * the FIRST_REV for being actually packed. That will also cover the + * special case of rev 0 never being packed. */ + if (!is_packed_revprop(fs, first_rev)) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Revprop pack for revision r%ld" + " starts at non-packed revisions r%ld"), + revprops->revision, (svn_revnum_t)first_rev); + + /* make PACKED_REVPROPS point to the first char after the header. + * This is where the serialized revprops are. */ + header_end = strstr(uncompressed->data, "\n\n"); + if (header_end == NULL) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Header end not found")); + + offset = header_end - uncompressed->data + 2; + + revprops->packed_revprops = svn_stringbuf_create_empty(pool); + revprops->packed_revprops->data = uncompressed->data + offset; + revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset); + revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset); + + /* STREAM still points to the first entry in the sizes list. + * Init / construct REVPROPS members. */ + revprops->start_revision = (svn_revnum_t)first_rev; + revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset)); + revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset)); + + /* Now parse, revision by revision, the size and content of each + * revisions' revprops. */ + for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i) + { + apr_int64_t size; + svn_string_t serialized; + apr_hash_t *properties; + svn_revnum_t revision = (svn_revnum_t)(first_rev + i); + + /* read & check the serialized size */ + SVN_ERR(read_number_from_stream(&size, NULL, stream, iterpool)); + if (size + offset > (apr_int64_t)revprops->packed_revprops->len) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Packed revprop size exceeds pack file size")); + + /* Parse this revprops list, if necessary */ + serialized.data = revprops->packed_revprops->data + offset; + serialized.len = (apr_size_t)size; + + if (revision == revprops->revision) + { + SVN_ERR(parse_revprop(&revprops->properties, fs, revision, + revprops->generation, &serialized, + pool, iterpool)); + revprops->serialized_size = serialized.len; + } + else + { + /* If revprop caching is enabled, parse any revprops. + * They will get cached as a side-effect of this. */ + if (has_revprop_cache(fs, pool)) + SVN_ERR(parse_revprop(&properties, fs, revision, + revprops->generation, &serialized, + iterpool, iterpool)); + } + + /* fill REVPROPS data structures */ + APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len; + APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset; + revprops->total_size += serialized.len; + + offset += serialized.len; + + svn_pool_clear(iterpool); + } + + return SVN_NO_ERROR; +} + +/* In filesystem FS, read the packed revprops for revision REV into + * *REVPROPS. Use GENERATION to populate the revprop cache, if enabled. + * Allocate data in POOL. + */ +static svn_error_t * +read_pack_revprop(packed_revprops_t **revprops, svn_fs_t *fs, svn_revnum_t rev, + apr_int64_t generation, apr_pool_t *pool) { - apr_hash_t *proplist; + apr_pool_t *iterpool = svn_pool_create(pool); + svn_boolean_t missing = FALSE; + svn_error_t *err; + packed_revprops_t *result; + int i; + + /* someone insisted that REV is packed. Double-check if necessary */ + if (!is_packed_revprop(fs, rev)) + SVN_ERR(update_min_unpacked_rev(fs, iterpool)); + + if (!is_packed_revprop(fs, rev)) + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("No such packed revision %ld"), rev); + + /* initialize the result data structure */ + result = apr_pcalloc(pool, sizeof(*result)); + result->revision = rev; + result->generation = generation; + + /* try to read the packed revprops. This may require retries if we have + * concurrent writers. */ + for (i = 0; i < RECOVERABLE_RETRY_COUNT && !result->packed_revprops; ++i) + { + const char *file_path; + + /* there might have been concurrent writes. + * Re-read the manifest and the pack file. + */ + SVN_ERR(get_revprop_packname(fs, result, pool, iterpool)); + file_path = svn_dirent_join(result->folder, + result->filename, + iterpool); + SVN_ERR(try_stringbuf_from_file(&result->packed_revprops, + &missing, + file_path, + i + 1 < RECOVERABLE_RETRY_COUNT, + pool)); + + /* If we could not find the file, there was a write. + * So, we should refresh our revprop generation info as well such + * that others may find data we will put into the cache. They would + * consider it outdated, otherwise. + */ + if (missing && has_revprop_cache(fs, pool)) + SVN_ERR(read_revprop_generation(&result->generation, fs, pool)); + + svn_pool_clear(iterpool); + } + + /* the file content should be available now */ + if (!result->packed_revprops) + return svn_error_createf(SVN_ERR_FS_PACKED_REVPROP_READ_FAILURE, NULL, + _("Failed to read revprop pack file for r%ld"), rev); + /* parse it. RESULT will be complete afterwards. */ + err = parse_packed_revprops(fs, result, pool, iterpool); + svn_pool_destroy(iterpool); + if (err) + return svn_error_createf(SVN_ERR_FS_CORRUPT, err, + _("Revprop pack file for r%ld is corrupt"), rev); + + *revprops = result; + + return SVN_NO_ERROR; +} + +/* Read the revprops for revision REV in FS and return them in *PROPERTIES_P. + * + * Allocations will be done in POOL. + */ +static svn_error_t * +get_revision_proplist(apr_hash_t **proplist_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + apr_int64_t generation = 0; + + /* not found, yet */ + *proplist_p = NULL; + + /* should they be available at all? */ SVN_ERR(ensure_revision_exists(fs, rev, pool)); - if (1) + /* Try cache lookup first. */ + if (has_revprop_cache(fs, pool)) { - apr_file_t *revprop_file = NULL; - svn_error_t *err = SVN_NO_ERROR; - int i; - apr_pool_t *iterpool; + svn_boolean_t is_cached; + pair_cache_key_t key = { 0 }; - proplist = apr_hash_make(pool); - iterpool = svn_pool_create(pool); - for (i = 0; i < RECOVERABLE_RETRY_COUNT; i++) + SVN_ERR(read_revprop_generation(&generation, fs, pool)); + + key.revision = rev; + key.second = generation; + SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached, + ffd->revprop_cache, &key, pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + /* if REV had not been packed when we began, try reading it from the + * non-packed shard. If that fails, we will fall through to packed + * shard reads. */ + if (!is_packed_revprop(fs, rev)) + { + svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev, + generation, pool); + if (err) { - svn_pool_clear(iterpool); + if (!APR_STATUS_IS_ENOENT(err->apr_err) + || ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) + return svn_error_trace(err); - /* Clear err here rather than after finding a recoverable error so - * we can return that error on the last iteration of the loop. */ svn_error_clear(err); - err = svn_io_file_open(&revprop_file, path_revprops(fs, rev, - iterpool), - APR_READ | APR_BUFFERED, APR_OS_DEFAULT, - iterpool); - if (err) - { - if (APR_STATUS_IS_ENOENT(err->apr_err)) - { - svn_error_clear(err); - return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, - _("No such revision %ld"), rev); - } -#ifdef ESTALE - else if (APR_TO_OS_ERROR(err->apr_err) == ESTALE - || APR_TO_OS_ERROR(err->apr_err) == EIO - || APR_TO_OS_ERROR(err->apr_err) == ENOENT) - continue; -#endif - return svn_error_trace(err); - } + *proplist_p = NULL; /* in case read_non_packed_revprop changed it */ + } + } - SVN_ERR(svn_hash__clear(proplist, iterpool)); - RETRY_RECOVERABLE(err, revprop_file, - svn_hash_read2(proplist, - svn_stream_from_aprfile2( - revprop_file, TRUE, iterpool), - SVN_HASH_TERMINATOR, pool)); + /* if revprop packing is available and we have not read the revprops, yet, + * try reading them from a packed shard. If that fails, REV is most + * likely invalid (or its revprops highly contested). */ + if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p) + { + packed_revprops_t *packed_revprops; + SVN_ERR(read_pack_revprop(&packed_revprops, fs, rev, generation, pool)); + *proplist_p = packed_revprops->properties; + } - IGNORE_RECOVERABLE(err, svn_io_file_close(revprop_file, iterpool)); + /* The revprops should have been there. Did we get them? */ + if (!*proplist_p) + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("Could not read revprops for revision %ld"), + rev); - break; + return SVN_NO_ERROR; +} + +/* Serialize the revision property list PROPLIST of revision REV in + * filesystem FS to a non-packed file. Return the name of that temporary + * file in *TMP_PATH and the file path that it must be moved to in + * *FINAL_PATH. + * + * Use POOL for allocations. + */ +static svn_error_t * +write_non_packed_revprop(const char **final_path, + const char **tmp_path, + svn_fs_t *fs, + svn_revnum_t rev, + apr_hash_t *proplist, + apr_pool_t *pool) +{ + svn_stream_t *stream; + *final_path = path_revprops(fs, rev, pool); + + /* ### do we have a directory sitting around already? we really shouldn't + ### have to get the dirname here. */ + SVN_ERR(svn_stream_open_unique(&stream, tmp_path, + svn_dirent_dirname(*final_path, pool), + svn_io_file_del_none, pool, pool)); + SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_stream_close(stream)); + + return SVN_NO_ERROR; +} + +/* After writing the new revprop file(s), call this function to move the + * file at TMP_PATH to FINAL_PATH and give it the permissions from + * PERMS_REFERENCE. + * + * If indicated in BUMP_GENERATION, increase FS' revprop generation. + * Finally, delete all the temporary files given in FILES_TO_DELETE. + * The latter may be NULL. + * + * Use POOL for temporary allocations. + */ +static svn_error_t * +switch_to_new_revprop(svn_fs_t *fs, + const char *final_path, + const char *tmp_path, + const char *perms_reference, + apr_array_header_t *files_to_delete, + svn_boolean_t bump_generation, + apr_pool_t *pool) +{ + /* Now, we may actually be replacing revprops. Make sure that all other + threads and processes will know about this. */ + if (bump_generation) + SVN_ERR(begin_revprop_change(fs, pool)); + + SVN_ERR(move_into_place(tmp_path, final_path, perms_reference, pool)); + + /* Indicate that the update (if relevant) has been completed. */ + if (bump_generation) + SVN_ERR(end_revprop_change(fs, pool)); + + /* Clean up temporary files, if necessary. */ + if (files_to_delete) + { + apr_pool_t *iterpool = svn_pool_create(pool); + int i; + + for (i = 0; i < files_to_delete->nelts; ++i) + { + const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*); + SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); + svn_pool_clear(iterpool); } - if (err) - return svn_error_trace(err); svn_pool_destroy(iterpool); } + return SVN_NO_ERROR; +} - *proplist_p = proplist; +/* Write a pack file header to STREAM that starts at revision START_REVISION + * and contains the indexes [START,END) of SIZES. + */ +static svn_error_t * +serialize_revprops_header(svn_stream_t *stream, + svn_revnum_t start_revision, + apr_array_header_t *sizes, + int start, + int end, + apr_pool_t *pool) +{ + apr_pool_t *iterpool = svn_pool_create(pool); + int i; + + SVN_ERR_ASSERT(start < end); + + /* start revision and entry count */ + SVN_ERR(svn_stream_printf(stream, pool, "%ld\n", start_revision)); + SVN_ERR(svn_stream_printf(stream, pool, "%d\n", end - start)); + + /* the sizes array */ + for (i = start; i < end; ++i) + { + apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t); + SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n", + size)); + } + + /* the double newline char indicates the end of the header */ + SVN_ERR(svn_stream_printf(stream, iterpool, "\n")); + + svn_pool_clear(iterpool); + return SVN_NO_ERROR; +} + +/* Writes the a pack file to FILE_STREAM. It copies the serialized data + * from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX. + * + * The data for the latter is taken from NEW_SERIALIZED. Note, that + * CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is + * taken in that case but only a subset of the old data will be copied. + * + * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size. + * POOL is used for temporary allocations. + */ +static svn_error_t * +repack_revprops(svn_fs_t *fs, + packed_revprops_t *revprops, + int start, + int end, + int changed_index, + svn_stringbuf_t *new_serialized, + apr_off_t new_total_size, + svn_stream_t *file_stream, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + svn_stream_t *stream; + int i; + + /* create data empty buffers and the stream object */ + svn_stringbuf_t *uncompressed + = svn_stringbuf_create_ensure((apr_size_t)new_total_size, pool); + svn_stringbuf_t *compressed + = svn_stringbuf_create_empty(pool); + stream = svn_stream_from_stringbuf(uncompressed, pool); + + /* write the header*/ + SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start, + revprops->sizes, start, end, pool)); + + /* append the serialized revprops */ + for (i = start; i < end; ++i) + if (i == changed_index) + { + SVN_ERR(svn_stream_write(stream, + new_serialized->data, + &new_serialized->len)); + } + else + { + apr_size_t size + = (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t); + apr_size_t offset + = (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t); + + SVN_ERR(svn_stream_write(stream, + revprops->packed_revprops->data + offset, + &size)); + } + + /* flush the stream buffer (if any) to our underlying data buffer */ + SVN_ERR(svn_stream_close(stream)); + + /* compress / store the data */ + SVN_ERR(svn__compress(svn_stringbuf__morph_into_string(uncompressed), + compressed, + ffd->compress_packed_revprops + ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT + : SVN_DELTA_COMPRESSION_LEVEL_NONE)); + + /* finally, write the content to the target stream and close it */ + SVN_ERR(svn_stream_write(file_stream, compressed->data, &compressed->len)); + SVN_ERR(svn_stream_close(file_stream)); + + return SVN_NO_ERROR; +} + +/* Allocate a new pack file name for revisions + * [REVPROPS->START_REVISION + START, REVPROPS->START_REVISION + END - 1] + * of REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE, + * auto-create that array if necessary. Return an open file stream to + * the new file in *STREAM allocated in POOL. + */ +static svn_error_t * +repack_stream_open(svn_stream_t **stream, + svn_fs_t *fs, + packed_revprops_t *revprops, + int start, + int end, + apr_array_header_t **files_to_delete, + apr_pool_t *pool) +{ + apr_int64_t tag; + const char *tag_string; + svn_string_t *new_filename; + int i; + apr_file_t *file; + int manifest_offset + = (int)(revprops->start_revision - revprops->manifest_start); + + /* get the old (= current) file name and enlist it for later deletion */ + const char *old_filename = APR_ARRAY_IDX(revprops->manifest, + start + manifest_offset, + const char*); + + if (*files_to_delete == NULL) + *files_to_delete = apr_array_make(pool, 3, sizeof(const char*)); + + APR_ARRAY_PUSH(*files_to_delete, const char*) + = svn_dirent_join(revprops->folder, old_filename, pool); + + /* increase the tag part, i.e. the counter after the dot */ + tag_string = strchr(old_filename, '.'); + if (tag_string == NULL) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Packed file '%s' misses a tag"), + old_filename); + + SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1)); + new_filename = svn_string_createf(pool, "%ld.%" APR_INT64_T_FMT, + revprops->start_revision + start, + ++tag); + + /* update the manifest to point to the new file */ + for (i = start; i < end; ++i) + APR_ARRAY_IDX(revprops->manifest, i + manifest_offset, const char*) + = new_filename->data; + + /* create a file stream for the new file */ + SVN_ERR(svn_io_file_open(&file, svn_dirent_join(revprops->folder, + new_filename->data, + pool), + APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool)); + *stream = svn_stream_from_aprfile2(file, FALSE, pool); + + return SVN_NO_ERROR; +} + +/* For revision REV in filesystem FS, set the revision properties to + * PROPLIST. Return a new file in *TMP_PATH that the caller shall move + * to *FINAL_PATH to make the change visible. Files to be deleted will + * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated. + * Use POOL for allocations. + */ +static svn_error_t * +write_packed_revprop(const char **final_path, + const char **tmp_path, + apr_array_header_t **files_to_delete, + svn_fs_t *fs, + svn_revnum_t rev, + apr_hash_t *proplist, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + packed_revprops_t *revprops; + apr_int64_t generation = 0; + svn_stream_t *stream; + svn_stringbuf_t *serialized; + apr_off_t new_total_size; + int changed_index; + + /* read the current revprop generation. This value will not change + * while we hold the global write lock to this FS. */ + if (has_revprop_cache(fs, pool)) + SVN_ERR(read_revprop_generation(&generation, fs, pool)); + + /* read contents of the current pack file */ + SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, pool)); + + /* serialize the new revprops */ + serialized = svn_stringbuf_create_empty(pool); + stream = svn_stream_from_stringbuf(serialized, pool); + SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_stream_close(stream)); + + /* calculate the size of the new data */ + changed_index = (int)(rev - revprops->start_revision); + new_total_size = revprops->total_size - revprops->serialized_size + + serialized->len + + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE; + + APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len; + + /* can we put the new data into the same pack as the before? */ + if ( new_total_size < ffd->revprop_pack_size + || revprops->sizes->nelts == 1) + { + /* simply replace the old pack file with new content as we do it + * in the non-packed case */ + + *final_path = svn_dirent_join(revprops->folder, revprops->filename, + pool); + SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder, + svn_io_file_del_none, pool, pool)); + SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts, + changed_index, serialized, new_total_size, + stream, pool)); + } + else + { + /* split the pack file into two of roughly equal size */ + int right_count, left_count, i; + + int left = 0; + int right = revprops->sizes->nelts - 1; + apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE; + apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE; + + /* let left and right side grow such that their size difference + * is minimal after each step. */ + while (left <= right) + if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) + < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)) + { + left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) + + SVN_INT64_BUFFER_SIZE; + ++left; + } + else + { + right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t) + + SVN_INT64_BUFFER_SIZE; + --right; + } + + /* since the items need much less than SVN_INT64_BUFFER_SIZE + * bytes to represent their length, the split may not be optimal */ + left_count = left; + right_count = revprops->sizes->nelts - left; + + /* if new_size is large, one side may exceed the pack size limit. + * In that case, split before and after the modified revprop.*/ + if ( left_size > ffd->revprop_pack_size + || right_size > ffd->revprop_pack_size) + { + left_count = changed_index; + right_count = revprops->sizes->nelts - left_count - 1; + } + + /* write the new, split files */ + if (left_count) + { + SVN_ERR(repack_stream_open(&stream, fs, revprops, 0, + left_count, files_to_delete, pool)); + SVN_ERR(repack_revprops(fs, revprops, 0, left_count, + changed_index, serialized, new_total_size, + stream, pool)); + } + + if (left_count + right_count < revprops->sizes->nelts) + { + SVN_ERR(repack_stream_open(&stream, fs, revprops, changed_index, + changed_index + 1, files_to_delete, + pool)); + SVN_ERR(repack_revprops(fs, revprops, changed_index, + changed_index + 1, + changed_index, serialized, new_total_size, + stream, pool)); + } + + if (right_count) + { + SVN_ERR(repack_stream_open(&stream, fs, revprops, + revprops->sizes->nelts - right_count, + revprops->sizes->nelts, + files_to_delete, pool)); + SVN_ERR(repack_revprops(fs, revprops, + revprops->sizes->nelts - right_count, + revprops->sizes->nelts, changed_index, + serialized, new_total_size, stream, + pool)); + } + + /* write the new manifest */ + *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); + SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder, + svn_io_file_del_none, pool, pool)); + + for (i = 0; i < revprops->manifest->nelts; ++i) + { + const char *filename = APR_ARRAY_IDX(revprops->manifest, i, + const char*); + SVN_ERR(svn_stream_printf(stream, pool, "%s\n", filename)); + } + + SVN_ERR(svn_stream_close(stream)); + } + + return SVN_NO_ERROR; +} + +/* Set the revision property list of revision REV in filesystem FS to + PROPLIST. Use POOL for temporary allocations. */ +static svn_error_t * +set_revision_proplist(svn_fs_t *fs, + svn_revnum_t rev, + apr_hash_t *proplist, + apr_pool_t *pool) +{ + svn_boolean_t is_packed; + svn_boolean_t bump_generation = FALSE; + const char *final_path; + const char *tmp_path; + const char *perms_reference; + apr_array_header_t *files_to_delete = NULL; + + SVN_ERR(ensure_revision_exists(fs, rev, pool)); + + /* this info will not change while we hold the global FS write lock */ + is_packed = is_packed_revprop(fs, rev); + + /* Test whether revprops already exist for this revision. + * Only then will we need to bump the revprop generation. */ + if (has_revprop_cache(fs, pool)) + { + if (is_packed) + { + bump_generation = TRUE; + } + else + { + svn_node_kind_t kind; + SVN_ERR(svn_io_check_path(path_revprops(fs, rev, pool), &kind, + pool)); + bump_generation = kind != svn_node_none; + } + } + + /* Serialize the new revprop data */ + if (is_packed) + SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete, + fs, rev, proplist, pool)); + else + SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path, + fs, rev, proplist, pool)); + + /* We use the rev file of this revision as the perms reference, + * because when setting revprops for the first time, the revprop + * file won't exist and therefore can't serve as its own reference. + * (Whereas the rev file should already exist at this point.) + */ + SVN_ERR(svn_fs_fs__path_rev_absolute(&perms_reference, fs, rev, pool)); + + /* Now, switch to the new revprop data. */ + SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference, + files_to_delete, bump_generation, pool)); return SVN_NO_ERROR; } @@ -3047,7 +4437,7 @@ svn_fs_fs__revision_proplist(apr_hash_t **proplist_p, svn_revnum_t rev, apr_pool_t *pool) { - SVN_ERR(revision_proplist(proplist_p, fs, rev, pool)); + SVN_ERR(get_revision_proplist(proplist_p, fs, rev, pool)); return SVN_NO_ERROR; } @@ -3059,6 +4449,8 @@ struct rep_state apr_file_t *file; /* The txdelta window cache to use or NULL. */ svn_cache__t *window_cache; + /* Caches un-deltified windows. May be NULL. */ + svn_cache__t *combined_cache; apr_off_t start; /* The starting offset for the raw svndiff/plaintext data minus header. */ apr_off_t off; /* The current offset into the file. */ @@ -3071,6 +4463,8 @@ struct rep_state static svn_error_t * create_rep_state_body(struct rep_state **rep_state, struct rep_args **rep_args, + apr_file_t **file_hint, + svn_revnum_t *rev_hint, representation_t *rep, svn_fs_t *fs, apr_pool_t *pool) @@ -3080,8 +4474,47 @@ create_rep_state_body(struct rep_state **rep_state, struct rep_args *ra; unsigned char buf[4]; - SVN_ERR(open_and_seek_representation(&rs->file, fs, rep, pool)); + /* If the hint is + * - given, + * - refers to a valid revision, + * - refers to a packed revision, + * - as does the rep we want to read, and + * - refers to the same pack file as the rep + * ... + */ + if ( file_hint && rev_hint && *file_hint + && SVN_IS_VALID_REVNUM(*rev_hint) + && *rev_hint < ffd->min_unpacked_rev + && rep->revision < ffd->min_unpacked_rev + && ( (*rev_hint / ffd->max_files_per_dir) + == (rep->revision / ffd->max_files_per_dir))) + { + /* ... we can re-use the same, already open file object + */ + apr_off_t offset; + SVN_ERR(get_packed_offset(&offset, fs, rep->revision, pool)); + + offset += rep->offset; + SVN_ERR(svn_io_file_seek(*file_hint, APR_SET, &offset, pool)); + + rs->file = *file_hint; + } + else + { + /* otherwise, create a new file object + */ + SVN_ERR(open_and_seek_representation(&rs->file, fs, rep, pool)); + } + + /* remember the current file, if suggested by the caller */ + if (file_hint) + *file_hint = rs->file; + if (rev_hint) + *rev_hint = rep->revision; + + /* continue constructing RS and RA */ rs->window_cache = ffd->txdelta_window_cache; + rs->combined_cache = ffd->combined_window_cache; SVN_ERR(read_rep_line(&ra, rs->file, pool)); SVN_ERR(get_file_offset(&rs->start, rs->file, pool)); @@ -3090,7 +4523,7 @@ create_rep_state_body(struct rep_state **rep_state, *rep_state = rs; *rep_args = ra; - if (ra->is_delta == FALSE) + if (!ra->is_delta) /* This is a plaintext, so just return the current rep_state. */ return SVN_NO_ERROR; @@ -3111,15 +4544,26 @@ create_rep_state_body(struct rep_state **rep_state, /* Read the rep args for REP in filesystem FS and create a rep_state for reading the representation. Return the rep_state in *REP_STATE - and the rep args in *REP_ARGS, both allocated in POOL. */ + and the rep args in *REP_ARGS, both allocated in POOL. + + When reading multiple reps, i.e. a skip delta chain, you may provide + non-NULL FILE_HINT and REV_HINT. (If FILE_HINT is not NULL, in the first + call it should be a pointer to NULL.) The function will use these variables + to store the previous call results and tries to re-use them. This may + result in significant savings in I/O for packed files. + */ static svn_error_t * create_rep_state(struct rep_state **rep_state, struct rep_args **rep_args, + apr_file_t **file_hint, + svn_revnum_t *rev_hint, representation_t *rep, svn_fs_t *fs, apr_pool_t *pool) { - svn_error_t *err = create_rep_state_body(rep_state, rep_args, rep, fs, pool); + svn_error_t *err = create_rep_state_body(rep_state, rep_args, + file_hint, rev_hint, + rep, fs, pool); if (err && err->apr_err == SVN_ERR_FS_CORRUPT) { fs_fs_data_t *ffd = fs->fsap_data; @@ -3133,66 +4577,23 @@ create_rep_state(struct rep_state **rep_state, ### going to jump straight to this comment anyway! */ return svn_error_createf(SVN_ERR_FS_CORRUPT, err, "Corrupt representation '%s'", - rep + rep ? representation_string(rep, ffd->format, TRUE, - pool) + TRUE, pool) : "(null)"); } /* ### Call representation_string() ? */ return svn_error_trace(err); } -/* Build an array of rep_state structures in *LIST giving the delta - reps from first_rep to a plain-text or self-compressed rep. Set - *SRC_STATE to the plain-text rep we find at the end of the chain, - or to NULL if the final delta representation is self-compressed. - The representation to start from is designated by filesystem FS, id - ID, and representation REP. */ -static svn_error_t * -build_rep_list(apr_array_header_t **list, - struct rep_state **src_state, - svn_fs_t *fs, - representation_t *first_rep, - apr_pool_t *pool) -{ - representation_t rep; - struct rep_state *rs; - struct rep_args *rep_args; - - *list = apr_array_make(pool, 1, sizeof(struct rep_state *)); - rep = *first_rep; - - while (1) - { - SVN_ERR(create_rep_state(&rs, &rep_args, &rep, fs, pool)); - if (rep_args->is_delta == FALSE) - { - /* This is a plaintext, so just return the current rep_state. */ - *src_state = rs; - return SVN_NO_ERROR; - } - - /* Push this rep onto the list. If it's self-compressed, we're done. */ - APR_ARRAY_PUSH(*list, struct rep_state *) = rs; - if (rep_args->is_delta_vs_empty) - { - *src_state = NULL; - return SVN_NO_ERROR; - } - - rep.revision = rep_args->base_revision; - rep.offset = rep_args->base_offset; - rep.size = rep_args->base_length; - rep.txn_id = NULL; - } -} - - struct rep_read_baton { /* The FS from which we're reading. */ svn_fs_t *fs; + /* If not NULL, this is the base for the first delta window in rs_list */ + svn_stringbuf_t *base_window; + /* The state of all prior delta representations. */ apr_array_header_t *rs_list; @@ -3225,7 +4626,7 @@ struct rep_read_baton /* The key for the fulltext cache for this rep, if there is a fulltext cache. */ - const char *fulltext_cache_key; + pair_cache_key_t fulltext_cache_key; /* The text we've been reading, if we're going to cache it. */ svn_stringbuf_t *current_fulltext; @@ -3237,51 +4638,9 @@ struct rep_read_baton apr_pool_t *filehandle_pool; }; -/* Create a rep_read_baton structure for node revision NODEREV in - filesystem FS and store it in *RB_P. If FULLTEXT_CACHE_KEY is not - NULL, it is the rep's key in the fulltext cache, and a stringbuf - must be allocated to store the text. Perform all allocations in - POOL. If rep is mutable, it must be for file contents. */ -static svn_error_t * -rep_read_get_baton(struct rep_read_baton **rb_p, - svn_fs_t *fs, - representation_t *rep, - const char *fulltext_cache_key, - apr_pool_t *pool) -{ - struct rep_read_baton *b; - - b = apr_pcalloc(pool, sizeof(*b)); - b->fs = fs; - b->chunk_index = 0; - b->buf = NULL; - b->md5_checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); - b->checksum_finalized = FALSE; - b->md5_checksum = svn_checksum_dup(rep->md5_checksum, pool); - b->len = rep->expanded_size ? rep->expanded_size : rep->size; - b->off = 0; - b->fulltext_cache_key = fulltext_cache_key; - b->pool = svn_pool_create(pool); - b->filehandle_pool = svn_pool_create(pool); - - if (fulltext_cache_key) - b->current_fulltext = svn_stringbuf_create_ensure - ((apr_size_t)b->len, - b->filehandle_pool); - else - b->current_fulltext = NULL; - - SVN_ERR(build_rep_list(&b->rs_list, &b->src_state, fs, rep, - b->filehandle_pool)); - - /* Save our output baton. */ - *rb_p = b; - - return SVN_NO_ERROR; -} - /* Combine the name of the rev file in RS with the given OFFSET to form - * a cache lookup key. Allocations will be made from POOL. */ + * a cache lookup key. Allocations will be made from POOL. May return + * NULL if the key cannot be constructed. */ static const char* get_window_key(struct rep_state *rs, apr_off_t offset, apr_pool_t *pool) { @@ -3295,7 +4654,7 @@ get_window_key(struct rep_state *rs, apr_off_t offset, apr_pool_t *pool) * comparison _will_ find them. */ if (apr_file_name_get(&name, rs->file)) - return ""; + return NULL; /* Handle packed files as well by scanning backwards until we find the * revision or pack number. */ @@ -3383,6 +4742,7 @@ set_cached_window(svn_txdelta_window_t *window, { /* store the window and the first offset _past_ it */ svn_fs_fs__txdelta_cached_window_t cached_window; + cached_window.window = window; cached_window.end_offset = rs->off; @@ -3397,11 +4757,200 @@ set_cached_window(svn_txdelta_window_t *window, return SVN_NO_ERROR; } +/* Read the WINDOW_P for the rep state RS from the current FSFS session's + * cache. This will be a no-op and IS_CACHED will be set to FALSE if no + * cache has been given. If a cache is available IS_CACHED will inform + * the caller about the success of the lookup. Allocations (of the window + * in particualar) will be made from POOL. + */ +static svn_error_t * +get_cached_combined_window(svn_stringbuf_t **window_p, + struct rep_state *rs, + svn_boolean_t *is_cached, + apr_pool_t *pool) +{ + if (! rs->combined_cache) + { + /* txdelta window has not been enabled */ + *is_cached = FALSE; + } + else + { + /* ask the cache for the desired txdelta window */ + return svn_cache__get((void **)window_p, + is_cached, + rs->combined_cache, + get_window_key(rs, rs->start, pool), + pool); + } + + return SVN_NO_ERROR; +} + +/* Store the WINDOW read at OFFSET for the rep state RS in the current + * FSFS session's cache. This will be a no-op if no cache has been given. + * Temporary allocations will be made from SCRATCH_POOL. */ +static svn_error_t * +set_cached_combined_window(svn_stringbuf_t *window, + struct rep_state *rs, + apr_off_t offset, + apr_pool_t *scratch_pool) +{ + if (rs->combined_cache) + { + /* but key it with the start offset because that is the known state + * when we will look it up */ + return svn_cache__set(rs->combined_cache, + get_window_key(rs, offset, scratch_pool), + window, + scratch_pool); + } + + return SVN_NO_ERROR; +} + +/* Build an array of rep_state structures in *LIST giving the delta + reps from first_rep to a plain-text or self-compressed rep. Set + *SRC_STATE to the plain-text rep we find at the end of the chain, + or to NULL if the final delta representation is self-compressed. + The representation to start from is designated by filesystem FS, id + ID, and representation REP. + Also, set *WINDOW_P to the base window content for *LIST, if it + could be found in cache. Otherwise, *LIST will contain the base + representation for the whole delta chain. + Finally, return the expanded size of the representation in + *EXPANDED_SIZE. It will take care of cases where only the on-disk + size is known. */ +static svn_error_t * +build_rep_list(apr_array_header_t **list, + svn_stringbuf_t **window_p, + struct rep_state **src_state, + svn_filesize_t *expanded_size, + svn_fs_t *fs, + representation_t *first_rep, + apr_pool_t *pool) +{ + representation_t rep; + struct rep_state *rs = NULL; + struct rep_args *rep_args; + svn_boolean_t is_cached = FALSE; + apr_file_t *last_file = NULL; + svn_revnum_t last_revision; + + *list = apr_array_make(pool, 1, sizeof(struct rep_state *)); + rep = *first_rep; + + /* The value as stored in the data struct. + 0 is either for unknown length or actually zero length. */ + *expanded_size = first_rep->expanded_size; + + /* for the top-level rep, we need the rep_args */ + SVN_ERR(create_rep_state(&rs, &rep_args, &last_file, + &last_revision, &rep, fs, pool)); + + /* Unknown size or empty representation? + That implies the this being the first iteration. + Usually size equals on-disk size, except for empty, + compressed representations (delta, size = 4). + Please note that for all non-empty deltas have + a 4-byte header _plus_ some data. */ + if (*expanded_size == 0) + if (! rep_args->is_delta || first_rep->size != 4) + *expanded_size = first_rep->size; + + while (1) + { + /* fetch state, if that has not been done already */ + if (!rs) + SVN_ERR(create_rep_state(&rs, &rep_args, &last_file, + &last_revision, &rep, fs, pool)); + + SVN_ERR(get_cached_combined_window(window_p, rs, &is_cached, pool)); + if (is_cached) + { + /* We already have a reconstructed window in our cache. + Write a pseudo rep_state with the full length. */ + rs->off = rs->start; + rs->end = rs->start + (*window_p)->len; + *src_state = rs; + return SVN_NO_ERROR; + } + + if (!rep_args->is_delta) + { + /* This is a plaintext, so just return the current rep_state. */ + *src_state = rs; + return SVN_NO_ERROR; + } + + /* Push this rep onto the list. If it's self-compressed, we're done. */ + APR_ARRAY_PUSH(*list, struct rep_state *) = rs; + if (rep_args->is_delta_vs_empty) + { + *src_state = NULL; + return SVN_NO_ERROR; + } + + rep.revision = rep_args->base_revision; + rep.offset = rep_args->base_offset; + rep.size = rep_args->base_length; + rep.txn_id = NULL; + + rs = NULL; + } +} + + +/* Create a rep_read_baton structure for node revision NODEREV in + filesystem FS and store it in *RB_P. If FULLTEXT_CACHE_KEY is not + NULL, it is the rep's key in the fulltext cache, and a stringbuf + must be allocated to store the text. Perform all allocations in + POOL. If rep is mutable, it must be for file contents. */ +static svn_error_t * +rep_read_get_baton(struct rep_read_baton **rb_p, + svn_fs_t *fs, + representation_t *rep, + pair_cache_key_t fulltext_cache_key, + apr_pool_t *pool) +{ + struct rep_read_baton *b; + + b = apr_pcalloc(pool, sizeof(*b)); + b->fs = fs; + b->base_window = NULL; + b->chunk_index = 0; + b->buf = NULL; + b->md5_checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); + b->checksum_finalized = FALSE; + b->md5_checksum = svn_checksum_dup(rep->md5_checksum, pool); + b->len = rep->expanded_size; + b->off = 0; + b->fulltext_cache_key = fulltext_cache_key; + b->pool = svn_pool_create(pool); + b->filehandle_pool = svn_pool_create(pool); + + SVN_ERR(build_rep_list(&b->rs_list, &b->base_window, + &b->src_state, &b->len, fs, rep, + b->filehandle_pool)); + + if (SVN_IS_VALID_REVNUM(fulltext_cache_key.revision)) + b->current_fulltext = svn_stringbuf_create_ensure + ((apr_size_t)b->len, + b->filehandle_pool); + else + b->current_fulltext = NULL; + + /* Save our output baton. */ + *rb_p = b; + + return SVN_NO_ERROR; +} + /* Skip forwards to THIS_CHUNK in REP_STATE and then read the next delta window into *NWIN. */ static svn_error_t * -read_window(svn_txdelta_window_t **nwin, int this_chunk, struct rep_state *rs, - apr_pool_t *pool) +read_delta_window(svn_txdelta_window_t **nwin, int this_chunk, + struct rep_state *rs, apr_pool_t *pool) { svn_stream_t *stream; svn_boolean_t is_cached; @@ -3409,6 +4958,10 @@ read_window(svn_txdelta_window_t **nwin, int this_chunk, struct rep_state *rs, SVN_ERR_ASSERT(rs->chunk_index <= this_chunk); + /* RS->FILE may be shared between RS instances -> make sure we point + * to the right data. */ + SVN_ERR(svn_io_file_seek(rs->file, APR_SET, &rs->off, pool)); + /* Skip windows to reach the current chunk if we aren't there yet. */ while (rs->chunk_index < this_chunk) { @@ -3444,45 +4997,103 @@ read_window(svn_txdelta_window_t **nwin, int this_chunk, struct rep_state *rs, return set_cached_window(*nwin, rs, old_offset, pool); } -/* Get one delta window that is a result of combining all but the last deltas - from the current desired representation identified in *RB, to its - final base representation. Store the window in *RESULT. */ +/* Read SIZE bytes from the representation RS and return it in *NWIN. */ static svn_error_t * -get_combined_window(svn_txdelta_window_t **result, +read_plain_window(svn_stringbuf_t **nwin, struct rep_state *rs, + apr_size_t size, apr_pool_t *pool) +{ + /* RS->FILE may be shared between RS instances -> make sure we point + * to the right data. */ + SVN_ERR(svn_io_file_seek(rs->file, APR_SET, &rs->off, pool)); + + /* Read the plain data. */ + *nwin = svn_stringbuf_create_ensure(size, pool); + SVN_ERR(svn_io_file_read_full2(rs->file, (*nwin)->data, size, NULL, NULL, + pool)); + (*nwin)->data[size] = 0; + + /* Update RS. */ + rs->off += (apr_off_t)size; + + return SVN_NO_ERROR; +} + +/* Get the undeltified window that is a result of combining all deltas + from the current desired representation identified in *RB with its + base representation. Store the window in *RESULT. */ +static svn_error_t * +get_combined_window(svn_stringbuf_t **result, struct rep_read_baton *rb) { - apr_pool_t *pool, *new_pool; + apr_pool_t *pool, *new_pool, *window_pool; int i; - svn_txdelta_window_t *window, *nwin; + svn_txdelta_window_t *window; + apr_array_header_t *windows; + svn_stringbuf_t *source, *buf = rb->base_window; struct rep_state *rs; - SVN_ERR_ASSERT(rb->rs_list->nelts >= 2); - - pool = svn_pool_create(rb->pool); + /* Read all windows that we need to combine. This is fine because + the size of each window is relatively small (100kB) and skip- + delta limits the number of deltas in a chain to well under 100. + Stop early if one of them does not depend on its predecessors. */ + window_pool = svn_pool_create(rb->pool); + windows = apr_array_make(window_pool, 0, sizeof(svn_txdelta_window_t *)); + for (i = 0; i < rb->rs_list->nelts; ++i) + { + rs = APR_ARRAY_IDX(rb->rs_list, i, struct rep_state *); + SVN_ERR(read_delta_window(&window, rb->chunk_index, rs, window_pool)); - /* Read the next window from the original rep. */ - rs = APR_ARRAY_IDX(rb->rs_list, 0, struct rep_state *); - SVN_ERR(read_window(&window, rb->chunk_index, rs, pool)); + APR_ARRAY_PUSH(windows, svn_txdelta_window_t *) = window; + if (window->src_ops == 0) + { + ++i; + break; + } + } - /* Combine in the windows from the other delta reps, if needed. */ - for (i = 1; i < rb->rs_list->nelts - 1; i++) + /* Combine in the windows from the other delta reps. */ + pool = svn_pool_create(rb->pool); + for (--i; i >= 0; --i) { - if (window->src_ops == 0) - break; rs = APR_ARRAY_IDX(rb->rs_list, i, struct rep_state *); + window = APR_ARRAY_IDX(windows, i, svn_txdelta_window_t *); + + /* Maybe, we've got a PLAIN start representation. If we do, read + as much data from it as the needed for the txdelta window's source + view. + Note that BUF / SOURCE may only be NULL in the first iteration. */ + source = buf; + if (source == NULL && rb->src_state != NULL) + SVN_ERR(read_plain_window(&source, rb->src_state, window->sview_len, + pool)); - SVN_ERR(read_window(&nwin, rb->chunk_index, rs, pool)); - - /* Combine this window with the current one. Cycle pools so that we - only need to hold three windows at a time. */ + /* Combine this window with the current one. */ new_pool = svn_pool_create(rb->pool); - window = svn_txdelta_compose_windows(nwin, window, new_pool); + buf = svn_stringbuf_create_ensure(window->tview_len, new_pool); + buf->len = window->tview_len; + + svn_txdelta_apply_instructions(window, source ? source->data : NULL, + buf->data, &buf->len); + if (buf->len != window->tview_len) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("svndiff window length is " + "corrupt")); + + /* Cache windows only if the whole rep content could be read as a + single chunk. Only then will no other chunk need a deeper RS + list than the cached chunk. */ + if ((rb->chunk_index == 0) && (rs->off == rs->end)) + SVN_ERR(set_cached_combined_window(buf, rs, rs->start, new_pool)); + + /* Cycle pools so that we only need to hold three windows at a time. */ svn_pool_destroy(pool); pool = new_pool; } - *result = window; + svn_pool_destroy(window_pool); + + *result = buf; return SVN_NO_ERROR; } @@ -3515,10 +5126,9 @@ get_contents(struct rep_read_baton *rb, char *buf, apr_size_t *len) { - apr_size_t copy_len, remaining = *len, tlen; - char *sbuf, *tbuf, *cur = buf; + apr_size_t copy_len, remaining = *len; + char *cur = buf; struct rep_state *rs; - svn_txdelta_window_t *cwindow, *lwindow; /* Special case for when there are no delta reps, only a plain text. */ @@ -3526,10 +5136,28 @@ get_contents(struct rep_read_baton *rb, { copy_len = remaining; rs = rb->src_state; - if (((apr_off_t) copy_len) > rs->end - rs->off) - copy_len = (apr_size_t) (rs->end - rs->off); - SVN_ERR(svn_io_file_read_full2(rs->file, cur, copy_len, NULL, - NULL, rb->pool)); + + if (rb->base_window != NULL) + { + /* We got the desired rep directly from the cache. + This is where we need the pseudo rep_state created + by build_rep_list(). */ + apr_size_t offset = (apr_size_t)(rs->off - rs->start); + if (copy_len + offset > rb->base_window->len) + copy_len = offset < rb->base_window->len + ? rb->base_window->len - offset + : 0ul; + + memcpy (cur, rb->base_window->data + offset, copy_len); + } + else + { + if (((apr_off_t) copy_len) > rs->end - rs->off) + copy_len = (apr_size_t) (rs->end - rs->off); + SVN_ERR(svn_io_file_read_full2(rs->file, cur, copy_len, NULL, + NULL, rb->pool)); + } + rs->off += copy_len; *len = copy_len; return SVN_NO_ERROR; @@ -3561,90 +5189,18 @@ get_contents(struct rep_read_baton *rb, } else { + svn_stringbuf_t *sbuf = NULL; rs = APR_ARRAY_IDX(rb->rs_list, 0, struct rep_state *); if (rs->off == rs->end) break; /* Get more buffered data by evaluating a chunk. */ - if (rb->rs_list->nelts > 1) - SVN_ERR(get_combined_window(&cwindow, rb)); - else - cwindow = NULL; - if (!cwindow || cwindow->src_ops > 0) - { - rs = APR_ARRAY_IDX(rb->rs_list, rb->rs_list->nelts - 1, - struct rep_state *); - /* Read window from last representation in list. */ - /* We apply this window directly instead of combining it - with the others. We do this because vdelta used to - be used for deltas against the empty stream, which - will trigger quadratic behaviour in the delta - combiner. It's still likely that we'll find such - deltas in an old repository; it may be worth - considering whether or not this special case is still - needed in the future, though. */ - SVN_ERR(read_window(&lwindow, rb->chunk_index, rs, rb->pool)); - - if (lwindow->src_ops > 0) - { - if (! rb->src_state) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("svndiff data requested " - "non-existent source")); - rs = rb->src_state; - sbuf = apr_palloc(rb->pool, lwindow->sview_len); - if (! ((rs->start + lwindow->sview_offset) < rs->end)) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("svndiff requested position " - "beyond end of stream")); - if ((rs->start + lwindow->sview_offset) != rs->off) - { - rs->off = rs->start + lwindow->sview_offset; - SVN_ERR(svn_io_file_seek(rs->file, APR_SET, &rs->off, - rb->pool)); - } - SVN_ERR(svn_io_file_read_full2(rs->file, sbuf, - lwindow->sview_len, - NULL, NULL, rb->pool)); - rs->off += lwindow->sview_len; - } - else - sbuf = NULL; - - /* Apply lwindow to source. */ - tlen = lwindow->tview_len; - tbuf = apr_palloc(rb->pool, tlen); - svn_txdelta_apply_instructions(lwindow, sbuf, tbuf, - &tlen); - if (tlen != lwindow->tview_len) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("svndiff window length is " - "corrupt")); - sbuf = tbuf; - } - else - sbuf = NULL; + SVN_ERR(get_combined_window(&sbuf, rb)); rb->chunk_index++; - - if (cwindow) - { - rb->buf_len = cwindow->tview_len; - rb->buf = apr_palloc(rb->pool, rb->buf_len); - svn_txdelta_apply_instructions(cwindow, sbuf, rb->buf, - &rb->buf_len); - if (rb->buf_len != cwindow->tview_len) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("svndiff window length is " - "corrupt")); - } - else - { - rb->buf_len = lwindow->tview_len; - rb->buf = sbuf; - } - + rb->buf_len = sbuf->len; + rb->buf = sbuf->data; rb->buf_pos = 0; } } @@ -3697,7 +5253,7 @@ rep_read_contents(void *baton, if (rb->off == rb->len && rb->current_fulltext) { fs_fs_data_t *ffd = rb->fs->fsap_data; - SVN_ERR(svn_cache__set(ffd->fulltext_cache, rb->fulltext_cache_key, + SVN_ERR(svn_cache__set(ffd->fulltext_cache, &rb->fulltext_cache_key, rb->current_fulltext, rb->pool)); rb->current_fulltext = NULL; } @@ -3728,27 +5284,30 @@ read_representation(svn_stream_t **contents_p, else { fs_fs_data_t *ffd = fs->fsap_data; - const char *fulltext_key = NULL; + pair_cache_key_t fulltext_cache_key = { 0 }; svn_filesize_t len = rep->expanded_size ? rep->expanded_size : rep->size; struct rep_read_baton *rb; + fulltext_cache_key.revision = rep->revision; + fulltext_cache_key.second = rep->offset; if (ffd->fulltext_cache && SVN_IS_VALID_REVNUM(rep->revision) && fulltext_size_is_cachable(ffd, len)) { - svn_string_t *fulltext; + svn_stringbuf_t *fulltext; svn_boolean_t is_cached; - fulltext_key = apr_psprintf(pool, "%ld/%" APR_OFF_T_FMT, - rep->revision, rep->offset); SVN_ERR(svn_cache__get((void **) &fulltext, &is_cached, - ffd->fulltext_cache, fulltext_key, pool)); + ffd->fulltext_cache, &fulltext_cache_key, + pool)); if (is_cached) { - *contents_p = svn_stream_from_string(fulltext, pool); + *contents_p = svn_stream_from_stringbuf(fulltext, pool); return SVN_NO_ERROR; } } + else + fulltext_cache_key.revision = SVN_INVALID_REVNUM; - SVN_ERR(rep_read_get_baton(&rb, fs, rep, fulltext_key, pool)); + SVN_ERR(rep_read_get_baton(&rb, fs, rep, fulltext_cache_key, pool)); *contents_p = svn_stream_create(rb, pool); svn_stream_set_read(*contents_p, rep_read_contents); @@ -3787,7 +5346,7 @@ delta_read_next_window(svn_txdelta_window_t **window, void *baton, return SVN_NO_ERROR; } - return read_window(window, drb->rs->chunk_index, drb->rs, pool); + return read_delta_window(window, drb->rs->chunk_index, drb->rs, pool); } /* This implements the svn_txdelta_md5_digest_fn_t interface. */ @@ -3819,13 +5378,15 @@ svn_fs_fs__get_file_delta_stream(svn_txdelta_stream_t **stream_p, struct rep_args *rep_args; /* Read target's base rep if any. */ - SVN_ERR(create_rep_state(&rep_state, &rep_args, target->data_rep, - fs, pool)); - /* If that matches source, then use this delta as is. */ + SVN_ERR(create_rep_state(&rep_state, &rep_args, NULL, NULL, + target->data_rep, fs, pool)); + + /* If that matches source, then use this delta as is. + Note that we want an actual delta here. E.g. a self-delta would + not be good enough. */ if (rep_args->is_delta - && (rep_args->is_delta_vs_empty - || (rep_args->base_revision == source->data_rep->revision - && rep_args->base_offset == source->data_rep->offset))) + && rep_args->base_revision == source->data_rep->revision + && rep_args->base_offset == source->data_rep->offset) { /* Create the delta read baton. */ struct delta_read_baton *drb = apr_pcalloc(pool, sizeof(*drb)); @@ -3846,11 +5407,83 @@ svn_fs_fs__get_file_delta_stream(svn_txdelta_stream_t **stream_p, else source_stream = svn_stream_empty(pool); SVN_ERR(read_representation(&target_stream, fs, target->data_rep, pool)); - svn_txdelta(stream_p, source_stream, target_stream, pool); + + /* Because source and target stream will already verify their content, + * there is no need to do this once more. In particular if the stream + * content is being fetched from cache. */ + svn_txdelta2(stream_p, source_stream, target_stream, FALSE, pool); return SVN_NO_ERROR; } +/* Baton for cache_access_wrapper. Wraps the original parameters of + * svn_fs_fs__try_process_file_content(). + */ +typedef struct cache_access_wrapper_baton_t +{ + svn_fs_process_contents_func_t func; + void* baton; +} cache_access_wrapper_baton_t; + +/* Wrapper to translate between svn_fs_process_contents_func_t and + * svn_cache__partial_getter_func_t. + */ +static svn_error_t * +cache_access_wrapper(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool) +{ + cache_access_wrapper_baton_t *wrapper_baton = baton; + + SVN_ERR(wrapper_baton->func((const unsigned char *)data, + data_len - 1, /* cache adds terminating 0 */ + wrapper_baton->baton, + pool)); + + /* non-NULL value to signal the calling cache that all went well */ + *out = baton; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_fs__try_process_file_contents(svn_boolean_t *success, + svn_fs_t *fs, + node_revision_t *noderev, + svn_fs_process_contents_func_t processor, + void* baton, + apr_pool_t *pool) +{ + representation_t *rep = noderev->data_rep; + if (rep) + { + fs_fs_data_t *ffd = fs->fsap_data; + pair_cache_key_t fulltext_cache_key = { 0 }; + + fulltext_cache_key.revision = rep->revision; + fulltext_cache_key.second = rep->offset; + if (ffd->fulltext_cache && SVN_IS_VALID_REVNUM(rep->revision) + && fulltext_size_is_cachable(ffd, rep->expanded_size)) + { + cache_access_wrapper_baton_t wrapper_baton; + void *dummy = NULL; + + wrapper_baton.func = processor; + wrapper_baton.baton = baton; + return svn_cache__get_partial(&dummy, success, + ffd->fulltext_cache, + &fulltext_cache_key, + cache_access_wrapper, + &wrapper_baton, + pool); + } + } + + *success = FALSE; + return SVN_NO_ERROR; +} /* Fetch the contents of a directory into ENTRIES. Values are stored as filename to string mappings; further conversion is necessary to @@ -3877,10 +5510,27 @@ get_dir_contents(apr_hash_t *entries, } else if (noderev->data_rep) { + /* use a temporary pool for temp objects. + * Also undeltify content before parsing it. Otherwise, we could only + * parse it byte-by-byte. + */ + apr_pool_t *text_pool = svn_pool_create(pool); + apr_size_t len = noderev->data_rep->expanded_size + ? (apr_size_t)noderev->data_rep->expanded_size + : (apr_size_t)noderev->data_rep->size; + svn_stringbuf_t *text = svn_stringbuf_create_ensure(len, text_pool); + text->len = len; + /* The representation is immutable. Read it normally. */ - SVN_ERR(read_representation(&contents, fs, noderev->data_rep, pool)); - SVN_ERR(svn_hash_read2(entries, contents, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(read_representation(&contents, fs, noderev->data_rep, text_pool)); + SVN_ERR(svn_stream_read(contents, text->data, &text->len)); SVN_ERR(svn_stream_close(contents)); + + /* de-serialize hash */ + contents = svn_stream_from_stringbuf(text, text_pool); + SVN_ERR(svn_hash_read2(entries, contents, SVN_HASH_TERMINATOR, pool)); + + svn_pool_destroy(text_pool); } return SVN_NO_ERROR; @@ -3907,7 +5557,14 @@ unparse_dir_entries(apr_hash_t **str_entries_p, { apr_hash_index_t *hi; - *str_entries_p = apr_hash_make(pool); + /* For now, we use a our own hash function to ensure that we get a + * (largely) stable order when serializing the data. It also gives + * us some performance improvement. + * + * ### TODO ### + * Use some sorted or other fixed order data container. + */ + *str_entries_p = svn_hash__make(pool); for (hi = apr_hash_first(pool, entries); hi; hi = apr_hash_next(hi)) { @@ -3947,10 +5604,10 @@ parse_dir_entries(apr_hash_t **entries_p, char *str, *last_str; svn_fs_dirent_t *dirent = apr_pcalloc(pool, sizeof(*dirent)); - str = apr_pstrdup(pool, str_val->data); + last_str = apr_pstrdup(pool, str_val->data); dirent->name = apr_pstrdup(pool, name); - str = apr_strtok(str, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Directory entry corrupt in '%s'"), @@ -3971,7 +5628,7 @@ parse_dir_entries(apr_hash_t **entries_p, unparsed_id); } - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Directory entry corrupt in '%s'"), @@ -3979,7 +5636,7 @@ parse_dir_entries(apr_hash_t **entries_p, dirent->id = svn_fs_fs__id_parse(str, strlen(str), pool); - apr_hash_set(*entries_p, dirent->name, APR_HASH_KEY_STRING, dirent); + svn_hash_sets(*entries_p, dirent->name, dirent); } return SVN_NO_ERROR; @@ -4038,7 +5695,8 @@ svn_fs_fs__rep_contents_dir_entry(svn_fs_dirent_t **dirent, svn_fs_t *fs, node_revision_t *noderev, const char *name, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { svn_boolean_t found = FALSE; @@ -4047,7 +5705,7 @@ svn_fs_fs__rep_contents_dir_entry(svn_fs_dirent_t **dirent, if (cache) { const char *unparsed_id = - svn_fs_fs__id_unparse(noderev->id, pool)->data; + svn_fs_fs__id_unparse(noderev->id, scratch_pool)->data; /* Cache lookup. */ SVN_ERR(svn_cache__get_partial((void **)dirent, @@ -4056,7 +5714,7 @@ svn_fs_fs__rep_contents_dir_entry(svn_fs_dirent_t **dirent, unparsed_id, svn_fs_fs__extract_dir_entry, (void*)name, - pool)); + result_pool)); } /* fetch data from disk if we did not find it in the cache */ @@ -4066,29 +5724,22 @@ svn_fs_fs__rep_contents_dir_entry(svn_fs_dirent_t **dirent, svn_fs_dirent_t *entry; svn_fs_dirent_t *entry_copy = NULL; - /* since we don't need the directory content later on, put it into - some sub-pool that will be reclaimed immedeately after exiting - this function successfully. Opon failure, it will live as long - as pool. - */ - apr_pool_t *sub_pool = svn_pool_create(pool); - /* read the dir from the file system. It will probably be put it into the cache for faster lookup in future calls. */ - SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, noderev, sub_pool)); + SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, noderev, + scratch_pool)); /* find desired entry and return a copy in POOL, if found */ - entry = apr_hash_get(entries, name, APR_HASH_KEY_STRING); + entry = svn_hash_gets(entries, name); if (entry != NULL) { - entry_copy = apr_palloc(pool, sizeof(*entry_copy)); - entry_copy->name = apr_pstrdup(pool, entry->name); - entry_copy->id = svn_fs_fs__id_copy(entry->id, pool); + entry_copy = apr_palloc(result_pool, sizeof(*entry_copy)); + entry_copy->name = apr_pstrdup(result_pool, entry->name); + entry_copy->id = svn_fs_fs__id_copy(entry->id, result_pool); entry_copy->kind = entry->kind; } *dirent = entry_copy; - apr_pool_destroy(sub_pool); } return SVN_NO_ERROR; @@ -4103,11 +5754,10 @@ svn_fs_fs__get_proplist(apr_hash_t **proplist_p, apr_hash_t *proplist; svn_stream_t *stream; - proplist = apr_hash_make(pool); - if (noderev->prop_rep && noderev->prop_rep->txn_id) { const char *filename = path_txn_node_props(fs, noderev->id, pool); + proplist = apr_hash_make(pool); SVN_ERR(svn_stream_open_readonly(&stream, filename, pool, pool)); SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool)); @@ -4115,9 +5765,33 @@ svn_fs_fs__get_proplist(apr_hash_t **proplist_p, } else if (noderev->prop_rep) { + fs_fs_data_t *ffd = fs->fsap_data; + representation_t *rep = noderev->prop_rep; + pair_cache_key_t key = { 0 }; + + key.revision = rep->revision; + key.second = rep->offset; + if (ffd->properties_cache && SVN_IS_VALID_REVNUM(rep->revision)) + { + svn_boolean_t is_cached; + SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached, + ffd->properties_cache, &key, pool)); + if (is_cached) + return SVN_NO_ERROR; + } + + proplist = apr_hash_make(pool); SVN_ERR(read_representation(&stream, fs, noderev->prop_rep, pool)); SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool)); SVN_ERR(svn_stream_close(stream)); + + if (ffd->properties_cache && SVN_IS_VALID_REVNUM(rep->revision)) + SVN_ERR(svn_cache__set(ffd->properties_cache, &key, proplist, pool)); + } + else + { + /* return an empty prop list if the node doesn't have any props */ + proplist = apr_hash_make(pool); } *proplist_p = proplist; @@ -4222,8 +5896,9 @@ fold_change(apr_hash_t *changes, apr_pool_t *pool = apr_hash_pool_get(changes); svn_fs_path_change2_t *old_change, *new_change; const char *path; + apr_size_t path_len = strlen(change->path); - if ((old_change = apr_hash_get(changes, change->path, APR_HASH_KEY_STRING))) + if ((old_change = apr_hash_get(changes, change->path, path_len))) { /* This path already exists in the hash, so we have to merge this change into the already existing one. */ @@ -4361,8 +6036,8 @@ fold_change(apr_hash_t *changes, re-use its value, but there is no way to fetch it. The API makes no guarantees that this (new) key will not be retained. Thus, we (again) copy the key into the target pool to ensure a proper lifetime. */ - path = apr_pstrdup(pool, change->path); - apr_hash_set(changes, path, APR_HASH_KEY_STRING, new_change); + path = apr_pstrmemdup(pool, change->path, path_len); + apr_hash_set(changes, path, path_len, new_change); /* Update the copyfrom cache, if any. */ if (copyfrom_cache) @@ -4380,10 +6055,12 @@ fold_change(apr_hash_t *changes, } /* We need to allocate a copy of the key in the copyfrom_pool if * we're not doing a deletion and if it isn't already there. */ - if (copyfrom_string && ! apr_hash_get(copyfrom_cache, copyfrom_key, - APR_HASH_KEY_STRING)) - copyfrom_key = apr_pstrdup(copyfrom_pool, copyfrom_key); - apr_hash_set(copyfrom_cache, copyfrom_key, APR_HASH_KEY_STRING, + if ( copyfrom_string + && ( ! apr_hash_count(copyfrom_cache) + || ! apr_hash_get(copyfrom_cache, copyfrom_key, path_len))) + copyfrom_key = apr_pstrmemdup(copyfrom_pool, copyfrom_key, path_len); + + apr_hash_set(copyfrom_cache, copyfrom_key, path_len, copyfrom_string); } @@ -4405,7 +6082,7 @@ read_change(change_t **change_p, char buf[MAX_CHANGE_LINE_LEN]; apr_size_t len = sizeof(buf); change_t *change; - char *str, *last_str, *kind_str; + char *str, *last_str = buf, *kind_str; svn_error_t *err; /* Default return value. */ @@ -4429,7 +6106,7 @@ read_change(change_t **change_p, change = apr_pcalloc(pool, sizeof(*change)); /* Get the node-id of the change. */ - str = apr_strtok(buf, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Invalid changes line in rev-file")); @@ -4440,7 +6117,7 @@ read_change(change_t **change_p, _("Invalid changes line in rev-file")); /* Get the change type. */ - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Invalid changes line in rev-file")); @@ -4490,7 +6167,7 @@ read_change(change_t **change_p, } /* Get the text-mod flag. */ - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Invalid changes line in rev-file")); @@ -4510,7 +6187,7 @@ read_change(change_t **change_p, } /* Get the prop-mod flag. */ - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Invalid changes line in rev-file")); @@ -4544,7 +6221,8 @@ read_change(change_t **change_p, } else { - str = apr_strtok(buf, " ", &last_str); + last_str = buf; + str = svn_cstring_tokenize(" ", &last_str); if (! str) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Invalid changes line in rev-file")); @@ -4562,32 +6240,32 @@ read_change(change_t **change_p, return SVN_NO_ERROR; } -/* Fetch all the changed path entries from FILE and store then in +/* Examine all the changed path entries in CHANGES and store them in *CHANGED_PATHS. Folding is done to remove redundant or unnecessary - *data. Store a hash of paths to copyfrom revisions/paths in + *data. Store a hash of paths to copyfrom "REV PATH" strings in COPYFROM_HASH if it is non-NULL. If PREFOLDED is true, assume that the changed-path entries have already been folded (by write_final_changed_path_info) and may be out of order, so we shouldn't remove children of replaced or deleted directories. Do all allocations in POOL. */ static svn_error_t * -fetch_all_changes(apr_hash_t *changed_paths, - apr_hash_t *copyfrom_hash, - apr_file_t *file, - svn_boolean_t prefolded, - apr_pool_t *pool) +process_changes(apr_hash_t *changed_paths, + apr_hash_t *copyfrom_cache, + apr_array_header_t *changes, + svn_boolean_t prefolded, + apr_pool_t *pool) { - change_t *change; apr_pool_t *iterpool = svn_pool_create(pool); + int i; /* Read in the changes one by one, folding them into our local hash as necessary. */ - SVN_ERR(read_change(&change, file, iterpool)); - - while (change) + for (i = 0; i < changes->nelts; ++i) { - SVN_ERR(fold_change(changed_paths, change, copyfrom_hash)); + change_t *change = APR_ARRAY_IDX(changes, i, change_t *); + + SVN_ERR(fold_change(changed_paths, change, copyfrom_cache)); /* Now, if our change was a deletion or replacement, we have to blow away any changes thus far on paths that are (or, were) @@ -4603,28 +6281,43 @@ fetch_all_changes(apr_hash_t *changed_paths, { apr_hash_index_t *hi; + /* a potential child path must contain at least 2 more chars + (the path separator plus at least one char for the name). + Also, we should not assume that all paths have been normalized + i.e. some might have trailing path separators. + */ + apr_ssize_t change_path_len = strlen(change->path); + apr_ssize_t min_child_len = change_path_len == 0 + ? 1 + : change->path[change_path_len-1] == '/' + ? change_path_len + 1 + : change_path_len + 2; + + /* CAUTION: This is the inner loop of an O(n^2) algorithm. + The number of changes to process may be >> 1000. + Therefore, keep the inner loop as tight as possible. + */ for (hi = apr_hash_first(iterpool, changed_paths); hi; hi = apr_hash_next(hi)) { /* KEY is the path. */ - const char *path = svn__apr_hash_index_key(hi); - apr_ssize_t klen = svn__apr_hash_index_klen(hi); - - /* If we come across our own path, ignore it. */ - if (strcmp(change->path, path) == 0) - continue; - - /* If we come across a child of our path, remove it. */ - if (svn_dirent_is_child(change->path, path, iterpool)) + const void *path; + apr_ssize_t klen; + apr_hash_this(hi, &path, &klen, NULL); + + /* If we come across a child of our path, remove it. + Call svn_dirent_is_child only if there is a chance that + this is actually a sub-path. + */ + if ( klen >= min_child_len + && svn_dirent_is_child(change->path, path, iterpool)) apr_hash_set(changed_paths, path, klen, NULL); } } /* Clear the per-iteration subpool. */ svn_pool_clear(iterpool); - - SVN_ERR(read_change(&change, file, iterpool)); } /* Destroy the per-iteration subpool. */ @@ -4633,6 +6326,29 @@ fetch_all_changes(apr_hash_t *changed_paths, return SVN_NO_ERROR; } +/* Fetch all the changes from FILE and store them in *CHANGES. Do all + allocations in POOL. */ +static svn_error_t * +read_all_changes(apr_array_header_t **changes, + apr_file_t *file, + apr_pool_t *pool) +{ + change_t *change; + + /* pre-allocate enough room for most change lists + (will be auto-expanded as necessary) */ + *changes = apr_array_make(pool, 30, sizeof(change_t *)); + + SVN_ERR(read_change(&change, file, pool)); + while (change) + { + APR_ARRAY_PUSH(*changes, change_t*) = change; + SVN_ERR(read_change(&change, file, pool)); + } + + return SVN_NO_ERROR; +} + svn_error_t * svn_fs_fs__txn_changes_fetch(apr_hash_t **changed_paths_p, svn_fs_t *fs, @@ -4641,11 +6357,15 @@ svn_fs_fs__txn_changes_fetch(apr_hash_t **changed_paths_p, { apr_file_t *file; apr_hash_t *changed_paths = apr_hash_make(pool); + apr_array_header_t *changes; + apr_pool_t *scratch_pool = svn_pool_create(pool); SVN_ERR(svn_io_file_open(&file, path_txn_changes(fs, txn_id, pool), APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool)); - SVN_ERR(fetch_all_changes(changed_paths, NULL, file, FALSE, pool)); + SVN_ERR(read_all_changes(&changes, file, scratch_pool)); + SVN_ERR(process_changes(changed_paths, NULL, changes, FALSE, pool)); + svn_pool_destroy(scratch_pool); SVN_ERR(svn_io_file_close(file, pool)); @@ -4654,16 +6374,31 @@ svn_fs_fs__txn_changes_fetch(apr_hash_t **changed_paths_p, return SVN_NO_ERROR; } -svn_error_t * -svn_fs_fs__paths_changed(apr_hash_t **changed_paths_p, - svn_fs_t *fs, - svn_revnum_t rev, - apr_hash_t *copyfrom_cache, - apr_pool_t *pool) +/* Fetch the list of change in revision REV in FS and return it in *CHANGES. + * Allocate the result in POOL. + */ +static svn_error_t * +get_changes(apr_array_header_t **changes, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *pool) { apr_off_t changes_offset; - apr_hash_t *changed_paths; apr_file_t *revision_file; + svn_boolean_t found; + fs_fs_data_t *ffd = fs->fsap_data; + + /* try cache lookup first */ + + if (ffd->changes_cache) + { + SVN_ERR(svn_cache__get((void **) changes, &found, ffd->changes_cache, + &rev, pool)); + if (found) + return SVN_NO_ERROR; + } + + /* read changes from revision file */ SVN_ERR(ensure_revision_exists(fs, rev, pool)); @@ -4673,14 +6408,37 @@ svn_fs_fs__paths_changed(apr_hash_t **changed_paths_p, rev, pool)); SVN_ERR(svn_io_file_seek(revision_file, APR_SET, &changes_offset, pool)); + SVN_ERR(read_all_changes(changes, revision_file, pool)); - changed_paths = apr_hash_make(pool); + SVN_ERR(svn_io_file_close(revision_file, pool)); - SVN_ERR(fetch_all_changes(changed_paths, copyfrom_cache, revision_file, - TRUE, pool)); + /* cache for future reference */ - /* Close the revision file. */ - SVN_ERR(svn_io_file_close(revision_file, pool)); + if (ffd->changes_cache) + SVN_ERR(svn_cache__set(ffd->changes_cache, &rev, *changes, pool)); + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_fs__paths_changed(apr_hash_t **changed_paths_p, + svn_fs_t *fs, + svn_revnum_t rev, + apr_hash_t *copyfrom_cache, + apr_pool_t *pool) +{ + apr_hash_t *changed_paths; + apr_array_header_t *changes; + apr_pool_t *scratch_pool = svn_pool_create(pool); + + SVN_ERR(get_changes(&changes, fs, rev, scratch_pool)); + + changed_paths = svn_hash__make(pool); + + SVN_ERR(process_changes(changed_paths, copyfrom_cache, changes, + TRUE, pool)); + svn_pool_destroy(scratch_pool); *changed_paths_p = changed_paths; @@ -4734,40 +6492,17 @@ get_and_increment_txn_key_body(void *baton, apr_pool_t *pool) { struct get_and_increment_txn_key_baton *cb = baton; const char *txn_current_filename = path_txn_current(cb->fs, pool); - apr_file_t *txn_current_file = NULL; const char *tmp_filename; char next_txn_id[MAX_KEY_SIZE+3]; - svn_error_t *err = SVN_NO_ERROR; - apr_pool_t *iterpool; apr_size_t len; - int i; - - cb->txn_id = apr_palloc(cb->pool, MAX_KEY_SIZE); - - iterpool = svn_pool_create(pool); - for (i = 0; i < RECOVERABLE_RETRY_COUNT; ++i) - { - svn_pool_clear(iterpool); - RETRY_RECOVERABLE(err, txn_current_file, - svn_io_file_open(&txn_current_file, - txn_current_filename, - APR_READ | APR_BUFFERED, - APR_OS_DEFAULT, iterpool)); - len = MAX_KEY_SIZE; - RETRY_RECOVERABLE(err, txn_current_file, - svn_io_read_length_line(txn_current_file, - cb->txn_id, - &len, - iterpool)); - IGNORE_RECOVERABLE(err, svn_io_file_close(txn_current_file, - iterpool)); + svn_stringbuf_t *buf; + SVN_ERR(read_content(&buf, txn_current_filename, cb->pool)); - break; - } - SVN_ERR(err); - - svn_pool_destroy(iterpool); + /* remove trailing newlines */ + svn_stringbuf_strip_whitespace(buf); + cb->txn_id = buf->data; + len = buf->len; /* Increment the key and add a trailing \n to the string so the txn-current file has a newline in it. */ @@ -4979,7 +6714,7 @@ svn_fs_fs__change_txn_props(svn_fs_txn_t *txn, { svn_prop_t *prop = &APR_ARRAY_IDX(props, i, svn_prop_t); - apr_hash_set(txn_prop, prop->name, APR_HASH_KEY_STRING, prop->value); + svn_hash_sets(txn_prop, prop->name, prop->value); } /* Create a new version of the file and write out the new props. */ @@ -5068,7 +6803,7 @@ read_next_ids(const char **node_id, apr_file_t *file; char buf[MAX_KEY_SIZE*2+3]; apr_size_t limit; - char *str, *last_str; + char *str, *last_str = buf; SVN_ERR(svn_io_file_open(&file, path_txn_next_ids(fs, txn_id, pool), APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool)); @@ -5080,14 +6815,14 @@ read_next_ids(const char **node_id, /* Parse this into two separate strings. */ - str = apr_strtok(buf, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (! str) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("next-id file corrupt")); *node_id = apr_pstrdup(pool, str); - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &last_str); if (! str) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("next-id file corrupt")); @@ -5207,35 +6942,37 @@ svn_fs_fs__set_entry(svn_fs_t *fs, apr_file_t *file; svn_stream_t *out; fs_fs_data_t *ffd = fs->fsap_data; + apr_pool_t *subpool = svn_pool_create(pool); if (!rep || !rep->txn_id) { const char *unique_suffix; + apr_hash_t *entries; - { - apr_hash_t *entries; - apr_pool_t *subpool = svn_pool_create(pool); - - /* Before we can modify the directory, we need to dump its old - contents into a mutable representation file. */ - SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, parent_noderev, - subpool)); - SVN_ERR(unparse_dir_entries(&entries, entries, subpool)); - SVN_ERR(svn_io_file_open(&file, filename, - APR_WRITE | APR_CREATE | APR_BUFFERED, - APR_OS_DEFAULT, pool)); - out = svn_stream_from_aprfile2(file, TRUE, pool); - SVN_ERR(svn_hash_write2(entries, out, SVN_HASH_TERMINATOR, subpool)); - - svn_pool_destroy(subpool); - } + /* Before we can modify the directory, we need to dump its old + contents into a mutable representation file. */ + SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, parent_noderev, + subpool)); + SVN_ERR(unparse_dir_entries(&entries, entries, subpool)); + SVN_ERR(svn_io_file_open(&file, filename, + APR_WRITE | APR_CREATE | APR_BUFFERED, + APR_OS_DEFAULT, pool)); + out = svn_stream_from_aprfile2(file, TRUE, pool); + SVN_ERR(svn_hash_write2(entries, out, SVN_HASH_TERMINATOR, subpool)); + + svn_pool_clear(subpool); /* Mark the node-rev's data rep as mutable. */ rep = apr_pcalloc(pool, sizeof(*rep)); rep->revision = SVN_INVALID_REVNUM; rep->txn_id = txn_id; - SVN_ERR(get_new_txn_node_id(&unique_suffix, fs, txn_id, pool)); - rep->uniquifier = apr_psprintf(pool, "%s/%s", txn_id, unique_suffix); + + if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) + { + SVN_ERR(get_new_txn_node_id(&unique_suffix, fs, txn_id, pool)); + rep->uniquifier = apr_psprintf(pool, "%s/%s", txn_id, unique_suffix); + } + parent_noderev->data_rep = rep; SVN_ERR(svn_fs_fs__put_node_revision(fs, parent_noderev->id, parent_noderev, FALSE, pool)); @@ -5251,12 +6988,11 @@ svn_fs_fs__set_entry(svn_fs_t *fs, /* if we have a directory cache for this transaction, update it */ if (ffd->txn_dir_cache) { - apr_pool_t *subpool = svn_pool_create(pool); - /* build parameters: (name, new entry) pair */ const char *key = svn_fs_fs__id_unparse(parent_noderev->id, subpool)->data; replace_baton_t baton; + baton.name = name; baton.new_entry = NULL; @@ -5269,28 +7005,31 @@ svn_fs_fs__set_entry(svn_fs_t *fs, } /* actually update the cached directory (if cached) */ - SVN_ERR(svn_cache__set_partial(ffd->txn_dir_cache, key, svn_fs_fs__replace_dir_entry, &baton, subpool)); - - svn_pool_destroy(subpool); + SVN_ERR(svn_cache__set_partial(ffd->txn_dir_cache, key, + svn_fs_fs__replace_dir_entry, &baton, + subpool)); } + svn_pool_clear(subpool); /* Append an incremental hash entry for the entry change. */ if (id) { - const char *val = unparse_dir_entry(kind, id, pool); + const char *val = unparse_dir_entry(kind, id, subpool); - SVN_ERR(svn_stream_printf(out, pool, "K %" APR_SIZE_T_FMT "\n%s\n" + SVN_ERR(svn_stream_printf(out, subpool, "K %" APR_SIZE_T_FMT "\n%s\n" "V %" APR_SIZE_T_FMT "\n%s\n", strlen(name), name, strlen(val), val)); } else { - SVN_ERR(svn_stream_printf(out, pool, "D %" APR_SIZE_T_FMT "\n%s\n", + SVN_ERR(svn_stream_printf(out, subpool, "D %" APR_SIZE_T_FMT "\n%s\n", strlen(name), name)); } - return svn_io_file_close(file, pool); + SVN_ERR(svn_io_file_close(file, subpool)); + svn_pool_destroy(subpool); + return SVN_NO_ERROR; } /* Write a single change entry, path PATH, change CHANGE, and copyfrom @@ -5458,15 +7197,21 @@ rep_write_contents(void *baton, /* Given a node-revision NODEREV in filesystem FS, return the representation in *REP to use as the base for a text representation - delta. Perform temporary allocations in *POOL. */ + delta if PROPS is FALSE. If PROPS has been set, a suitable props + base representation will be returned. Perform temporary allocations + in *POOL. */ static svn_error_t * choose_delta_base(representation_t **rep, svn_fs_t *fs, node_revision_t *noderev, + svn_boolean_t props, apr_pool_t *pool) { int count; + int walk; node_revision_t *base; + fs_fs_data_t *ffd = fs->fsap_data; + svn_boolean_t maybe_shared_rep = FALSE; /* If we have no predecessors, then use the empty stream as a base. */ @@ -5483,20 +7228,141 @@ choose_delta_base(representation_t **rep, count = noderev->predecessor_count; count = count & (count - 1); + /* We use skip delta for limiting the number of delta operations + along very long node histories. Close to HEAD however, we create + a linear history to minimize delta size. */ + walk = noderev->predecessor_count - count; + if (walk < (int)ffd->max_linear_deltification) + count = noderev->predecessor_count - 1; + + /* Finding the delta base over a very long distance can become extremely + expensive for very deep histories, possibly causing client timeouts etc. + OTOH, this is a rare operation and its gains are minimal. Lets simply + start deltification anew close every other 1000 changes or so. */ + if (walk > (int)ffd->max_deltification_walk) + { + *rep = NULL; + return SVN_NO_ERROR; + } + /* Walk back a number of predecessors equal to the difference between count and the original predecessor count. (For example, if noderev has ten predecessors and we want the eighth file rev, walk back two predecessors.) */ base = noderev; while ((count++) < noderev->predecessor_count) - SVN_ERR(svn_fs_fs__get_node_revision(&base, fs, - base->predecessor_id, pool)); + { + SVN_ERR(svn_fs_fs__get_node_revision(&base, fs, + base->predecessor_id, pool)); + + /* If there is a shared rep along the way, we need to limit the + * length of the deltification chain. + * + * Please note that copied nodes - such as branch directories - will + * look the same (false positive) while reps shared within the same + * revision will not be caught (false negative). + */ + if (props) + { + if ( base->prop_rep + && svn_fs_fs__id_rev(base->id) > base->prop_rep->revision) + maybe_shared_rep = TRUE; + } + else + { + if ( base->data_rep + && svn_fs_fs__id_rev(base->id) > base->data_rep->revision) + maybe_shared_rep = TRUE; + } + } + + /* return a suitable base representation */ + *rep = props ? base->prop_rep : base->data_rep; + + /* if we encountered a shared rep, it's parent chain may be different + * from the node-rev parent chain. */ + if (*rep && maybe_shared_rep) + { + /* Check whether the length of the deltification chain is acceptable. + * Otherwise, shared reps may form a non-skipping delta chain in + * extreme cases. */ + apr_pool_t *sub_pool = svn_pool_create(pool); + representation_t base_rep = **rep; + + /* Some reasonable limit, depending on how acceptable longer linear + * chains are in this repo. Also, allow for some minimal chain. */ + int max_chain_length = 2 * (int)ffd->max_linear_deltification + 2; - *rep = base->data_rep; + /* re-use open files between iterations */ + svn_revnum_t rev_hint = SVN_INVALID_REVNUM; + apr_file_t *file_hint = NULL; + /* follow the delta chain towards the end but for at most + * MAX_CHAIN_LENGTH steps. */ + for (; max_chain_length; --max_chain_length) + { + struct rep_state *rep_state; + struct rep_args *rep_args; + + SVN_ERR(create_rep_state_body(&rep_state, + &rep_args, + &file_hint, + &rev_hint, + &base_rep, + fs, + sub_pool)); + if (!rep_args->is_delta || !rep_args->base_revision) + break; + + base_rep.revision = rep_args->base_revision; + base_rep.offset = rep_args->base_offset; + base_rep.size = rep_args->base_length; + base_rep.txn_id = NULL; + } + + /* start new delta chain if the current one has grown too long */ + if (max_chain_length == 0) + *rep = NULL; + + svn_pool_destroy(sub_pool); + } + + /* verify that the reps don't form a degenerated '*/ return SVN_NO_ERROR; } +/* Something went wrong and the pool for the rep write is being + cleared before we've finished writing the rep. So we need + to remove the rep from the protorevfile and we need to unlock + the protorevfile. */ +static apr_status_t +rep_write_cleanup(void *data) +{ + struct rep_write_baton *b = data; + const char *txn_id = svn_fs_fs__id_txn_id(b->noderev->id); + svn_error_t *err; + + /* Truncate and close the protorevfile. */ + err = svn_io_file_trunc(b->file, b->rep_offset, b->pool); + err = svn_error_compose_create(err, svn_io_file_close(b->file, b->pool)); + + /* Remove our lock regardless of any preceeding errors so that the + being_written flag is always removed and stays consistent with the + file lock which will be removed no matter what since the pool is + going away. */ + err = svn_error_compose_create(err, unlock_proto_rev(b->fs, txn_id, + b->lockcookie, b->pool)); + if (err) + { + apr_status_t rc = err->apr_err; + svn_error_clear(err); + return rc; + } + + return APR_SUCCESS; +} + + /* Get a rep_write_baton and store it in *WB_P for the representation indicated by NODEREV in filesystem FS. Perform allocations in POOL. Only appropriate for file contents, not for props or @@ -5539,7 +7405,7 @@ rep_write_get_baton(struct rep_write_baton **wb_p, SVN_ERR(get_file_offset(&b->rep_offset, file, b->pool)); /* Get the base for this delta. */ - SVN_ERR(choose_delta_base(&base_rep, fs, noderev, b->pool)); + SVN_ERR(choose_delta_base(&base_rep, fs, noderev, FALSE, b->pool)); SVN_ERR(read_representation(&source, fs, base_rep, b->pool)); /* Write out the rep header. */ @@ -5560,6 +7426,10 @@ rep_write_get_baton(struct rep_write_baton **wb_p, /* Now determine the offset of the actual svndiff data. */ SVN_ERR(get_file_offset(&b->delta_start, file, b->pool)); + /* Cleanup in case something goes wrong. */ + apr_pool_cleanup_register(b->pool, b, rep_write_cleanup, + apr_pool_cleanup_null); + /* Prepare to write the svndiff data. */ svn_txdelta_to_svndiff3(&wh, &whb, @@ -5575,6 +7445,106 @@ rep_write_get_baton(struct rep_write_baton **wb_p, return SVN_NO_ERROR; } +/* For the hash REP->SHA1, try to find an already existing representation + in FS and return it in *OUT_REP. If no such representation exists or + if rep sharing has been disabled for FS, NULL will be returned. Since + there may be new duplicate representations within the same uncommitted + revision, those can be passed in REPS_HASH (maps a sha1 digest onto + representation_t*), otherwise pass in NULL for REPS_HASH. + POOL will be used for allocations. The lifetime of the returned rep is + limited by both, POOL and REP lifetime. + */ +static svn_error_t * +get_shared_rep(representation_t **old_rep, + svn_fs_t *fs, + representation_t *rep, + apr_hash_t *reps_hash, + apr_pool_t *pool) +{ + svn_error_t *err; + fs_fs_data_t *ffd = fs->fsap_data; + + /* Return NULL, if rep sharing has been disabled. */ + *old_rep = NULL; + if (!ffd->rep_sharing_allowed) + return SVN_NO_ERROR; + + /* Check and see if we already have a representation somewhere that's + identical to the one we just wrote out. Start with the hash lookup + because it is cheepest. */ + if (reps_hash) + *old_rep = apr_hash_get(reps_hash, + rep->sha1_checksum->digest, + APR_SHA1_DIGESTSIZE); + + /* If we haven't found anything yet, try harder and consult our DB. */ + if (*old_rep == NULL) + { + err = svn_fs_fs__get_rep_reference(old_rep, fs, rep->sha1_checksum, + pool); + /* ### Other error codes that we shouldn't mask out? */ + if (err == SVN_NO_ERROR) + { + if (*old_rep) + SVN_ERR(verify_walker(*old_rep, NULL, fs, pool)); + } + else if (err->apr_err == SVN_ERR_FS_CORRUPT + || SVN_ERROR_IN_CATEGORY(err->apr_err, + SVN_ERR_MALFUNC_CATEGORY_START)) + { + /* Fatal error; don't mask it. + + In particular, this block is triggered when the rep-cache refers + to revisions in the future. We signal that as a corruption situation + since, once those revisions are less than youngest (because of more + commits), the rep-cache would be invalid. + */ + SVN_ERR(err); + } + else + { + /* Something's wrong with the rep-sharing index. We can continue + without rep-sharing, but warn. + */ + (fs->warning)(fs->warning_baton, err); + svn_error_clear(err); + *old_rep = NULL; + } + } + + /* look for intra-revision matches (usually data reps but not limited + to them in case props happen to look like some data rep) + */ + if (*old_rep == NULL && rep->txn_id) + { + svn_node_kind_t kind; + const char *file_name + = path_txn_sha1(fs, rep->txn_id, rep->sha1_checksum, pool); + + /* in our txn, is there a rep file named with the wanted SHA1? + If so, read it and use that rep. + */ + SVN_ERR(svn_io_check_path(file_name, &kind, pool)); + if (kind == svn_node_file) + { + svn_stringbuf_t *rep_string; + SVN_ERR(svn_stringbuf_from_file2(&rep_string, file_name, pool)); + SVN_ERR(read_rep_offsets_body(old_rep, rep_string->data, + rep->txn_id, FALSE, pool)); + } + } + + /* Add information that is missing in the cached data. */ + if (*old_rep) + { + /* Use the old rep for this content. */ + (*old_rep)->md5_checksum = rep->md5_checksum; + (*old_rep)->uniquifier = rep->uniquifier; + } + + return SVN_NO_ERROR; +} + /* Close handler for the representation write stream. BATON is a rep_write_baton. Writes out a new node-rev that correctly references the representation we just finished writing. */ @@ -5582,11 +7552,11 @@ static svn_error_t * rep_write_contents_close(void *baton) { struct rep_write_baton *b = baton; - fs_fs_data_t *ffd = b->fs->fsap_data; const char *unique_suffix; representation_t *rep; representation_t *old_rep; apr_off_t offset; + fs_fs_data_t *ffd = b->fs->fsap_data; rep = apr_pcalloc(b->parent_pool, sizeof(*rep)); rep->offset = b->rep_offset; @@ -5603,9 +7573,13 @@ rep_write_contents_close(void *baton) /* Fill in the rest of the representation field. */ rep->expanded_size = b->rep_size; rep->txn_id = svn_fs_fs__id_txn_id(b->noderev->id); - SVN_ERR(get_new_txn_node_id(&unique_suffix, b->fs, rep->txn_id, b->pool)); - rep->uniquifier = apr_psprintf(b->parent_pool, "%s/%s", rep->txn_id, - unique_suffix); + + if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) + { + SVN_ERR(get_new_txn_node_id(&unique_suffix, b->fs, rep->txn_id, b->pool)); + rep->uniquifier = apr_psprintf(b->parent_pool, "%s/%s", rep->txn_id, + unique_suffix); + } rep->revision = SVN_INVALID_REVNUM; /* Finalize the checksum. */ @@ -5616,38 +7590,7 @@ rep_write_contents_close(void *baton) /* Check and see if we already have a representation somewhere that's identical to the one we just wrote out. */ - if (ffd->rep_sharing_allowed) - { - svn_error_t *err; - err = svn_fs_fs__get_rep_reference(&old_rep, b->fs, rep->sha1_checksum, - b->parent_pool); - /* ### Other error codes that we shouldn't mask out? */ - if (err == SVN_NO_ERROR - || err->apr_err == SVN_ERR_FS_CORRUPT - || SVN_ERROR_IN_CATEGORY(err->apr_err, - SVN_ERR_MALFUNC_CATEGORY_START)) - { - /* Fatal error; don't mask it. - - In particular, this block is triggered when the rep-cache refers - to revisions in the future. We signal that as a corruption situation - since, once those revisions are less than youngest (because of more - commits), the rep-cache would be invalid. - */ - SVN_ERR(err); - } - else - { - /* Something's wrong with the rep-sharing index. We can continue - without rep-sharing, but warn. - */ - (b->fs->warning)(b->fs->warning_baton, err); - svn_error_clear(err); - old_rep = NULL; - } - } - else - old_rep = NULL; + SVN_ERR(get_shared_rep(&old_rep, b->fs, rep, NULL, b->parent_pool)); if (old_rep) { @@ -5655,21 +7598,24 @@ rep_write_contents_close(void *baton) SVN_ERR(svn_io_file_trunc(b->file, b->rep_offset, b->pool)); /* Use the old rep for this content. */ - old_rep->md5_checksum = rep->md5_checksum; - old_rep->uniquifier = rep->uniquifier; b->noderev->data_rep = old_rep; } else { /* Write out our cosmetic end marker. */ - SVN_ERR(svn_stream_printf(b->rep_stream, b->pool, "ENDREP\n")); + SVN_ERR(svn_stream_puts(b->rep_stream, "ENDREP\n")); b->noderev->data_rep = rep; } + /* Remove cleanup callback. */ + apr_pool_cleanup_kill(b->pool, b, rep_write_cleanup); + /* Write out the new node-rev information. */ SVN_ERR(svn_fs_fs__put_node_revision(b->fs, b->noderev->id, b->noderev, FALSE, b->pool)); + if (!old_rep) + SVN_ERR(store_sha1_rep_mapping(b->fs, b->noderev, b->pool)); SVN_ERR(svn_io_file_close(b->file, b->pool)); SVN_ERR(unlock_proto_rev(b->fs, rep->txn_id, b->lockcookie, b->pool)); @@ -5790,23 +7736,25 @@ get_next_revision_ids(const char **node_id, apr_pool_t *pool) { char *buf; - char *str, *last_str; + char *str; + svn_stringbuf_t *content; - SVN_ERR(read_current(svn_fs_fs__path_current(fs, pool), &buf, pool)); + SVN_ERR(read_content(&content, svn_fs_fs__path_current(fs, pool), pool)); + buf = content->data; - str = apr_strtok(buf, " ", &last_str); + str = svn_cstring_tokenize(" ", &buf); if (! str) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Corrupt 'current' file")); - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &buf); if (! str) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Corrupt 'current' file")); *node_id = apr_pstrdup(pool, str); - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" \n", &buf); if (! str) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Corrupt 'current' file")); @@ -5823,7 +7771,8 @@ struct write_hash_baton apr_size_t size; - svn_checksum_ctx_t *checksum_ctx; + svn_checksum_ctx_t *md5_ctx; + svn_checksum_ctx_t *sha1_ctx; }; /* The handler for the write_hash_rep stream. BATON is a @@ -5836,7 +7785,8 @@ write_hash_handler(void *baton, { struct write_hash_baton *whb = baton; - SVN_ERR(svn_checksum_update(whb->checksum_ctx, data, *len)); + SVN_ERR(svn_checksum_update(whb->md5_ctx, data, *len)); + SVN_ERR(svn_checksum_update(whb->sha1_ctx, data, *len)); SVN_ERR(svn_stream_write(whb->stream, data, len)); whb->size += *len; @@ -5845,41 +7795,184 @@ write_hash_handler(void *baton, } /* Write out the hash HASH as a text representation to file FILE. In - the process, record the total size of the dump in *SIZE, and the - md5 digest in CHECKSUM. Perform temporary allocations in POOL. */ + the process, record position, the total size of the dump and MD5 as + well as SHA1 in REP. If rep sharing has been enabled and REPS_HASH + is not NULL, it will be used in addition to the on-disk cache to find + earlier reps with the same content. When such existing reps can be + found, we will truncate the one just written from the file and return + the existing rep. Perform temporary allocations in POOL. */ static svn_error_t * -write_hash_rep(svn_filesize_t *size, - svn_checksum_t **checksum, +write_hash_rep(representation_t *rep, apr_file_t *file, apr_hash_t *hash, + svn_fs_t *fs, + apr_hash_t *reps_hash, apr_pool_t *pool) { svn_stream_t *stream; struct write_hash_baton *whb; + representation_t *old_rep; + + SVN_ERR(get_file_offset(&rep->offset, file, pool)); whb = apr_pcalloc(pool, sizeof(*whb)); whb->stream = svn_stream_from_aprfile2(file, TRUE, pool); whb->size = 0; - whb->checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); + whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); + whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool); stream = svn_stream_create(whb, pool); svn_stream_set_write(stream, write_hash_handler); - SVN_ERR(svn_stream_printf(whb->stream, pool, "PLAIN\n")); + SVN_ERR(svn_stream_puts(whb->stream, "PLAIN\n")); SVN_ERR(svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, pool)); /* Store the results. */ - SVN_ERR(svn_checksum_final(checksum, whb->checksum_ctx, pool)); - *size = whb->size; + SVN_ERR(svn_checksum_final(&rep->md5_checksum, whb->md5_ctx, pool)); + SVN_ERR(svn_checksum_final(&rep->sha1_checksum, whb->sha1_ctx, pool)); + + /* Check and see if we already have a representation somewhere that's + identical to the one we just wrote out. */ + SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, pool)); + + if (old_rep) + { + /* We need to erase from the protorev the data we just wrote. */ + SVN_ERR(svn_io_file_trunc(file, rep->offset, pool)); + + /* Use the old rep for this content. */ + memcpy(rep, old_rep, sizeof (*rep)); + } + else + { + /* Write out our cosmetic end marker. */ + SVN_ERR(svn_stream_puts(whb->stream, "ENDREP\n")); + + /* update the representation */ + rep->size = whb->size; + rep->expanded_size = whb->size; + } + + return SVN_NO_ERROR; +} + +/* Write out the hash HASH pertaining to the NODEREV in FS as a deltified + text representation to file FILE. In the process, record the total size + and the md5 digest in REP. If rep sharing has been enabled and REPS_HASH + is not NULL, it will be used in addition to the on-disk cache to find + earlier reps with the same content. When such existing reps can be found, + we will truncate the one just written from the file and return the existing + rep. If PROPS is set, assume that we want to a props representation as + the base for our delta. Perform temporary allocations in POOL. */ +static svn_error_t * +write_hash_delta_rep(representation_t *rep, + apr_file_t *file, + apr_hash_t *hash, + svn_fs_t *fs, + node_revision_t *noderev, + apr_hash_t *reps_hash, + svn_boolean_t props, + apr_pool_t *pool) +{ + svn_txdelta_window_handler_t diff_wh; + void *diff_whb; + + svn_stream_t *file_stream; + svn_stream_t *stream; + representation_t *base_rep; + representation_t *old_rep; + svn_stream_t *source; + const char *header; + + apr_off_t rep_end = 0; + apr_off_t delta_start = 0; + + struct write_hash_baton *whb; + fs_fs_data_t *ffd = fs->fsap_data; + int diff_version = ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT ? 1 : 0; + + /* Get the base for this delta. */ + SVN_ERR(choose_delta_base(&base_rep, fs, noderev, props, pool)); + SVN_ERR(read_representation(&source, fs, base_rep, pool)); + + SVN_ERR(get_file_offset(&rep->offset, file, pool)); + + /* Write out the rep header. */ + if (base_rep) + { + header = apr_psprintf(pool, REP_DELTA " %ld %" APR_OFF_T_FMT " %" + SVN_FILESIZE_T_FMT "\n", + base_rep->revision, base_rep->offset, + base_rep->size); + } + else + { + header = REP_DELTA "\n"; + } + SVN_ERR(svn_io_file_write_full(file, header, strlen(header), NULL, + pool)); + + SVN_ERR(get_file_offset(&delta_start, file, pool)); + file_stream = svn_stream_from_aprfile2(file, TRUE, pool); + + /* Prepare to write the svndiff data. */ + svn_txdelta_to_svndiff3(&diff_wh, + &diff_whb, + file_stream, + diff_version, + SVN_DELTA_COMPRESSION_LEVEL_DEFAULT, + pool); + + whb = apr_pcalloc(pool, sizeof(*whb)); + whb->stream = svn_txdelta_target_push(diff_wh, diff_whb, source, pool); + whb->size = 0; + whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); + whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool); + + /* serialize the hash */ + stream = svn_stream_create(whb, pool); + svn_stream_set_write(stream, write_hash_handler); + + SVN_ERR(svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR(svn_stream_close(whb->stream)); + + /* Store the results. */ + SVN_ERR(svn_checksum_final(&rep->md5_checksum, whb->md5_ctx, pool)); + SVN_ERR(svn_checksum_final(&rep->sha1_checksum, whb->sha1_ctx, pool)); + + /* Check and see if we already have a representation somewhere that's + identical to the one we just wrote out. */ + SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, pool)); + + if (old_rep) + { + /* We need to erase from the protorev the data we just wrote. */ + SVN_ERR(svn_io_file_trunc(file, rep->offset, pool)); + + /* Use the old rep for this content. */ + memcpy(rep, old_rep, sizeof (*rep)); + } + else + { + /* Write out our cosmetic end marker. */ + SVN_ERR(get_file_offset(&rep_end, file, pool)); + SVN_ERR(svn_stream_puts(file_stream, "ENDREP\n")); + + /* update the representation */ + rep->expanded_size = whb->size; + rep->size = rep_end - delta_start; + } - return svn_stream_printf(whb->stream, pool, "ENDREP\n"); + return SVN_NO_ERROR; } /* Sanity check ROOT_NODEREV, a candidate for being the root node-revision of (not yet committed) revision REV in FS. Use POOL for temporary allocations. + + If you change this function, consider updating svn_fs_fs__verify() too. */ static svn_error_t * validate_root_noderev(svn_fs_t *fs, @@ -5911,7 +8004,7 @@ validate_root_noderev(svn_fs_t *fs, This kind of corruption was seen on svn.apache.org (both on the root noderev and on other fspaths' noderevs); see - http://mid.gmane.org/20111002202833.GA12373@daniel3.local + issue #4129. Normally (rev == root_noderev->predecessor_count), but here we use a more roundabout check that should only trigger on new instances @@ -5953,6 +8046,10 @@ validate_root_noderev(svn_fs_t *fs, If REPS_TO_CACHE is not NULL, append to it a copy (allocated in REPS_POOL) of each data rep that is new in this revision. + If REPS_HASH is not NULL, append copies (allocated in REPS_POOL) + of the representations of each property rep that is new in this + revision. + AT_ROOT is true if the node revision being written is the root node-revision. It is only controls additional sanity checking logic. @@ -5968,6 +8065,7 @@ write_final_rev(const svn_fs_id_t **new_id_p, const char *start_copy_id, apr_off_t initial_offset, apr_array_header_t *reps_to_cache, + apr_hash_t *reps_hash, apr_pool_t *reps_pool, svn_boolean_t at_root, apr_pool_t *pool) @@ -6013,7 +8111,7 @@ write_final_rev(const svn_fs_id_t **new_id_p, svn_pool_clear(subpool); SVN_ERR(write_final_rev(&new_id, file, rev, fs, dirent->id, start_node_id, start_copy_id, initial_offset, - reps_to_cache, reps_pool, FALSE, + reps_to_cache, reps_hash, reps_pool, FALSE, subpool)); if (new_id && (svn_fs_fs__id_rev(new_id) == rev)) dirent->id = svn_fs_fs__id_copy(new_id, pool); @@ -6027,11 +8125,14 @@ write_final_rev(const svn_fs_id_t **new_id_p, noderev->data_rep->txn_id = NULL; noderev->data_rep->revision = rev; - SVN_ERR(get_file_offset(&noderev->data_rep->offset, file, pool)); - SVN_ERR(write_hash_rep(&noderev->data_rep->size, - &noderev->data_rep->md5_checksum, file, - str_entries, pool)); - noderev->data_rep->expanded_size = noderev->data_rep->size; + + if (ffd->deltify_directories) + SVN_ERR(write_hash_delta_rep(noderev->data_rep, file, + str_entries, fs, noderev, NULL, + FALSE, pool)); + else + SVN_ERR(write_hash_rep(noderev->data_rep, file, str_entries, + fs, NULL, pool)); } } else @@ -6059,15 +8160,18 @@ write_final_rev(const svn_fs_id_t **new_id_p, if (noderev->prop_rep && noderev->prop_rep->txn_id) { apr_hash_t *proplist; - SVN_ERR(svn_fs_fs__get_proplist(&proplist, fs, noderev, pool)); - SVN_ERR(get_file_offset(&noderev->prop_rep->offset, file, pool)); - SVN_ERR(write_hash_rep(&noderev->prop_rep->size, - &noderev->prop_rep->md5_checksum, file, - proplist, pool)); noderev->prop_rep->txn_id = NULL; noderev->prop_rep->revision = rev; + + if (ffd->deltify_properties) + SVN_ERR(write_hash_delta_rep(noderev->prop_rep, file, + proplist, fs, noderev, reps_hash, + TRUE, pool)); + else + SVN_ERR(write_hash_rep(noderev->prop_rep, file, proplist, + fs, reps_hash, pool)); } @@ -6110,24 +8214,53 @@ write_final_rev(const svn_fs_id_t **new_id_p, noderev->id = new_id; + if (ffd->rep_sharing_allowed) + { + /* Save the data representation's hash in the rep cache. */ + if ( noderev->data_rep && noderev->kind == svn_node_file + && noderev->data_rep->revision == rev) + { + SVN_ERR_ASSERT(reps_to_cache && reps_pool); + APR_ARRAY_PUSH(reps_to_cache, representation_t *) + = svn_fs_fs__rep_copy(noderev->data_rep, reps_pool); + } + + if (noderev->prop_rep && noderev->prop_rep->revision == rev) + { + /* Add new property reps to hash and on-disk cache. */ + representation_t *copy + = svn_fs_fs__rep_copy(noderev->prop_rep, reps_pool); + + SVN_ERR_ASSERT(reps_to_cache && reps_pool); + APR_ARRAY_PUSH(reps_to_cache, representation_t *) = copy; + + apr_hash_set(reps_hash, + copy->sha1_checksum->digest, + APR_SHA1_DIGESTSIZE, + copy); + } + } + + /* don't serialize SHA1 for dirs to disk (waste of space) */ + if (noderev->data_rep && noderev->kind == svn_node_dir) + noderev->data_rep->sha1_checksum = NULL; + + /* don't serialize SHA1 for props to disk (waste of space) */ + if (noderev->prop_rep) + noderev->prop_rep->sha1_checksum = NULL; + + /* Workaround issue #4031: is-fresh-txn-root in revision files. */ + noderev->is_fresh_txn_root = FALSE; + /* Write out our new node-revision. */ if (at_root) SVN_ERR(validate_root_noderev(fs, noderev, rev, pool)); + SVN_ERR(svn_fs_fs__write_noderev(svn_stream_from_aprfile2(file, TRUE, pool), noderev, ffd->format, svn_fs_fs__fs_supports_mergeinfo(fs), pool)); - /* Save the data representation's hash in the rep cache. */ - if (ffd->rep_sharing_allowed - && noderev->data_rep && noderev->kind == svn_node_file - && noderev->data_rep->revision == rev) - { - SVN_ERR_ASSERT(reps_to_cache && reps_pool); - APR_ARRAY_PUSH(reps_to_cache, representation_t *) - = svn_fs_fs__rep_copy(noderev->data_rep, reps_pool); - } - /* Return our ID that references the revision file. */ *new_id_p = noderev->id; @@ -6231,6 +8364,51 @@ write_current(svn_fs_t *fs, svn_revnum_t rev, const char *next_node_id, return move_into_place(tmp_name, name, name, pool); } +/* Open a new svn_fs_t handle to FS, set that handle's concept of "current + youngest revision" to NEW_REV, and call svn_fs_fs__verify_root() on + NEW_REV's revision root. + + Intended to be called as the very last step in a commit before 'current' + is bumped. This implies that we are holding the write lock. */ +static svn_error_t * +verify_as_revision_before_current_plus_plus(svn_fs_t *fs, + svn_revnum_t new_rev, + apr_pool_t *pool) +{ +#ifdef SVN_DEBUG + fs_fs_data_t *ffd = fs->fsap_data; + svn_fs_t *ft; /* fs++ == ft */ + svn_fs_root_t *root; + fs_fs_data_t *ft_ffd; + apr_hash_t *fs_config; + + SVN_ERR_ASSERT(ffd->svn_fs_open_); + + /* make sure FT does not simply return data cached by other instances + * but actually retrieves it from disk at least once. + */ + fs_config = apr_hash_make(pool); + svn_hash_sets(fs_config, SVN_FS_CONFIG_FSFS_CACHE_NS, + svn_uuid_generate(pool)); + SVN_ERR(ffd->svn_fs_open_(&ft, fs->path, + fs_config, + pool)); + ft_ffd = ft->fsap_data; + /* Don't let FT consult rep-cache.db, either. */ + ft_ffd->rep_sharing_allowed = FALSE; + + /* Time travel! */ + ft_ffd->youngest_rev_cache = new_rev; + + SVN_ERR(svn_fs_fs__revision_root(&root, ft, new_rev, pool)); + SVN_ERR_ASSERT(root->is_txn_root == FALSE && root->rev == new_rev); + SVN_ERR_ASSERT(ft_ffd->youngest_rev_cache == new_rev); + SVN_ERR(svn_fs_fs__verify_root(root, pool)); +#endif /* SVN_DEBUG */ + + return SVN_NO_ERROR; +} + /* Update the 'current' file to hold the correct next node and copy_ids from transaction TXN_ID in filesystem FS. The current revision is set to REV. Perform temporary allocations in POOL. */ @@ -6305,7 +8483,7 @@ verify_locks(svn_fs_t *fs, continue; /* Fetch the change associated with our path. */ - change = apr_hash_get(changes, path, APR_HASH_KEY_STRING); + change = svn_hash_gets(changes, path); /* What does it mean to succeed at lock verification for a given path? For an existing file or directory getting modified @@ -6341,6 +8519,7 @@ struct commit_baton { svn_fs_t *fs; svn_fs_txn_t *txn; apr_array_header_t *reps_to_cache; + apr_hash_t *reps_hash; apr_pool_t *reps_pool; }; @@ -6398,8 +8577,8 @@ commit_body(void *baton, apr_pool_t *pool) root_id = svn_fs_fs__id_txn_create("0", "0", cb->txn->id, pool); SVN_ERR(write_final_rev(&new_root_id, proto_file, new_rev, cb->fs, root_id, start_node_id, start_copy_id, initial_offset, - cb->reps_to_cache, cb->reps_pool, TRUE, - pool)); + cb->reps_to_cache, cb->reps_hash, cb->reps_pool, + TRUE, pool)); /* Write the changed-path information. */ SVN_ERR(write_final_changed_path_info(&changed_path_offset, proto_file, @@ -6423,14 +8602,13 @@ commit_body(void *baton, apr_pool_t *pool) txnprop_list = apr_array_make(pool, 3, sizeof(svn_prop_t)); prop.value = NULL; - if (apr_hash_get(txnprops, SVN_FS__PROP_TXN_CHECK_OOD, APR_HASH_KEY_STRING)) + if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_OOD)) { prop.name = SVN_FS__PROP_TXN_CHECK_OOD; APR_ARRAY_PUSH(txnprop_list, svn_prop_t) = prop; } - if (apr_hash_get(txnprops, SVN_FS__PROP_TXN_CHECK_LOCKS, - APR_HASH_KEY_STRING)) + if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_LOCKS)) { prop.name = SVN_FS__PROP_TXN_CHECK_LOCKS; APR_ARRAY_PUSH(txnprop_list, svn_prop_t) = prop; @@ -6444,7 +8622,7 @@ commit_body(void *baton, apr_pool_t *pool) fails because the shard already existed for some reason. */ if (ffd->max_files_per_dir && new_rev % ffd->max_files_per_dir == 0) { - if (1) + /* Create the revs shard. */ { const char *new_dir = path_rev_shard(cb->fs, new_rev, pool); svn_error_t *err = svn_io_dir_make(new_dir, APR_OS_DEFAULT, pool); @@ -6501,6 +8679,7 @@ commit_body(void *baton, apr_pool_t *pool) old_rev_filename, pool)); /* Update the 'current' file. */ + SVN_ERR(verify_as_revision_before_current_plus_plus(cb->fs, new_rev, pool)); SVN_ERR(write_final_current(cb->fs, cb->txn->id, new_rev, start_node_id, start_copy_id, pool)); @@ -6540,23 +8719,6 @@ write_reps_to_cache(svn_fs_t *fs, return SVN_NO_ERROR; } -/* Implements svn_sqlite__transaction_callback_t. */ -static svn_error_t * -commit_sqlite_txn_callback(void *baton, svn_sqlite__db_t *db, - apr_pool_t *scratch_pool) -{ - struct commit_baton *cb = baton; - - /* Write new entries to the rep-sharing database. - * - * We use an sqlite transcation to speed things up; - * see <http://www.sqlite.org/faq.html#q19>. - */ - SVN_ERR(write_reps_to_cache(cb->fs, cb->reps_to_cache, scratch_pool)); - - return SVN_NO_ERROR; -} - svn_error_t * svn_fs_fs__commit(svn_revnum_t *new_rev_p, svn_fs_t *fs, @@ -6573,24 +8735,33 @@ svn_fs_fs__commit(svn_revnum_t *new_rev_p, if (ffd->rep_sharing_allowed) { cb.reps_to_cache = apr_array_make(pool, 5, sizeof(representation_t *)); + cb.reps_hash = apr_hash_make(pool); cb.reps_pool = pool; } else { cb.reps_to_cache = NULL; + cb.reps_hash = NULL; cb.reps_pool = NULL; } SVN_ERR(svn_fs_fs__with_write_lock(fs, commit_body, &cb, pool)); + /* At this point, *NEW_REV_P has been set, so errors below won't affect + the success of the commit. (See svn_fs_commit_txn().) */ + if (ffd->rep_sharing_allowed) { - /* At this point, *NEW_REV_P has been set, so errors here won't affect - the success of the commit. (See svn_fs_commit_txn().) */ SVN_ERR(svn_fs_fs__open_rep_cache(fs, pool)); - SVN_ERR(svn_sqlite__with_transaction(ffd->rep_cache_db, - commit_sqlite_txn_callback, - &cb, pool)); + + /* Write new entries to the rep-sharing database. + * + * We use an sqlite transaction to speed things up; + * see <http://www.sqlite.org/faq.html#q19>. + */ + SVN_SQLITE__WITH_TXN( + write_reps_to_cache(fs, cb.reps_to_cache, pool), + ffd->rep_cache_db); } return SVN_NO_ERROR; @@ -6646,7 +8817,7 @@ write_revision_zero(svn_fs_t *fs) date.data = svn_time_to_cstring(apr_time_now(), fs->pool); date.len = strlen(date.data); proplist = apr_hash_make(fs->pool); - apr_hash_set(proplist, SVN_PROP_REVISION_DATE, APR_HASH_KEY_STRING, &date); + svn_hash_sets(proplist, SVN_PROP_REVISION_DATE, &date); return set_revision_proplist(fs, 0, proplist, fs->pool); } @@ -6662,15 +8833,14 @@ svn_fs_fs__create(svn_fs_t *fs, /* See if compatibility with older versions was explicitly requested. */ if (fs->config) { - if (apr_hash_get(fs->config, SVN_FS_CONFIG_PRE_1_4_COMPATIBLE, - APR_HASH_KEY_STRING)) + if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_4_COMPATIBLE)) format = 1; - else if (apr_hash_get(fs->config, SVN_FS_CONFIG_PRE_1_5_COMPATIBLE, - APR_HASH_KEY_STRING)) + else if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_5_COMPATIBLE)) format = 2; - else if (apr_hash_get(fs->config, SVN_FS_CONFIG_PRE_1_6_COMPATIBLE, - APR_HASH_KEY_STRING)) + else if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_6_COMPATIBLE)) format = 3; + else if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_8_COMPATIBLE)) + format = 4; } ffd->format = format; @@ -6713,13 +8883,18 @@ svn_fs_fs__create(svn_fs_t *fs, ? "0\n" : "0 1 1\n"), pool)); SVN_ERR(svn_io_file_create(path_lock(fs, pool), "", pool)); - SVN_ERR(svn_fs_fs__set_uuid(fs, svn_uuid_generate(pool), pool)); + SVN_ERR(svn_fs_fs__set_uuid(fs, NULL, pool)); SVN_ERR(write_revision_zero(fs)); - SVN_ERR(write_config(fs, pool)); + /* Create the fsfs.conf file if supported. Older server versions would + simply ignore the file but that might result in a different behavior + than with the later releases. Also, hotcopy would ignore, i.e. not + copy, a fsfs.conf with old formats. */ + if (ffd->format >= SVN_FS_FS__MIN_CONFIG_FILE) + SVN_ERR(write_config(fs, pool)); - SVN_ERR(read_config(fs, pool)); + SVN_ERR(read_config(ffd, fs->path, pool)); /* Create the min unpacked rev file. */ if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) @@ -6873,13 +9048,13 @@ recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev, pool)); /* Check that this is a directory. It should be. */ - value = apr_hash_get(headers, HEADER_TYPE, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_TYPE); if (value == NULL || strcmp(value, KIND_DIR) != 0) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Recovery encountered a non-directory node")); /* Get the data location. No data location indicates an empty directory. */ - value = apr_hash_get(headers, HEADER_TEXT, APR_HASH_KEY_STRING); + value = svn_hash_gets(headers, HEADER_TEXT); if (!value) return SVN_NO_ERROR; SVN_ERR(read_rep_offsets(&data_rep, value, NULL, FALSE, pool)); @@ -6905,7 +9080,9 @@ recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev, stored in the representation. */ baton.file = rev_file; baton.pool = pool; - baton.remaining = data_rep->expanded_size; + baton.remaining = data_rep->expanded_size + ? data_rep->expanded_size + : data_rep->size; stream = svn_stream_create(&baton, pool); svn_stream_set_read(stream, read_handler_recover); @@ -6920,7 +9097,7 @@ recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev, for (hi = apr_hash_first(pool, entries); hi; hi = apr_hash_next(hi)) { char *str_val; - char *str, *last_str; + char *str; svn_node_kind_t kind; svn_fs_id_t *id; const char *node_id, *copy_id; @@ -6931,7 +9108,7 @@ recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev, str_val = apr_pstrdup(iterpool, path->data); - str = apr_strtok(str_val, " ", &last_str); + str = svn_cstring_tokenize(" ", &str_val); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Directory entry corrupt")); @@ -6946,7 +9123,7 @@ recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev, _("Directory entry corrupt")); } - str = apr_strtok(NULL, " ", &last_str); + str = svn_cstring_tokenize(" ", &str_val); if (str == NULL) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Directory entry corrupt")); @@ -6986,6 +9163,77 @@ recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev, return SVN_NO_ERROR; } +/* Return TRUE, if for REVISION in FS, we can find the revprop pack file. + * Use POOL for temporary allocations. + * Set *MISSING, if the reason is a missing manifest or pack file. + */ +static svn_boolean_t +packed_revprop_available(svn_boolean_t *missing, + svn_fs_t *fs, + svn_revnum_t revision, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + svn_stringbuf_t *content = NULL; + + /* try to read the manifest file */ + const char *folder = path_revprops_pack_shard(fs, revision, pool); + const char *manifest_path = svn_dirent_join(folder, PATH_MANIFEST, pool); + + svn_error_t *err = try_stringbuf_from_file(&content, + missing, + manifest_path, + FALSE, + pool); + + /* if the manifest cannot be read, consider the pack files inaccessible + * even if the file itself exists. */ + if (err) + { + svn_error_clear(err); + return FALSE; + } + + if (*missing) + return FALSE; + + /* parse manifest content until we find the entry for REVISION. + * Revision 0 is never packed. */ + revision = revision < ffd->max_files_per_dir + ? revision - 1 + : revision % ffd->max_files_per_dir; + while (content->data) + { + char *next = strchr(content->data, '\n'); + if (next) + { + *next = 0; + ++next; + } + + if (revision-- == 0) + { + /* the respective pack file must exist (and be a file) */ + svn_node_kind_t kind; + err = svn_io_check_path(svn_dirent_join(folder, content->data, + pool), + &kind, pool); + if (err) + { + svn_error_clear(err); + return FALSE; + } + + *missing = kind == svn_node_none; + return kind == svn_node_file; + } + + content->data = next; + } + + return FALSE; +} + /* Baton used for recover_body below. */ struct recover_baton { svn_fs_t *fs; @@ -7008,7 +9256,10 @@ recover_body(void *baton, apr_pool_t *pool) svn_revnum_t youngest_rev; svn_node_kind_t youngest_revprops_kind; - /* First, we need to know the largest revision in the filesystem. */ + /* Lose potentially corrupted data in temp files */ + SVN_ERR(cleanup_revprop_namespace(fs)); + + /* We need to know the largest revision in the filesystem. */ SVN_ERR(recover_get_largest_revision(fs, &max_rev, pool)); /* Get the expected youngest revision */ @@ -7099,13 +9350,24 @@ recover_body(void *baton, apr_pool_t *pool) &youngest_revprops_kind, pool)); if (youngest_revprops_kind == svn_node_none) { - if (1) + svn_boolean_t missing = TRUE; + if (!packed_revprop_available(&missing, fs, max_rev, pool)) { - return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Revision %ld has a revs file but no " - "revprops file"), - max_rev); - } + if (missing) + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Revision %ld has a revs file but no " + "revprops file"), + max_rev); + } + else + { + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Revision %ld has a revs file but the " + "revprops file is inaccessible"), + max_rev); + } + } } else if (youngest_revprops_kind != svn_node_file) { @@ -7115,9 +9377,17 @@ recover_body(void *baton, apr_pool_t *pool) max_rev); } - /* Prune younger-than-(newfound-youngest) revisions from the rep cache. */ - if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) - SVN_ERR(svn_fs_fs__del_rep_reference(fs, max_rev, pool)); + /* Prune younger-than-(newfound-youngest) revisions from the rep + cache if sharing is enabled taking care not to create the cache + if it does not exist. */ + if (ffd->rep_sharing_allowed) + { + svn_boolean_t rep_cache_exists; + + SVN_ERR(svn_fs_fs__exists_rep_cache(&rep_cache_exists, fs, pool)); + if (rep_cache_exists) + SVN_ERR(svn_fs_fs__del_rep_reference(fs, max_rev, pool)); + } /* Now store the discovered youngest revision, and the next IDs if relevant, in a new 'current' file. */ @@ -7143,17 +9413,6 @@ svn_fs_fs__recover(svn_fs_t *fs, } svn_error_t * -svn_fs_fs__get_uuid(svn_fs_t *fs, - const char **uuid_p, - apr_pool_t *pool) -{ - fs_fs_data_t *ffd = fs->fsap_data; - - *uuid_p = apr_pstrdup(pool, ffd->uuid); - return SVN_NO_ERROR; -} - -svn_error_t * svn_fs_fs__set_uuid(svn_fs_t *fs, const char *uuid, apr_pool_t *pool) @@ -7162,7 +9421,6 @@ svn_fs_fs__set_uuid(svn_fs_t *fs, apr_size_t my_uuid_len; const char *tmp_path; const char *uuid_path = path_uuid(fs, pool); - fs_fs_data_t *ffd = fs->fsap_data; if (! uuid) uuid = svn_uuid_generate(pool); @@ -7183,7 +9441,7 @@ svn_fs_fs__set_uuid(svn_fs_t *fs, /* Remove the newline we added, and stash the UUID. */ my_uuid[my_uuid_len - 1] = '\0'; - ffd->uuid = my_uuid; + fs->uuid = my_uuid; return SVN_NO_ERROR; } @@ -7191,7 +9449,7 @@ svn_fs_fs__set_uuid(svn_fs_t *fs, /** Node origin lazy cache. */ /* If directory PATH does not exist, create it and give it the same - permissions as FS->path.*/ + permissions as FS_path.*/ svn_error_t * svn_fs_fs__ensure_dir_exists(const char *path, const char *fs_path, @@ -7253,7 +9511,7 @@ svn_fs_fs__get_node_origin(const svn_fs_id_t **origin_id, if (node_origins) { svn_string_t *origin_id_str = - apr_hash_get(node_origins, node_id, APR_HASH_KEY_STRING); + svn_hash_gets(node_origins, node_id); if (origin_id_str) *origin_id = svn_fs_fs__id_parse(origin_id_str->data, origin_id_str->len, pool); @@ -7288,7 +9546,7 @@ set_node_origins_for_file(svn_fs_t *fs, if (! origins_hash) origins_hash = apr_hash_make(pool); - old_node_rev_id = apr_hash_get(origins_hash, node_id, APR_HASH_KEY_STRING); + old_node_rev_id = svn_hash_gets(origins_hash, node_id); if (old_node_rev_id && !svn_string_compare(node_rev_id, old_node_rev_id)) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, @@ -7297,7 +9555,7 @@ set_node_origins_for_file(svn_fs_t *fs, "(%s)"), node_id, old_node_rev_id->data, node_rev_id->data); - apr_hash_set(origins_hash, node_id, APR_HASH_KEY_STRING, node_rev_id); + svn_hash_sets(origins_hash, node_id, node_rev_id); /* Sure, there's a race condition here. Two processes could be trying to add different cache elements to the same file at the @@ -7478,7 +9736,7 @@ svn_fs_fs__revision_prop(svn_string_t **value_p, SVN_ERR(svn_fs__check_fs(fs, TRUE)); SVN_ERR(svn_fs_fs__revision_proplist(&table, fs, rev, pool)); - *value_p = apr_hash_get(table, propname, APR_HASH_KEY_STRING); + *value_p = svn_hash_gets(table, propname); return SVN_NO_ERROR; } @@ -7496,7 +9754,6 @@ struct change_rev_prop_baton { /* The work-horse for svn_fs_fs__change_rev_prop, called with the FS write lock. This implements the svn_fs_fs__with_write_lock() 'body' callback type. BATON is a 'struct change_rev_prop_baton *'. */ - static svn_error_t * change_rev_prop_body(void *baton, apr_pool_t *pool) { @@ -7508,8 +9765,7 @@ change_rev_prop_body(void *baton, apr_pool_t *pool) if (cb->old_value_p) { const svn_string_t *wanted_value = *cb->old_value_p; - const svn_string_t *present_value = apr_hash_get(table, cb->name, - APR_HASH_KEY_STRING); + const svn_string_t *present_value = svn_hash_gets(table, cb->name); if ((!wanted_value != !present_value) || (wanted_value && present_value && !svn_string_compare(wanted_value, present_value))) @@ -7522,7 +9778,7 @@ change_rev_prop_body(void *baton, apr_pool_t *pool) } /* Fall through. */ } - apr_hash_set(table, cb->name, APR_HASH_KEY_STRING, cb->value); + svn_hash_sets(table, cb->name, cb->value); return set_revision_proplist(cb->fs, cb->rev, table, pool); } @@ -7581,7 +9837,7 @@ svn_fs_fs__txn_prop(svn_string_t **value_p, SVN_ERR(svn_fs__check_fs(fs, TRUE)); SVN_ERR(svn_fs_fs__txn_proplist(&table, txn, pool)); - *value_p = apr_hash_get(table, propname, APR_HASH_KEY_STRING); + *value_p = svn_hash_gets(table, propname); return SVN_NO_ERROR; } @@ -7659,43 +9915,31 @@ write_revnum_file(const char *fs_path, return SVN_NO_ERROR; } -/* Pack a single shard SHARD in REVS_DIR, using POOL for allocations. - CANCEL_FUNC and CANCEL_BATON are what you think they are. - - If for some reason we detect a partial packing already performed, we - remove the pack file and start again. */ +/* Pack the revision SHARD containing exactly MAX_FILES_PER_DIR revisions + * from SHARD_PATH into the PACK_FILE_DIR, using POOL for allocations. + * CANCEL_FUNC and CANCEL_BATON are what you think they are. + * + * If for some reason we detect a partial packing already performed, we + * remove the pack file and start again. + */ static svn_error_t * -pack_shard(const char *revs_dir, - const char *fs_path, - apr_int64_t shard, - int max_files_per_dir, - svn_fs_pack_notify_t notify_func, - void *notify_baton, - svn_cancel_func_t cancel_func, - void *cancel_baton, - apr_pool_t *pool) +pack_rev_shard(const char *pack_file_dir, + const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) { - const char *pack_file_path, *manifest_file_path, *shard_path; - const char *pack_file_dir; + const char *pack_file_path, *manifest_file_path; svn_stream_t *pack_stream, *manifest_stream; svn_revnum_t start_rev, end_rev, rev; apr_off_t next_offset; apr_pool_t *iterpool; /* Some useful paths. */ - pack_file_dir = svn_dirent_join(revs_dir, - apr_psprintf(pool, "%" APR_INT64_T_FMT ".pack", shard), - pool); - pack_file_path = svn_dirent_join(pack_file_dir, "pack", pool); - manifest_file_path = svn_dirent_join(pack_file_dir, "manifest", pool); - shard_path = svn_dirent_join(revs_dir, - apr_psprintf(pool, "%" APR_INT64_T_FMT, shard), - pool); - - /* Notify caller we're starting to pack this shard. */ - if (notify_func) - SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start, - pool)); + pack_file_path = svn_dirent_join(pack_file_dir, PATH_PACKED, pool); + manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, pool); /* Remove any existing pack file for this shard, since it is incomplete. */ SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton, @@ -7741,21 +9985,338 @@ pack_shard(const char *revs_dir, SVN_ERR(svn_stream_close(manifest_stream)); SVN_ERR(svn_stream_close(pack_stream)); - SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, pool)); - SVN_ERR(svn_io_set_file_read_only(pack_file_path, FALSE, pool)); - SVN_ERR(svn_io_set_file_read_only(manifest_file_path, FALSE, pool)); + SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); + SVN_ERR(svn_io_set_file_read_only(pack_file_path, FALSE, iterpool)); + SVN_ERR(svn_io_set_file_read_only(manifest_file_path, FALSE, iterpool)); + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Copy revprop files for revisions [START_REV, END_REV) from SHARD_PATH + * to the pack file at PACK_FILE_NAME in PACK_FILE_DIR. + * + * The file sizes have already been determined and written to SIZES. + * Please note that this function will be executed while the filesystem + * has been locked and that revprops files will therefore not be modified + * while the pack is in progress. + * + * COMPRESSION_LEVEL defines how well the resulting pack file shall be + * compressed or whether is shall be compressed at all. TOTAL_SIZE is + * a hint on which initial buffer size we should use to hold the pack file + * content. + * + * CANCEL_FUNC and CANCEL_BATON are used as usual. Temporary allocations + * are done in SCRATCH_POOL. + */ +static svn_error_t * +copy_revprops(const char *pack_file_dir, + const char *pack_filename, + const char *shard_path, + svn_revnum_t start_rev, + svn_revnum_t end_rev, + apr_array_header_t *sizes, + apr_size_t total_size, + int compression_level, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + svn_stream_t *pack_stream; + apr_file_t *pack_file; + svn_revnum_t rev; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + svn_stream_t *stream; + + /* create empty data buffer and a write stream on top of it */ + svn_stringbuf_t *uncompressed + = svn_stringbuf_create_ensure(total_size, scratch_pool); + svn_stringbuf_t *compressed + = svn_stringbuf_create_empty(scratch_pool); + pack_stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); + + /* write the pack file header */ + SVN_ERR(serialize_revprops_header(pack_stream, start_rev, sizes, 0, + sizes->nelts, iterpool)); + + /* Some useful paths. */ + SVN_ERR(svn_io_file_open(&pack_file, svn_dirent_join(pack_file_dir, + pack_filename, + scratch_pool), + APR_WRITE | APR_CREATE, APR_OS_DEFAULT, + scratch_pool)); + + /* Iterate over the revisions in this shard, squashing them together. */ + for (rev = start_rev; rev <= end_rev; rev++) + { + const char *path; + + svn_pool_clear(iterpool); + + /* Construct the file name. */ + path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), + iterpool); + + /* Copy all the bits from the non-packed revprop file to the end of + * the pack file. */ + SVN_ERR(svn_stream_open_readonly(&stream, path, iterpool, iterpool)); + SVN_ERR(svn_stream_copy3(stream, pack_stream, + cancel_func, cancel_baton, iterpool)); + } + + /* flush stream buffers to content buffer */ + SVN_ERR(svn_stream_close(pack_stream)); + + /* compress the content (or just store it for COMPRESSION_LEVEL 0) */ + SVN_ERR(svn__compress(svn_stringbuf__morph_into_string(uncompressed), + compressed, compression_level)); + + /* write the pack file content to disk */ + stream = svn_stream_from_aprfile2(pack_file, FALSE, scratch_pool); + SVN_ERR(svn_stream_write(stream, compressed->data, &compressed->len)); + SVN_ERR(svn_stream_close(stream)); + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* For the revprop SHARD at SHARD_PATH with exactly MAX_FILES_PER_DIR + * revprop files in it, create a packed shared at PACK_FILE_DIR. + * + * COMPRESSION_LEVEL defines how well the resulting pack file shall be + * compressed or whether is shall be compressed at all. Individual pack + * file containing more than one revision will be limited to a size of + * MAX_PACK_SIZE bytes before compression. + * + * CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary + * allocations are done in SCRATCH_POOL. + */ +static svn_error_t * +pack_revprops_shard(const char *pack_file_dir, + const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + apr_off_t max_pack_size, + int compression_level, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + const char *manifest_file_path, *pack_filename = NULL; + svn_stream_t *manifest_stream; + svn_revnum_t start_rev, end_rev, rev; + apr_off_t total_size; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + apr_array_header_t *sizes; + + /* Some useful paths. */ + manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, + scratch_pool); + + /* Remove any existing pack file for this shard, since it is incomplete. */ + SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton, + scratch_pool)); + + /* Create the new directory and manifest file stream. */ + SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool)); + SVN_ERR(svn_stream_open_writable(&manifest_stream, manifest_file_path, + scratch_pool, scratch_pool)); + + /* revisions to handle. Special case: revision 0 */ + start_rev = (svn_revnum_t) (shard * max_files_per_dir); + end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1); + if (start_rev == 0) + ++start_rev; + + /* initialize the revprop size info */ + sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_off_t)); + total_size = 2 * SVN_INT64_BUFFER_SIZE; + + /* Iterate over the revisions in this shard, determine their size and + * squashing them together into pack files. */ + for (rev = start_rev; rev <= end_rev; rev++) + { + apr_finfo_t finfo; + const char *path; + + svn_pool_clear(iterpool); + + /* Get the size of the file. */ + path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), + iterpool); + SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool)); + + /* if we already have started a pack file and this revprop cannot be + * appended to it, write the previous pack file. */ + if (sizes->nelts != 0 && + total_size + SVN_INT64_BUFFER_SIZE + finfo.size > max_pack_size) + { + SVN_ERR(copy_revprops(pack_file_dir, pack_filename, shard_path, + start_rev, rev-1, sizes, (apr_size_t)total_size, + compression_level, cancel_func, cancel_baton, + iterpool)); + + /* next pack file starts empty again */ + apr_array_clear(sizes); + total_size = 2 * SVN_INT64_BUFFER_SIZE; + start_rev = rev; + } + + /* Update the manifest. Allocate a file name for the current pack + * file if it is a new one */ + if (sizes->nelts == 0) + pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev); + + SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%s\n", + pack_filename)); + + /* add to list of files to put into the current pack file */ + APR_ARRAY_PUSH(sizes, apr_off_t) = finfo.size; + total_size += SVN_INT64_BUFFER_SIZE + finfo.size; + } + + /* write the last pack file */ + if (sizes->nelts != 0) + SVN_ERR(copy_revprops(pack_file_dir, pack_filename, shard_path, + start_rev, rev-1, sizes, (apr_size_t)total_size, + compression_level, cancel_func, cancel_baton, + iterpool)); + + /* flush the manifest file and update permissions */ + SVN_ERR(svn_stream_close(manifest_stream)); + SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Delete the non-packed revprop SHARD at SHARD_PATH with exactly + * MAX_FILES_PER_DIR revprop files in it. If this is shard 0, keep the + * revprop file for revision 0. + * + * CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary + * allocations are done in SCRATCH_POOL. + */ +static svn_error_t * +delete_revprops_shard(const char *shard_path, + apr_int64_t shard, + int max_files_per_dir, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + if (shard == 0) + { + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + int i; + + /* delete all files except the one for revision 0 */ + for (i = 1; i < max_files_per_dir; ++i) + { + const char *path = svn_dirent_join(shard_path, + apr_psprintf(iterpool, "%d", i), + iterpool); + if (cancel_func) + SVN_ERR((*cancel_func)(cancel_baton)); + + SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); + svn_pool_clear(iterpool); + } + + svn_pool_destroy(iterpool); + } + else + SVN_ERR(svn_io_remove_dir2(shard_path, TRUE, + cancel_func, cancel_baton, scratch_pool)); + + return SVN_NO_ERROR; +} + +/* In the file system at FS_PATH, pack the SHARD in REVS_DIR and + * REVPROPS_DIR containing exactly MAX_FILES_PER_DIR revisions, using POOL + * for allocations. REVPROPS_DIR will be NULL if revprop packing is not + * supported. COMPRESSION_LEVEL and MAX_PACK_SIZE will be ignored in that + * case. + * + * CANCEL_FUNC and CANCEL_BATON are what you think they are; similarly + * NOTIFY_FUNC and NOTIFY_BATON. + * + * If for some reason we detect a partial packing already performed, we + * remove the pack file and start again. + */ +static svn_error_t * +pack_shard(const char *revs_dir, + const char *revsprops_dir, + const char *fs_path, + apr_int64_t shard, + int max_files_per_dir, + apr_off_t max_pack_size, + int compression_level, + svn_fs_pack_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + const char *rev_shard_path, *rev_pack_file_dir; + const char *revprops_shard_path, *revprops_pack_file_dir; + + /* Notify caller we're starting to pack this shard. */ + if (notify_func) + SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start, + pool)); + + /* Some useful paths. */ + rev_pack_file_dir = svn_dirent_join(revs_dir, + apr_psprintf(pool, + "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, + shard), + pool); + rev_shard_path = svn_dirent_join(revs_dir, + apr_psprintf(pool, "%" APR_INT64_T_FMT, shard), + pool); + + /* pack the revision content */ + SVN_ERR(pack_rev_shard(rev_pack_file_dir, rev_shard_path, + shard, max_files_per_dir, + cancel_func, cancel_baton, pool)); + + /* if enabled, pack the revprops in an equivalent way */ + if (revsprops_dir) + { + revprops_pack_file_dir = svn_dirent_join(revsprops_dir, + apr_psprintf(pool, + "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, + shard), + pool); + revprops_shard_path = svn_dirent_join(revsprops_dir, + apr_psprintf(pool, "%" APR_INT64_T_FMT, shard), + pool); + + SVN_ERR(pack_revprops_shard(revprops_pack_file_dir, revprops_shard_path, + shard, max_files_per_dir, + (int)(0.9 * max_pack_size), + compression_level, + cancel_func, cancel_baton, pool)); + } /* Update the min-unpacked-rev file to reflect our newly packed shard. * (This doesn't update ffd->min_unpacked_rev. That will be updated by * update_min_unpacked_rev() when necessary.) */ SVN_ERR(write_revnum_file(fs_path, PATH_MIN_UNPACKED_REV, (svn_revnum_t)((shard + 1) * max_files_per_dir), - iterpool)); - svn_pool_destroy(iterpool); + pool)); - /* Finally, remove the existing shard directory. */ - SVN_ERR(svn_io_remove_dir2(shard_path, TRUE, cancel_func, cancel_baton, - pool)); + /* Finally, remove the existing shard directories. */ + SVN_ERR(svn_io_remove_dir2(rev_shard_path, TRUE, + cancel_func, cancel_baton, pool)); + if (revsprops_dir) + SVN_ERR(delete_revprops_shard(revprops_shard_path, + shard, max_files_per_dir, + cancel_func, cancel_baton, pool)); /* Notify caller we're starting to pack this shard. */ if (notify_func) @@ -7786,8 +10347,7 @@ struct pack_baton extension, on not having to use a retry when calling svn_fs_fs__path_rev_absolute() and friends). If you add a call to this function, consider whether you have to call - update_min_unpacked_rev() and update_min_unpacked_revprop() - afterwards. + update_min_unpacked_rev(). See this thread: http://thread.gmane.org/1291206765.3782.3309.camel@edith */ static svn_error_t * @@ -7795,51 +10355,63 @@ pack_body(void *baton, apr_pool_t *pool) { struct pack_baton *pb = baton; - int format, max_files_per_dir; + fs_fs_data_t ffd = {0}; apr_int64_t completed_shards; apr_int64_t i; svn_revnum_t youngest; apr_pool_t *iterpool; - const char *data_path; - svn_revnum_t min_unpacked_rev; + const char *rev_data_path; + const char *revprops_data_path = NULL; - SVN_ERR(read_format(&format, &max_files_per_dir, path_format(pb->fs, pool), - pool)); - SVN_ERR(check_format(format)); + /* read repository settings */ + SVN_ERR(read_format(&ffd.format, &ffd.max_files_per_dir, + path_format(pb->fs, pool), pool)); + SVN_ERR(check_format(ffd.format)); + SVN_ERR(read_config(&ffd, pb->fs->path, pool)); /* If the repository isn't a new enough format, we don't support packing. Return a friendly error to that effect. */ - if (format < SVN_FS_FS__MIN_PACKED_FORMAT) + if (ffd.format < SVN_FS_FS__MIN_PACKED_FORMAT) return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, _("FSFS format (%d) too old to pack; please upgrade the filesystem."), - format); + ffd.format); /* If we aren't using sharding, we can't do any packing, so quit. */ - if (!max_files_per_dir) + if (!ffd.max_files_per_dir) return SVN_NO_ERROR; - SVN_ERR(read_min_unpacked_rev(&min_unpacked_rev, + SVN_ERR(read_min_unpacked_rev(&ffd.min_unpacked_rev, path_min_unpacked_rev(pb->fs, pool), pool)); SVN_ERR(get_youngest(&youngest, pb->fs->path, pool)); - completed_shards = (youngest + 1) / max_files_per_dir; + completed_shards = (youngest + 1) / ffd.max_files_per_dir; /* See if we've already completed all possible shards thus far. */ - if (min_unpacked_rev == (completed_shards * max_files_per_dir)) + if (ffd.min_unpacked_rev == (completed_shards * ffd.max_files_per_dir)) return SVN_NO_ERROR; - data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, pool); + rev_data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, pool); + if (ffd.format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) + revprops_data_path = svn_dirent_join(pb->fs->path, PATH_REVPROPS_DIR, + pool); iterpool = svn_pool_create(pool); - for (i = min_unpacked_rev / max_files_per_dir; i < completed_shards; i++) + for (i = ffd.min_unpacked_rev / ffd.max_files_per_dir; + i < completed_shards; + i++) { svn_pool_clear(iterpool); if (pb->cancel_func) SVN_ERR(pb->cancel_func(pb->cancel_baton)); - SVN_ERR(pack_shard(data_path, pb->fs->path, i, max_files_per_dir, + SVN_ERR(pack_shard(rev_data_path, revprops_data_path, + pb->fs->path, i, ffd.max_files_per_dir, + ffd.revprop_pack_size, + ffd.compress_packed_revprops + ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT + : SVN_DELTA_COMPRESSION_LEVEL_NONE, pb->notify_func, pb->notify_baton, pb->cancel_func, pb->cancel_baton, iterpool)); } @@ -7864,3 +10436,1217 @@ svn_fs_fs__pack(svn_fs_t *fs, pb.cancel_baton = cancel_baton; return svn_fs_fs__with_write_lock(fs, pack_body, &pb, pool); } + + +/** Verifying. **/ + +/* Baton type expected by verify_walker(). The purpose is to reuse open + * rev / pack file handles between calls. Its contents need to be cleaned + * periodically to limit resource usage. + */ +typedef struct verify_walker_baton_t +{ + /* number of calls to verify_walker() since the last clean */ + int iteration_count; + + /* number of files opened since the last clean */ + int file_count; + + /* progress notification callback to invoke periodically (may be NULL) */ + svn_fs_progress_notify_func_t notify_func; + + /* baton to use with NOTIFY_FUNC */ + void *notify_baton; + + /* remember the last revision for which we called notify_func */ + svn_revnum_t last_notified_revision; + + /* current file handle (or NULL) */ + apr_file_t *file_hint; + + /* corresponding revision (or SVN_INVALID_REVNUM) */ + svn_revnum_t rev_hint; + + /* pool to use for the file handles etc. */ + apr_pool_t *pool; +} verify_walker_baton_t; + +/* Used by svn_fs_fs__verify(). + Implements svn_fs_fs__walk_rep_reference().walker. */ +static svn_error_t * +verify_walker(representation_t *rep, + void *baton, + svn_fs_t *fs, + apr_pool_t *scratch_pool) +{ + struct rep_state *rs; + struct rep_args *rep_args; + + if (baton) + { + verify_walker_baton_t *walker_baton = baton; + apr_file_t * previous_file; + + /* notify and free resources periodically */ + if ( walker_baton->iteration_count > 1000 + || walker_baton->file_count > 16) + { + if ( walker_baton->notify_func + && rep->revision != walker_baton->last_notified_revision) + { + walker_baton->notify_func(rep->revision, + walker_baton->notify_baton, + scratch_pool); + walker_baton->last_notified_revision = rep->revision; + } + + svn_pool_clear(walker_baton->pool); + + walker_baton->iteration_count = 0; + walker_baton->file_count = 0; + walker_baton->file_hint = NULL; + walker_baton->rev_hint = SVN_INVALID_REVNUM; + } + + /* access the repo data */ + previous_file = walker_baton->file_hint; + SVN_ERR(create_rep_state(&rs, &rep_args, &walker_baton->file_hint, + &walker_baton->rev_hint, rep, fs, + walker_baton->pool)); + + /* update resource usage counters */ + walker_baton->iteration_count++; + if (previous_file != walker_baton->file_hint) + walker_baton->file_count++; + } + else + { + /* ### Should this be using read_rep_line() directly? */ + SVN_ERR(create_rep_state(&rs, &rep_args, NULL, NULL, rep, fs, + scratch_pool)); + } + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_fs__verify(svn_fs_t *fs, + svn_revnum_t start, + svn_revnum_t end, + svn_fs_progress_notify_func_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + svn_boolean_t exists; + svn_revnum_t youngest = ffd->youngest_rev_cache; /* cache is current */ + + if (ffd->format < SVN_FS_FS__MIN_REP_SHARING_FORMAT) + return SVN_NO_ERROR; + + /* Input validation. */ + if (! SVN_IS_VALID_REVNUM(start)) + start = 0; + if (! SVN_IS_VALID_REVNUM(end)) + end = youngest; + SVN_ERR(ensure_revision_exists(fs, start, pool)); + SVN_ERR(ensure_revision_exists(fs, end, pool)); + + /* rep-cache verification. */ + SVN_ERR(svn_fs_fs__exists_rep_cache(&exists, fs, pool)); + if (exists) + { + /* provide a baton to allow the reuse of open file handles between + iterations (saves 2/3 of OS level file operations). */ + verify_walker_baton_t *baton = apr_pcalloc(pool, sizeof(*baton)); + baton->rev_hint = SVN_INVALID_REVNUM; + baton->pool = svn_pool_create(pool); + baton->last_notified_revision = SVN_INVALID_REVNUM; + baton->notify_func = notify_func; + baton->notify_baton = notify_baton; + + /* tell the user that we are now ready to do *something* */ + if (notify_func) + notify_func(SVN_INVALID_REVNUM, notify_baton, baton->pool); + + /* Do not attempt to walk the rep-cache database if its file does + not exist, since doing so would create it --- which may confuse + the administrator. Don't take any lock. */ + SVN_ERR(svn_fs_fs__walk_rep_reference(fs, start, end, + verify_walker, baton, + cancel_func, cancel_baton, + pool)); + + /* walker resource cleanup */ + svn_pool_destroy(baton->pool); + } + + return SVN_NO_ERROR; +} + + +/** Hotcopy. **/ + +/* Like svn_io_dir_file_copy(), but doesn't copy files that exist at + * the destination and do not differ in terms of kind, size, and mtime. */ +static svn_error_t * +hotcopy_io_dir_file_copy(const char *src_path, + const char *dst_path, + const char *file, + apr_pool_t *scratch_pool) +{ + const svn_io_dirent2_t *src_dirent; + const svn_io_dirent2_t *dst_dirent; + const char *src_target; + const char *dst_target; + + /* Does the destination already exist? If not, we must copy it. */ + dst_target = svn_dirent_join(dst_path, file, scratch_pool); + SVN_ERR(svn_io_stat_dirent2(&dst_dirent, dst_target, FALSE, TRUE, + scratch_pool, scratch_pool)); + if (dst_dirent->kind != svn_node_none) + { + /* If the destination's stat information indicates that the file + * is equal to the source, don't bother copying the file again. */ + src_target = svn_dirent_join(src_path, file, scratch_pool); + SVN_ERR(svn_io_stat_dirent2(&src_dirent, src_target, FALSE, FALSE, + scratch_pool, scratch_pool)); + if (src_dirent->kind == dst_dirent->kind && + src_dirent->special == dst_dirent->special && + src_dirent->filesize == dst_dirent->filesize && + src_dirent->mtime <= dst_dirent->mtime) + return SVN_NO_ERROR; + } + + return svn_error_trace(svn_io_dir_file_copy(src_path, dst_path, file, + scratch_pool)); +} + +/* Set *NAME_P to the UTF-8 representation of directory entry NAME. + * NAME is in the internal encoding used by APR; PARENT is in + * UTF-8 and in internal (not local) style. + * + * Use PARENT only for generating an error string if the conversion + * fails because NAME could not be represented in UTF-8. In that + * case, return a two-level error in which the outer error's message + * mentions PARENT, but the inner error's message does not mention + * NAME (except possibly in hex) since NAME may not be printable. + * Such a compound error at least allows the user to go looking in the + * right directory for the problem. + * + * If there is any other error, just return that error directly. + * + * If there is any error, the effect on *NAME_P is undefined. + * + * *NAME_P and NAME may refer to the same storage. + */ +static svn_error_t * +entry_name_to_utf8(const char **name_p, + const char *name, + const char *parent, + apr_pool_t *pool) +{ + svn_error_t *err = svn_path_cstring_to_utf8(name_p, name, pool); + if (err && err->apr_err == APR_EINVAL) + { + return svn_error_createf(err->apr_err, err, + _("Error converting entry " + "in directory '%s' to UTF-8"), + svn_dirent_local_style(parent, pool)); + } + return err; +} + +/* Like svn_io_copy_dir_recursively() but doesn't copy regular files that + * exist in the destination and do not differ from the source in terms of + * kind, size, and mtime. */ +static svn_error_t * +hotcopy_io_copy_dir_recursively(const char *src, + const char *dst_parent, + const char *dst_basename, + svn_boolean_t copy_perms, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + svn_node_kind_t kind; + apr_status_t status; + const char *dst_path; + apr_dir_t *this_dir; + apr_finfo_t this_entry; + apr_int32_t flags = APR_FINFO_TYPE | APR_FINFO_NAME; + + /* Make a subpool for recursion */ + apr_pool_t *subpool = svn_pool_create(pool); + + /* The 'dst_path' is simply dst_parent/dst_basename */ + dst_path = svn_dirent_join(dst_parent, dst_basename, pool); + + /* Sanity checks: SRC and DST_PARENT are directories, and + DST_BASENAME doesn't already exist in DST_PARENT. */ + SVN_ERR(svn_io_check_path(src, &kind, subpool)); + if (kind != svn_node_dir) + return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL, + _("Source '%s' is not a directory"), + svn_dirent_local_style(src, pool)); + + SVN_ERR(svn_io_check_path(dst_parent, &kind, subpool)); + if (kind != svn_node_dir) + return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL, + _("Destination '%s' is not a directory"), + svn_dirent_local_style(dst_parent, pool)); + + SVN_ERR(svn_io_check_path(dst_path, &kind, subpool)); + + /* Create the new directory. */ + /* ### TODO: copy permissions (needs apr_file_attrs_get()) */ + SVN_ERR(svn_io_make_dir_recursively(dst_path, pool)); + + /* Loop over the dirents in SRC. ('.' and '..' are auto-excluded) */ + SVN_ERR(svn_io_dir_open(&this_dir, src, subpool)); + + for (status = apr_dir_read(&this_entry, flags, this_dir); + status == APR_SUCCESS; + status = apr_dir_read(&this_entry, flags, this_dir)) + { + if ((this_entry.name[0] == '.') + && ((this_entry.name[1] == '\0') + || ((this_entry.name[1] == '.') + && (this_entry.name[2] == '\0')))) + { + continue; + } + else + { + const char *entryname_utf8; + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + SVN_ERR(entry_name_to_utf8(&entryname_utf8, this_entry.name, + src, subpool)); + if (this_entry.filetype == APR_REG) /* regular file */ + { + SVN_ERR(hotcopy_io_dir_file_copy(src, dst_path, entryname_utf8, + subpool)); + } + else if (this_entry.filetype == APR_LNK) /* symlink */ + { + const char *src_target = svn_dirent_join(src, entryname_utf8, + subpool); + const char *dst_target = svn_dirent_join(dst_path, + entryname_utf8, + subpool); + SVN_ERR(svn_io_copy_link(src_target, dst_target, + subpool)); + } + else if (this_entry.filetype == APR_DIR) /* recurse */ + { + const char *src_target; + + /* Prevent infinite recursion by filtering off our + newly created destination path. */ + if (strcmp(src, dst_parent) == 0 + && strcmp(entryname_utf8, dst_basename) == 0) + continue; + + src_target = svn_dirent_join(src, entryname_utf8, subpool); + SVN_ERR(hotcopy_io_copy_dir_recursively(src_target, + dst_path, + entryname_utf8, + copy_perms, + cancel_func, + cancel_baton, + subpool)); + } + /* ### support other APR node types someday?? */ + + } + } + + if (! (APR_STATUS_IS_ENOENT(status))) + return svn_error_wrap_apr(status, _("Can't read directory '%s'"), + svn_dirent_local_style(src, pool)); + + status = apr_dir_close(this_dir); + if (status) + return svn_error_wrap_apr(status, _("Error closing directory '%s'"), + svn_dirent_local_style(src, pool)); + + /* Free any memory used by recursion */ + svn_pool_destroy(subpool); + + return SVN_NO_ERROR; +} + +/* Copy an un-packed revision or revprop file for revision REV from SRC_SUBDIR + * to DST_SUBDIR. Assume a sharding layout based on MAX_FILES_PER_DIR. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_copy_shard_file(const char *src_subdir, + const char *dst_subdir, + svn_revnum_t rev, + int max_files_per_dir, + apr_pool_t *scratch_pool) +{ + const char *src_subdir_shard = src_subdir, + *dst_subdir_shard = dst_subdir; + + if (max_files_per_dir) + { + const char *shard = apr_psprintf(scratch_pool, "%ld", + rev / max_files_per_dir); + src_subdir_shard = svn_dirent_join(src_subdir, shard, scratch_pool); + dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); + + if (rev % max_files_per_dir == 0) + { + SVN_ERR(svn_io_make_dir_recursively(dst_subdir_shard, scratch_pool)); + SVN_ERR(svn_io_copy_perms(dst_subdir, dst_subdir_shard, + scratch_pool)); + } + } + + SVN_ERR(hotcopy_io_dir_file_copy(src_subdir_shard, dst_subdir_shard, + apr_psprintf(scratch_pool, "%ld", rev), + scratch_pool)); + return SVN_NO_ERROR; +} + + +/* Copy a packed shard containing revision REV, and which contains + * MAX_FILES_PER_DIR revisions, from SRC_FS to DST_FS. + * Update *DST_MIN_UNPACKED_REV in case the shard is new in DST_FS. + * Do not re-copy data which already exists in DST_FS. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_copy_packed_shard(svn_revnum_t *dst_min_unpacked_rev, + svn_fs_t *src_fs, + svn_fs_t *dst_fs, + svn_revnum_t rev, + int max_files_per_dir, + apr_pool_t *scratch_pool) +{ + const char *src_subdir; + const char *dst_subdir; + const char *packed_shard; + const char *src_subdir_packed_shard; + svn_revnum_t revprop_rev; + apr_pool_t *iterpool; + fs_fs_data_t *src_ffd = src_fs->fsap_data; + + /* Copy the packed shard. */ + src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, scratch_pool); + dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, scratch_pool); + packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD, + rev / max_files_per_dir); + src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, + scratch_pool); + SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard, + dst_subdir, packed_shard, + TRUE /* copy_perms */, + NULL /* cancel_func */, NULL, + scratch_pool)); + + /* Copy revprops belonging to revisions in this pack. */ + src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, scratch_pool); + dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, scratch_pool); + + if ( src_ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT + || src_ffd->min_unpacked_rev < rev + max_files_per_dir) + { + /* copy unpacked revprops rev by rev */ + iterpool = svn_pool_create(scratch_pool); + for (revprop_rev = rev; + revprop_rev < rev + max_files_per_dir; + revprop_rev++) + { + svn_pool_clear(iterpool); + + SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir, + revprop_rev, max_files_per_dir, + iterpool)); + } + svn_pool_destroy(iterpool); + } + else + { + /* revprop for revision 0 will never be packed */ + if (rev == 0) + SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir, + 0, max_files_per_dir, + scratch_pool)); + + /* packed revprops folder */ + packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD, + rev / max_files_per_dir); + src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, + scratch_pool); + SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard, + dst_subdir, packed_shard, + TRUE /* copy_perms */, + NULL /* cancel_func */, NULL, + scratch_pool)); + } + + /* If necessary, update the min-unpacked rev file in the hotcopy. */ + if (*dst_min_unpacked_rev < rev + max_files_per_dir) + { + *dst_min_unpacked_rev = rev + max_files_per_dir; + SVN_ERR(write_revnum_file(dst_fs->path, PATH_MIN_UNPACKED_REV, + *dst_min_unpacked_rev, + scratch_pool)); + } + + return SVN_NO_ERROR; +} + +/* If NEW_YOUNGEST is younger than *DST_YOUNGEST, update the 'current' + * file in DST_FS and set *DST_YOUNGEST to NEW_YOUNGEST. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_update_current(svn_revnum_t *dst_youngest, + svn_fs_t *dst_fs, + svn_revnum_t new_youngest, + apr_pool_t *scratch_pool) +{ + char next_node_id[MAX_KEY_SIZE] = "0"; + char next_copy_id[MAX_KEY_SIZE] = "0"; + fs_fs_data_t *dst_ffd = dst_fs->fsap_data; + + if (*dst_youngest >= new_youngest) + return SVN_NO_ERROR; + + /* If necessary, get new current next_node and next_copy IDs. */ + if (dst_ffd->format < SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT) + { + apr_off_t root_offset; + apr_file_t *rev_file; + char max_node_id[MAX_KEY_SIZE] = "0"; + char max_copy_id[MAX_KEY_SIZE] = "0"; + apr_size_t len; + + if (dst_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) + SVN_ERR(update_min_unpacked_rev(dst_fs, scratch_pool)); + + SVN_ERR(open_pack_or_rev_file(&rev_file, dst_fs, new_youngest, + scratch_pool)); + SVN_ERR(get_root_changes_offset(&root_offset, NULL, rev_file, + dst_fs, new_youngest, scratch_pool)); + SVN_ERR(recover_find_max_ids(dst_fs, new_youngest, rev_file, + root_offset, max_node_id, max_copy_id, + scratch_pool)); + SVN_ERR(svn_io_file_close(rev_file, scratch_pool)); + + /* We store the _next_ ids. */ + len = strlen(max_node_id); + svn_fs_fs__next_key(max_node_id, &len, next_node_id); + len = strlen(max_copy_id); + svn_fs_fs__next_key(max_copy_id, &len, next_copy_id); + } + + /* Update 'current'. */ + SVN_ERR(write_current(dst_fs, new_youngest, next_node_id, next_copy_id, + scratch_pool)); + + *dst_youngest = new_youngest; + + return SVN_NO_ERROR; +} + + +/* Remove revision or revprop files between START_REV (inclusive) and + * END_REV (non-inclusive) from folder DST_SUBDIR in DST_FS. Assume + * sharding as per MAX_FILES_PER_DIR. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_remove_files(svn_fs_t *dst_fs, + const char *dst_subdir, + svn_revnum_t start_rev, + svn_revnum_t end_rev, + int max_files_per_dir, + apr_pool_t *scratch_pool) +{ + const char *shard; + const char *dst_subdir_shard; + svn_revnum_t rev; + apr_pool_t *iterpool; + + /* Pre-compute paths for initial shard. */ + shard = apr_psprintf(scratch_pool, "%ld", start_rev / max_files_per_dir); + dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); + + iterpool = svn_pool_create(scratch_pool); + for (rev = start_rev; rev < end_rev; rev++) + { + const char *path; + svn_pool_clear(iterpool); + + /* If necessary, update paths for shard. */ + if (rev != start_rev && rev % max_files_per_dir == 0) + { + shard = apr_psprintf(iterpool, "%ld", rev / max_files_per_dir); + dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); + } + + /* remove files for REV */ + path = svn_dirent_join(dst_subdir_shard, + apr_psprintf(iterpool, "%ld", rev), + iterpool); + + /* Make the rev file writable and remove it. */ + SVN_ERR(svn_io_set_file_read_write(path, TRUE, iterpool)); + SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Remove revisions between START_REV (inclusive) and END_REV (non-inclusive) + * from DST_FS. Assume sharding as per MAX_FILES_PER_DIR. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +hotcopy_remove_rev_files(svn_fs_t *dst_fs, + svn_revnum_t start_rev, + svn_revnum_t end_rev, + int max_files_per_dir, + apr_pool_t *scratch_pool) +{ + SVN_ERR_ASSERT(start_rev <= end_rev); + SVN_ERR(hotcopy_remove_files(dst_fs, + svn_dirent_join(dst_fs->path, + PATH_REVS_DIR, + scratch_pool), + start_rev, end_rev, + max_files_per_dir, scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Remove revision properties between START_REV (inclusive) and END_REV + * (non-inclusive) from DST_FS. Assume sharding as per MAX_FILES_PER_DIR. + * Use SCRATCH_POOL for temporary allocations. Revision 0 revprops will + * not be deleted. */ +static svn_error_t * +hotcopy_remove_revprop_files(svn_fs_t *dst_fs, + svn_revnum_t start_rev, + svn_revnum_t end_rev, + int max_files_per_dir, + apr_pool_t *scratch_pool) +{ + SVN_ERR_ASSERT(start_rev <= end_rev); + + /* don't delete rev 0 props */ + SVN_ERR(hotcopy_remove_files(dst_fs, + svn_dirent_join(dst_fs->path, + PATH_REVPROPS_DIR, + scratch_pool), + start_rev ? start_rev : 1, end_rev, + max_files_per_dir, scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Verify that DST_FS is a suitable destination for an incremental + * hotcopy from SRC_FS. */ +static svn_error_t * +hotcopy_incremental_check_preconditions(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + apr_pool_t *pool) +{ + fs_fs_data_t *src_ffd = src_fs->fsap_data; + fs_fs_data_t *dst_ffd = dst_fs->fsap_data; + + /* We only support incremental hotcopy between the same format. */ + if (src_ffd->format != dst_ffd->format) + return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("The FSFS format (%d) of the hotcopy source does not match the " + "FSFS format (%d) of the hotcopy destination; please upgrade " + "both repositories to the same format"), + src_ffd->format, dst_ffd->format); + + /* Make sure the UUID of source and destination match up. + * We don't want to copy over a different repository. */ + if (strcmp(src_fs->uuid, dst_fs->uuid) != 0) + return svn_error_create(SVN_ERR_RA_UUID_MISMATCH, NULL, + _("The UUID of the hotcopy source does " + "not match the UUID of the hotcopy " + "destination")); + + /* Also require same shard size. */ + if (src_ffd->max_files_per_dir != dst_ffd->max_files_per_dir) + return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("The sharding layout configuration " + "of the hotcopy source does not match " + "the sharding layout configuration of " + "the hotcopy destination")); + return SVN_NO_ERROR; +} + +/* Remove folder PATH. Ignore errors due to the sub-tree not being empty. + * CANCEL_FUNC and CANCEL_BATON do the usual thing. + * Use POOL for temporary allocations. + */ +static svn_error_t * +remove_folder(const char *path, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + svn_error_t *err = svn_io_remove_dir2(path, TRUE, + cancel_func, cancel_baton, pool); + + if (err && APR_STATUS_IS_ENOTEMPTY(err->apr_err)) + { + svn_error_clear(err); + err = SVN_NO_ERROR; + } + + return svn_error_trace(err); +} + +/* Baton for hotcopy_body(). */ +struct hotcopy_body_baton { + svn_fs_t *src_fs; + svn_fs_t *dst_fs; + svn_boolean_t incremental; + svn_cancel_func_t cancel_func; + void *cancel_baton; +} hotcopy_body_baton; + +/* Perform a hotcopy, either normal or incremental. + * + * Normal hotcopy assumes that the destination exists as an empty + * directory. It behaves like an incremental hotcopy except that + * none of the copied files already exist in the destination. + * + * An incremental hotcopy copies only changed or new files to the destination, + * and removes files from the destination no longer present in the source. + * While the incremental hotcopy is running, readers should still be able + * to access the destintation repository without error and should not see + * revisions currently in progress of being copied. Readers are able to see + * new fully copied revisions even if the entire incremental hotcopy procedure + * has not yet completed. + * + * Writers are blocked out completely during the entire incremental hotcopy + * process to ensure consistency. This function assumes that the repository + * write-lock is held. + */ +static svn_error_t * +hotcopy_body(void *baton, apr_pool_t *pool) +{ + struct hotcopy_body_baton *hbb = baton; + svn_fs_t *src_fs = hbb->src_fs; + fs_fs_data_t *src_ffd = src_fs->fsap_data; + svn_fs_t *dst_fs = hbb->dst_fs; + fs_fs_data_t *dst_ffd = dst_fs->fsap_data; + int max_files_per_dir = src_ffd->max_files_per_dir; + svn_boolean_t incremental = hbb->incremental; + svn_cancel_func_t cancel_func = hbb->cancel_func; + void* cancel_baton = hbb->cancel_baton; + svn_revnum_t src_youngest; + svn_revnum_t dst_youngest; + svn_revnum_t rev; + svn_revnum_t src_min_unpacked_rev; + svn_revnum_t dst_min_unpacked_rev; + const char *src_subdir; + const char *dst_subdir; + const char *revprop_src_subdir; + const char *revprop_dst_subdir; + apr_pool_t *iterpool; + svn_node_kind_t kind; + + /* Try to copy the config. + * + * ### We try copying the config file before doing anything else, + * ### because higher layers will abort the hotcopy if we throw + * ### an error from this function, and that renders the hotcopy + * ### unusable anyway. */ + if (src_ffd->format >= SVN_FS_FS__MIN_CONFIG_FILE) + { + svn_error_t *err; + + err = svn_io_dir_file_copy(src_fs->path, dst_fs->path, PATH_CONFIG, + pool); + if (err) + { + if (APR_STATUS_IS_ENOENT(err->apr_err)) + { + /* 1.6.0 to 1.6.11 did not copy the configuration file during + * hotcopy. So if we're hotcopying a repository which has been + * created as a hotcopy itself, it's possible that fsfs.conf + * does not exist. Ask the user to re-create it. + * + * ### It would be nice to make this a non-fatal error, + * ### but this function does not get an svn_fs_t object + * ### so we have no way of just printing a warning via + * ### the fs->warning() callback. */ + + const char *msg; + const char *src_abspath; + const char *dst_abspath; + const char *config_relpath; + svn_error_t *err2; + + config_relpath = svn_dirent_join(src_fs->path, PATH_CONFIG, pool); + err2 = svn_dirent_get_absolute(&src_abspath, src_fs->path, pool); + if (err2) + return svn_error_trace(svn_error_compose_create(err, err2)); + err2 = svn_dirent_get_absolute(&dst_abspath, dst_fs->path, pool); + if (err2) + return svn_error_trace(svn_error_compose_create(err, err2)); + + /* ### hack: strip off the 'db/' directory from paths so + * ### they make sense to the user */ + src_abspath = svn_dirent_dirname(src_abspath, pool); + dst_abspath = svn_dirent_dirname(dst_abspath, pool); + + msg = apr_psprintf(pool, + _("Failed to create hotcopy at '%s'. " + "The file '%s' is missing from the source " + "repository. Please create this file, for " + "instance by running 'svnadmin upgrade %s'"), + dst_abspath, config_relpath, src_abspath); + return svn_error_quick_wrap(err, msg); + } + else + return svn_error_trace(err); + } + } + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Find the youngest revision in the source and destination. + * We only support hotcopies from sources with an equal or greater amount + * of revisions than the destination. + * This also catches the case where users accidentally swap the + * source and destination arguments. */ + SVN_ERR(get_youngest(&src_youngest, src_fs->path, pool)); + if (incremental) + { + SVN_ERR(get_youngest(&dst_youngest, dst_fs->path, pool)); + if (src_youngest < dst_youngest) + return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("The hotcopy destination already contains more revisions " + "(%lu) than the hotcopy source contains (%lu); are source " + "and destination swapped?"), + dst_youngest, src_youngest); + } + else + dst_youngest = 0; + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Copy the min unpacked rev, and read its value. */ + if (src_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) + { + const char *min_unpacked_rev_path; + + min_unpacked_rev_path = svn_dirent_join(src_fs->path, + PATH_MIN_UNPACKED_REV, + pool); + SVN_ERR(read_min_unpacked_rev(&src_min_unpacked_rev, + min_unpacked_rev_path, + pool)); + + min_unpacked_rev_path = svn_dirent_join(dst_fs->path, + PATH_MIN_UNPACKED_REV, + pool); + SVN_ERR(read_min_unpacked_rev(&dst_min_unpacked_rev, + min_unpacked_rev_path, + pool)); + + /* We only support packs coming from the hotcopy source. + * The destination should not be packed independently from + * the source. This also catches the case where users accidentally + * swap the source and destination arguments. */ + if (src_min_unpacked_rev < dst_min_unpacked_rev) + return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, + _("The hotcopy destination already contains " + "more packed revisions (%lu) than the " + "hotcopy source contains (%lu)"), + dst_min_unpacked_rev - 1, + src_min_unpacked_rev - 1); + + SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path, + PATH_MIN_UNPACKED_REV, pool)); + } + else + { + src_min_unpacked_rev = 0; + dst_min_unpacked_rev = 0; + } + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* + * Copy the necessary rev files. + */ + + src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, pool); + dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, pool); + SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool)); + + iterpool = svn_pool_create(pool); + /* First, copy packed shards. */ + for (rev = 0; rev < src_min_unpacked_rev; rev += max_files_per_dir) + { + svn_pool_clear(iterpool); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Copy the packed shard. */ + SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev, + src_fs, dst_fs, + rev, max_files_per_dir, + iterpool)); + + /* If necessary, update 'current' to the most recent packed rev, + * so readers can see new revisions which arrived in this pack. */ + SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, + rev + max_files_per_dir - 1, + iterpool)); + + /* Remove revision files which are now packed. */ + if (incremental) + { + SVN_ERR(hotcopy_remove_rev_files(dst_fs, rev, + rev + max_files_per_dir, + max_files_per_dir, iterpool)); + if (dst_ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) + SVN_ERR(hotcopy_remove_revprop_files(dst_fs, rev, + rev + max_files_per_dir, + max_files_per_dir, + iterpool)); + } + + /* Now that all revisions have moved into the pack, the original + * rev dir can be removed. */ + SVN_ERR(remove_folder(path_rev_shard(dst_fs, rev, iterpool), + cancel_func, cancel_baton, iterpool)); + if (rev > 0 && dst_ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) + SVN_ERR(remove_folder(path_revprops_shard(dst_fs, rev, iterpool), + cancel_func, cancel_baton, iterpool)); + } + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Now, copy pairs of non-packed revisions and revprop files. + * If necessary, update 'current' after copying all files from a shard. */ + SVN_ERR_ASSERT(rev == src_min_unpacked_rev); + SVN_ERR_ASSERT(src_min_unpacked_rev == dst_min_unpacked_rev); + revprop_src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, pool); + revprop_dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, pool); + SVN_ERR(svn_io_make_dir_recursively(revprop_dst_subdir, pool)); + for (; rev <= src_youngest; rev++) + { + svn_error_t *err; + + svn_pool_clear(iterpool); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* Copy the rev file. */ + err = hotcopy_copy_shard_file(src_subdir, dst_subdir, + rev, max_files_per_dir, + iterpool); + if (err) + { + if (APR_STATUS_IS_ENOENT(err->apr_err) && + src_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) + { + svn_error_clear(err); + + /* The source rev file does not exist. This can happen if the + * source repository is being packed concurrently with this + * hotcopy operation. + * + * If the new revision is now packed, and the youngest revision + * we're interested in is not inside this pack, try to copy the + * pack instead. + * + * If the youngest revision ended up being packed, don't try + * to be smart and work around this. Just abort the hotcopy. */ + SVN_ERR(update_min_unpacked_rev(src_fs, pool)); + if (is_packed_rev(src_fs, rev)) + { + if (is_packed_rev(src_fs, src_youngest)) + return svn_error_createf( + SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("The assumed HEAD revision (%lu) of the " + "hotcopy source has been packed while the " + "hotcopy was in progress; please restart " + "the hotcopy operation"), + src_youngest); + + SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev, + src_fs, dst_fs, + rev, max_files_per_dir, + iterpool)); + rev = dst_min_unpacked_rev; + continue; + } + else + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, + _("Revision %lu disappeared from the " + "hotcopy source while hotcopy was " + "in progress"), rev); + } + else + return svn_error_trace(err); + } + + /* Copy the revprop file. */ + SVN_ERR(hotcopy_copy_shard_file(revprop_src_subdir, + revprop_dst_subdir, + rev, max_files_per_dir, + iterpool)); + + /* After completing a full shard, update 'current'. */ + if (max_files_per_dir && rev % max_files_per_dir == 0) + SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, rev, iterpool)); + } + svn_pool_destroy(iterpool); + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + /* We assume that all revisions were copied now, i.e. we didn't exit the + * above loop early. 'rev' was last incremented during exit of the loop. */ + SVN_ERR_ASSERT(rev == src_youngest + 1); + + /* All revisions were copied. Update 'current'. */ + SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, src_youngest, pool)); + + /* Replace the locks tree. + * This is racy in case readers are currently trying to list locks in + * the destination. However, we need to get rid of stale locks. + * This is the simplest way of doing this, so we accept this small race. */ + dst_subdir = svn_dirent_join(dst_fs->path, PATH_LOCKS_DIR, pool); + SVN_ERR(svn_io_remove_dir2(dst_subdir, TRUE, cancel_func, cancel_baton, + pool)); + src_subdir = svn_dirent_join(src_fs->path, PATH_LOCKS_DIR, pool); + SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); + if (kind == svn_node_dir) + SVN_ERR(svn_io_copy_dir_recursively(src_subdir, dst_fs->path, + PATH_LOCKS_DIR, TRUE, + cancel_func, cancel_baton, pool)); + + /* Now copy the node-origins cache tree. */ + src_subdir = svn_dirent_join(src_fs->path, PATH_NODE_ORIGINS_DIR, pool); + SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); + if (kind == svn_node_dir) + SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir, dst_fs->path, + PATH_NODE_ORIGINS_DIR, TRUE, + cancel_func, cancel_baton, pool)); + + /* + * NB: Data copied below is only read by writers, not readers. + * Writers are still locked out at this point. + */ + + if (dst_ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) + { + /* Copy the rep cache and then remove entries for revisions + * younger than the destination's youngest revision. */ + src_subdir = svn_dirent_join(src_fs->path, REP_CACHE_DB_NAME, pool); + dst_subdir = svn_dirent_join(dst_fs->path, REP_CACHE_DB_NAME, pool); + SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); + if (kind == svn_node_file) + { + SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, pool)); + SVN_ERR(svn_fs_fs__del_rep_reference(dst_fs, dst_youngest, pool)); + } + } + + /* Copy the txn-current file. */ + if (dst_ffd->format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT) + SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path, + PATH_TXN_CURRENT, pool)); + + /* If a revprop generation file exists in the source filesystem, + * reset it to zero (since this is on a different path, it will not + * overlap with data already in cache). Also, clean up stale files + * used for the named atomics implementation. */ + SVN_ERR(svn_io_check_path(path_revprop_generation(src_fs, pool), + &kind, pool)); + if (kind == svn_node_file) + SVN_ERR(write_revprop_generation_file(dst_fs, 0, pool)); + + SVN_ERR(cleanup_revprop_namespace(dst_fs)); + + /* Hotcopied FS is complete. Stamp it with a format file. */ + SVN_ERR(write_format(svn_dirent_join(dst_fs->path, PATH_FORMAT, pool), + dst_ffd->format, max_files_per_dir, TRUE, pool)); + + return SVN_NO_ERROR; +} + + +/* Set up shared data between SRC_FS and DST_FS. */ +static void +hotcopy_setup_shared_fs_data(svn_fs_t *src_fs, svn_fs_t *dst_fs) +{ + fs_fs_data_t *src_ffd = src_fs->fsap_data; + fs_fs_data_t *dst_ffd = dst_fs->fsap_data; + + /* The common pool and mutexes are shared between src and dst filesystems. + * During hotcopy we only grab the mutexes for the destination, so there + * is no risk of dead-lock. We don't write to the src filesystem. Shared + * data for the src_fs has already been initialised in fs_hotcopy(). */ + dst_ffd->shared = src_ffd->shared; +} + +/* Create an empty filesystem at DST_FS at DST_PATH with the same + * configuration as SRC_FS (uuid, format, and other parameters). + * After creation DST_FS has no revisions, not even revision zero. */ +static svn_error_t * +hotcopy_create_empty_dest(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + const char *dst_path, + apr_pool_t *pool) +{ + fs_fs_data_t *src_ffd = src_fs->fsap_data; + fs_fs_data_t *dst_ffd = dst_fs->fsap_data; + + dst_fs->path = apr_pstrdup(pool, dst_path); + + dst_ffd->max_files_per_dir = src_ffd->max_files_per_dir; + dst_ffd->config = src_ffd->config; + dst_ffd->format = src_ffd->format; + + /* Create the revision data directories. */ + if (dst_ffd->max_files_per_dir) + SVN_ERR(svn_io_make_dir_recursively(path_rev_shard(dst_fs, 0, pool), + pool)); + else + SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, + PATH_REVS_DIR, pool), + pool)); + + /* Create the revprops directory. */ + if (src_ffd->max_files_per_dir) + SVN_ERR(svn_io_make_dir_recursively(path_revprops_shard(dst_fs, 0, pool), + pool)); + else + SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, + PATH_REVPROPS_DIR, + pool), + pool)); + + /* Create the transaction directory. */ + SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, PATH_TXNS_DIR, + pool), + pool)); + + /* Create the protorevs directory. */ + if (dst_ffd->format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT) + SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, + PATH_TXN_PROTOS_DIR, + pool), + pool)); + + /* Create the 'current' file. */ + SVN_ERR(svn_io_file_create(svn_fs_fs__path_current(dst_fs, pool), + (dst_ffd->format >= + SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT + ? "0\n" : "0 1 1\n"), + pool)); + + /* Create lock file and UUID. */ + SVN_ERR(svn_io_file_create(path_lock(dst_fs, pool), "", pool)); + SVN_ERR(svn_fs_fs__set_uuid(dst_fs, src_fs->uuid, pool)); + + /* Create the min unpacked rev file. */ + if (dst_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) + SVN_ERR(svn_io_file_create(path_min_unpacked_rev(dst_fs, pool), + "0\n", pool)); + /* Create the txn-current file if the repository supports + the transaction sequence file. */ + if (dst_ffd->format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT) + { + SVN_ERR(svn_io_file_create(path_txn_current(dst_fs, pool), + "0\n", pool)); + SVN_ERR(svn_io_file_create(path_txn_current_lock(dst_fs, pool), + "", pool)); + } + + dst_ffd->youngest_rev_cache = 0; + + hotcopy_setup_shared_fs_data(src_fs, dst_fs); + SVN_ERR(svn_fs_fs__initialize_caches(dst_fs, pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_fs__hotcopy(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + const char *src_path, + const char *dst_path, + svn_boolean_t incremental, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *pool) +{ + struct hotcopy_body_baton hbb; + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + SVN_ERR(svn_fs_fs__open(src_fs, src_path, pool)); + + if (incremental) + { + const char *dst_format_abspath; + svn_node_kind_t dst_format_kind; + + /* Check destination format to be sure we know how to incrementally + * hotcopy to the destination FS. */ + dst_format_abspath = svn_dirent_join(dst_path, PATH_FORMAT, pool); + SVN_ERR(svn_io_check_path(dst_format_abspath, &dst_format_kind, pool)); + if (dst_format_kind == svn_node_none) + { + /* Destination doesn't exist yet. Perform a normal hotcopy to a + * empty destination using the same configuration as the source. */ + SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path, pool)); + } + else + { + /* Check the existing repository. */ + SVN_ERR(svn_fs_fs__open(dst_fs, dst_path, pool)); + SVN_ERR(hotcopy_incremental_check_preconditions(src_fs, dst_fs, + pool)); + hotcopy_setup_shared_fs_data(src_fs, dst_fs); + SVN_ERR(svn_fs_fs__initialize_caches(dst_fs, pool)); + } + } + else + { + /* Start out with an empty destination using the same configuration + * as the source. */ + SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path, pool)); + } + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + + hbb.src_fs = src_fs; + hbb.dst_fs = dst_fs; + hbb.incremental = incremental; + hbb.cancel_func = cancel_func; + hbb.cancel_baton = cancel_baton; + SVN_ERR(svn_fs_fs__with_write_lock(dst_fs, hotcopy_body, &hbb, pool)); + + return SVN_NO_ERROR; +} |