diff options
| author | Jindřich Makovička <makovick@gmail.com> | 2021-10-01 21:04:59 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-10-01 12:04:59 -0700 |
| commit | ae1f002999fc060178ef00d10cc021404a428e41 (patch) | |
| tree | 8e5562766de7e5b63362d00424f2f7c3ddb8b19c /flist.c | |
| parent | 3911c2386623e1fbd7e985f641c1cee6ac21e067 (diff) | |
| download | rsync-ae1f002999fc060178ef00d10cc021404a428e41.tar.gz | |
Reduce memory usage (#235)
In 2004, an allocation optimization has been added to the file
list handling code, that preallocates 32k of file_struct pointers
in a file_list. This optimization predates the incremental
recursion feature, for which it is not appropriate anymore. When
copying a tree containing a large number of small directories,
using the incremental recursion, rsync allocates many short
file_lists. Suddenly, the unused file_struct pointers can easily
take 90-95% of the memory allocated by rsync.
Diffstat (limited to 'flist.c')
| -rw-r--r-- | flist.c | 11 |
1 files changed, 9 insertions, 2 deletions
@@ -295,6 +295,8 @@ static void flist_expand(struct file_list *flist, int extra) flist->malloced = FLIST_START; else if (flist->malloced >= FLIST_LINEAR) flist->malloced += FLIST_LINEAR; + else if (flist->malloced < FLIST_START_LARGE/16) + flist->malloced *= 4; else flist->malloced *= 2; @@ -305,7 +307,7 @@ static void flist_expand(struct file_list *flist, int extra) new_ptr = realloc_array(flist->files, struct file_struct *, flist->malloced); - if (DEBUG_GTE(FLIST, 1) && flist->malloced != FLIST_START) { + if (DEBUG_GTE(FLIST, 1) && flist->files) { rprintf(FCLIENT, "[%s] expand file_list pointer array to %s bytes, did%s move\n", who_am_i(), big_num(sizeof flist->files[0] * flist->malloced), @@ -2186,8 +2188,10 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) #endif flist = cur_flist = flist_new(0, "send_file_list"); + flist_expand(flist, FLIST_START_LARGE); if (inc_recurse) { dir_flist = flist_new(FLIST_TEMP, "send_file_list"); + flist_expand(dir_flist, FLIST_START_LARGE); flags |= FLAG_DIVERT_DIRS; } else dir_flist = cur_flist; @@ -2541,10 +2545,13 @@ struct file_list *recv_file_list(int f, int dir_ndx) #endif flist = flist_new(0, "recv_file_list"); + flist_expand(flist, FLIST_START_LARGE); if (inc_recurse) { - if (flist->ndx_start == 1) + if (flist->ndx_start == 1) { dir_flist = flist_new(FLIST_TEMP, "recv_file_list"); + flist_expand(dir_flist, FLIST_START_LARGE); + } dstart = dir_flist->used; } else { dir_flist = flist; |