diff options
author | Anders Björklund <anders.f.bjorklund@gmail.com> | 2022-05-23 21:40:32 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-05-23 21:40:32 +0200 |
commit | d09df3b94a1e32a1b4993d1b3a7a1f081726208a (patch) | |
tree | b2d47a0a9572afd404a4a32aa15bd6e4a96faac5 /misc/upload-redis | |
parent | 077e2a6c279755f6521ab78f4a1a049ae6a93990 (diff) | |
download | ccache-d09df3b94a1e32a1b4993d1b3a7a1f081726208a.tar.gz |
fix: Fix and optimize Redis upload script (#1081)
Diffstat (limited to 'misc/upload-redis')
-rwxr-xr-x | misc/upload-redis | 8 |
1 files changed, 7 insertions, 1 deletions
diff --git a/misc/upload-redis b/misc/upload-redis index 1fed79d6..87a34b36 100755 --- a/misc/upload-redis +++ b/misc/upload-redis @@ -34,6 +34,7 @@ for dirpath, dirnames, filenames in os.walk(ccache): filelist.sort() files = result = manifest = objects = 0 size = 0 +batchsize = 0 columns = os.get_terminal_size()[0] width = min(columns - 22, 100) bar = progress.bar.Bar( @@ -57,11 +58,16 @@ for mtime, dirpath, filename, filesize in filelist: val = open(os.path.join(dirpath, filename), "rb").read() or None if val: # print("%s: %s %d" % (key, ext, len(val))) - pipe.set(key, val) + pipe.setnx(key, val) objects = objects + 1 files = files + 1 size = size + filesize + batchsize = batchsize + filesize + if batchsize > 64 * 1024 * 1024: + pipe.execute() + batchsize = 0 bar.next() +pipe.execute() bar.finish() print( "%d files, %d result (%d manifest) = %d objects (%s)" |