From 66c9735cc1944b9e23fa0dc60cee6d759b22cebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20Bj=C3=B6rklund?= Date: Wed, 22 Feb 2023 19:38:26 +0100 Subject: chore: Clean up Redis helper scripts a bit (#1254) - Support any version of manifest and result. - Skip over tmp dir and allow avoiding setnx. --- misc/upload-redis | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'misc/upload-redis') diff --git a/misc/upload-redis b/misc/upload-redis index b3595c5a..a1f9cb08 100755 --- a/misc/upload-redis +++ b/misc/upload-redis @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# This script uploads the contents of the local cache to a Redis remote storage. +# This script uploads the contents of the local cache +# to a Redis remote storage. import redis import os @@ -16,20 +17,24 @@ elif config.startswith("/"): host, port, sock = None, None, config else: host, port, sock = config, 6379, None + username = os.getenv("REDIS_USERNAME") password = os.getenv("REDIS_PASSWORD") context = redis.Redis(host=host, port=port, unix_socket_path=sock, password=password) pipe = context.pipeline(transaction=False) +use_setnx = True + ccache = os.getenv("CCACHE_DIR", os.path.expanduser("~/.cache/ccache")) filelist = [] -for dirpath, dirnames, filenames in os.walk(ccache): +for dirpath, dirnames, filenames in os.walk(ccache, topdown=True): # sort by modification time, most recently used last for filename in filenames: if filename.endswith(".lock"): continue stat = os.stat(os.path.join(dirpath, filename)) filelist.append((stat.st_mtime, dirpath, filename, stat.st_size)) + dirnames[:] = [d for d in dirnames if d != "tmp"] filelist.sort() files = result = manifest = objects = 0 @@ -42,12 +47,7 @@ bar = progress.bar.Bar( ) for mtime, dirpath, filename, filesize in filelist: dirname = dirpath.replace(ccache + os.path.sep, "") - if dirname == "tmp": - continue - elif filename == "CACHEDIR.TAG" or filename == "stats": - # ignore these - files += 1 - else: + if filename != "ccache.conf" and filename != "CACHEDIR.TAG" and filename != "stats": (base, ext) = filename[:-1], filename[-1:] if ext == "R" or ext == "M": if ext == "R": @@ -58,7 +58,10 @@ for mtime, dirpath, filename, filesize in filelist: val = open(os.path.join(dirpath, filename), "rb").read() or None if val: # print("%s: %s %d" % (key, ext, len(val))) - pipe.setnx(key, val) + if use_setnx: + pipe.setnx(key, val) + else: + pipe.set(key, val) objects += 1 files += 1 size += filesize -- cgit v1.2.1