From f19efa182d0e53d97e163f980d6e221c55dd9c9b Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Mon, 10 Dec 2012 17:55:23 +0000 Subject: Add support for ccache This is set up so that each individual project repository has its own ccache, all under one defined directory. The top-level ccache directory is added as the setting 'compiler-cache-dir', and defaults to $cachedir/ccache. When a build is performed, this will bind-mount a project's ccache into the /tmp/ccache of the staging-area and set up the environment variables so that ccache will be used (if appropriate executables are installed to /usr/lib/ccache in the staging-area). In addition, this removes code for ccache-remotedir, as it is unrelated to this implementation of ccache. Reviewed-by: Jannis Pohlmann Reviewed-by: Lars Wirzenius --- morphlib/app.py | 19 +++++-------------- morphlib/buildenvironment.py | 4 +--- morphlib/builder2.py | 24 ++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/morphlib/app.py b/morphlib/app.py index d154f8b9..a13d26a1 100755 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -43,8 +43,6 @@ defaults = { 'max-jobs': morphlib.util.make_concurrency(), 'prefix': '/usr', 'toolchain-target': '%s-baserock-linux-gnu' % os.uname()[4], - 'ccache-remotedir': '', - 'ccache-remotenlevels': 2, 'build-ref-prefix': 'baserock/builds' } @@ -61,6 +59,11 @@ class Morph(cliapp.Application): 'cache git repositories and build results in DIR', metavar='DIR', default=defaults['cachedir']) + self.settings.string(['compiler-cache-dir'], + 'cache compiled objects in CCDIR/REPO_NAME', + metavar='CCDIR', + default=os.path.join(self.settings['cachedir'], + 'ccache')) self.settings.string(['build-ref-prefix'], 'Prefix to use for temporary build refs', metavar='PREFIX', @@ -109,18 +112,6 @@ class Morph(cliapp.Application): 'DANGEROUS and will install stuff on your ' 'system', group=group_build) - self.settings.string(['ccache-remotedir'], - 'allow ccache to download objects from REMOTEDIR ' - 'if they are not cached locally', - metavar='REMOTEDIR', - default=defaults['ccache-remotedir'], - group=group_build) - self.settings.integer(['ccache-remotenlevels'], - 'assume ccache directory objects are split into ' - 'NLEVELS levels of subdirectories', - metavar='NLEVELS', - default=defaults['ccache-remotenlevels'], - group=group_build) self.settings.boolean(['keep-path'], 'do not touch the PATH environment variable', group=group_build) diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index c9c12cc4..d9e3210f 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -88,14 +88,12 @@ class BuildEnvironment(): # FIXME: we should set CCACHE_BASEDIR so any objects that refer to their # current directory get corrected. This improve the cache hit rate # env['CCACHE_BASEDIR'] = self.tempdir.dirname + env['CCACHE_DIR'] = '/tmp/ccache' env['CCACHE_EXTRAFILES'] = ':'.join( f for f in ('/baserock/binutils.meta', '/baserock/eglibc.meta', '/baserock/gcc.meta') if os.path.exists(f) ) - if settings['ccache-remotedir'] != '': - env['CCACHE_REMOTEDIR'] = settings['ccache-remotedir'] - env['CCACHE_REMOTENLEVELS'] = str(settings['ccache-remotenlevels']) if not settings['no-distcc']: env['CCACHE_PREFIX'] = 'distcc' diff --git a/morphlib/builder2.py b/morphlib/builder2.py index c7d25e1a..0c1acfde 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -289,6 +289,28 @@ class ChunkBuilder(BuilderBase): ('dev/shm', 'tmpfs', 'none'), ) + def mount_ccachedir(self): #pragma: no cover + ccache_dir = self.app.settings['compiler-cache-dir'] + if not os.path.isdir(ccache_dir): + os.makedirs(ccache_dir) + # Get a path for the repo's ccache + ccache_repobase = os.path.basename(self.artifact.source.repo.path) + ccache_repodir = os.path.join(ccache_dir, + ccache_repobase) + # Make sure that directory exists + if not os.path.isdir(ccache_repodir): + os.mkdir(ccache_repodir) + # Get the destination path + ccache_destdir= os.path.join(self.staging_area.tempdir, + 'tmp', 'ccache') + # Make sure that the destination exists + if not os.path.isdir(ccache_destdir): + os.mkdir(ccache_destdir) + # Mount it into the staging-area + self.app.runcmd(['mount', '--bind', ccache_repodir, + ccache_destdir]) + return ccache_destdir + def do_mounts(self): # pragma: no cover mounted = [] if not self.setup_mounts: @@ -300,6 +322,8 @@ class ChunkBuilder(BuilderBase): os.makedirs(path) self.app.runcmd(['mount', '-t', mount_type, source, path]) mounted.append(path) + if not self.app.settings['no-ccache']: + mounted.append(self.mount_ccachedir()) return mounted def do_unmounts(self, mounted): # pragma: no cover -- cgit v1.2.1