diff options
Diffstat (limited to 'lorry')
| -rwxr-xr-x | lorry | 145 |
1 files changed, 98 insertions, 47 deletions
@@ -41,6 +41,9 @@ __version__ = '0.0' lorry_path = os.path.realpath(__file__) +UPDATE_COUNT_NAME = 'lorry-update-count' + + def file_missing_or_empty(filename): ''' A more comprehensive alternative to os.path.exists(). ''' return (not os.path.isfile(filename)) or (os.path.getsize(filename) <= 0) @@ -350,15 +353,12 @@ class Lorry(cliapp.Application): dirname = self.dirname(name) if not os.path.exists(dirname): os.mkdir(dirname) - gitdir = os.path.join(dirname, 'git') - - time = datetime.now().strftime('%F-%T') - pre_update_name = 'git-pre-update' - pre_update_backup_suffix = (pre_update_name + '-' + time - if self.settings['keep-multiple-backups'] else pre_update_name) + self.migrate_oldstyle_repos(dirname) + temp_repo, active_repo, next_update_count = \ + self.prepare_working_repos(dirname) - pre_update_backup_dir = os.path.join(dirname, pre_update_backup_suffix) + time = datetime.now().strftime('%F-%T') post_fail_name = 'git-post-fail' post_fail_backup_suffix = (post_fail_name + '-' + time @@ -367,79 +367,130 @@ class Lorry(cliapp.Application): post_fail_backup_dir = os.path.join(dirname, post_fail_backup_suffix) if not self.settings['keep-multiple-backups']: - # remove previous backups if they exist - if os.path.exists(pre_update_backup_dir): - shutil.rmtree(pre_update_backup_dir) - if os.path.exists(post_fail_backup_dir): shutil.rmtree(post_fail_backup_dir) - backupdir = self.backup_gitdir(name, gitdir, pre_update_backup_dir) - try: self.needs_aggressive = False - table[vcstype](name, dirname, gitdir, spec) + table[vcstype](name, dirname, temp_repo, spec) if self.settings['repack']: self.progress('.. repacking %s git repository' % name) self.run_program(['git', 'config', - 'pack.windowMemory', '128M'], cwd=gitdir) + 'pack.windowMemory', '128M'], cwd=temp_repo) args = ['git', 'gc'] if self.needs_aggressive: args += ['--aggressive'] - self.run_program(args, cwd=gitdir) + self.run_program(args, cwd=temp_repo) - self.bundle(name, gitdir) - self.make_tarball(name, gitdir) + self.bundle(name, temp_repo) + self.make_tarball(name, temp_repo) + + self.write_update_count(temp_repo, next_update_count) + active_repo = temp_repo except: - if backupdir is not None: - faildir = self.backup_gitdir(name, gitdir, post_fail_backup_dir) - self.restore_backup(name, backupdir, gitdir) + if active_repo is not None: + os.rename(temp_repo, post_fail_backup_dir) self.output.write('Mirror of %s failed, state before mirror ' 'is saved at %s and state after mirror is ' - 'saved at %s\n' % (name, backupdir, faildir)) + 'saved at %s\n' + % (name, active_repo, post_fail_backup_dir)) logging.debug('Mirror of %s failed, state before mirror ' 'is saved at %s and state after mirror is ' - 'saved at %s\n', name, backupdir, faildir) + 'saved at %s\n', + name, active_repo, post_fail_backup_dir) raise if not self.settings['pull-only']: if len(self.settings['mirror-base-url-push']) > 0: if 'refspecs' in spec: - self.push_to_mirror_server(name, gitdir, spec['refspecs']) + self.push_to_mirror_server(name, active_repo, spec['refspecs']) else: - self.push_to_mirror_server(name, gitdir) - - if backupdir is not None: - self.progress('.. removing %s git repository backup' % name) - shutil.rmtree(backupdir) - - def restore_backup(self, name, backupdir, gitdir): - self.progress('.. restoring %s good git repository' % name) - dotgit = os.path.join(gitdir, '.git') - if not os.path.exists(dotgit): - dotgit = gitdir - shutil.rmtree(dotgit) - self.copy_gitdir(backupdir, dotgit) + self.push_to_mirror_server(name, active_repo) + + def migrate_oldstyle_repos(self, dirname): + # Migrate old-style active repository + old_repo = os.path.join(dirname, 'git') + if os.path.exists(old_repo): + new_repo = os.path.join(dirname, 'git-a') + if os.path.exists(new_repo): + msg = ('Found both old %s and new %s directories; ' + 'not migrating\n' % (old_repo, new_repo)) + self.output.write(msg) + logging.warning(msg) + else: + # If it has a .git subdirectory, use that + old_gitdir = os.path.join(old_repo, '.git') + if not os.path.exists(old_gitdir): + old_gitdir = old_repo + + self.write_update_count(old_gitdir, 1) + + # Move it to new name, and remove top-level directory if we + # moved the .git subdirectory + os.rename(old_gitdir, new_repo) + if old_repo != old_gitdir: + shutil.rmtree(old_repo) + + # Remove old-style backup repository + old_repo = os.path.join(dirname, 'git-pre-update') + if os.path.exists(old_repo): + shutil.rmtree(old_repo) + + def prepare_working_repos(self, dirname): + # Determine which repository is active (has highest update + # count) and which we will create or replace + repos = [] + for repo in [os.path.join(dirname, 'git-a'), + os.path.join(dirname, 'git-b')]: + tstamp = -1 + count = 0 + try: + count_name = os.path.join(repo, UPDATE_COUNT_NAME) + with open(count_name, 'r') as count_file: + tstamp = os.stat(count_file.fileno()).st_mtime + count = int(count_file.readline()) + except (FileNotFoundError, ValueError): + pass + repos.append((count, tstamp, repo)) + repos.sort() + temp_count, _, temp_repo = repos[0] + active_count, active_tstamp, active_repo = repos[1] + + # Remove/rename temporary repository + if os.path.exists(temp_repo): + # If this was the result of a successful conversion, and + # multiple backups are enabled, rename it. We name it + # using the timestamp of the active repository, i.e. the + # time that this repository became inactive. + if temp_count > 0 and self.settings['keep-multiple-backups']: + time = datetime.fromtimestamp(active_tstamp) \ + .strftime('%F-%T') + os.rename(temp_repo, + os.path.join(dirname, 'git-pre-update-' + time)) + else: + shutil.rmtree(temp_repo) - def backup_gitdir(self, name, gitdir, backupdir): - dotgit = os.path.join(gitdir, '.git') - if not os.path.exists(dotgit): - dotgit = gitdir + if active_count == 0: + return temp_repo, None, 1 - self.progress('.. backing up %s git repository to %s' % (name, backupdir)) + self.copy_gitdir(active_repo, temp_repo) + return temp_repo, active_repo, active_count + 1 - return self.copy_gitdir(dotgit, backupdir) + def write_update_count(self, gitdir, count): + count_name = os.path.join(gitdir, UPDATE_COUNT_NAME) + with open(count_name, 'w') as count_file: + count_file.write('%d\n' % count) def copy_gitdir(self, source, dest): if not os.path.exists(source): return None - # copy everything except the objects dir - def ignoreobjects(dirname, filenames): + # copy everything except the objects dir and update count + def ignore_filter(dirname, filenames): if dirname.endswith(source): - return ['objects'] + return ['objects', UPDATE_COUNT_NAME] return [] - shutil.copytree(source, dest, ignore=ignoreobjects) + shutil.copytree(source, dest, ignore=ignore_filter) # hardlink the objects sourceobjects = os.path.join(source, 'objects') |