From 6676c4ac025954a7c5c6a38c88350838c45be8b6 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 24 Oct 2017 10:03:12 +0000 Subject: Ignore all hidden directories when looking for definitions This un-does a change done in commit f072299d04 which caused YBD to stat() every directory under the current working directory when looking for definitions. This was causing issues on GitLab CI, where we're required to put the YBD cache inside our checkout of definitions.git. The cache contains 10,000s of files and calling stat() on each one is a massive slowdown. This change also causes YBD to ignore any hidden directory when searching for definitions rather than just `.git`. This allows us to put the cache at `./.cache` without issue. --- ybd/morphs.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/ybd/morphs.py b/ybd/morphs.py index 6ded04d..d910a27 100644 --- a/ybd/morphs.py +++ b/ybd/morphs.py @@ -15,12 +15,15 @@ # =*= License: GPL-2 =*= import yaml -import glob import os from app import chdir, config, log from defaults import Defaults +def is_not_hidden_dir(d): + return not d.startswith('.') + + class Morphs(object): def __init__(self, directory='.'): @@ -29,18 +32,19 @@ class Morphs(object): self.defaults = Defaults() self.fields = self.defaults.build_steps + self.defaults.fields - directories = [d[0] for d in os.walk(directory) if '/.' not in d[0]] - for d in sorted(directories): - files = glob.glob(os.path.join(d, '*.morph')) - for path in sorted(files): - data = self._load(path) - if data is not None: - path = self._demorph(path) - if path.startswith('./'): - path = path[2:] - data['path'] = path - self._fix_keys(data) - self._tidy_and_insert_recursively(data) + with chdir(directory): + for outer_dirname, dirnames, filenames in os.walk('.'): + dirnames[:] = filter(is_not_hidden_dir, dirnames) + filenames.sort() + dirnames.sort() + for filename in filenames: + if filename.endswith('.morph'): + path = os.path.join(outer_dirname, filename) + data = self._load(path) + if data is not None: + data['path'] = self._demorph(path[2:]) + self._fix_keys(data) + self._tidy_and_insert_recursively(data) for x in self._data: dn = self._data[x] -- cgit v1.2.1