storage: Add filter_callback parameter to Directory.import_files

3 files changed, 12 insertions, 0 deletions

diff --git a/buildstream/storage/_casbaseddirectory.py b/buildstream/storage/_casbaseddirectory.py
index 3477eeb3a..68b0ded6d 100644
--- a/buildstream/storage/_casbaseddirectory.py
+++ b/buildstream/storage/_casbaseddirectory.py
@@ -411,6 +411,7 @@ class CasBasedDirectory(Directory):
         return result
 
     def import_files(self, external_pathspec, *, files=None,
+                     filter_callback=None,
                      report_written=True, update_mtime=False,
                      can_link=False):
         """Imports some or all files from external_path into this directory.
@@ -439,6 +440,9 @@ class CasBasedDirectory(Directory):
                 assert isinstance(external_pathspec, Directory)
                 files = external_pathspec.list_relative_paths()
 
+        if filter_callback:
+            files = [path for path in files if filter_callback(path)]
+
         if isinstance(external_pathspec, FileBasedDirectory):
             source_directory = external_pathspec._get_underlying_directory()
             result = self._import_files_from_directory(source_directory, files=files)
diff --git a/buildstream/storage/_filebaseddirectory.py b/buildstream/storage/_filebaseddirectory.py
index 2872812b6..5e0b9e2ac 100644
--- a/buildstream/storage/_filebaseddirectory.py
+++ b/buildstream/storage/_filebaseddirectory.py
@@ -75,6 +75,7 @@ class FileBasedDirectory(Directory):
         return FileBasedDirectory(new_path).descend(subdirectory_spec[1:], create)
 
     def import_files(self, external_pathspec, *, files=None,
+                     filter_callback=None,
                      report_written=True, update_mtime=False,
                      can_link=False):
         """ See superclass Directory for arguments """
@@ -86,9 +87,11 @@ class FileBasedDirectory(Directory):
 
         if can_link and not update_mtime:
             import_result = link_files(source_directory, self.external_directory, files=files,
+                                       filter_callback=filter_callback,
                                        ignore_missing=False, report_written=report_written)
         else:
             import_result = copy_files(source_directory, self.external_directory, files=files,
+                                       filter_callback=filter_callback,
                                        ignore_missing=False, report_written=report_written)
         if update_mtime:
             cur_time = time.time()
diff --git a/buildstream/storage/directory.py b/buildstream/storage/directory.py
index 9022b8770..f19a8bed6 100644
--- a/buildstream/storage/directory.py
+++ b/buildstream/storage/directory.py
@@ -74,6 +74,7 @@ class Directory():
 
     # Import and export of files and links
     def import_files(self, external_pathspec, *, files=None,
+                     filter_callback=None,
                      report_written=True, update_mtime=False,
                      can_link=False):
         """Imports some or all files from external_path into this directory.
@@ -84,6 +85,10 @@ class Directory():
           files (list of str): A list of all the files relative to
             the external_pathspec to copy. If 'None' is supplied, all
             files are copied.
+          filter_callback (callable): Optional filter callback. Called with the
+            relative path as argument for every file in the source directory.
+            The file is imported only if the callable returns True.
+            If no filter callback is specified, all files will be imported.
           report_written (bool): Return the full list of files
             written. Defaults to true. If false, only a list of
             overwritten files is returned.