Merge branch 'tristan/symlinks-and-staging' into 'master'

Tristan/symlinks and staging

See merge request !308

1 files changed, 51 insertions, 9 deletions

diff --git a/ybd/utils.py b/ybd/utils.py
index 975d8ae..aa49f0a 100644
--- a/ybd/utils.py
+++ b/ybd/utils.py
@@ -118,6 +118,34 @@ def hardlink_all_files(srcpath, destpath):
     _process_tree(destpath, srcpath, destpath, os.link)
 
 
+def _ensure_real_directory(root, destpath):
+    # The realpath in the sandbox may refer to a file outside of the
+    # sandbox when any of the direcory branches are a symlink to an
+    # absolute path.
+    #
+    # This should not happen as we rely on relative_symlink_target() below
+    # when staging the actual symlinks which may lead up to this path.
+    #
+    realpath = os.path.realpath(destpath)
+    if not realpath.startswith(os.path.realpath(root)):
+        raise IOError('Destination path resolves to a path outside ' +
+                      'of the staging area\n\n' +
+                      '  Destination path: %s\n' % destpath +
+                      '  Real path: %s' % realpath)
+
+    # Ensure the real destination path exists before trying to get the mode
+    # of the real destination path.
+    #
+    # It is acceptable that chunks create symlinks inside artifacts which
+    # refer to non-existing directories, they will be created on demand here
+    # at staging time.
+    #
+    if not os.path.exists(realpath):
+        os.makedirs(realpath)
+
+    return realpath
+
+
 def _process_tree(root, srcpath, destpath, actionfunc):
     if os.path.lexists(destpath):
         app.log('OVERLAPS', 'WARNING: overlap at', destpath, verbose=True)
@@ -127,18 +155,18 @@ def _process_tree(root, srcpath, destpath, actionfunc):
 
     if stat.S_ISDIR(mode):
         # Ensure directory exists in destination, then recurse.
+
+        # os.path.lexists() returns True for broken symlinks
+        #
         if not os.path.lexists(destpath):
             os.makedirs(destpath)
-        try:
-            realpath = os.path.realpath(destpath)
-            dest_stat = os.stat(realpath)
-        except:
-            import traceback
-            traceback.print_exc()
-            print 'destpath is', destpath
-            print 'realpath is', realpath
 
-            app.log('UTILS', 'ERROR: file operation failed', exit=True)
+        # This creates the realpath safely, the above line can
+        # probably be removed
+        realpath = _ensure_real_directory(root, destpath)
+
+        # At this point we know it exists, stat() should not fail here
+        dest_stat = os.stat(realpath)
 
         if not stat.S_ISDIR(dest_stat.st_mode):
             raise IOError('Destination not a directory: source has %s'
@@ -254,6 +282,20 @@ def _process_list(srcdir, destdir, filelist, actionfunc):
         # The destination directory may not have been created separately
         _copy_directories(srcdir, destdir, path)
 
+        # Ensure that broken symlinks to directories have their targets
+        # created before attempting to stage files across broken
+        # symlink boundaries
+        _ensure_real_directory(destdir, os.path.dirname(destpath))
+
+        if not os.path.lexists(srcpath):
+            app.log('UTILS',
+                    'WARNING: Ignoring missing source file while moving '
+                    'split artifacts: %s\n\n' % srcpath +
+                    '  Hint: This file is probably a broken symlink in\n' +
+                    '        the artifact, if it is a library symlink\n' +
+                    '        then it would have been removed by ldconfig\n')
+            continue
+
         try:
             file_stat = os.lstat(srcpath)
             mode = file_stat.st_mode