ensure_binary_mode: python3 and python2/win32 compatibility

Python-3 needs binary mode so that it doesn't try to read into unicode
strings. Python-2 just uses bytes on Linux, but needs the mode set to binary
on Windows.

The smudge filter must also read binary because we can have files with a
"managed" extension that is not actually managed by git-fat. In that
case, we get raw binary data on stdin. It will not match our cookie, but
we must not corrupt its contents in the working tree, thus we have to
treat it as binary.

1 files changed, 18 insertions, 1 deletions

diff --git a/git-fat b/git-fat
index e99559a..aa90dbf 100755
--- a/git-fat
+++ b/git-fat
@@ -21,6 +21,17 @@ def verbose_stderr(*args, **kwargs):
 def verbose_ignore(*args, **kwargs):
     pass
 
+def ensure_binary_mode(stream):
+    try:       # Attempt the Python-3 way, also needed to handle unicode
+        return stream.detach()
+    except:
+        pass
+    if sys.platform == "win32":
+        # Fall back to Python-2 way, only needed on Windows
+        import msvcrt
+        msvcrt.setmode(stream.fileno(), os.O_BINARY)
+    return stream
+
 def mkdir_p(path):
     import errno
     try:
@@ -162,7 +173,7 @@ class GitFat(object):
         if stat.st_size != self.magiclen:
             return False, None
         # read file
-        digest, bytes = self.decode_stream(open(fname))
+        digest, bytes = self.decode_stream(open(fname, 'rb'))
         if isinstance(digest, str):
             return digest, bytes
         else:
@@ -218,10 +229,16 @@ class GitFat(object):
         version of the file on stdin and produces the "clean" (repository) version on stdout.
         '''
         self.setup()
+        # Set stdin and stdout to binary mode
+        sys.stdin = ensure_binary_mode(sys.stdin)
+        sys.stdout = ensure_binary_mode(sys.stdout)
         self.filter_clean(sys.stdin, sys.stdout)
 
     def cmd_filter_smudge(self):
         self.setup()
+        # Ensure streams are treated as binary
+        sys.stdin = ensure_binary_mode(sys.stdin)
+        sys.stdout = ensure_binary_mode(sys.stdout)
         result, bytes = self.decode_stream(sys.stdin)
         if isinstance(result, str):       # We got a digest
             objfile = os.path.join(self.objdir, result)