From a5c03feacbbd9361eb3e2abe367b75529c83459b Mon Sep 17 00:00:00 2001 From: bescoto Date: Mon, 31 Oct 2005 04:53:31 +0000 Subject: Added various compare options like --compare-full and --compare-hash git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@664 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/rdiff_backup/iterfile.py | 71 +++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 32 deletions(-) (limited to 'rdiff-backup/rdiff_backup/iterfile.py') diff --git a/rdiff-backup/rdiff_backup/iterfile.py b/rdiff-backup/rdiff_backup/iterfile.py index 0ae998e..608f251 100644 --- a/rdiff-backup/rdiff_backup/iterfile.py +++ b/rdiff-backup/rdiff_backup/iterfile.py @@ -41,14 +41,14 @@ class UnwrapFile: """Return pair (type, data) next in line on the file type is a single character which is either - "o" for object, + "o" for an object, "f" for file, "c" for a continution of a file, "e" for an exception, or None if no more data can be read. Data is either the file's data, if type is "c" or "f", or the - actual object if the type is "o" or "e". + actual object if the type is "o", "e", or "r" """ header = self.file.read(8) @@ -57,8 +57,10 @@ class UnwrapFile: assert None, "Header %s is only %d bytes" % (header, len(header)) type, length = header[0], C.str2long(header[1:]) buf = self.file.read(length) - if type == "o" or type == "e": return type, cPickle.loads(buf) - else: return type, buf + if type in ("o", "e"): return type, cPickle.loads(buf) + else: + assert type in ("f", "c") + return type, buf class IterWrappingFile(UnwrapFile): @@ -213,7 +215,7 @@ class FileWrappingIter: self.currently_in_file.read, [Globals.blocksize]) if buf == "" or buf is None: - assert not self.currently_in_file.close() + self.currently_in_file.close() self.currently_in_file = None if buf is None: # error occurred above, encode exception prefix_letter = "e" @@ -238,33 +240,37 @@ class FileWrappingIter: def close(self): self.closed = 1 -class RORPIterFlush: - """Used to signal that a RORPIterToFile should flush buffer""" +class MiscIterFlush: + """Used to signal that a MiscIterToFile should flush buffer""" pass -class RORPIterFlushRepeat(RORPIterFlush): - """Flush, but then cause RORPIter to yield this same object +class MiscIterFlushRepeat(MiscIterFlush): + """Flush, but then cause Misc Iter to yield this same object - Thus if we put together a pipeline of these, one RORPIterContFlush + Thus if we put together a pipeline of these, one MiscIterFlushRepeat can cause all the segments to flush in sequence. """ pass -class RORPIterToFile(FileWrappingIter): - """Take a RORPIter and give it a file-ish interface +class MiscIterToFile(FileWrappingIter): + """Take an iter and give it a file-ish interface + + This expands on the FileWrappingIter by understanding how to + process RORPaths with file objects attached. It adds a new + character "r" to mark these. This is how we send signatures and diffs across the line. As sending each one separately via a read() call would result in a lot of latency, the read()'s are buffered - a read() call with no arguments will return a variable length string (possibly empty). - To flush the RORPIterToFile, have the iterator yield a - RORPIterFlush class. + To flush the MiscIterToFile, have the iterator yield a + MiscIterFlush class. """ def __init__(self, rpiter, max_buffer_bytes = None, max_buffer_rps = None): - """RORPIterToFile initializer + """MiscIterToFile initializer max_buffer_bytes is the maximum size of the buffer in bytes. max_buffer_rps is the maximum size of the buffer in rorps. @@ -313,17 +319,18 @@ class RORPIterToFile(FileWrappingIter): if hasattr(currentobj, "read") and hasattr(currentobj, "close"): self.currently_in_file = currentobj self.addfromfile("f") - elif (type(currentobj) is types.ClassType and - issubclass(currentobj, iterfile.RORPIterFlush)): - if currentobj is iterfile.RORPIterFlushRepeat: - self.add_flush_repeater() + elif currentobj is iterfile.MiscIterFlush: return None + elif currentobj is iterfile.MiscIterFlushRepeat: + self.add_misc(currentobj) return None - else: self.addrorp(currentobj) + elif isinstance(currentobj, rpath.RORPath): + self.addrorp(currentobj) + else: self.add_misc(currentobj) return 1 - def add_flush_repeater(self): - """Add a RORPIterFlushRepeat object to the buffer""" - pickle = cPickle.dumps(iterfile.RORPIterFlushRepeat, 1) + def add_misc(self, obj): + """Add an arbitrary pickleable object to the buffer""" + pickle = cPickle.dumps(obj, 1) self.array_buf.fromstring("o") self.array_buf.fromstring(C.long2str(long(len(pickle)))) self.array_buf.fromstring(pickle) @@ -336,7 +343,7 @@ class RORPIterToFile(FileWrappingIter): else: pickle = cPickle.dumps((rorp.index, rorp.data, 0), 1) self.rorps_in_buffer += 1 - self.array_buf.fromstring("o") + self.array_buf.fromstring("r") self.array_buf.fromstring(C.long2str(long(len(pickle)))) self.array_buf.fromstring(pickle) @@ -348,8 +355,8 @@ class RORPIterToFile(FileWrappingIter): def close(self): self.closed = 1 -class FileToRORPIter(IterWrappingFile): - """Take a RORPIterToFile and turn it back into a RORPIter""" +class FileToMiscIter(IterWrappingFile): + """Take a MiscIterToFile and turn it back into a iterator""" def __init__(self, file): IterWrappingFile.__init__(self, file) self.buf = "" @@ -363,9 +370,8 @@ class FileToRORPIter(IterWrappingFile): type = None while not type: type, data = self._get() if type == "z": raise StopIteration - elif type == "o": - if data is iterfile.RORPIterFlushRepeat: return data - else: return self.get_rorp(data) + elif type == "r": return self.get_rorp(data) + elif type == "o": return data else: raise IterFileException("Bad file type %s" % (type,)) def get_rorp(self, pickled_tuple): @@ -401,20 +407,21 @@ class FileToRORPIter(IterWrappingFile): if not self.buf: self.buf += self.file.read() if not self.buf: return None, None - assert len(self.buf) >= 8, "Unexpected end of RORPIter file" + assert len(self.buf) >= 8, "Unexpected end of MiscIter file" type, length = self.buf[0], C.str2long(self.buf[1:8]) data = self.buf[8:8+length] self.buf = self.buf[8+length:] - if type == "o" or type == "e": return type, cPickle.loads(data) + if type in "oer": return type, cPickle.loads(data) else: return type, data class ErrorFile: - """File-like that just raises error (used by FileToRORPIter above)""" + """File-like that just raises error (used by FileToMiscIter above)""" def __init__(self, exc): """Initialize new ErrorFile. exc is the exception to raise on read""" self.exc = exc def read(self, l=-1): raise self.exc def close(self): return None + import iterfile -- cgit v1.2.1