Try read/write fsync for cygwin, also add --preserve-numerical-ids

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@638 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

3 files changed, 161 insertions, 103 deletions

diff --git a/rdiff-backup/rdiff_backup/Main.py b/rdiff-backup/rdiff_backup/Main.py
index 62ce7fc..4a6450d 100644
--- a/rdiff-backup/rdiff_backup/Main.py
+++ b/rdiff-backup/rdiff_backup/Main.py
@@ -33,7 +33,9 @@ remote_cmd, remote_schema = None, None
 force = None
 select_opts = []
 select_files = []
-user_mapping_filename, group_mapping_filename = None, None
+user_mapping_filename, group_mapping_filename, preserve_numerical_ids = \
+					   None, None, None
+
 # These are global because they are set while we are trying to figure
 # whether to restore or to backup
 restore_root, restore_index, restore_root_set = None, None, 0
@@ -43,7 +45,9 @@ def parse_cmdlineoptions(arglist):
 	"""Parse argument list and set global preferences"""
 	global args, action, create_full_path, force, restore_timestr, remote_cmd
 	global remote_schema, remove_older_than_string
-	global user_mapping_filename, group_mapping_filename
+	global user_mapping_filename, group_mapping_filename, \
+		   preserve_numerical_ids
+
 	def sel_fl(filename):
 		"""Helper function for including/excluding filelists below"""
 		try: return open(filename, "r")
@@ -73,7 +77,8 @@ def parse_cmdlineoptions(arglist):
 		  "no-compression-regexp=", "no-eas", "no-file-statistics",
 		  "no-hard-links", "null-separator",
 		  "override-chars-to-quote=", "parsable-output",
-		  "print-statistics", "remote-cmd=", "remote-schema=",
+		  "preserve-numerical-ids", "print-statistics",
+		  "remote-cmd=", "remote-schema=",
 		  "remove-older-than=", "restore-as-of=", "restrict=",
 		  "restrict-read-only=", "restrict-update-only=", "server",
 		  "ssh-no-compression", "terminal-verbosity=", "test-server",
@@ -156,6 +161,7 @@ def parse_cmdlineoptions(arglist):
 		elif opt == "--override-chars-to-quote":
 			Globals.set('chars_to_quote', arg)
 		elif opt == "--parsable-output": Globals.set('parsable_output', 1)
+		elif opt == "--preserve-numerical-ids": preserve_numerical_ids = 1
 		elif opt == "--print-statistics": Globals.set('print_statistics', 1)
 		elif opt == "-r" or opt == "--restore-as-of":
 			restore_timestr, action = arg, "restore-as-of"
@@ -233,7 +239,8 @@ def misc_setup(rps):
 
 def init_user_group_mapping(destination_conn):
 	"""Initialize user and group mapping on destination connection"""
-	global user_mapping_filename, group_mapping_filename
+	global user_mapping_filename, group_mapping_filename, \
+		   preserve_numerical_ids
 	def get_string_from_file(filename):
 		if not filename: return None
 		rp = rpath.RPath(Globals.local_connection, filename)
@@ -242,9 +249,11 @@ def init_user_group_mapping(destination_conn):
 			Log.FatalError("Error '%s' reading mapping file '%s'" %
 						   (str(e), filename))
 	user_mapping_string = get_string_from_file(user_mapping_filename)
-	destination_conn.user_group.init_user_mapping(user_mapping_string)
+	destination_conn.user_group.init_user_mapping(user_mapping_string,
+												  preserve_numerical_ids)
 	group_mapping_string = get_string_from_file(group_mapping_filename)
-	destination_conn.user_group.init_group_mapping(group_mapping_string)
+	destination_conn.user_group.init_group_mapping(group_mapping_string,
+												   preserve_numerical_ids)
 
 def take_action(rps):
 	"""Do whatever action says"""
diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py
index 95dd54f..8213a23 100644
--- a/rdiff-backup/rdiff_backup/rpath.py
+++ b/rdiff-backup/rdiff_backup/rpath.py
@@ -152,7 +152,8 @@ def copy_attribs(rpin, rpout):
 	"""
 	log.Log("Copying attributes from %s to %s" % (rpin.index, rpout.path), 7)
 	assert rpin.lstat() == rpout.lstat() or rpin.isspecial()
-	if Globals.change_ownership: rpout.chown(*user_group.map_rpath(rpin))
+	if Globals.change_ownership:
+		rpout.chown(*rpout.conn.user_group.map_rpath(rpin))
 	if rpin.issym(): return # symlinks don't have times or perms
 	if Globals.resource_forks_write and rpin.isreg():
 		rpout.write_resource_fork(rpin.get_resource_fork())
@@ -1068,12 +1069,33 @@ class RPath(RORPath):
 		if not fp: self.conn.rpath.RPath.fsync_local(self)
 		else: os.fsync(fp.fileno())
 
-	def fsync_local(self):
-		"""fsync current file, run locally"""
+	def fsync_local(self, thunk = None):
+		"""fsync current file, run locally
+
+		If thunk is given, run it before syncing but after gathering
+		the file's file descriptor.
+
+		"""
 		assert self.conn is Globals.local_connection
-		fd = os.open(self.path, os.O_RDONLY)
-		os.fsync(fd)
-		os.close(fd)
+		try:
+			fd = os.open(self.path, os.O_RDONLY)
+			os.fsync(fd)
+			os.close(fd)
+		except OSError, e:
+			if e.errno != errno.EPERM or self.isdir(): raise
+
+			# Maybe the system doesn't like read-only fsyncing.
+			# However, to open RDWR, we may need to alter permissions
+			# temporarily.
+			if self.hasfullperms(): oldperms = None
+			else:
+				oldperms = self.getperms()
+				self.chmod(0700)
+			fd = os.open(self.path, os.O_RDWR)
+			if oldperms is not None: self.chmod(oldperms)
+			if thunk: thunk()
+			os.fsync(fd) # Sync after we switch back permissions!
+			os.close(fd)
 
 	def fsync_with_dir(self, fp = None):
 		"""fsync self and directory self is under"""
@@ -1087,11 +1109,7 @@ class RPath(RORPath):
 		file and the directory to make sure.
 
 		"""
-		if self.lstat() and not self.issym():
-			fp = self.open("rb")
-			self.delete()
-			os.fsync(fp.fileno())
-		assert not fp.close()
+		if self.lstat() and not self.issym(): self.fsync_local(self.delete)
 		if Globals.fsync_directories: self.get_parent_rp().fsync()
 
 	def get_data(self):
@@ -1224,6 +1242,7 @@ def setdata_local(rpath):
 		rpath.get_resource_fork()
 	if Globals.carbonfile_conn and rpath.isreg(): rpath.get_carbonfile()
 
+
 # These two are overwritten by the eas_acls.py module.  We can't
 # import that module directly because of circular dependency problems.
 def acl_get(rp): assert 0
diff --git a/rdiff-backup/rdiff_backup/user_group.py b/rdiff-backup/rdiff_backup/user_group.py
index 9209daf..186e1b6 100644
--- a/rdiff-backup/rdiff_backup/user_group.py
+++ b/rdiff-backup/rdiff_backup/user_group.py
@@ -25,13 +25,16 @@ this.
 
 On the destination connection only, if necessary have a separate
 dictionary of mappings, which specify how to map users/groups on one
-connection to the users/groups on the other.
+connection to the users/groups on the other.  The UserMap and GroupMap
+objects should only be used on the destination.
 
 """
 
 import grp, pwd
 import log, Globals
 
+############ "Private" section - don't use outside user_group ###########
+
 # This should be set to the user UserMap class object if using
 # user-defined user mapping, and a Map class object otherwise.
 UserMap = None
@@ -40,73 +43,55 @@ UserMap = None
 # user-defined group mapping, and a Map class object otherwise.
 GroupMap = None
 
-
+# Used to cache by uid2uname and gid2gname below
 uid2uname_dict = {}; gid2gname_dict = {}
-def uid2uname(uid):
-	"""Given uid, return uname or None if cannot find"""
-	try: return uid2uname_dict[uid]
-	except KeyError:
-		try: uname = pwd.getpwuid(uid)[0]
-		except (KeyError, OverflowError), e: uname = None
-		uid2uname_dict[uid] = uname
-		return uname
-
-def gid2gname(gid):
-	"""Given gid, return group name or None if cannot find"""
-	try: return gid2gname_dict[gid]
-	except KeyError:
-		try: gname = grp.getgrgid(gid)[0]
-		except (KeyError, OverflowError), e: gname = None
-		gid2gname_dict[gid] = gname
-		return gname
 
+uname2uid_dict = {}
 def uname2uid(uname):
 	"""Given uname, return uid or None if cannot find"""
-	try: uname = pwd.getpwnam(uname)[2]
-	except KeyError: return None
+	try: return uname2uid_dict[uname]
+	except KeyError:
+		try: uid = pwd.getpwnam(uname)[2]
+		except KeyError: uid = None
+		uname2uid_dict[uname] = uid
+		return uid
 
+gname2gid_dict = {}
 def gname2gid(gname):
 	"""Given gname, return gid or None if cannot find"""
-	try: gname = grp.getgrnam(gname)[2]
-	except KeyError: return None
+	try: return gname2gid_dict[gname]
+	except KeyError:
+		try: gid = grp.getgrnam(gname)[2]
+		except KeyError: gid = None
+		gname2gid_dict[gname] = gid
+		return gid
 
 
 class Map:
 	"""Used for mapping names and id on source side to dest side"""
-	def __init__(self, name2id_func):
-		"""Map initializer, set dictionaries"""
-		self.name2id_dict = {}
-		self.name2id_func = name2id_func
+	def __init__(self, is_user):
+		"""Initialize, user is true for users, false for groups"""
+		self.name2id = (is_user and uname2uid) or gname2gid
 
-	def get_id(self, id, name = None):
+	def __call__(self, id, name = None):
 		"""Return mapped id from id and, if available, name"""
-		if not name: return self.get_id_from_id(id)
-		try: return self.name2id_dict[name]
-		except KeyError:
-			out_id = self.find_id(id, name)
-			self.name2id_dict[name] = out_id
-			return out_id
-
-	def get_id_from_name(self, name):
-		"""Return mapped id from name only, or None if cannot"""
-		try: return self.name2id_dict[name]
-		except KeyError:
-			out_id = self.find_id_from_name(name)
-			self.name2id_dict[name] = out_id
-			return out_id
-
-	def get_id_from_id(self, id): return id
-
-	def find_id(self, id, name):
-		"""Find the proper id to use with given id and name"""
-		try: return self.name2id_func(name)
-		except KeyError: return id
-
-	def find_id_from_name(self, name):
-		"""Look up proper id to use with name, or None"""
-		try: return self.name2id_func(name)
-		except KeyError: return None
-			
+		if not name: return id
+		newid = self.name2id(name)
+		if newid is None: return id
+		else: return newid
+
+	def map_acl(self, id, name = None):
+		"""Like get_id, but use this for ACLs.  Return id or None
+
+		Unlike ordinary user/group ownership, ACLs are not required
+		and can be dropped.  If we cannot map the name over, return
+		None.
+
+		"""
+		if not name: return id
+		return self.name2id(name)
+		
+
 class DefinedMap(Map):
 	"""Map names and ids on source side to appropriate ids on dest side
 
@@ -114,7 +99,7 @@ class DefinedMap(Map):
 	supersedes Map.
 
 	"""
-	def __init__(self, name2id_func, mapping_string):
+	def __init__(self, is_user, mapping_string):
 		"""Initialize object with given mapping string
 
 		The mapping_string should consist of a number of lines, each which
@@ -122,7 +107,7 @@ class DefinedMap(Map):
 		mapping unless user is false, then do group.
 
 		"""
-		Map.__init__(self, name2id_func)
+		Map.__init__(self, is_user)
 		self.name_mapping_dict = {}; self.id_mapping_dict = {}
 
 		for line in mapping_string.split('\n'):
@@ -142,44 +127,89 @@ class DefinedMap(Map):
 		"""Return id of id_or_name, failing if cannot.  Used in __init__"""
 		try: return int(id_or_name)
 		except ValueError:
-			try: id = self.name2id_func(id_or_name)
+			try: return self.name2id(id_or_name)
 			except KeyError:
 				log.Log.FatalError("Cannot get id for user or group name "
 								   + id_or_name)
-			return id
 
-	def get_id_from_id(self, id): return self.id_mapping_dict.get(id, id)
+	def __call__(self, id, name = None):
+		"""Return new id given old id and name"""
+		newid = self.map_acl(id, name)
+		if newid is None: return id
+		else: return newid
+
+	def map_acl(self, id, name = None):
+		"""Return new id or None given old and name (used for ACLs)"""
+		if name:
+			try: return self.name_mapping_dict[name]
+			except KeyError: pass
+			newid = self.name2id(name)
+			if newid is not None: return newid
+		try: return self.id_mapping_dict[id]
+		except KeyError: return None
+
+
+class NumericalMap:
+	"""Simple Map replacement that just keeps numerical uid or gid"""
+	def __call__(self, id, name = None): return id
+	def map_acl(self, id, name = None): return id
+
 
-	def find_id(self, id, name):
-		"""Find proper id to use when source file has give id and name"""
-		try: return self.name_mapping_dict[name]
-		except KeyError:
-			try: return self.id_mapping_dict[id]
-			except KeyError: return Map.find_id(self, id, name)
+############ Public section - don't use outside user_group ###########
 
-	def find_id_from_name(self, name):
-		"""Find id to map name to, or None if we can't"""
-		try: return self.name_mapping_dict[name]
-		except KeyError: return Map.find_id_from_name(name)
 
-def init_user_mapping(mapping_string = None):
-	"""Initialize user mapping with given mapping string or None"""
+def uid2uname(uid):
+	"""Given uid, return uname from passwd file, or None if cannot find"""
+	try: return uid2uname_dict[uid]
+	except KeyError:
+		try: uname = pwd.getpwuid(uid)[0]
+		except (KeyError, OverflowError), e: uname = None
+		uid2uname_dict[uid] = uname
+		return uname
+
+def gid2gname(gid):
+	"""Given gid, return group name from group file or None if cannot find"""
+	try: return gid2gname_dict[gid]
+	except KeyError:
+		try: gname = grp.getgrgid(gid)[0]
+		except (KeyError, OverflowError), e: gname = None
+		gid2gname_dict[gid] = gname
+		return gname
+
+
+def init_user_mapping(mapping_string = None, numerical_ids = None):
+	"""Initialize user mapping with given mapping string
+
+	If numerical_ids is set, just keep the same uid.  If either
+	argument is None, default to preserving uname where possible.
+
+	"""
 	global UserMap
-	name2id_func = lambda name: pwd.getpwnam(name)[2]
-	if mapping_string: UserMap = DefinedMap(name2id_func, mapping_string)
-	else: UserMap = Map(name2id_func)
+	if numerical_ids: UserMap = NumericalMap()
+	elif mapping_string: UserMap = DefinedMap(1, mapping_string)
+	else: UserMap = Map(1)
+
+def init_group_mapping(mapping_string = None, numerical_ids = None):
+	"""Initialize group mapping with given mapping string
 
-def init_group_mapping(mapping_string = None):
-	"""Initialize the group mapping dictionary with given mapping string"""
+	If numerical_ids is set, just keep the same gid.  If either
+	argument is None, default to preserving gname where possible.
+
+	"""
 	global GroupMap
-	name2id_func = lambda name: grp.getgrnam(name)[2]
-	if mapping_string: GroupMap = DefinedMap(name2id_func, mapping_string)
-	else: GroupMap = Map(name2id_func)
+	if numerical_ids: GroupMap = NumericalMap()
+	elif mapping_string: GroupMap = DefinedMap(0, mapping_string)
+	else: GroupMap = Map(0)
+
 
-	
 def map_rpath(rp):
-	"""Return (uid, gid) of mapped ownership of given rpath"""
-	old_uid, old_gid = rp.getuidgid()
-	new_uid = UserMap.get_id(old_uid, rp.getuname())
-	new_gid = GroupMap.get_id(old_gid, rp.getgname())
-	return (new_uid, new_gid)
+	"""Return mapped (newuid, newgid) from rpath's initial info
+
+	This is the main function exported by the user_group module.  Note
+	that it is connection specific.
+
+	"""
+	uid, gid = rp.getuidgid()
+	uname, gname = rp.getuname(), rp.getgname()
+	return (UserMap(uid, uname), GroupMap(gid, gname))
+