Allows to configure the rsync modules where the replicators will send data

Currently, the rsync module where the replicators send data is static. It
forbids administrators to set rsync configuration based on their current
deployment or needs.

As an example, the rsyncd configuration example encourages to set a connections
limit for the modules account, container and object. It permits to protect
devices from excessives parallels connections, because it would impact
performances.

On a server with many devices, it is tempting to increase this number
proportionally, but nothing guarantees that the distribution of the connections
will be balanced. In the worst scenario, a single device can receive all the
connections, which is a severe impact on performances.

This commit adds a new option named 'rsync_module' to the *-replicator sections
of the *-server configuration file. This configuration variable can be
extrapolated with device attributes like ip, port, device, zone, ... by using
the format {NAME}. eg:
    rsync_module = {replication_ip}::object_{device}

With this configuration, an administrators can solve the problem of connections
distribution by creating one module per device in rsyncd configuration.

The default values are backward compatible:
    {replication_ip}::account
    {replication_ip}::container
    {replication_ip}::object

Option vm_test_mode is deprecated by this commit, but backward compatibility is
maintained. The option is only effective when rsync_module is not set. In that
case, {replication_port} is appended to the default value of rsync_module.

Change-Id: Iad91df50dadbe96c921181797799b4444323ce2e

27 files changed, 247 insertions, 76 deletions

diff --git a/doc/manpages/account-server.conf.5 b/doc/manpages/account-server.conf.5
index cec075299..955eedfa7 100644
--- a/doc/manpages/account-server.conf.5
+++ b/doc/manpages/account-server.conf.5
@@ -176,8 +176,6 @@ Syslog log facility. The default is LOG_LOCAL0.
 Logging level. The default is INFO.
 .IP \fBlog_address\fR
 Logging address. The default is /dev/log.
-.IP \fBvm_test_mode\fR
-Indicates that you are using a VM environment. The default is no.
 .IP \fBper_diff\fR
 The default is 1000.
 .IP \fBmax_diffs\fR
diff --git a/doc/manpages/container-server.conf.5 b/doc/manpages/container-server.conf.5
index 2cd1623dc..65981308b 100644
--- a/doc/manpages/container-server.conf.5
+++ b/doc/manpages/container-server.conf.5
@@ -182,8 +182,6 @@ Syslog log facility. The default is LOG_LOCAL0.
 Logging level. The default is INFO.
 .IP \fBlog_address\fR
 Logging address. The default is /dev/log.
-.IP \fBvm_test_mode\fR
-Indicates that you are using a VM environment. The default is no.
 .IP \fBer_diff\fR
 The default is 1000.
 .IP \fBmax_diffs\fR
diff --git a/doc/manpages/object-server.conf.5 b/doc/manpages/object-server.conf.5
index e82a56ffb..c790729c1 100644
--- a/doc/manpages/object-server.conf.5
+++ b/doc/manpages/object-server.conf.5
@@ -185,8 +185,6 @@ Syslog log facility. The default is LOG_LOCAL0.
 Logging level. The default is INFO.
 .IP \fBlog_address\fR
 Logging address. The default is /dev/log.
-.IP \fBvm_test_mode\fR
-Indicates that you are using a VM environment. The default is no.
 .IP \fBdaemonize\fR
 Whether or not to run replication as a daemon. The default is yes.
 .IP "\fBrun_pause [deprecated]\fR"
diff --git a/doc/saio/swift/account-server/1.conf b/doc/saio/swift/account-server/1.conf
index ce599a7c8..91ded8905 100644
--- a/doc/saio/swift/account-server/1.conf
+++ b/doc/saio/swift/account-server/1.conf
@@ -20,7 +20,7 @@ use = egg:swift#account
 use = egg:swift#recon
 
 [account-replicator]
-vm_test_mode = yes
+rsync_module = {replication_ip}::account{replication_port}
 
 [account-auditor]
 
diff --git a/doc/saio/swift/account-server/2.conf b/doc/saio/swift/account-server/2.conf
index 0593e9ad3..70f8fae89 100644
--- a/doc/saio/swift/account-server/2.conf
+++ b/doc/saio/swift/account-server/2.conf
@@ -20,7 +20,7 @@ use = egg:swift#account
 use = egg:swift#recon
 
 [account-replicator]
-vm_test_mode = yes
+rsync_module = {replication_ip}::account{replication_port}
 
 [account-auditor]
 
diff --git a/doc/saio/swift/account-server/3.conf b/doc/saio/swift/account-server/3.conf
index abc27c258..6090883bd 100644
--- a/doc/saio/swift/account-server/3.conf
+++ b/doc/saio/swift/account-server/3.conf
@@ -20,7 +20,7 @@ use = egg:swift#account
 use = egg:swift#recon
 
 [account-replicator]
-vm_test_mode = yes
+rsync_module = {replication_ip}::account{replication_port}
 
 [account-auditor]
 
diff --git a/doc/saio/swift/account-server/4.conf b/doc/saio/swift/account-server/4.conf
index cda9ac87b..e435842d1 100644
--- a/doc/saio/swift/account-server/4.conf
+++ b/doc/saio/swift/account-server/4.conf
@@ -20,7 +20,7 @@ use = egg:swift#account
 use = egg:swift#recon
 
 [account-replicator]
-vm_test_mode = yes
+rsync_module = {replication_ip}::account{replication_port}
 
 [account-auditor]
 
diff --git a/doc/saio/swift/container-server/1.conf b/doc/saio/swift/container-server/1.conf
index 176096dbe..5bf3c0f28 100644
--- a/doc/saio/swift/container-server/1.conf
+++ b/doc/saio/swift/container-server/1.conf
@@ -20,7 +20,7 @@ use = egg:swift#container
 use = egg:swift#recon
 
 [container-replicator]
-vm_test_mode = yes
+rsync_module = {replication_ip}::container{replication_port}
 
 [container-updater]
 
diff --git a/doc/saio/swift/container-server/2.conf b/doc/saio/swift/container-server/2.conf
index 7100710b3..7046236fa 100644
--- a/doc/saio/swift/container-server/2.conf
+++ b/doc/saio/swift/container-server/2.conf
@@ -20,7 +20,7 @@ use = egg:swift#container
 use = egg:swift#recon
 
 [container-replicator]
-vm_test_mode = yes
+rsync_module = {replication_ip}::container{replication_port}
 
 [container-updater]
 
diff --git a/doc/saio/swift/container-server/3.conf b/doc/saio/swift/container-server/3.conf
index 06ec47414..c7f726199 100644
--- a/doc/saio/swift/container-server/3.conf
+++ b/doc/saio/swift/container-server/3.conf
@@ -20,7 +20,7 @@ use = egg:swift#container
 use = egg:swift#recon
 
 [container-replicator]
-vm_test_mode = yes
+rsync_module = {replication_ip}::container{replication_port}
 
 [container-updater]
 
diff --git a/doc/saio/swift/container-server/4.conf b/doc/saio/swift/container-server/4.conf
index 1acc3b5c5..ff9c0e806 100644
--- a/doc/saio/swift/container-server/4.conf
+++ b/doc/saio/swift/container-server/4.conf
@@ -20,7 +20,7 @@ use = egg:swift#container
 use = egg:swift#recon
 
 [container-replicator]
-vm_test_mode = yes
+rsync_module = {replication_ip}::container{replication_port}
 
 [container-updater]
 
diff --git a/doc/saio/swift/object-server/1.conf b/doc/saio/swift/object-server/1.conf
index 178e3fcba..c6d1b1464 100644
--- a/doc/saio/swift/object-server/1.conf
+++ b/doc/saio/swift/object-server/1.conf
@@ -20,7 +20,7 @@ use = egg:swift#object
 use = egg:swift#recon
 
 [object-replicator]
-vm_test_mode = yes
+rsync_module = {replication_ip}::object{replication_port}
 
 [object-reconstructor]
 
diff --git a/doc/saio/swift/object-server/2.conf b/doc/saio/swift/object-server/2.conf
index 6b611ca25..3d863cf59 100644
--- a/doc/saio/swift/object-server/2.conf
+++ b/doc/saio/swift/object-server/2.conf
@@ -20,7 +20,7 @@ use = egg:swift#object
 use = egg:swift#recon
 
 [object-replicator]
-vm_test_mode = yes
+rsync_module = {replication_ip}::object{replication_port}
 
 [object-reconstructor]
 
diff --git a/doc/saio/swift/object-server/3.conf b/doc/saio/swift/object-server/3.conf
index 735259231..d99995bb5 100644
--- a/doc/saio/swift/object-server/3.conf
+++ b/doc/saio/swift/object-server/3.conf
@@ -20,7 +20,7 @@ use = egg:swift#object
 use = egg:swift#recon
 
 [object-replicator]
-vm_test_mode = yes
+rsync_module = {replication_ip}::object{replication_port}
 
 [object-reconstructor]
 
diff --git a/doc/saio/swift/object-server/4.conf b/doc/saio/swift/object-server/4.conf
index be1211047..9f6fd1fc1 100644
--- a/doc/saio/swift/object-server/4.conf
+++ b/doc/saio/swift/object-server/4.conf
@@ -20,7 +20,7 @@ use = egg:swift#object
 use = egg:swift#recon
 
 [object-replicator]
-vm_test_mode = yes
+rsync_module = {replication_ip}::object{replication_port}
 
 [object-reconstructor]
 
diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst
index da9d0b4fa..1516d083c 100644
--- a/doc/source/deployment_guide.rst
+++ b/doc/source/deployment_guide.rst
@@ -594,6 +594,17 @@ node_timeout        DEFAULT or 10      Request timeout to external services.
                                        in the DEFAULT section, or 10 (though
                                        other sections use 3 as the final
                                        default).
+rsync_module        {replication_ip}::object
+                                       Format of the rsync module where the
+                                       replicator will send data. The
+                                       configuration value can include some
+                                       variables that will be extracted from
+                                       the ring. Variables must follow the
+                                       format {NAME} where NAME is one of:
+                                       ip, port, replication_ip,
+                                       replication_port, region, zone, device,
+                                       meta. See etc/rsyncd.conf-sample for
+                                       some examples.
 ==================  =================  =======================================
 
 [object-updater]
@@ -723,6 +734,18 @@ conn_timeout        0.5                   Connection timeout to external
                                           services
 reclaim_age         604800                Time elapsed in seconds before a
                                           container can be reclaimed
+rsync_module        {replication_ip}::container
+                                          Format of the rsync module where the
+                                          replicator will send data. The
+                                          configuration value can include some
+                                          variables that will be extracted from
+                                          the ring. Variables must follow the
+                                          format {NAME} where NAME is one of:
+                                          ip, port, replication_ip,
+                                          replication_port, region, zone,
+                                          device, meta. See
+                                          etc/rsyncd.conf-sample for some
+                                          examples.
 ==================  ====================  ====================================
 
 [container-updater]
@@ -850,6 +873,18 @@ node_timeout        10                  Request timeout to external services
 conn_timeout        0.5                 Connection timeout to external services
 reclaim_age         604800              Time elapsed in seconds before an
                                         account can be reclaimed
+rsync_module        {replication_ip}::account
+                                        Format of the rsync module where the
+                                        replicator will send data. The
+                                        configuration value can include some
+                                        variables that will be extracted from
+                                        the ring. Variables must follow the
+                                        format {NAME} where NAME is one of:
+                                        ip, port, replication_ip,
+                                        replication_port, region, zone,
+                                        device, meta. See
+                                        etc/rsyncd.conf-sample for some
+                                        examples.
 ==================  ==================  ======================================
 
 [account-auditor]
diff --git a/doc/source/replication_network.rst b/doc/source/replication_network.rst
index 138cb4218..609ebc968 100644
--- a/doc/source/replication_network.rst
+++ b/doc/source/replication_network.rst
@@ -146,7 +146,7 @@ For SAIO replication
 
    delete all configuration options in section [<*>-replicator]
 
-#. Add configuration files for object-server, in /etc/swift/objec-server/
+#. Add configuration files for object-server, in /etc/swift/object-server/
 
    * 5.conf::
 
@@ -170,7 +170,7 @@ For SAIO replication
         use = egg:swift#recon
 
         [object-replicator]
-        vm_test_mode = yes
+        rsync_module = {replication_ip}::object{replication_port}
 
    * 6.conf::
 
@@ -194,7 +194,7 @@ For SAIO replication
         use = egg:swift#recon
 
         [object-replicator]
-        vm_test_mode = yes
+        rsync_module = {replication_ip}::object{replication_port}
 
    * 7.conf::
 
@@ -218,7 +218,7 @@ For SAIO replication
         use = egg:swift#recon
 
         [object-replicator]
-        vm_test_mode = yes
+        rsync_module = {replication_ip}::object{replication_port}
 
    * 8.conf::
 
@@ -242,7 +242,7 @@ For SAIO replication
         use = egg:swift#recon
 
         [object-replicator]
-        vm_test_mode = yes
+        rsync_module = {replication_ip}::object{replication_port}
 
 #. Add configuration files for container-server, in /etc/swift/container-server/
 
@@ -268,7 +268,7 @@ For SAIO replication
         use = egg:swift#recon
 
         [container-replicator]
-        vm_test_mode = yes
+        rsync_module = {replication_ip}::container{replication_port}
 
    * 6.conf::
 
@@ -292,7 +292,7 @@ For SAIO replication
         use = egg:swift#recon
 
         [container-replicator]
-        vm_test_mode = yes
+        rsync_module = {replication_ip}::container{replication_port}
 
    * 7.conf::
 
@@ -316,7 +316,7 @@ For SAIO replication
         use = egg:swift#recon
 
         [container-replicator]
-        vm_test_mode = yes
+        rsync_module = {replication_ip}::container{replication_port}
 
    * 8.conf::
 
@@ -340,7 +340,7 @@ For SAIO replication
         use = egg:swift#recon
 
         [container-replicator]
-        vm_test_mode = yes
+        rsync_module = {replication_ip}::container{replication_port}
 
 #. Add configuration files for account-server, in /etc/swift/account-server/
 
@@ -366,7 +366,7 @@ For SAIO replication
         use = egg:swift#recon
 
         [account-replicator]
-        vm_test_mode = yes
+        rsync_module = {replication_ip}::account{replication_port}
 
    * 6.conf::
 
@@ -390,7 +390,7 @@ For SAIO replication
         use = egg:swift#recon
 
         [account-replicator]
-        vm_test_mode = yes
+        rsync_module = {replication_ip}::account{replication_port}
 
    * 7.conf::
 
@@ -414,7 +414,7 @@ For SAIO replication
         use = egg:swift#recon
 
         [account-replicator]
-        vm_test_mode = yes
+        rsync_module = {replication_ip}::account{replication_port}
 
    * 8.conf::
 
@@ -438,7 +438,7 @@ For SAIO replication
         use = egg:swift#recon
 
         [account-replicator]
-        vm_test_mode = yes
+        rsync_module = {replication_ip}::account{replication_port}
 
 
 ---------------------------------
diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample
index 8cfbfa009..9eb61a9d9 100644
--- a/etc/account-server.conf-sample
+++ b/etc/account-server.conf-sample
@@ -90,7 +90,6 @@ use = egg:swift#recon
 # log_level = INFO
 # log_address = /dev/log
 #
-# vm_test_mode = no
 # per_diff = 1000
 # max_diffs = 100
 # concurrency = 8
@@ -111,6 +110,10 @@ use = egg:swift#recon
 # a different region than the local one.
 # rsync_compress = no
 #
+# Format of the rysnc module where the replicator will send data. See
+# etc/rsyncd.conf-sample for some usage examples.
+# rsync_module = {replication_ip}::account
+#
 # recon_cache_path = /var/cache/swift
 
 [account-auditor]
diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample
index 54daee1e8..0eebf366e 100644
--- a/etc/container-server.conf-sample
+++ b/etc/container-server.conf-sample
@@ -99,7 +99,6 @@ use = egg:swift#recon
 # log_level = INFO
 # log_address = /dev/log
 #
-# vm_test_mode = no
 # per_diff = 1000
 # max_diffs = 100
 # concurrency = 8
@@ -120,6 +119,10 @@ use = egg:swift#recon
 # a different region than the local one.
 # rsync_compress = no
 #
+# Format of the rysnc module where the replicator will send data. See
+# etc/rsyncd.conf-sample for some usage examples.
+# rsync_module = {replication_ip}::container
+#
 # recon_cache_path = /var/cache/swift
 
 [container-updater]
diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample
index 31bd160a3..84328b0c3 100644
--- a/etc/object-server.conf-sample
+++ b/etc/object-server.conf-sample
@@ -162,7 +162,6 @@ use = egg:swift#recon
 # log_level = INFO
 # log_address = /dev/log
 #
-# vm_test_mode = no
 # daemonize = on
 #
 # Time in seconds to wait between replication passes
@@ -195,6 +194,10 @@ use = egg:swift#recon
 # slow down the syncing process.
 # rsync_compress = no
 #
+# Format of the rysnc module where the replicator will send data. See
+# etc/rsyncd.conf-sample for some usage examples.
+# rsync_module = {replication_ip}::object
+#
 # node_timeout = <whatever's in the DEFAULT section or 10>
 # max duration of an http request; this is for REPLICATE finalization calls and
 # so should be longer than node_timeout
diff --git a/etc/rsyncd.conf-sample b/etc/rsyncd.conf-sample
index c3b9952b1..103a85a57 100644
--- a/etc/rsyncd.conf-sample
+++ b/etc/rsyncd.conf-sample
@@ -20,3 +20,59 @@ max connections = 8
 path = /srv/node
 read only = false
 lock file = /var/lock/object.lock
+
+
+# If rsync_module includes the device, you can tune rsyncd to permit 4
+# connections per device instead of simply allowing 8 connections for all
+# devices:
+# rsync_module = {replication_ip}::object_{device}
+#
+# (if devices in your object ring are named sda, sdb and sdc)
+#
+#[object_sda]
+#max connections = 4
+#path = /srv/node
+#read only = false
+#lock file = /var/lock/object_sda.lock
+#
+#[object_sdb]
+#max connections = 4
+#path = /srv/node
+#read only = false
+#lock file = /var/lock/object_sdb.lock
+#
+#[object_sdc]
+#max connections = 4
+#path = /srv/node
+#read only = false
+#lock file = /var/lock/object_sdc.lock
+
+
+# To emulate the deprecated option vm_test_mode = yes, set:
+# rsync_module = {replication_ip}::object{replication_port}
+#
+# So, on your SAIO, you have to set the following rsyncd configuration:
+#
+#[object6010]
+#max connections = 25
+#path = /srv/1/node/
+#read only = false
+#lock file = /var/lock/object6010.lock
+#
+#[object6020]
+#max connections = 25
+#path = /srv/2/node/
+#read only = false
+#lock file = /var/lock/object6020.lock
+#
+#[object6030]
+#max connections = 25
+#path = /srv/3/node/
+#read only = false
+#lock file = /var/lock/object6030.lock
+#
+#[object6040]
+#max connections = 25
+#path = /srv/4/node/
+#read only = false
+#lock file = /var/lock/object6040.lock
diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py
index 7a6e8d549..71d1634c4 100644
--- a/swift/common/db_replicator.py
+++ b/swift/common/db_replicator.py
@@ -31,7 +31,8 @@ import swift.common.db
 from swift.common.direct_client import quote
 from swift.common.utils import get_logger, whataremyips, storage_directory, \
     renamer, mkdirs, lock_parent_directory, config_true_value, \
-    unlink_older_than, dump_recon_cache, rsync_ip, ismount, json, Timestamp
+    unlink_older_than, dump_recon_cache, rsync_module_interpolation, ismount, \
+    json, Timestamp
 from swift.common import ring
 from swift.common.ring.utils import is_local_device
 from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE
@@ -165,11 +166,20 @@ class Replicator(Daemon):
         self.max_diffs = int(conf.get('max_diffs') or 100)
         self.interval = int(conf.get('interval') or
                             conf.get('run_pause') or 30)
-        self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no'))
         self.node_timeout = int(conf.get('node_timeout', 10))
         self.conn_timeout = float(conf.get('conn_timeout', 0.5))
         self.rsync_compress = config_true_value(
             conf.get('rsync_compress', 'no'))
+        self.rsync_module = conf.get('rsync_module', '').rstrip('/')
+        if not self.rsync_module:
+            self.rsync_module = '{replication_ip}::%s' % self.server_type
+            if config_true_value(conf.get('vm_test_mode', 'no')):
+                self.logger.warn('Option %(type)s-replicator/vm_test_mode is '
+                                 'deprecated and will be removed in a future '
+                                 'version. Update your configuration to use '
+                                 'option %(type)s-replicator/rsync_module.'
+                                 % {'type': self.server_type})
+                self.rsync_module += '{replication_port}'
         self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7))
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
@@ -267,14 +277,9 @@ class Replicator(Daemon):
         :param different_region: if True, the destination node is in a
                                  different region
         """
-        device_ip = rsync_ip(device['replication_ip'])
-        if self.vm_test_mode:
-            remote_file = '%s::%s%s/%s/tmp/%s' % (
-                device_ip, self.server_type, device['replication_port'],
-                device['device'], local_id)
-        else:
-            remote_file = '%s::%s/%s/tmp/%s' % (
-                device_ip, self.server_type, device['device'], local_id)
+        rsync_module = rsync_module_interpolation(self.rsync_module, device)
+        rsync_path = '%s/tmp/%s' % (device['device'], local_id)
+        remote_file = '%s/%s' % (rsync_module, rsync_path)
         mtime = os.path.getmtime(broker.db_file)
         if not self._rsync_file(broker.db_file, remote_file,
                                 different_region=different_region):
diff --git a/swift/common/utils.py b/swift/common/utils.py
index 0dbbda6c0..996486916 100644
--- a/swift/common/utils.py
+++ b/swift/common/utils.py
@@ -2703,6 +2703,33 @@ def rsync_ip(ip):
         return '[%s]' % ip
 
 
+def rsync_module_interpolation(template, device):
+    """
+    Interpolate devices variables inside a rsync module template
+
+    :param template: rsync module template as a string
+    :param device: a device from a ring
+
+    :returns: a string with all variables replaced by device attributes
+    """
+    replacements = {
+        'ip': rsync_ip(device.get('ip', '')),
+        'port': device.get('port', ''),
+        'replication_ip': rsync_ip(device.get('replication_ip', '')),
+        'replication_port': device.get('replication_port', ''),
+        'region': device.get('region', ''),
+        'zone': device.get('zone', ''),
+        'device': device.get('device', ''),
+        'meta': device.get('meta', ''),
+    }
+    try:
+        module = template.format(**replacements)
+    except KeyError as e:
+        raise ValueError('Cannot interpolate rsync_module, invalid variable: '
+                         '%s' % e)
+    return module
+
+
 def get_valid_utf8_str(str_or_unicode):
     """
     Get valid parts of utf-8 str from str, unicode and even invalid utf-8 str
diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py
index 842a2a859..124df3e78 100644
--- a/swift/obj/replicator.py
+++ b/swift/obj/replicator.py
@@ -31,8 +31,8 @@ from eventlet.support.greenlets import GreenletExit
 from swift.common.ring.utils import is_local_device
 from swift.common.utils import whataremyips, unlink_older_than, \
     compute_eta, get_logger, dump_recon_cache, ismount, \
-    rsync_ip, mkdirs, config_true_value, list_from_csv, get_hub, \
-    tpool_reraise, config_auto_int_value, storage_directory
+    rsync_module_interpolation, mkdirs, config_true_value, list_from_csv, \
+    get_hub, tpool_reraise, config_auto_int_value, storage_directory
 from swift.common.bufferedhttp import http_connect
 from swift.common.daemon import Daemon
 from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE
@@ -62,7 +62,6 @@ class ObjectReplicator(Daemon):
         self.logger = logger or get_logger(conf, log_route='object-replicator')
         self.devices_dir = conf.get('devices', '/srv/node')
         self.mount_check = config_true_value(conf.get('mount_check', 'true'))
-        self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no'))
         self.swift_dir = conf.get('swift_dir', '/etc/swift')
         self.bind_ip = conf.get('bind_ip', '0.0.0.0')
         self.servers_per_port = int(conf.get('servers_per_port', '0') or 0)
@@ -81,6 +80,15 @@ class ObjectReplicator(Daemon):
         self.rsync_bwlimit = conf.get('rsync_bwlimit', '0')
         self.rsync_compress = config_true_value(
             conf.get('rsync_compress', 'no'))
+        self.rsync_module = conf.get('rsync_module', '').rstrip('/')
+        if not self.rsync_module:
+            self.rsync_module = '{replication_ip}::object'
+            if config_true_value(conf.get('vm_test_mode', 'no')):
+                self.logger.warn('Option object-replicator/vm_test_mode is '
+                                 'deprecated and will be removed in a future '
+                                 'version. Update your configuration to use '
+                                 'option object-replicator/rsync_module.')
+                self.rsync_module += '{replication_port}'
         self.http_timeout = int(conf.get('http_timeout', 60))
         self.lockup_timeout = int(conf.get('lockup_timeout', 1800))
         self.recon_cache_path = conf.get('recon_cache_path',
@@ -223,11 +231,7 @@ class ObjectReplicator(Daemon):
             # Allow for compression, but only if the remote node is in
             # a different region than the local one.
             args.append('--compress')
-        node_ip = rsync_ip(node['replication_ip'])
-        if self.vm_test_mode:
-            rsync_module = '%s::object%s' % (node_ip, node['replication_port'])
-        else:
-            rsync_module = '%s::object' % node_ip
+        rsync_module = rsync_module_interpolation(self.rsync_module, node)
         had_any = False
         for suffix in suffixes:
             spath = join(job['path'], suffix)
diff --git a/test/probe/common.py b/test/probe/common.py
index 45a907444..37bf88553 100644
--- a/test/probe/common.py
+++ b/test/probe/common.py
@@ -29,7 +29,8 @@ from six.moves.http_client import HTTPConnection
 from swiftclient import get_auth, head_account
 from swift.obj.diskfile import get_data_dir
 from swift.common.ring import Ring
-from swift.common.utils import readconf, renamer
+from swift.common.utils import readconf, renamer, \
+    config_true_value, rsync_module_interpolation
 from swift.common.manager import Manager
 from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY
 
@@ -219,11 +220,12 @@ def get_ring(ring_name, required_replicas, required_devices,
                 "unable to find ring device %s under %s's devices (%s)" % (
                     dev['device'], server, conf['devices']))
         # verify server is exposing rsync device
-        if conf.get('vm_test_mode', False):
-            rsync_export = '%s%s' % (server, dev['replication_port'])
-        else:
-            rsync_export = server
-        cmd = "rsync rsync://localhost/%s" % rsync_export
+        rsync_export = conf.get('rsync_module', '').rstrip('/')
+        if not rsync_export:
+            rsync_export = '{replication_ip}::%s' % server
+            if config_true_value(conf.get('vm_test_mode', 'no')):
+                rsync_export += '{replication_port}'
+        cmd = "rsync %s" % rsync_module_interpolation(rsync_export, dev)
         p = Popen(cmd, shell=True, stdout=PIPE)
         stdout, _stderr = p.communicate()
         if p.returncode:
diff --git a/test/unit/common/test_db_replicator.py b/test/unit/common/test_db_replicator.py
index 91a5adfbf..03c386435 100644
--- a/test/unit/common/test_db_replicator.py
+++ b/test/unit/common/test_db_replicator.py
@@ -364,10 +364,6 @@ class TestDBReplicator(unittest.TestCase):
                        'replication_ip': '127.0.0.1', 'replication_port': '0',
                        'device': 'sda1'}
 
-        def mock_rsync_ip(ip):
-            self.assertEquals(fake_device['ip'], ip)
-            return 'rsync_ip(%s)' % ip
-
         class MyTestReplicator(TestReplicator):
             def __init__(self, db_file, remote_file):
                 super(MyTestReplicator, self).__init__({})
@@ -381,20 +377,11 @@ class TestDBReplicator(unittest.TestCase):
                 self_._rsync_file_called = True
                 return False
 
-        with patch('swift.common.db_replicator.rsync_ip', mock_rsync_ip):
-            broker = FakeBroker()
-            remote_file = 'rsync_ip(127.0.0.1)::container/sda1/tmp/abcd'
-            replicator = MyTestReplicator(broker.db_file, remote_file)
-            replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd')
-            self.assertTrue(replicator._rsync_file_called)
-
-        with patch('swift.common.db_replicator.rsync_ip', mock_rsync_ip):
-            broker = FakeBroker()
-            remote_file = 'rsync_ip(127.0.0.1)::container0/sda1/tmp/abcd'
-            replicator = MyTestReplicator(broker.db_file, remote_file)
-            replicator.vm_test_mode = True
-            replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd')
-            self.assertTrue(replicator._rsync_file_called)
+        broker = FakeBroker()
+        remote_file = '127.0.0.1::container/sda1/tmp/abcd'
+        replicator = MyTestReplicator(broker.db_file, remote_file)
+        replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd')
+        self.assertTrue(replicator._rsync_file_called)
 
     def test_rsync_db_rsync_file_failure(self):
         class MyTestReplicator(TestReplicator):
diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py
index 653d939cc..d1fdb1f41 100644
--- a/test/unit/common/test_utils.py
+++ b/test/unit/common/test_utils.py
@@ -2244,6 +2244,58 @@ cluster_dfw1 = http://dfw1.host/v1/
         self.assertEqual(
             utils.rsync_ip('::ffff:192.0.2.128'), '[::ffff:192.0.2.128]')
 
+    def test_rsync_module_interpolation(self):
+        fake_device = {'ip': '127.0.0.1', 'port': 11,
+                       'replication_ip': '127.0.0.2', 'replication_port': 12,
+                       'region': '1', 'zone': '2', 'device': 'sda1',
+                       'meta': 'just_a_string'}
+
+        self.assertEqual(
+            utils.rsync_module_interpolation('{ip}', fake_device),
+            '127.0.0.1')
+        self.assertEqual(
+            utils.rsync_module_interpolation('{port}', fake_device),
+            '11')
+        self.assertEqual(
+            utils.rsync_module_interpolation('{replication_ip}', fake_device),
+            '127.0.0.2')
+        self.assertEqual(
+            utils.rsync_module_interpolation('{replication_port}',
+                                             fake_device),
+            '12')
+        self.assertEqual(
+            utils.rsync_module_interpolation('{region}', fake_device),
+            '1')
+        self.assertEqual(
+            utils.rsync_module_interpolation('{zone}', fake_device),
+            '2')
+        self.assertEqual(
+            utils.rsync_module_interpolation('{device}', fake_device),
+            'sda1')
+        self.assertEqual(
+            utils.rsync_module_interpolation('{meta}', fake_device),
+            'just_a_string')
+
+        self.assertEqual(
+            utils.rsync_module_interpolation('{replication_ip}::object',
+                                             fake_device),
+            '127.0.0.2::object')
+        self.assertEqual(
+            utils.rsync_module_interpolation('{ip}::container{port}',
+                                             fake_device),
+            '127.0.0.1::container11')
+        self.assertEqual(
+            utils.rsync_module_interpolation(
+                '{replication_ip}::object_{device}', fake_device),
+            '127.0.0.2::object_sda1')
+        self.assertEqual(
+            utils.rsync_module_interpolation(
+                '127.0.0.3::object_{replication_port}', fake_device),
+            '127.0.0.3::object_12')
+
+        self.assertRaises(ValueError, utils.rsync_module_interpolation,
+                          '{replication_ip}::object_{deivce}', fake_device)
+
     def test_fallocate_reserve(self):
 
         class StatVFS(object):