QPID-3603: Update HA documentatiion

Also renamed --ha-replicate-default to --ha-replicate

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1310026 13f79535-47bb-0310-9956-ffa450edef68

3 files changed, 269 insertions, 242 deletions

diff --git a/qpid/cpp/src/qpid/ha/HaPlugin.cpp b/qpid/cpp/src/qpid/ha/HaPlugin.cpp
index 419ee962da..4da3b0d7d2 100644
--- a/qpid/cpp/src/qpid/ha/HaPlugin.cpp
+++ b/qpid/cpp/src/qpid/ha/HaPlugin.cpp
@@ -37,7 +37,7 @@ struct Options : public qpid::Options {
              "URL that backup brokers use to connect and fail over.")
             ("ha-public-brokers", optValue(settings.clientUrl,"URL"),
              "URL that clients use to connect and fail over, defaults to ha-brokers.")
-            ("ha-replicate-default",
+            ("ha-replicate",
              optValue(settings.replicateDefault, "LEVEL"),
             "Replication level for creating queues and exchanges if there is no qpid.replicate argument supplied. LEVEL is 'none', 'configuration' or 'all'")
             ("ha-expected-backups", optValue(settings.expectedBackups, "N"),
diff --git a/qpid/cpp/src/tests/ha_tests.py b/qpid/cpp/src/tests/ha_tests.py
index 10d5fc0db2..a9771350aa 100755
--- a/qpid/cpp/src/tests/ha_tests.py
+++ b/qpid/cpp/src/tests/ha_tests.py
@@ -30,7 +30,7 @@ log = getLogger("qpid.ha-tests")
 
 class HaBroker(Broker):
     def __init__(self, test, args=[], broker_url=None, ha_cluster=True,
-                 ha_replicate_default="all", **kwargs):
+                 ha_replicate="all", **kwargs):
         assert BrokerTest.ha_lib, "Cannot locate HA plug-in"
         args = copy(args)
         args += ["--load-module", BrokerTest.ha_lib,
@@ -38,8 +38,8 @@ class HaBroker(Broker):
                  # FIXME aconway 2012-02-13: workaround slow link failover.
                  "--link-maintenace-interval=0.1",
                  "--ha-cluster=%s"%ha_cluster]
-        if ha_replicate_default is not None:
-            args += [ "--ha-replicate-default=%s"%ha_replicate_default ]
+        if ha_replicate is not None:
+            args += [ "--ha-replicate=%s"%ha_replicate ]
         if broker_url: args.extend([ "--ha-brokers", broker_url ])
         Broker.__init__(self, test, args, **kwargs)
         self.commands=os.getenv("PYTHON_COMMANDS")
@@ -533,10 +533,10 @@ class ReplicationTests(HaTest):
         for t in tests: t.verify(self, backup2)
 
     def test_replicate_default(self):
-        """Make sure we don't replicate if ha-replicate-default is unspecified or none"""
-        cluster1 = HaCluster(self, 2, ha_replicate_default=None)
+        """Make sure we don't replicate if ha-replicate is unspecified or none"""
+        cluster1 = HaCluster(self, 2, ha_replicate=None)
         c1 = cluster1[0].connect().session().sender("q;{create:always}")
-        cluster2 = HaCluster(self, 2, ha_replicate_default="none")
+        cluster2 = HaCluster(self, 2, ha_replicate="none")
         cluster2[0].connect().session().sender("q;{create:always}")
         time.sleep(.1)               # Give replication a chance.
         try:
@@ -550,7 +550,7 @@ class ReplicationTests(HaTest):
 
     def test_invalid_default(self):
         """Verify that a queue with an invalid qpid.replicate gets default treatment"""
-        cluster = HaCluster(self, 2, ha_replicate_default="all")
+        cluster = HaCluster(self, 2, ha_replicate="all")
         c = cluster[0].connect().session().sender("q;{create:always, node:{x-declare:{arguments:{'qpid.replicate':XXinvalidXX}}}}")
         self.wait_backup(cluster[1], "q")
 
diff --git a/qpid/doc/book/src/Active-Passive-Cluster.xml b/qpid/doc/book/src/Active-Passive-Cluster.xml
index 52748b2570..4e7c72d6e0 100644
--- a/qpid/doc/book/src/Active-Passive-Cluster.xml
+++ b/qpid/doc/book/src/Active-Passive-Cluster.xml
@@ -51,7 +51,7 @@ under the License.
       This approach depends on an external <firstterm>cluster resource manager</firstterm> to detect
       failures and choose the primary. <ulink
       url="https://fedorahosted.org/cluster/wiki/RGManager">Rgmanager</ulink> is supported
-      initially, but others may be supported in future future.
+      initially, but others may be supported in the future.
     </para>
     <section>
       <title>Why the new approach?</title>
@@ -162,11 +162,12 @@ under the License.
       virtual IP addresses for clients or brokers.
     </para>
   </section>
+  
   <section>
     <title>Configuring the Brokers</title>
     <para>
-      The broker must load the <filename>ha</filename> module, it is loaded by default. The
-      following broker options are available for the HA module.
+      The broker must load the <filename>ha</filename> module, it is loaded by
+      default. The following broker options are available for the HA module.
     </para>
     <table frame="all" id="ha-broker-options">
       <title>Options for High Availability Messaging Cluster</title>
@@ -194,31 +195,50 @@ under the License.
 	      <literal>--ha-brokers <replaceable>URL</replaceable></literal>
 	    </entry>
 	    <entry>
-	      URL use by brokers to connect to each other. The URL lists the addresses of
-	      all the brokers in the cluster
-	      <footnote>
-		<para>
-		  If the resource manager supports virtual IP addresses then the URL can
-		  contain just the single virtual IP.
-		</para>
-	      </footnote>
-	      in the following form:
-	      <programlisting>
-		url = ["amqp:"][ user ["/" password] "@" ] addr ("," addr)*
-		addr = tcp_addr / rmda_addr / ssl_addr / ...
-		tcp_addr = ["tcp:"] host [":" port]
-		rdma_addr = "rdma:" host [":" port]
-		ssl_addr = "ssl:" host [":" port]'
-	      </programlisting>
+	      <para>
+		The URL
+		<footnote>
+		  <para>
+		  The full format of the URL is given by this grammar:
+		  <programlisting>
+		    url = ["amqp:"][ user ["/" password] "@" ] addr ("," addr)*
+		    addr = tcp_addr / rmda_addr / ssl_addr / ...
+		    tcp_addr = ["tcp:"] host [":" port]
+		    rdma_addr = "rdma:" host [":" port]
+		    ssl_addr = "ssl:" host [":" port]'
+		  </programlisting>
+		  </para>
+		</footnote>
+		used by brokers to connect to each other. If you use a virtual IP address
+		then this is a single address, for example
+		<literal>amqp:20.0.20.200</literal>. If you do not use a virtual IP
+		address then the URL must list all the addresses of brokers in the
+		cluster, for example <literal>amqp:node1,node2,node3</literal>
+	      </para>
 	    </entry>
 	  </row>
 	  <row>
-	    <entry> <literal>--ha-public-brokers <replaceable>URL</replaceable></literal> </entry>
+	    <entry><literal>--ha-public-brokers <replaceable>URL</replaceable></literal> </entry>
 	    <entry>
-	      URL used by clients to connect to the brokers in the same format as
-	      <literal>--ha-brokers</literal> above. Use this option if you want client
-	      traffic on a different network from broker replication traffic. If this
-	      option is not set, clients will use the same URL as brokers.
+	      <para>
+		The URL used by clients to connect to the brokers. This has the same
+		format as the <literal>--ha-brokers</literal> URL above.
+	      </para>
+	      <para>
+		If this option is not set, clients will use the same URL as brokers.
+		This option allows you to put client traffic on a different network from
+		broker traffic, which is recommended.
+	      </para>
+	    </entry>
+	  </row>
+	  <row>
+	    <entry><literal>--ha-replicate</literal></entry>
+	    <foo/>
+	    <entry>
+	      <para> 
+		Specifies whether queues and exchanges are replicated by default.
+		For details see <xref linkend="creating-replicated"/>
+	      </para>
 	    </entry>
 	  </row>
 	  <row>
@@ -228,52 +248,253 @@ under the License.
 	      <para><literal>--ha-mechanism <replaceable>MECH</replaceable></literal></para>
 	    </entry>
 	    <entry>
-	      Brokers use <replaceable>USER</replaceable>,
-	      <replaceable>PASS</replaceable>, <replaceable>MECH</replaceable> to
-	      authenticate when connecting to each other.
+	      Authentication settings used by brokers to connect to each other.
 	    </entry>
 	  </row>
 	</tbody>
       </tgroup>
     </table>
     <para>
-      To configure a cluster you must set at least <literal>ha-cluster</literal> and <literal>ha-brokers</literal>.
+      To configure a HA cluster you must set at least <literal>ha-cluster</literal> and
+      <literal>ha-brokers</literal>.
+    </para>
+  </section>
+
+  <section>
+    <title>The Cluster Resource Manager</title>
+    <para>
+      Broker fail-over is managed by a <firstterm>cluster resource
+      manager</firstterm>.  An integration with <ulink
+      url="https://fedorahosted.org/cluster/wiki/RGManager">rgmanager</ulink> is
+      provided, but it is possible to integrate with other resource managers.
+    </para>
+    <para>
+      The resource manager is responsible for starting the <command>qpidd</command> broker
+      on each node in the cluster. The resource manager <firstterm>promotes</firstterm>
+      one of the brokers to be the primary. The other brokers connect to the primary as
+      backups, using the URL provided in the <literal>ha-brokers</literal> configuration
+      option.
+    </para>
+    <para>
+      Once connected, the backup brokers synchronize their state with the
+      primary.  When a backup is synchronized, or "hot", it is ready to take
+      over if the primary fails.  Backup brokers continually receive updates
+      from the primary in order to stay synchronized.
+    </para>
+    <para>
+      If the primary fails, backup brokers go into fail-over mode. The resource
+      manager must detect the failure and promote one of the backups to be the
+      new primary.  The other backups connect to the new primary and synchronize
+      their state with it.
+    </para>
+    <para>
+      The resource manager is also responsible for protecting the cluster from
+      <firstterm>split-brain</firstterm> conditions resulting from a network partition.  A
+      network partition divide a cluster into two sub-groups which cannot see each other.
+      Usually a <firstterm>quorum</firstterm> voting algorithm is used that disables nodes
+      in the inquorate sub-group.
     </para>
   </section>
 
   <section>
+    <title>Configuring <command>rgmanager</command> as resource manager</title>
+    <para>
+      This section assumes that you are already familiar with setting up and configuring
+      clustered services using <command>cman</command> and
+      <command>rgmanager</command>. It will show you how to configure an active-passive,
+      hot-standby <command>qpidd</command> HA cluster with <command>rgmanager</command>.
+    </para>
+    <para>
+      You must provide a <literal>cluster.conf</literal> file to configure
+      <command>cman</command> and <command>rgmanager</command>.  Here is
+      an example <literal>cluster.conf</literal> file for a cluster of 3 nodes named
+      node1, node2 and node3. We will go through the configuration step-by-step.
+    </para>
+    <programlisting>
+      <![CDATA[
+<?xml version="1.0"?>
+<!--
+This is an example of a cluster.conf file to run qpidd HA under rgmanager.
+This example assumes a 3 node cluster, with nodes named node1, node2 and node3.
+-->
+
+<cluster name="qpid-test" config_version="18">
+  <!-- The cluster has 3 nodes. Each has a unique nodid and one vote for quorum. -->
+  <clusternodes>
+    <clusternode name="node1" nodeid="1">
+      <fence/>
+    </clusternode>
+    <clusternode name="node2" nodeid="2">
+      <fence/>
+    </clusternode>
+    <clusternode name="node3" nodeid="3">
+      <fence/>
+    </clusternode>
+  </clusternodes>
+  <!-- Resouce Manager configuration. -->
+  <rm log_level="7">		<!-- Verbose logging -->
+    <!--
+	There is a failoverdomain for each node containing just that node.
+	This lets us stipulate that the qpidd service should always run on all nodes.
+    -->
+    <failoverdomains>
+      <failoverdomain name="node1-domain" restricted="1">
+	<failoverdomainnode name="node1"/>
+      </failoverdomain>
+      <failoverdomain name="node2-domain" restricted="1">
+	<failoverdomainnode name="node2"/>
+      </failoverdomain>
+      <failoverdomain name="node3-domain" restricted="1">
+	<failoverdomainnode name="node3"/>
+      </failoverdomain>
+    </failoverdomains>
+
+    <resources>
+      <!-- This script starts a qpidd broker acting as a backup. -->
+      <script file="!!sysconfdir!!/init.d/qpidd" name="qpidd"/>
+
+      <!-- This script promotes the qpidd broker on this node to primary. -->
+      <script file="!!sysconfdir!!/init.d/qpidd-primary" name="qpidd-primary"/>
+
+      <!-- This is a virtual IP address for broker replication traffic. -->
+      <ip address="20.0.10.200" monitor_link="1"/>
+
+      <!-- This is a virtual IP address on a seprate network for client traffic. -->
+      <ip address="20.0.20.200" monitor_link="1"/>
+    </resources>
+
+    <!-- There is a qpidd service on each node, it should be restarted if it fails. -->
+    <service name="node1-qpidd-service" domain="node1-domain" recovery="restart">
+      <script ref="qpidd"/>
+    </service>
+    <service name="node2-qpidd-service" domain="node2-domain" recovery="restart">
+      <script ref="qpidd"/>
+    </service>
+    <service name="node3-qpidd-service" domain="node3-domain"  recovery="restart">
+      <script ref="qpidd"/>
+    </service>
+
+    <!-- There should always be a single qpidd-primary service, it can run on any node. -->
+    <service name="qpidd-primary-service" autostart="1" exclusive="0" recovery="relocate">
+      <script ref="qpidd-primary"/>
+      <!-- The primary has the IP addresses for brokers and clients to connect. -->
+      <ip ref="20.0.10.200"/>
+      <ip ref="20.0.20.200"/>
+    </service>
+  </rm>
+  <fencedevices/>
+  <fence_daemon clean_start="0" post_fail_delay="0" post_join_delay="3"/>
+</cluster>
+      ]]>
+    </programlisting>
+
+    <para>
+      There is a <literal>failoverdomain</literal> for each node containing just that
+      one node.  This lets us stipulate that the qpidd service should always run on all
+      nodes.
+    </para>
+    <para>
+      The <literal>resources</literal> section defines the <command>qpidd</command>
+      script used to start the <command>qpidd</command> service. It also defines the
+      <command>qpid-primary</command> script which does not
+      actually start a new service, rather it promotes the existing
+      <command>qpidd</command> broker to primary status.
+    </para>
+    <para>
+      The <literal>resources</literal> section also defines a pair of virtual IP
+      addresses on different sub-nets. One will be used for broker-to-broker
+      communication, the other for client-to-broker. 
+    </para>
+    <para>
+      To take advantage of the virtual IP addresses, <filename>qpidd.conf</filename>
+      should contain these  lines:
+    </para>
+    <programlisting>
+      ha-cluster=yes
+      ha-brokers=20.0.20.200
+      ha-public-brokers=20.0.10.200
+    </programlisting>
+    <para>
+      This configuration specifies that backup brokers will use 20.0.20.200
+      to connect to the primary and will advertise 20.0.10.200 to clients.
+      Clients should connect to 20.0.10.200.
+    </para>
+    <para>
+      The <literal>service</literal> section defines 3 <literal>qpidd</literal>
+      services, one for each node. Each service is in a restricted fail-over
+      domain containing just that node, and has the <literal>restart</literal>
+      recovery policy. The effect of this is that rgmanager will run
+      <command>qpidd</command> on each node, restarting if it fails.
+    </para>
+    <para>
+      There is a single <literal>qpidd-primary-service</literal> using the
+      <command>qpidd-primary</command> script which is not restricted to a
+      domain and has the <literal>relocate</literal> recovery policy. This means
+      rgmanager will start <command>qpidd-primary</command> on one of the nodes
+      when the cluster starts and will relocate it to another node if the
+      original node fails. Running the <literal>qpidd-primary</literal> script
+      does not start a new broker process, it promotes the existing broker to
+      become the primary.
+    </para>
+  </section>
+
+  <section>
+    <title>Broker Administration Tools</title>
+    <para>
+      Normally, clients are not allowed to connect to a backup broker. However management tools are
+      allowed to connect to a backup brokers. If you use these tools you <emphasis>must
+      not</emphasis> add or remove messages from replicated queues, or delete replicated queues or
+      exchanges as this will corrupt the replication process and may cause message loss.
+    </para>
+    <para>
+      <command>qpid-ha</command> allows you to view and change HA configuration settings.
+    </para>
+    <para>
+      The tools <command>qpid-config</command>, <command>qpid-route</command> and
+      <command>qpid-stat</command> will connect to a backup if you pass the flag <command>--ha-admin</command> on the
+      command line.
+    </para>
+  </section>
+  
+  <section id="ha-creating-replicated">
     <title>Creating replicated queues and exchanges</title>
     <para>
-      To create a replicated queue or exchange, pass the argument
-      <literal>qpid.replicate</literal> when creating the queue or exchange. It should
-      have one of the following three values:
+      By default, queues and exchanges are not replicated automatically. You can change
+      the default behavior by setting the <literal>ha-replicate</literal> configuration
+      option. It has one of the following values:
       <itemizedlist>
 	<listitem>
-	  <firstterm>all</firstterm>: Replicate the queue or exchange, messages and bindings.
+	  <firstterm>all</firstterm>: Replicate everything automatically: queues, exchanges, bindings and messages.
 	</listitem>
 	<listitem>
-	  <firstterm>configuration</firstterm>: Replicate the existence of the queue or
+	  <firstterm>configuration</firstterm>: Replicate the existence of queues,
 	  exchange and bindings but don't replicate messages.
 	</listitem>
 	<listitem>
-	  <firstterm>none</firstterm>: Don't replicate, this is the default.
+	  <firstterm>none</firstterm>: Don't replicate anything, this is the default.
 	</listitem>
       </itemizedlist>
     </para>
     <para>
-      Bindings are automatically replicated if the queue and exchange being bound both have
-      replication argument of <literal>all</literal> or <literal>configuration</literal>, they are
-      not replicated otherwise.
+      You can over-ride the default for a particular queue or exchange by passing the
+      argument <literal>qpid.replicate</literal> when creating the queue or exchange. It
+      takes the same values as <literal>ha-replicate</literal>
+    </para>
+    <para>
+      Bindings are automatically replicated if the queue and exchange being bound both
+      have replication <literal>all</literal> or <literal>configuration</literal>, they
+      are not replicated otherwise.
     </para>
     <para>
-      You can create replicated queues and exchanges with the <command>qpid-config</command>
-      management tool like this:
+      You can create replicated queues and exchanges with the
+      <command>qpid-config</command> management tool like this:
     </para>
     <programlisting>
       qpid-config add queue myqueue --replicate all
     </programlisting>
     <para>
-      To create replicated queues and exchanges via the client API, add a <literal>node</literal> entry to the address like this:
+      To create replicated queues and exchanges via the client API, add a
+      <literal>node</literal> entry to the address like this:
     </para>
     <programlisting>
       "myqueue;{create:always,node:{x-declare:{arguments:{'qpid.replicate':all}}}}"
@@ -439,200 +660,6 @@ under the License.
       </screen>
     </section>
   </section>
-
-  <section>
-    <title>The Cluster Resource Manager</title>
-    <para>
-      Broker fail-over is managed by a <firstterm>cluster resource
-      manager</firstterm>.  An integration with <ulink
-      url="https://fedorahosted.org/cluster/wiki/RGManager">rgmanager</ulink> is
-      provided, but it is possible to integrate with other resource managers.
-    </para>
-    <para>
-      The resource manager is responsible for starting a on each node in the
-      cluster.  The resource manager then <firstterm>promotes</firstterm> one of
-      the brokers to be the primary. The other brokers connect to the primary as
-      backups, using the URL provided in the <literal>ha-brokers</literal>
-      configuration option.
-    </para>
-    <para>
-      Once connected, the backup brokers synchronize their state with the
-      primary.  When a backup is synchronized, or "hot", it is ready to take
-      over if the primary fails.  Backup brokers continually receive updates
-      from the primary in order to stay synchronized.
-    </para>
-    <para>
-      If the primary fails, backup brokers go into fail-over mode. The resource
-      manager must detect the failure and promote one of the backups to be the
-      new primary.  The other backups connect to the new primary and synchronize
-      their state so they can be backups for it.
-    </para>
-    <para>
-      The resource manager is also responsible for protecting the cluster from
-      <firstterm>split-brain</firstterm> conditions resulting from a network partition.
-      A network partition divide a cluster into two sub-groups which cannot see each other.
-      Usually a <firstterm>quorum</firstterm> voting algorithm is used that disables
-      nodes in the inquorate sub-group.
-    </para>
-  </section>
-  <section>
-    <title>Configuring <command>rgmanager</command> as resource manager</title>
-    <para>
-      This section assumes that you are already familiar with setting up and
-      configuring clustered services using <command>cman</command> and
-      <command>rgmanager</command>. It will show you how to configure an
-      active-passive, hot-standby <command>qpidd</command> HA cluster.
-    </para>
-    <para>
-      Here is an example <literal>cluster.conf</literal> file for a cluster of 3
-      nodes named node1, node2 and node3. We will go through the configuration
-      step-by-step.
-    </para>
-    <programlisting>
-      <![CDATA[
-<?xml version="1.0"?>
-<!--
-This is an example of a cluster.conf file to run qpidd HA under rgmanager.
-This example assumes a 3 node cluster, with nodes named node1, node2 and node3.
--->
-
-<cluster name="qpid-test" config_version="18">
-  <!-- The cluster has 3 nodes. Each has a unique nodid and one vote for quorum. -->
-  <clusternodes>
-    <clusternode name="node1" nodeid="1">
-      <fence/>
-    </clusternode>
-    <clusternode name="node2" nodeid="2">
-      <fence/>
-    </clusternode>
-    <clusternode name="node3" nodeid="3">
-      <fence/>
-    </clusternode>
-  </clusternodes>
-  <!-- Resouce Manager configuration. -->
-  <rm log_level="7">		<!-- Verbose logging -->
-    <!--
-	There is a failoverdomain for each node containing just that node.
-	This lets us stipulate that the qpidd service should always run on all nodes.
-    -->
-    <failoverdomains>
-      <failoverdomain name="node1-domain" restricted="1">
-	<failoverdomainnode name="node1"/>
-      </failoverdomain>
-      <failoverdomain name="node2-domain" restricted="1">
-	<failoverdomainnode name="node2"/>
-      </failoverdomain>
-      <failoverdomain name="node3-domain" restricted="1">
-	<failoverdomainnode name="node3"/>
-      </failoverdomain>
-    </failoverdomains>
-
-    <resources>
-      <!-- This script starts a qpidd broker acting as a backup. -->
-      <script file="!!sysconfdir!!/init.d/qpidd" name="qpidd"/>
-
-      <!-- This script promotes the qpidd broker on this node to primary. -->
-      <script file="!!sysconfdir!!/init.d/qpidd-primary" name="qpidd-primary"/>
-
-      <!-- This is a virtual IP address for broker replication traffic. -->
-      <ip address="20.0.10.200" monitor_link="1"/>
-
-      <!-- This is a virtual IP address on a seprate network for client traffic. -->
-      <ip address="20.0.20.200" monitor_link="1"/>
-    </resources>
-
-    <!-- There is a qpidd service on each node, it should be restarted if it fails. -->
-    <service name="node1-qpidd-service" domain="node1-domain" recovery="restart">
-      <script ref="qpidd"/>
-    </service>
-    <service name="node2-qpidd-service" domain="node2-domain" recovery="restart">
-      <script ref="qpidd"/>
-    </service>
-    <service name="node3-qpidd-service" domain="node3-domain"  recovery="restart">
-      <script ref="qpidd"/>
-    </service>
-
-    <!-- There should always be a single qpidd-primary service, it can run on any node. -->
-    <service name="qpidd-primary-service" autostart="1" exclusive="0" recovery="relocate">
-      <script ref="qpidd-primary"/>
-      <!-- The primary has the IP addresses for brokers and clients to connect. -->
-      <ip ref="20.0.10.200"/>
-      <ip ref="20.0.20.200"/>
-    </service>
-  </rm>
-  <fencedevices/>
-  <fence_daemon clean_start="0" post_fail_delay="0" post_join_delay="3"/>
-</cluster>
-      ]]>
-    </programlisting>
-    <para>
-      There is a <literal>failoverdomain</literal> for each node containing just that
-      one node.  This lets us stipulate that the qpidd service should always run on all
-      nodes.
-    </para>
-    <para>
-      The <literal>resources</literal> section defines the usual initialization
-      script to start the <command>qpidd</command> service.
-      <command>qpidd</command>. It also defines the
-      <command>qpid-primary</command> script. Starting this script does not
-      actually start a new service, rather it promotes the existing
-      <command>qpidd</command> broker to primary status.
-    </para>
-    <para>
-      The <literal>resources</literal> section also defines a pair of virtual IP
-      addresses on different sub-nets. One will be used for broker-to-broker
-      communication, the other for client-to-broker. 
-    </para>
-    <para>
-      To take advantage of the virtual IP addresses, <filename>qpidd.conf</filename>
-      should contain these  lines:
-    </para>
-    <programlisting>
-      ha-cluster=yes
-      ha-brokers=20.0.20.200
-      ha-public-brokers=20.0.10.200
-    </programlisting>
-    <para>
-      This configuration specifies that backup brokers will use 20.0.20.200
-      to connect to the primary and will advertise 20.0.10.200 to clients.
-      Clients should connect to 20.0.10.200.
-    </para>
-    <para>
-      The <literal>service</literal> section defines 3 <command>qpidd</command>
-      services, one for each node. Each service is in a restricted fail-over
-      domain containing just that node, and has the <literal>restart</literal>
-      recovery policy. The effect of this is that rgmanager will run
-      <command>qpidd</command> on each node, restarting if it fails.
-    </para>
-    <para>
-      There is a single <literal>qpidd-primary-service</literal> running the
-      <command>qpidd-primary</command> script which is not restricted to a
-      domain and has the <literal>relocate</literal> recovery policy. This means
-      rgmanager will start <command>qpidd-primary</command> on one of the nodes
-      when the cluster starts and will relocate it to another node if the
-      original node fails. Running the <literal>qpidd-primary</literal> script
-      does not start a new broker process, it promotes the existing broker to
-      become the primary.
-    </para>
-  </section>
-
-  <section>
-    <title>Broker Administration</title>
-    <para>
-      Normally, clients are not allowed to connect to a backup broker. However management tools are
-      allowed to connect to a backup brokers. If you use these tools you <emphasis>must
-      not</emphasis> add or remove messages from replicated queues, or delete replicated queues or
-      exchanges as this will corrupt the replication process and may cause message loss.
-    </para>
-    <para>
-      <command>qpid-ha</command> allows you to view and change HA configuration settings.
-    </para>
-    <para>
-      The tools <command>qpid-config</command>, <command>qpid-route</command> and
-      <command>qpid-stat</command> will connect to a backup if you pass the flag <command>--ha-admin</command> on the
-      command line.
-    </para>
-  </section>
 </section>
 
 <!-- LocalWords:  scalability rgmanager multicast RGManager mailto LVQ qpidd IP dequeued Transactional username