ovn-ctl: Support starting clustered OVN dbs

This patch adds the options to start clustered OVN db servers in ovn-ctl.
To support this, following options are added - '--db-(nb/sb)-cluster-local-addr',
'--db-(nb/sb)-cluster-local-port', '--db-(nb/sb)-cluster-local-proto',
'--db-(nb/sb)-cluster-remote-addr', '--db-(nb/sb)-cluster-remote-port' and
'--db-(nb/sb)-cluster-remote-proto'.

If only '--db-(nb/sb)-cluster-local-addr' is defined then clustered db is created
(using ovsdb-tool create-cluster). If both are defined, then the db is added to
the cluster (using ovsdb-tool join-cluster)

This patch also adds the support to configure ovn-northd to point to all the servers
in the cluster using the options - '--ovn-northd-nb-db' and 'ovn-northd-sb-db'.

Presently this patch doesn't handle the schema update scenario when restarting the
clustered ovsdb-servers. This will be handled in a separate patch.

The initial versions of these commands are tested by Aliasgar <aginwala <aginwala@asu.edu>
and the discussion on this can be found here -
https://mail.openvswitch.org/pipermail/ovs-discuss/2018-March/046470.html

(There are 4 checkpatch warnings 'Line length is >79-characters long' in ovn-ctl.8.xml
which I couldn't resolve without losing proper rendering when "man ovn-ctl" is run.)

Co-authored-by: Ben Pfaff <blp@ovn.org>
Signed-off-by: Numan Siddique <nusiddiq@redhat.com>
Signed-off-by: Ben Pfaff <blp@ovn.org>

2 files changed, 198 insertions, 64 deletions

diff --git a/ovn/utilities/ovn-ctl b/ovn/utilities/ovn-ctl
index dc0c26159..25dda524c 100755
--- a/ovn/utilities/ovn-ctl
+++ b/ovn/utilities/ovn-ctl
@@ -93,84 +93,103 @@ promote_ovnsb() {
     ovs-appctl -t $rundir/ovnsb_db.ctl ovsdb-server/disconnect-active-ovsdb-server
 }
 
-start_nb_ovsdb() {
-    # Check and eventually start ovsdb-server for Northbound DB
-    if ! pidfile_is_running $DB_NB_PID; then
-        upgrade_db "$DB_NB_FILE" "$DB_NB_SCHEMA" 1>/dev/null 2>/dev/null
+start_ovsdb__() {
+    local DB=$1 db=$2 schema_name=$3 table_name=$4
+    eval local pid=\$DB_${DB}_PID
+    eval local cluster_local_addr=\$DB_${DB}_CLUSTER_LOCAL_ADDR
+    eval local cluster_local_port=\$DB_${DB}_CLUSTER_LOCAL_PORT
+    eval local cluster_local_proto=\$DB_${DB}_CLUSTER_LOCAL_PROTO
+    eval local cluster_remote_addr=\$DB_${DB}_CLUSTER_REMOTE_ADDR
+    eval local cluster_remote_port=\$DB_${DB}_CLUSTER_REMOTE_PORT
+    eval local cluster_remote_proto=\$DB_${DB}_CLUSTER_REMOTE_PROTO
+    eval local sync_from_proto=\$DB_${DB}_SYNC_FROM_PROTO
+    eval local sync_from_addr=\$DB_${DB}_SYNC_FROM_ADDR
+    eval local sync_from_port=\$DB_${DB}_SYNC_FROM_PORT
+    eval local file=\$DB_${DB}_FILE
+    eval local schema=\$DB_${DB}_SCHEMA
+    eval local logfile=\$OVN_${DB}_LOGFILE
+    eval local log=\$OVN_${DB}_LOG
+    eval local sock=\$DB_${DB}_SOCK
+    eval local detach=\$DB_${DB}_DETACH
+    eval local create_insecure_remote=\$DB_${DB}_CREATE_INSECURE_REMOTE
+    eval local port=\$DB_${DB}_PORT
+    eval local addr=\$DB_${DB}_ADDR
+    eval local active_conf_file=\$ovn${db}_active_conf_file
+
+    # Check and eventually start ovsdb-server for DB
+    if pidfile_is_running $pid; then
+        return
+    fi
 
-        set ovsdb-server
+    if test ! -z "$cluster_local_addr"; then
+        mode=cluster
+    elif test ! -z "$sync_from_addr"; then
+        mode=active_passive
+        echo "$sync_from_proto:$sync_from_addr:\
+$sync_from_port" > $active_conf_file
+    else
+        mode=standalone
+    fi
 
-        if test X"$DB_NB_DETACH" != Xno; then
-            set "$@" --detach --monitor
+    if test $mode = cluster; then
+        local local=$cluster_local_proto:$cluster_local_addr:\
+$cluster_local_port
+        local remote=$cluster_remote_proto:$cluster_remote_addr:\
+$cluster_remote_port
+        if test -n "$cluster_remote_addr"; then
+            join_cluster "$file" "$schema_name" "$local" "$remote"
         else
-            set exec "$@"
+            create_cluster "$file" "$schema" "$local"
         fi
+    else
+        upgrade_db "$file" "$schema"
+    fi
 
-        set "$@" $OVN_NB_LOG --log-file=$OVN_NB_LOGFILE
-        set "$@" --remote=punix:$DB_NB_SOCK --pidfile=$DB_NB_PID
-        set "$@" --remote=db:OVN_Northbound,NB_Global,connections
-        set "$@" --unixctl=ovnnb_db.ctl
-        set "$@" --private-key=db:OVN_Northbound,SSL,private_key
-        set "$@" --certificate=db:OVN_Northbound,SSL,certificate
-        set "$@" --ca-cert=db:OVN_Northbound,SSL,ca_cert
-        set "$@" --ssl-protocols=db:OVN_Northbound,SSL,ssl_protocols
-        set "$@" --ssl-ciphers=db:OVN_Northbound,SSL,ssl_ciphers
-
-        if test X"$DB_NB_CREATE_INSECURE_REMOTE" = Xyes; then
-            set "$@" --remote=ptcp:$DB_NB_PORT:$DB_NB_ADDR
-        fi
+    set ovsdb-server
+    set "$@" $log --log-file=$logfile
+    set "$@" --remote=punix:$sock --pidfile=$pid
+    set "$@" --unixctl=ovn${db}_db.ctl
 
-        if test ! -z "$DB_NB_SYNC_FROM_ADDR"; then
-            echo "$DB_NB_SYNC_FROM_PROTO:$DB_NB_SYNC_FROM_ADDR:$DB_NB_SYNC_FROM_PORT" > $ovnnb_active_conf_file
-        fi
+    if test X"$detach" != Xno; then
+        set "$@" --detach --monitor
+    else
+        set exec "$@"
+    fi
 
-        if test -e $ovnnb_active_conf_file; then
-            set "$@" --sync-from=`cat $ovnnb_active_conf_file`
-        fi
+    set "$@" --remote=db:$schema_name,$table_name,connections
+    set "$@" --private-key=db:$schema_name,SSL,private_key
+    set "$@" --certificate=db:$schema_name,SSL,certificate
+    set "$@" --ca-cert=db:$schema_name,SSL,ca_cert
+    set "$@" --ssl-protocols=db:$schema_name,SSL,ssl_protocols
+    set "$@" --ssl-ciphers=db:$schema_name,SSL,ssl_ciphers
 
-        $@ $DB_NB_FILE
-        ovn-nbctl init
+    if test X"$create_insecure_remote" = Xyes; then
+        set "$@" --remote=ptcp:$port:$addr
     fi
-}
 
-start_sb_ovsdb() {
-    # Check and eventually start ovsdb-server for Southbound DB
-    if ! pidfile_is_running $DB_SB_PID; then
-        upgrade_db "$DB_SB_FILE" "$DB_SB_SCHEMA" 1>/dev/null 2>/dev/null
-
-        set ovsdb-server
+    if test $mode = active_passive; then
+        set "$@" --sync-from=`cat $active_conf_file`
+    fi
 
-        if test X"$DB_SB_DETACH" != Xno; then
-            set "$@" --detach --monitor
-        else
-            set exec "$@"
-        fi
+    "$@" "$file"
 
-        set "$@" $OVN_SB_LOG --log-file=$OVN_SB_LOGFILE
-        set "$@" --remote=punix:$DB_SB_SOCK --pidfile=$DB_SB_PID
-        set "$@" --remote=db:OVN_Southbound,SB_Global,connections
-        set "$@" --unixctl=ovnsb_db.ctl
-        set "$@" --private-key=db:OVN_Southbound,SSL,private_key
-        set "$@" --certificate=db:OVN_Southbound,SSL,certificate
-        set "$@" --ca-cert=db:OVN_Southbound,SSL,ca_cert
-        set "$@" --ssl-protocols=db:OVN_Southbound,SSL,ssl_protocols
-        set "$@" --ssl-ciphers=db:OVN_Southbound,SSL,ssl_ciphers
-
-        if test X"$DB_SB_CREATE_INSECURE_REMOTE" = Xyes; then
-            set "$@" --remote=ptcp:$DB_SB_PORT:$DB_SB_ADDR
-        fi
+    # Initialize the database if it's running standalone,
+    # active-passive, or is the first server in a cluster.
+    if test -z "$cluster_remote_addr"; then
+        ovn-nbctl init
+    fi
 
-        if test ! -z "$DB_SB_SYNC_FROM_ADDR"; then
-            echo "$DB_SB_SYNC_FROM_PROTO:$DB_SB_SYNC_FROM_ADDR:$DB_SB_SYNC_FROM_PORT" > $ovnsb_active_conf_file
-        fi
+    if test $mode = cluster; then
+        upgrade_cluster "$schema" "unix:$sock"
+    fi
+}
 
-        if test -e $ovnsb_active_conf_file; then
-            set "$@" --sync-from=`cat $ovnsb_active_conf_file`
-        fi
+start_nb_ovsdb() {
+    start_ovsdb__ NB nb OVN_Northbound NB_Global
+}
 
-        $@ $DB_SB_FILE
-        ovn-sbctl init
-    fi
+start_sb_ovsdb() {
+    start_ovsdb__ SB sb OVN_Southbound SB_Global
 }
 
 start_ovsdb () {
@@ -236,7 +255,8 @@ start_northd () {
                 exit
             fi
         fi
-        ovn_northd_params="--ovnnb-db=unix:$DB_NB_SOCK --ovnsb-db=unix:$DB_SB_SOCK"
+        ovn_northd_params="--ovnnb-db=$OVN_NORTHD_NB_DB \
+        --ovnsb-db=$OVN_NORTHD_SB_DB"
     else
         ovn_northd_params="`cat $ovn_northd_db_conf_file`"
     fi
@@ -406,6 +426,23 @@ set_defaults () {
 
     DB_NB_DETACH="yes"
     DB_SB_DETACH="yes"
+
+    DB_NB_CLUSTER_LOCAL_ADDR=""
+    DB_NB_CLUSTER_LOCAL_PROTO="tcp"
+    DB_NB_CLUSTER_LOCAL_PORT=6643
+    DB_NB_CLUSTER_REMOTE_ADDR=""
+    DB_NB_CLUSTER_REMOTE_PROTO="tcp"
+    DB_NB_CLUSTER_REMOTE_PORT=6643
+
+    DB_SB_CLUSTER_LOCAL_ADDR=""
+    DB_SB_CLUSTER_LOCAL_PROTO="tcp"
+    DB_SB_CLUSTER_LOCAL_PORT=6644
+    DB_SB_CLUSTER_REMOTE_ADDR=""
+    DB_SB_CLUSTER_REMOTE_PROTO="tcp"
+    DB_SB_CLUSTER_REMOTE_PORT=6644
+
+    OVN_NORTHD_NB_DB="unix:$DB_NB_SOCK"
+    OVN_NORTHD_SB_DB="unix:$DB_SB_SOCK"
 }
 
 set_option () {
@@ -494,6 +531,32 @@ File location options:
   --db-sb-sync-from-port=ADDR OVN Southbound active db tcp port (default: $DB_SB_SYNC_FROM_PORT)
   --db-sb-sync-from-proto=PROTO OVN Southbound active db transport (default: $DB_SB_SYNC_FROM_PROTO)
   --db-sb-create-insecure-remote=yes|no Create ptcp OVN Southbound remote (default: $DB_SB_CREATE_INSECURE_REMOTE)
+  --db-nb-cluster-local-addr=ADDR OVN_Northbound cluster local address \
+  (default: $DB_NB_CLUSTER_LOCAL_ADDR)
+  --db-nb-cluster-local-port=PORT OVN_Northbound cluster local tcp port \
+  (default: $DB_NB_CLUSTER_LOCAL_PORT)
+  --db-nb-cluster-local-proto=PROTO OVN_Northbound cluster local db transport \
+  (default: $DB_NB_CLUSTER_LOCAL_PROTO)
+  --db-nb-cluster-remote-addr=ADDR OVN_Northbound cluster remote address \
+  (default: $DB_NB_CLUSTER_REMOTE_ADDR)
+  --db-nb-cluster-remote-port=PORT OVN_Northbound cluster remote tcp port \
+  (default: $DB_NB_CLUSTER_REMOTE_PORT)
+  --db-nb-cluster-remote-proto=PROTO OVN_Northbound cluster remote db \
+  transport (default: $DB_NB_CLUSTER_REMOTE_PROTO)
+  --db-sb-cluster-local-addr=ADDR OVN_Southbound cluster local address \
+  (default: $DB_SB_CLUSTER_LOCAL_ADDR)
+  --db-sb-cluster-local-port=PORT OVN_Southbound cluster local tcp port \
+  (default: $DB_SB_CLUSTER_LOCAL_PORT)
+  --db-sb-cluster-local-proto=PROTO OVN_Southbound cluster local db transport \
+  (default: $DB_SB_CLUSTER_LOCAL_PROTO)
+  --db-sb-cluster-remote-addr=ADDR OVN_Southbound cluster remote address \
+  (default: $DB_SB_CLUSTER_REMOTE_ADDR)
+  --db-sb-cluster-remote-port=PORT OVN_Southbound cluster remote tcp port \
+  (default: $DB_SB_CLUSTER_REMOTE_PORT)
+  --db-sb-cluster-remote-proto=PROTO OVN_Southbound cluster remote db \
+  transport (default: $DB_SB_CLUSTER_REMOTE_PROTO)
+  --ovn-northd-nb-db=NB DB address(es) (default: $OVN_NORTHD_NB_DB)
+  --ovn-northd-sb-db=SB DB address(es) (default: $OVN_NORTHD_SB_DB)
 
 Default directories with "configure" option and environment variable override:
   logs: /usr/local/var/log/openvswitch (--with-logdir, OVS_LOGDIR)
diff --git a/ovn/utilities/ovn-ctl.8.xml b/ovn/utilities/ovn-ctl.8.xml
index 40defc9ec..02235fe1e 100644
--- a/ovn/utilities/ovn-ctl.8.xml
+++ b/ovn/utilities/ovn-ctl.8.xml
@@ -66,6 +66,31 @@
     <p><code>--db-sb-sync-from-addr=<var>IP ADDRESS</var></code></p>
     <p><code>--db-sb-sync-from-port=<var>PORT NUMBER</var></code></p>
     <p><code>--db-sb-sync-from-proto=<var>PROTO</var></code></p>
+    <p>
+      <code>
+        --ovn-northd-nb-db=<var>PROTO</var>:<var>IP ADDRESS</var>:
+        <var>PORT</var>..
+      </code>
+    </p>
+    <p>
+      <code>
+        --ovn-northd-sb-db=<var>PROTO</var>:<var>IP ADDRESS</var>:
+        <var>PORT</var>..
+      </code>
+    </p>
+    <h1> Clustering options </h1>
+    <p><code>--db-nb-cluster-local-addr=<var>IP ADDRESS</var></code></p>
+    <p><code>--db-nb-cluster-local-port=<var>PORT NUMBER</var></code></p>
+    <p><code>--db-nb-cluster-local-proto=<var>PROTO (tcp/ssl)</var></code></p>
+    <p><code>--db-nb-cluster-remote-addr=<var>IP ADDRESS</var></code></p>
+    <p><code>--db-nb-cluster-remote-port=<var>PORT NUMBER</var></code></p>
+    <p><code>--db-nb-cluster-remote-proto=<var>PROTO (tcp/ssl)</var></code></p>
+    <p><code>--db-sb-cluster-local-addr=<var>IP ADDRESS</var></code></p>
+    <p><code>--db-sb-cluster-local-port=<var>PORT NUMBER</var></code></p>
+    <p><code>--db-sb-cluster-local-proto=<var>PROTO (tcp/ssl)</var></code></p>
+    <p><code>--db-sb-cluster-remote-addr=<var>IP ADDRESS</var></code></p>
+    <p><code>--db-sb-cluster-remote-port=<var>PORT NUMBER</var></code></p>
+    <p><code>--db-sb-cluster-remote-proto=<var>PROTO (tcp/ssl)</var></code></p>
 
     <h1>Configuration files</h1>
     <p>Following are the optional configuration files. If present, it should be located in the etc dir</p>
@@ -125,4 +150,50 @@
     <p><code># ovn-ctl promote_ovnsb</code></p>
     <p><code># ovn-ctl --db-nb-sync-from-addr=x.x.x.x --db-nb-sync-from-port=6641 demote_ovnnb</code></p>
     <p><code># ovn-ctl --db-sb-sync-from-addr=x.x.x.x --db-sb-sync-from-port=6642 demote_ovnsb</code></p>
+
+    <h2>Creating a clustered db on 3 nodes with IPs x.x.x.x, y.y.y.y and z.z.z.z</h2>
+    <h3>Starting OVN ovsdb servers and ovn-northd on the node with IP x.x.x.x</h3>
+    <p>
+      <code>
+        # ovn-ctl --db-nb-addr=x.x.x.x --db-nb-create-insecure-remote=yes
+        --db-sb-addr=x.x.x.x --db-sb-create-insecure-remote=yes
+        --db-nb-cluster-local-addr=x.x.x.x
+        --db-sb-cluster-local-addr=x.x.x.x
+        --ovn-northd-nb-db=tcp:x.x.x.x:6641,tcp:y.y.y.y:6641,tcp:z.z.z.z:6641
+        --ovn-northd-sb-db=tcp:x.x.x.x:6642,tcp:y.y.y.y:6642,tcp:z.z.z.z:6642
+        start_northd
+      </code>
+    </p>
+
+    <h3>Starting OVN ovsdb-servers and ovn-northd on the node with IP y.y.y.y and joining the cluster started at x.x.x.x</h3>
+    <p>
+      <code>
+        # ovn-ctl --db-nb-addr=y.y.y.y --db-nb-create-insecure-remote=yes
+        --db-sb-addr=y.y.y.y --db-sb-create-insecure-remote=yes
+        --db-nb-cluster-local-addr=y.y.y.y
+        --db-sb-cluster-local-addr=y.y.y.y
+        --db-nb-cluster-remote-addr=x.x.x.x
+        --db-sb-cluster-remote-addr=x.x.x.x
+        --ovn-northd-nb-db=tcp:x.x.x.x:6641,tcp:y.y.y.y:6641,tcp:z.z.z.z:6641
+        --ovn-northd-sb-db=tcp:x.x.x.x:6642,tcp:y.y.y.y:6642,tcp:z.z.z.z:6642
+        start_northd
+      </code>
+    </p>
+
+    <h3>Starting OVN ovsdb-servers and ovn-northd on the node with IP z.z.z.z and joining the cluster started at x.x.x.x</h3>
+    <p>
+      <code>
+        # ovn-ctl --db-nb-addr=z.z.z.z
+          --db-nb-create-insecure-remote=yes
+          --db-nb-cluster-local-addr=z.z.z.z
+          --db-sb-addr=z.z.z.z
+          --db-sb-create-insecure-remote=yes
+          --db-sb-cluster-local-addr=z.z.z.z
+          --db-nb-cluster-remote-addr=x.x.x.x
+          --db-sb-cluster-remote-addr=x.x.x.x
+          --ovn-northd-nb-db=tcp:x.x.x.x:6641,tcp:y.y.y.y:6641,tcp:z.z.z.z:6641
+          --ovn-northd-sb-db=tcp:x.x.x.x:6642,tcp:y.y.y.y:6642,tcp:z.z.z.z:6642
+          start_northd
+      </code>
+    </p>
 </manpage>