ovs-sim: New utility for multi-node OVS and OVN simulation.

Please see the examples in ovs-sim(1) for some examples of how this
can be useful.

Signed-off-by: Ben Pfaff <blp@nicira.com>

3 files changed, 682 insertions, 0 deletions

diff --git a/utilities/automake.mk b/utilities/automake.mk
index 6083b4b68..3b43cb412 100644
--- a/utilities/automake.mk
+++ b/utilities/automake.mk
@@ -30,6 +30,10 @@ check_SCRIPTS += \
 	utilities/ovs-appctl-bashcomp.bash \
 	utilities/ovs-vsctl-bashcomp.bash
 
+EXTRA_DIST += utilities/ovs-sim.in utilities/ovs-sim.1.xml
+man_MANS += utilities/ovs-sim.1
+noinst_SCRIPTS += utilities/ovs-sim
+
 utilities/ovs-lib: $(top_builddir)/config.status
 
 docs += utilities/ovs-command-bashcomp.INSTALL.md
diff --git a/utilities/ovs-sim.1.xml b/utilities/ovs-sim.1.xml
new file mode 100644
index 000000000..cedf66138
--- /dev/null
+++ b/utilities/ovs-sim.1.xml
@@ -0,0 +1,323 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manpage program="ovs-sim" section="1" title="ovs-sim">
+    <h1>Name</h1>
+    <p>ovs-sim -- Open vSwitch simulator environment</p>
+
+    <h1>Synopsis</h1>
+    <p><code>ovs-sim</code> [<var>option</var>]... [<var>script</var>]...</p>
+
+    <h1>Description</h1>
+    <p>
+      <code>ovs-sim</code> provides a convenient environment for running one or
+      more Open vSwitch instances and related software in a sandboxed
+      simulation environment.
+    </p>
+
+    <p>
+      To use <code>ovs-sim</code>, first build Open vSwitch, then invoke it
+      directly from the build directory, e.g.:
+    </p>
+
+    <pre>
+git clone https://github.com/openvswitch/ovs.git
+cd ovs
+./configure
+make
+utilities/ovs-sim
+    </pre>
+
+    <p>
+      When invoked in the most ordinary way as shown above,
+      <code>ovs-sim</code> does the following:
+    </p>
+
+    <ol>
+      <li>
+        Creates a directory <code>sandbox</code> as a subdirectory of the
+        current directory (first destroying such a directory if it already
+        exists) and <code>cd</code>s into that directory.
+      </li>
+
+      <li>
+        Installs all of the Open vSwitch manpages into a <code>man</code>
+        subdirectory of <code>sandbox</code> and adjusts the <env>MANPATH</env>
+        environment variable so that <code>man</code> and other manpage viewers
+        can find them.
+      </li>
+
+      <li>
+        <p>
+          Creates a simulated Open vSwitch named <code>main</code> and sets it
+          up as the default target for OVS commands, as if the following
+          <code>ovs-sim</code> commands had been run:
+        </p>
+
+        <pre>
+          sim_add main
+          as main
+        </pre>
+
+        <p>
+          See <code>Commands</code>, below, for an explanation.
+        </p>
+      </li>
+
+      <li>
+        Runs any scripts specified on the command line (see
+        <code>Options</code> below).  The scripts can use arbitrary Bash
+        syntax, plus the additional commands described under
+        <code>Commands</code>, below.
+      </li>
+
+      <li>
+        If no scripts were specified, or if <option>-i</option> or
+        <option>--interactive</option> was specified, invokes an interactive
+        Bash subshell.  The user can use arbitrary Bash commands, plus the
+        additional commands described under <code>Commands</code>, below.
+      </li>
+    </ol>
+
+    <p>
+      <code>ovs-sim</code> and the sandbox environment that it creates does not
+      require superuser or other special privileges.  Generally, it should not
+      be run with such privileges.
+    </p>
+
+    <h1>Options</h1>
+
+    <p>
+      <code>ovs-sim</code> accepts the following options and arguments:
+    </p>
+
+    <dl>
+      <dt><var>script</var></dt>
+      <dd>
+        Runs <var>script</var>, which should be a Bash script, within a
+        subshell after initializing.  If multiple <var>script</var> arguments
+        are given, then they are run in the order given.  If any
+        <var>script</var> exits with a nonzero exit code, then
+        <code>ovs-sim</code> exits immediately with the same exit code.
+      </dd>
+
+      <dt><option>-i</option></dt>
+      <dt><option>--interactive</option></dt>
+      <dd>
+        By default, if any <var>script</var> is specified, <code>ovs-sim</code>
+        exits as soon as the scripts finish executing.  With this option, or if
+        no scripts are specified, <code>ovs-sim</code> instead starts an
+        interactive Bash session.
+      </dd>
+    </dl>
+
+    <h1>Commands</h1>
+
+    <p>
+      Scripts and interactive usage may use the following commands implemented
+      by <code>ovs-sim</code>.  They are implemented as Bash shell functions
+      exported to subshells.
+    </p>
+
+    <h2>Basic Commands</h2>
+
+    <p>
+      These are the basic commands for working with sandboxed Open vSwitch
+      instances.
+    </p>
+
+    <dl>
+      <dt><code>sim_add</code> <var>sandbox</var></dt>
+      <dd>
+        <p>
+          Starts a new simulated Open vSwitch instance named
+          <var>sandbox</var>.  Files related to the instance, such as logs,
+          databases, sockets, and pidfiles, are created in a subdirectory also
+          named <var>sandbox</var>.  Afterward, the <code>as</code> command
+          (see below) can be used to run Open vSwitch utilities in the context
+          of the new sandbox.
+        </p>
+
+        <p>
+          The new sandbox starts out without any bridges.  Use
+          <code>ovs-vsctl</code> in the context of the new sandbox to create a
+          bridge, e.g.:
+        </p>
+
+        <pre>
+sim_add hv0           # Create sandbox hv0.  
+as hv0                # Set hv0 as default sandbox.
+ovs-vsctl add-br br0  # Add bridge br0 inside hv0.
+        </pre>
+
+        <p>
+          The Open vSwitch instances that <code>sim_add</code> create enable
+          ``dummy'' devices.  This means that bridges and interfaces can be
+          created with type <code>dummy</code> to indicate that they should be
+          totally simulated, without any reference to system entities.  In
+          fact, <code>ovs-sim</code> also configures Open vSwitch so that the
+          default <code>system</code> type of bridges and interfaces are
+          replaced by <code>dummy</code> devices.  Other types of devices,
+          however, retain their usual functions, which means that, e.g.,
+          <code>vxlan</code> tunnels still act as tunnels (see
+          <code>README-native-tunneling.md</code>).
+        </p>
+      </dd>
+
+      <dt><code>as</code> <var>sandbox</var></dt>
+      <dd>
+        <p>
+          Sets <var>sandbox</var> as the default simulation target for Open
+          vSwitch commands (e.g. <code>ovs-vsctl</code>,
+          <code>ovs-ofctl</code>, <code>ovs-appctl</code>).
+        </p>
+
+        <p>
+          This command updates the beginning of the shell prompt to indicate
+          the new default target.
+        </p>
+      </dd>
+
+      <dt><code>as</code> <var>sandbox</var> <var>command</var> <var>arg</var>...</dt>
+      <dd>
+        Runs the given <var>command</var> with <var>sandbox</var> as the
+        simulation target, e.g. <code>as hv0 ovs-vsctl add-br br0</code> runs
+        <code>ovs-vsctl add-br br0</code> within sandbox <code>hv0</code>.
+        The default target is unchanged.
+      </dd>
+    </dl>
+
+    <h2>Interconnection Network Commands</h2>
+
+    <p>
+      When multiple sandboxed Open vSwitch instances exist, one will inevitably
+      want to connect them together.  These commands allow for that.
+      Conceptually, an interconnection network is a switch that
+      <code>ovs-sim</code> makes it easy to plug into other switches in other
+      sandboxed Open vSwitch instances.  Interconnection networks are
+      implemented as bridges in the <code>main</code> switch that
+      <code>ovs-sim</code> creates by default, so to use interconnection
+      networks please avoid working with <code>main</code> directly.
+    </p>
+
+    <dl>
+      <dt><code>net_add</code> <var>network</var></dt>
+      <dd>
+        Creates a new interconnection network named <var>network</var>.
+      </dd>
+
+      <dt><code>net_attach</code> <var>network</var> <var>bridge</var></dt>
+      <dd>
+        Adds a new port to <var>bridge</var> in the default sandbox (as set
+        with <code>as</code>) and plugs it into the <var>network</var>
+        interconnection network.  <var>network</var> must already have been
+        created by a previous invocation of <code>net_add</code>.  The default
+        sandbox must not be <code>main</code>.
+      </dd>
+    </dl>
+
+    <h2>OVN Commands</h2>
+
+    <p>
+      These commands interact with OVN, the Open Virtual Network.
+    </p>
+
+    <dl>
+      <dt><code>ovn_start</code></dt>
+      <dd>
+        Creates and initializes the central OVN databases (both
+        <code>ovn-sb</code>(5) and <code>ovn-nb</code>) and starts an instance
+        of <code>ovsdb-server</code> for each one.  Also starts an instance of
+        <code>ovn-northd</code>.
+      </dd>
+
+      <dt><code>ovn_attach</code> <var>network</var> <var>bridge</var> <var>ip</var> [<var>masklen</var>]</dt>
+      <dd>
+        First, this command attaches <var>bridge</var> to interconnection
+        network <var>network</var>, just like <code>net_attach</code>
+        <var>network</var> <var>bridge</var>.  Second, it configures
+        (simulated) IP address <var>ip</var> (with network mask length
+        <code>masklen</code>, which defaults to 24) on <var>bridge</var>.
+        Finally, it configures the Open vSwitch database to work with OVN and
+        starts <code>ovn-controller</code>.
+      </dd>
+    </dl>
+
+    <h1>Examples</h1>
+
+    <p>
+      The following creates a pair of Open vSwitch instances
+      <code>hv0</code> and <code>hv1</code>, adds a port named
+      <code>vif0</code> or <code>vif1</code>, respectively, to each
+      one, and then connects the two through an interconnection
+      network <code>n1</code>:
+    </p>
+
+    <pre>
+net_add n1
+for i in 0 1; do
+    sim_add hv$i
+    as hv$i ovs-vsctl add-br br0 -- add-port br0 vif$i
+    as hv$i net_attach n1 br0
+done
+    </pre>
+
+    <p>
+      Here's an extended version that also starts OVN:
+    </p>
+
+    <pre>
+ovn_start
+ovn-nbctl lswitch-add lsw0
+
+net_add n1
+for i in 0 1; do
+    sim_add hv$i
+    as hv$i
+    ovs-vsctl add-br br-phys
+    ovn_attach n1 br-phys 192.168.0.`expr $i + 1`
+    ovs-vsctl add-port br-int vif$i -- set Interface vif$i external-ids:iface-id=lp$i
+    ovn-nbctl lport-add lsw0 lp$i
+    ovn-nbctl lport-set-macs lp$i f0:00:00:00:00:0$i
+done
+    </pre>
+    
+    <p>
+      Here's a primitive OVN ``scale test'' (adjust the scale by
+      changing <var>n</var> in the first line :
+    </p>
+
+    <pre>
+n=200; export n
+ovn_start
+net_add n1
+ovn-nbctl lswitch-add br0
+for i in `seq $n`; do
+    (sim_add hv$i
+    as hv$i
+    ovs-vsctl add-br br-phys
+    y=$(expr $i / 256)
+    x=$(expr $i % 256)
+    ovn_attach n1 br-phys 192.168.$y.$x
+    ovs-vsctl add-port br-int vif$i -- set Interface vif$i external-ids:iface-id=lp$i) &amp;
+    case $i in
+    	 *50|*00) echo $i; wait ;;
+    esac
+done
+wait
+for i in `seq $n`; do
+    yy=$(printf %02x $(expr $i / 256))
+    xx=$(printf $02x $(expr $i % 256))
+    ovn-nbctl lport-add br0 lp$i
+    ovn-nbctl lport-set-macs lp$i f0:00:00:00:$yy:$xx
+done
+    </pre>
+
+    <p>
+      When the scale test has finished initializing, you can watch the
+      logical ports come up with a command like this:
+    </p>
+
+    <pre>
+watch 'for i in `seq $n`; do if test `ovn-nbctl lport-get-up lp$i` != up; then echo $i; fi; done'
+    </pre>
+
+</manpage>
diff --git a/utilities/ovs-sim.in b/utilities/ovs-sim.in
new file mode 100755
index 000000000..f77b5d1df
--- /dev/null
+++ b/utilities/ovs-sim.in
@@ -0,0 +1,355 @@
+#! /usr/bin/env bash
+#
+# Copyright (c) 2013, 2015 Nicira, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+sim_builddir='@abs_builddir@'; export sim_builddir
+sim_srcdir='@abs_top_srcdir@'; export sim_srcdir
+interactive=false
+scripts=
+
+for option; do
+    case $option in
+        -h|--help)
+            cat <<EOF
+$0, for starting sandboxed dummy Open vSwitch environments
+usage: $0 [OPTION...] [SCRIPT...]
+
+Options:
+  -i, --interactive    Prompt for interactive input (default if no SCRIPTs)
+  -h, --help           Print this usage message.
+EOF
+            exit 0
+            ;;
+
+        -i|--i*)
+            interactive=:
+            ;;
+
+        -*)
+            echo "unrecognized option $option (use --help for help)" >&2
+            exit 1
+            ;;
+        *)
+	    case $option in
+		/*) ;;
+		*) option=`pwd`/$option ;;
+	    esac
+            scripts="$scripts $option"
+            ;;
+    esac
+    shift
+done
+
+if test -z "$scripts"; then
+    interactive=:
+fi
+
+# Check that we've got proper builddir and srcdir.
+if test ! -e "$sim_builddir"/vswitchd/ovs-vswitchd; then
+    echo "$sim_builddir/vswitchd/ovs-vswitchd does not exist (need to run \"make\"?)" >&2
+    exit 1
+fi
+if test ! -e "$sim_srcdir"/WHY-OVS.md; then
+    echo "$sim_srcdir/WHY-OVS.md does not exist" >&2
+    exit 1
+fi
+
+# Put built tools early in $PATH.
+PATH=$sim_builddir/ovsdb:$sim_builddir/vswitchd:$sim_builddir/utilities:$PATH
+PATH=$sim_builddir/ovn:$sim_srcdir/ovn:$sim_builddir/ovn/controller:$sim_builddir/ovn/northd:$PATH
+export PATH
+
+rm -rf sandbox
+mkdir sandbox
+cd sandbox
+sim_base=`pwd`; export sim_base
+
+trap_signals() {
+    for signal in 0 1 2 3 13 14 15; do
+	trap "
+	    set +e
+	    cd '$sim_base' && (kill \`cat */*.pid\`) >/dev/null 2>&1
+	    trap - $signal
+	    kill -$signal $$" $signal
+    done
+}
+export -f trap_signals
+trap_signals
+
+sim_setvars() {
+    sandbox=$1
+    OVS_RUNDIR=$sim_base/$1; export OVS_RUNDIR
+    OVS_LOGDIR=$sim_base/$1; export OVS_LOGDIR
+    OVS_DBDIR=$sim_base/$1; export OVS_DBDIR
+    OVS_SYSCONFDIR=$sim_base/$1; export OVS_SYSCONFDIR
+    PS1="|$1: $sim_PS1"
+}
+export -f sim_setvars
+
+as() {
+    case $# in
+	0)
+            echo >&2 "$FUNCNAME: missing arguments (use --help for help)"
+            return 1
+	    ;;
+	1) 
+	    if test "$1" != --help; then
+		sim_setvars $1
+	    else
+		cat <<EOF
+$FUNCNAME: set the default sandbox for Open vSwitch commands
+usage: $FUNCNAME SANDBOX [COMMAND ARG...]
+where SANDBOX is the name of the desired sandbox.
+
+With COMMAND arguments, this command sets the default target for that
+single command, which it runs directly.  Otherwise, it sets the default
+target for all following commands.
+EOF
+	    fi
+	    ;;
+	*)
+	    (sim_setvars $1; shift; $@)
+	    ;;
+    esac
+}
+export -f as
+
+sim_add() {
+    if test "$1" == --help; then
+        cat <<EOF
+$FUNCNAME: create a new sandboxed Open vSwitch instance
+usage: $FUNCNAME SANDBOX
+
+where SANDBOX is the name of the new sandbox, which will be created in
+a directory named $sim_base/SANDBOX.
+Afterward, use "as SANDBOX" to execute OVS commands in the sandbox's
+context.
+EOF
+	return 0
+    fi
+    if test $# != 1; then
+        echo >&2 "$FUNCNAME: missing argument (use --help for help)"
+        return 1
+    fi
+
+    set X $1; shift
+    if test $# != 1; then
+        echo >&2 "$FUNCNAME: sandbox name must be a single word"
+        return 1
+    fi
+
+    if test -e "$sim_base/$1"; then
+	echo >&2 "$1 already exists"
+	return 1
+    fi
+
+    # Create sandbox.
+    mkdir "$sim_base"/$1 || return 1
+
+    daemon_opts="--detach --no-chdir --pidfile -vconsole:off --log-file"
+
+    # Create database and start ovsdb-server.
+    touch $sim_base/$1/.conf.db.~lock~
+    as $1 ovsdb-tool create $sim_base/$1/conf.db "$sim_srcdir/vswitchd/vswitch.ovsschema"
+    as $1 ovsdb-server $daemon_opts --remote=punix:"$sim_base"/$1/db.sock
+
+    # Initialize database.
+    as $1 ovs-vsctl --no-wait -- init
+
+    # Start ovs-vswitchd.
+    as $1 ovs-vswitchd $daemon_opts --enable-dummy=system -vvconn -vnetdev_dummy
+}
+export -f sim_add
+
+net_add() {
+    if test "$1" == --help; then
+        cat <<EOF
+$FUNCNAME: create a new interconnection network
+usage: $FUNCNAME NETWORK
+
+where NETWORK is the name of the new network.  Interconnection networks
+are used with net_attach and ovn_attach.
+EOF
+	return 0
+    fi
+    if test $# != 1; then
+        echo >&2 "$FUNCNAME: missing argument (use --help for help)"
+        return 1
+    fi
+
+    as main ovs-vsctl add-br "$1"
+}
+export -f net_add
+
+net_attach() {
+    if test "$1" == --help; then
+        cat <<EOF
+$FUNCNAME: attach the default sandbox to an interconnection network
+usage: $FUNCNAME NETWORK BRIDGE
+
+Adds a port to BRIDGE within the default sandbox that connects BRIDGE
+to the interconnection network NETWORK.  (Use "as" to set the default
+sandbox.)
+EOF
+	return 0
+    fi
+    if test $# != 2; then
+        echo >&2 "$FUNCNAME: wrong number of arguments (use --help for help)"
+        return 1
+    fi
+    if test $sandbox = main; then
+	echo >&2 "$FUNCNAME: can only attach interconnection networks to sandboxes other than main"
+	return 1
+    fi
+
+    local net=$1 bridge=$2
+
+    port=${sandbox}_$bridge
+    as main ovs-vsctl \
+	-- add-port $net "$port" \
+	-- set Interface "$port" options:pstream="punix:$sim_base/main/$port.sock" options:rxq_pcap="$sim_base/main/$port-rx.pcap" options:tx_pcap="$sim_base/main/$port-tx.pcap" options:header=extended
+
+    ovs-vsctl \
+	-- set Interface $bridge options:tx_pcap="$sim_base/$sandbox/$bridge-tx.pcap" options:rxq_pcap="$sim_base/$sandbox/$bridge-rx.pcap" \
+	-- add-port $bridge ${bridge}_$net \
+	-- set Interface ${bridge}_$net options:stream="unix:$sim_base/main/$port.sock" options:rxq_pcap="$sim_base/$sandbox/${bridge}_$net-rx.pcap" options:tx_pcap="$sim_base/$sandbox/${bridge}_$net-tx.pcap" options:header=extended
+}
+export -f net_attach
+
+ovn_start() {
+    if test "$1" == --help; then
+        cat <<EOF
+$FUNCNAME: start OVN central databases and daemons
+usage: $FUNCNAME
+
+This creates and initializes the central OVN databases (northbound and
+southbound), starts their ovsdb-server daemons, and starts the ovn-northd
+daemon.
+EOF
+	return 0
+    fi
+    if test $# != 0; then
+        echo >&2 "$FUNCNAME: no arguments accepted (use --help for help)"
+        return 1
+    fi
+
+    if test -d ovn-sb || test -d ovn-nb; then
+	echo >&2 "OVN already started"
+	exit 1
+    fi
+
+    daemon_opts="--detach --no-chdir --pidfile -vconsole:off --log-file"
+    for db in ovn-sb ovn-nb; do
+	mkdir "$sim_base"/$db
+	touch "$sim_base"/$db/.$db.db.~lock~
+	as $db ovsdb-tool create "$sim_base"/$db/$db.db "$sim_srcdir"/ovn/$db.ovsschema
+	as $db ovsdb-server $daemon_opts --remote=punix:"$sim_base"/$db/$db.sock "$sim_base"/$db/$db.db
+    done
+
+    OVN_NB_DB=unix:$sim_base/ovn-nb/ovn-nb.sock; export OVN_NB_DB
+
+    mkdir "$sim_base"/northd
+    as northd ovn-northd $daemon_opts \
+	       --ovnnb-db=unix:"$sim_base"/ovn-nb/ovn-nb.sock \
+	       --ovnsb-db=unix:"$sim_base"/ovn-sb/ovn-sb.sock
+}
+export -f ovn_start
+
+ovn_attach() {
+    if test "$1" == --help; then
+        cat <<EOF
+$FUNCNAME: attach default sandbox to an interconnection network for OVN
+usage: $FUNCNAME NETWORK BRIDGE IP [MASKLEN]
+
+This starts by doing everything that net_attach does.  Then it configures the
+specified IP and MASKLEN (e.g. 192.168.0.1 and 24) on BRIDGE and starts
+and configures ovn-controller.
+
+MASKLEN defaults to 24 if it is not specified.
+EOF
+	return 0
+    fi
+    if test $# != 3 && test $# != 4; then
+        echo >&2 "$FUNCNAME: wrong number of arguments (use --help for help)"
+        return 1
+    fi
+
+    local net=$1 bridge=$2 ip=$3 masklen=${4-24}
+    net_attach $net $bridge || return $?
+
+    ovs-appctl netdev-dummy/ip4addr $bridge $ip/$masklen >/dev/null
+    ovs-appctl ovs/route/add $ip/$masklen $bridge > /dev/null
+    ovs-vsctl \
+	-- set Open_vSwitch . external-ids:system-id=$sandbox \
+        -- set Open_vSwitch . external-ids:ovn-remote=unix:$sim_base/ovn-sb/ovn-sb.sock \
+	-- set Open_vSwitch . external-ids:ovn-encap-type=geneve \
+	-- set Open_vSwitch . external-ids:ovn-encap-ip=$ip\
+	-- add-br br-int \
+	-- set bridge br-int fail-mode=secure other-config:disable-in-band=true
+    ovn-controller --detach --no-chdir --pidfile -vconsole:off --log-file
+}
+export -f ovn_attach
+
+# Easy access to OVS manpages.
+mkdir $sim_base/man
+mandir=`cd $sim_base/man && pwd`
+(cd "$sim_builddir" && ${MAKE-make} install-man mandir=$mandir >/dev/null)
+MANPATH=$mandir:; export MANPATH
+
+export scripts
+export interactive
+rc='
+    if [ -f /etc/bashrc ]; then
+	. /etc/bashrc
+    fi
+    if [ -f ~/.bashrc ]; then
+	. ~/.bashrc
+    fi
+
+    trap_signals
+    sim_PS1=$PS1
+    sim_add main
+    as main
+
+    for script in $scripts; do
+        . $script || exit $?
+    done
+
+    $interactive || exit 0
+
+    cat <<EOF
+ ______________________________________________________________________
+|
+| You are running in a nested shell environment meant for Open vSwitch
+| and OVN testing in simulation.   The OVS manpages are available via
+| "man".  Please see ovs-sim(1) for more information.
+|
+| Exit the shell to kill the running daemons and leave the simulation
+| environment.
+EOF
+'
+
+status=0; bash --rcfile <(echo "$rc") || status=$?
+
+if $interactive; then
+    cat <<EOF
+|______________________________________________________________________
+
+EOF
+fi
+
+exit $status