Initial skeleton for WEBAPP and yarns for it

8 files changed, 523 insertions, 0 deletions

diff --git a/check b/check
new file mode 100755
index 0000000..70aa608
--- /dev/null
+++ b/check
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -eu
+
+# This is currently being developed on a BR-13 devel-system instance, which
+# has neither bottle, nor flup. I've checked these out so I can use these,
+# but really, this will need to go away and development should be done on
+# a system with the necessary dependencies installed properly. (BR-13 also
+# doesn't have a working sqlite3 Python binding, and that's not so easy to
+# fix without a fixed system deployed.)
+export PYTHONPATH="/home/root/new-lorry-controller/bottle:/home/root/new-lorry-controller/flup"
+
+yarn -s yarns.webapp/yarn.sh yarns.webapp/*.yarn --env PYTHONPATH="$PYTHONPATH" "$@"
diff --git a/lorry-controller-webapp b/lorry-controller-webapp
new file mode 100755
index 0000000..6acbac0
--- /dev/null
+++ b/lorry-controller-webapp
@@ -0,0 +1,193 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import logging
+import os
+import sqlite3
+import time
+
+import bottle
+import cliapp
+from flup.server.fcgi import WSGIServer
+
+
+class StateDB(object):
+
+    '''A wrapper around raw Sqlite for STATEDB.'''
+
+    def __init__(self, filename):
+        self._filename = filename
+        self._conn = None
+
+    def _open(self):
+        if self._conn is None:
+            self._conn = sqlite3.connect(self._filename)
+            self._initialise_tables()
+
+    def _initialise_tables(self):
+        c = self._conn.cursor()
+        c.execute('BEGIN TRANSACTION')
+        c.execute('CREATE TABLE IF NOT EXISTS running_queue (running INT)')
+        c.execute('INSERT INTO running_queue VALUES (0)')
+        self._conn.commit()
+
+    def get_running_queue(self):
+        self._open()
+        c = self._conn.cursor()
+        for (running,) in c.execute('SELECT running FROM running_queue'):
+            return running
+
+    def set_running_queue(self, new_status):
+        self._open()
+        c = self._conn.cursor()
+        c.execute('UPDATE running_queue SET running = ?', str(new_status))
+        self._conn.commit()
+
+
+class LorryControllerRoute(object):
+
+    '''Base class for Lorry Controller HTTP API routes.
+
+    A route is an HTTP request that the Bottle web application
+    recognises as satisfied by a particular callback. To make it
+    easier to implement them and get them added automagically to a
+    Bottle instance, we define the callbacks as subclasses of this
+    base class.
+
+    Subclasses MUST define the attributes ``http_method`` and
+    ``path``, which are given the bottle.Bottle.route method as the
+    arguments ``method`` and ``path``, respectively.
+
+    '''
+
+    def __init__(self, statedb):
+        self.statedb = statedb
+
+    def run(self, **kwargs):
+        raise NotImplementedError()
+
+
+class Status(LorryControllerRoute):
+
+    http_method = 'GET'
+    path = '/1.0/status'
+
+    def run(self, **kwargs):
+        quotes = [
+            "Never get drunk unless you're willing to pay for it - "
+            "the next day.",
+            "I'm giving her all she's got, Captain!",
+            ]
+        import random
+        return {
+            'quote': '%s\n' % random.choice(quotes),
+            'running-queue': self.statedb.get_running_queue(),
+            }
+
+
+class StartQueue(LorryControllerRoute):
+
+    http_method = 'GET'
+    path = '/1.0/start-queue'
+
+    def run(self, **kwargs):
+        self.statedb.set_running_queue(1)
+        return 'Queue set to run'
+
+
+class StopQueue(LorryControllerRoute):
+
+    http_method = 'GET'
+    path = '/1.0/stop-queue'
+
+    def run(self, **kwargs):
+        self.statedb.set_running_queue(0)
+        return 'Queue set to not run'
+
+
+class WEBAPP(cliapp.Application):
+
+    def add_settings(self):
+        self.settings.string(
+            ['statedb'],
+            'use FILE as the state database',
+            metavar='FILE')
+
+        self.settings.boolean(
+            ['wsgi'],
+            'run in wsgi mode (default is debug mode, for development)')
+
+        self.settings.integer(
+            ['debug-port'],
+            'listen on PORT when in debug mode (i.e., not running under WSGI)',
+            metavar='PORT',
+            default=8888)
+
+        self.settings.string(
+            ['debug-host'],
+            'listen on HOST when in debug mode (i.e., not running under WSGI)',
+            metavar='HOST',
+            default='0.0.0.0')
+
+    def find_routes(self):
+        '''Return all classes that are API routes.
+
+        This is a generator.
+
+        '''
+
+        # This is a bit tricky and magic. globals() returns a dict
+        # that contains all objects in the global namespace. We
+        # iterate over the objects and pick the ones that are
+        # subclasses of our superclass (no duck typing here), but ARE
+        # NOT the superclass itself.
+
+        for x in globals().values():
+            is_route = (
+                type(x) == type and # it must be class, for issubclass
+                issubclass(x, LorryControllerRoute) and
+                x != LorryControllerRoute)
+            if is_route:
+                yield x
+
+    def process_args(self, args):
+        self.settings.require('statedb')
+        statedb = StateDB(self.settings['statedb'])
+
+        webapp = bottle.Bottle()
+
+        for route_class in self.find_routes():
+            route = route_class(statedb)
+            webapp.route(
+                path=route.path,
+                method=route.http_method,
+                callback=route.run)
+
+        logging.info('Starting server')
+        if self.settings['wsgi']:
+            WSGIServer(webapp).run()
+        else:
+            bottle.run(
+                webapp,
+                host=self.settings['debug-host'],
+                port=self.settings['debug-port'],
+                quiet=True,
+                debug=True)
+
+
+WEBAPP().run()
diff --git a/test-wait-for-port b/test-wait-for-port
new file mode 100755
index 0000000..22e07be
--- /dev/null
+++ b/test-wait-for-port
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+'''Wait for a given port to be open.
+
+WARNING: This may wait for quite a long time. There is no timeout. Or
+spoon.
+
+'''
+
+import sys, socket, errno
+
+host = sys.argv[1]
+port = int(sys.argv[2])
+
+while True:
+    print "Trying %s port %s" % (host, port)
+    s = socket.socket()
+    try:
+        s.connect((host, port))
+    except socket.error as e:
+        if e.errno == errno.ECONNREFUSED:
+            continue
+        raise
+    s.close()
+    break
diff --git a/yarns.webapp/010-introduction.yarn b/yarns.webapp/010-introduction.yarn
new file mode 100644
index 0000000..ae3af58
--- /dev/null
+++ b/yarns.webapp/010-introduction.yarn
@@ -0,0 +1,77 @@
+% Lorry Controller WEBAPP integration test suite
+% Codethink Ltd
+
+
+Introduction
+============
+
+This is an integration test suite for the WEBAPP component of Lorry
+Controller. It is implemented using the [yarn] tool and uses a style
+of automated testing called "scenario testing" by the tool authors.
+
+[yarn]: http://liw.fi/cmdtest/README.yarn/
+
+As an example, here is a scenario that verifies that the Lorry
+Controller WEBAPP can be started at all:
+
+    SCENARIO WEBAPP can be started at all
+    WHEN WEBAPP --help is requested
+    THEN WEBAPP --help exited with a zero exit code
+
+A scenario consists of a sequence of steps that can be executed by a
+computer. The steps are then defined using IMPLEMENTS:
+
+    IMPLEMENTS WHEN WEBAPP --help is requested
+    if "$SRCDIR/lorry-controller-webapp" --help
+    then
+        exit=0
+    else
+        exit=$?
+    fi
+    echo "$exit" > "$DATADIR/webapp.exit"
+
+And another:
+
+    IMPLEMENTS THEN WEBAPP --help exited with a zero exit code
+    grep -Fx 0 "$DATADIR/webapp.exit"
+
+Yarn will run each scenario in the order it finds them. If all steps
+in a scenario succeed, the scenario succeeds.
+
+Scenarios, though not their implementations, are intended to be
+understandable by people who aren't programmers, though some
+understanding of the technology is required.
+
+For more information, see the documentation for yarn.
+
+
+Test environment and setup
+==========================
+
+In this chapter, we discuss how the environment is set up for tests to
+run in. Yarn provides a temporary directory in which tests can create
+temporary directories, and sets the environment variable `$DATADIR` to
+point at that directory. Yarn also deletes the directory and all of
+its contents at the end, so the test suite itself does not need to do
+that.
+
+We put several files into `$DATADIR`.
+
+* The WEBAPP STATEDB database file.
+* Responses from HTTP queries to WEBAPP.
+* PID of the running WEBAPP.
+
+The purpose of each file is documented with the IMPLEMENTS sections
+that use it, typically with the one that creates it.
+
+Since many scenarios will start an instance of WEBAPP, they also need
+to make sure it gets killed. There are steps for these (`GIVEN a
+running WEBAPP` and `FINALLY WEBAPP is terminated`), which MUST be
+used as a pair in each scenario: having only one of these steps is
+always a bug in the scenario, whereas having neither is OK.
+
+WEBAPP has stores its persistent state in STATEDB, which is an Sqlite
+database on disk. Our tests do _not_ touch it directly, only via WEBAPP,
+so that we do not encode in our tests internals of the database, such
+as the database schema. We do not care: we only care that WEBAPP
+works, and the database schema of STATEDB is _not_ a public interface.
diff --git a/yarns.webapp/020-status.yarn b/yarns.webapp/020-status.yarn
new file mode 100644
index 0000000..ba8c903
--- /dev/null
+++ b/yarns.webapp/020-status.yarn
@@ -0,0 +1,14 @@
+WEBAPP status reporting
+=======================
+
+WEBAPP reports it status via an HTTP request. We verify that when it
+starts up, the status is that it is doing nothing: there are no jobs,
+it has no Lorry or Trove specs.
+
+    SCENARIO WEBAPP is idle when it starts
+    GIVEN a running WEBAPP
+    WHEN admin makes request GET /1.0/status
+    THEN response is JSON
+    AND response has running-queue set to False
+    FINALLY WEBAPP terminates
+
diff --git a/yarns.webapp/030-queue-management.yarn b/yarns.webapp/030-queue-management.yarn
new file mode 100644
index 0000000..75b17e3
--- /dev/null
+++ b/yarns.webapp/030-queue-management.yarn
@@ -0,0 +1,35 @@
+Run queue management
+====================
+
+This chapter contains tests meant for managing the run-queue in
+WEBAPP.
+
+Start and stop job scheduling
+-----------------------------
+
+The administrator needs to be able to stop WEBAPP from scheduling any
+new jobs, and later to start it again.
+
+    SCENARIO admin can start and stop WEBAPP job scheduling
+    GIVEN a running WEBAPP
+    WHEN admin makes request GET /1.0/status
+    THEN response has running-queue set to False
+
+    WHEN admin makes request GET /1.0/start-queue
+    AND admin makes request GET /1.0/status
+    THEN response has running-queue set to True
+
+Further, the state change needs to be persistent across WEBAPP
+instances, so we kill the WEBAPP that's currently running, and start a
+new one, and verify that the `running-queue` status is still `True`.
+
+    WHEN WEBAPP is terminated
+    THEN WEBAPP isn't running
+
+    GIVEN a running WEBAPP
+    WHEN admin makes request GET /1.0/status
+    THEN response has running-queue set to True
+
+Finally, clean up.
+
+    FINALLY WEBAPP terminates
diff --git a/yarns.webapp/900-implementations.yarn b/yarns.webapp/900-implementations.yarn
new file mode 100644
index 0000000..7bfb6bb
--- /dev/null
+++ b/yarns.webapp/900-implementations.yarn
@@ -0,0 +1,119 @@
+Implementations
+===============
+
+This chapter includes IMPLEMENTS sections for the various steps used
+in scenarios.
+
+Managing a WEBAPP instance
+--------------------------
+
+We're testing a web application (convenivently named WEBAPP, though
+the executable is `lorry-controller-webapp`), so we need to be able to
+start it and stop it in scenarios. We start it as a background
+process, and keep its PID in `$DATADIR/webapp.pid`. When it's time to
+kill it, we kill the process with the PID in that file. This is not
+perfect, though it's good enough for our purposes. It doesn't handle
+running multiple instances at the same time, which we don't need, and
+doens't handle the case of the process dying and the kernel re-using
+the PID for something else, which is quite unlikely.
+
+Start an instance of the WEBAPP, using a random port. Record the PID
+and the port. Listen only on localhost. We use `start-stop-daemon` to
+start the process, so that it can keep running in the background,
+but the shell doesn't wait for it to terminate. This way, WEBAPP will
+be running until it crashes or is explicitly killed.
+
+    IMPLEMENTS GIVEN a running WEBAPP
+    # Pick a random port beyond 1024 (i.e., an unreserved one).
+    port=0
+    while [ "$port" -le 1024 ]
+    do
+        port=$RANDOM
+    done
+    echo "$port" > "$DATADIR/webapp.port"
+
+    start-stop-daemon -S -x "$SRCDIR/lorry-controller-webapp" \
+        -b -p "$DATADIR/webapp.pid" -m --verbose \
+        -- \
+        --statedb "$DATADIR/webapp.db" \
+        --log-level debug \
+        --log "$DATADIR/webapp.log" \
+        --debug-host 127.0.0.1 \
+        --debug-port "$port"
+
+    # Wait for the WEBAPP to actually be ready, i.e., that it's
+    # listening on its assigned port.
+    "$SRCDIR/test-wait-for-port" 127.0.0.1 "$port"
+    
+Kill the running WEBAPP, using the recorded PID. We need to do this
+both as a WHEN and a FINALLY step.
+
+    IMPLEMENTS WHEN WEBAPP is terminated
+    kill_daemon_using_pid_file "$DATADIR/webapp.pid"
+
+    IMPLEMENTS FINALLY WEBAPP terminates
+    kill_daemon_using_pid_file "$DATADIR/webapp.pid"
+
+Also test that WEBAPP isn't running.
+
+    IMPLEMENTS THEN WEBAPP isn't running
+    pid=$(head -n1 "$DATADIR/webapp.pid")
+    if kill -0 "$pid"
+    then
+        echo "process $pid is still running, but should'nt be" 1>&2
+        exit 1
+    fi
+
+Making and analysing GET requests
+---------------------------------
+
+Simple HTTP GET requests are simple. We make the request, and capture
+the response: HTTP status code, response headers, response body.
+
+We make the request using the `curl` command line program, which makes
+capturing the response quite convenient.
+
+HTTP requests can be made by various entities. For now, we assume
+they're all made by the admin.
+
+We check that the HTTP status indicates success, so that every
+scenario doesn't need ot check that separately.
+
+    IMPLEMENTS WHEN admin makes request GET (\S+)
+    rm -f "$DATADIR/response.headers"
+    rm -f "$DATADIR/response.body"
+    port=$(cat "$DATADIR/webapp.port")
+    curl \
+        -D "$DATADIR/response.headers" \
+        -o "$DATADIR/response.body" \
+        --silent --show-error \
+        "http://127.0.0.1:$port$MATCH_1"
+    cat "$DATADIR/response.headers"
+    cat "$DATADIR/response.body"
+    head -n1 "$DATADIR/response.headers" | grep '^HTTP/1\.[01] 200 '
+
+Check the Content-Type of the response is JSON.
+
+    IMPLEMENTS THEN response is JSON
+    cat "$DATADIR/response.headers"
+    grep -i '^Content-Type: application/json' "$DATADIR/response.headers"
+
+A JSON response can then be queried further. The JSON is expected to
+be a dict, so that values are accessed by name from the dict. The
+value is expresssed as a Python value in the step.
+
+    IMPLEMENTS THEN response has (\S+) set to (\S+)
+    cat "$DATADIR/response.body"
+    python -c "
+    import json, os, sys
+    data = json.load(sys.stdin)
+    key = os.environ['MATCH_1']
+    expected = eval(os.environ['MATCH_2']) # I feel dirty and evil.
+    value = data[key]
+    if value != expected:
+        sys.stderr.write(
+            'Key {key} has value {value}, but '
+            '{expected} was expected'.format(
+                key=key, value=value, expected=expected))
+        sys.exit(1)
+    " < "$DATADIR/response.body"
diff --git a/yarns.webapp/yarn.sh b/yarns.webapp/yarn.sh
new file mode 100644
index 0000000..58af354
--- /dev/null
+++ b/yarns.webapp/yarn.sh
@@ -0,0 +1,32 @@
+# Copyright (C) 2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+# This file is a yarn shell library for testing Lorry Controller.
+
+
+# Kill a daemon given its pid file. Report whether it got killed or not.
+
+kill_daemon_using_pid_file()
+{
+    local pid=$(head -n1 "$1")
+    if kill -9 "$pid"
+    then
+        echo "Killed daemon running as $pid"
+    else
+        echo "Error killing daemon running as pid $pid"
+    fi
+}