proxy: new integration tests.

uses mocked backend servers so we can test:
- end to end client to backend proxying
- lua API functions
- configuration reload
- various error conditions

5 files changed, 933 insertions, 0 deletions

diff --git a/t/lib/MemcachedTest.pm b/t/lib/MemcachedTest.pm
index 4b2d409..a9557e5 100644
--- a/t/lib/MemcachedTest.pm
+++ b/t/lib/MemcachedTest.pm
@@ -427,6 +427,11 @@ sub graceful_stop {
     kill 'SIGUSR1', $self->{pid};
 }
 
+sub reload {
+    my $self = shift;
+    kill 'SIGHUP', $self->{pid};
+}
+
 # -1 if the pid is actually dead.
 sub is_running {
     my $self = shift;
diff --git a/t/proxyconfig.lua b/t/proxyconfig.lua
new file mode 100644
index 0000000..18a35de
--- /dev/null
+++ b/t/proxyconfig.lua
@@ -0,0 +1,31 @@
+-- get some information about the test being run from an external file
+-- so we can modify ourselves.
+local mode = dofile("/tmp/proxyconfigmode.lua")
+
+function mcp_config_pools(old)
+    if mode == "none" then
+        return {}
+    elseif mode == "start" then
+        local b1 = mcp.backend('b1', '127.0.0.1', 11511)
+        local b2 = mcp.backend('b2', '127.0.0.1', 11512)
+        local b3 = mcp.backend('b3', '127.0.0.1', 11513)
+
+        local pools = {
+            test = mcp.pool({b1, b2, b3})
+        }
+        return pools
+    end
+end
+
+-- At least to start we don't need to test every command, but we should do
+-- some tests against the two broad types of commands (gets vs sets with
+-- payloads)
+function mcp_config_routes(zones)
+    if mode == "none" then
+        mcp.attach(mcp.CMD_MG, function(r) return "SERVER_ERROR no mg route\r\n" end)
+        mcp.attach(mcp.CMD_MS, function(r) return "SERVER_ERROR no ms route\r\n" end)
+    elseif mode == "start" then
+        mcp.attach(mcp.CMD_MG, function(r) return zones["test"](r) end)
+        mcp.attach(mcp.CMD_MS, function(r) return zones["test"](r) end)
+    end
+end
diff --git a/t/proxyconfig.t b/t/proxyconfig.t
new file mode 100644
index 0000000..2380997
--- /dev/null
+++ b/t/proxyconfig.t
@@ -0,0 +1,147 @@
+#!/usr/bin/env perl
+
+# NOTE: These tests cover the act of reloading the configuration; changing
+# backends, pools, routes, etc. It doesn't cover ensuring the code of the main
+# file changes naturally, which is fine: there isn't any real way that can
+# fail and it can be covered specifically in a different test file.
+
+use strict;
+use warnings;
+use Test::More;
+use FindBin qw($Bin);
+use lib "$Bin/lib";
+use Carp qw(croak);
+use MemcachedTest;
+use IO::Select;
+use IO::Socket qw(AF_INET SOCK_STREAM);
+
+if (!supports_proxy()) {
+    plan skip_all => 'proxy not enabled';
+    exit 0;
+}
+
+# TODO: possibly... set env var to a generated temp filename before starting
+# the server so we can pass that in?
+my $modefile = "/tmp/proxyconfigmode.lua";
+
+# Set up some server sockets.
+sub mock_server {
+    my $port = shift;
+    my $srv = IO::Socket->new(
+        Domain => AF_INET,
+        Type => SOCK_STREAM,
+        Proto => 'tcp',
+        LocalHost => '127.0.0.1',
+        LocalPort => $port,
+        ReusePort => 1,
+        Listen => 5) || die "IO::Socket: $@";
+    return $srv;
+}
+
+# Put a version command down the pipe to ensure the socket is clear.
+# client version commands skip the proxy code
+sub check_version {
+    my $ps = shift;
+    print $ps "version\r\n";
+    like(<$ps>, qr/VERSION /, "version received");
+}
+
+sub write_modefile {
+    my $cmd = shift;
+    open(my $fh, "> $modefile") or die "Couldn't overwrite $modefile: $!";
+    print $fh $cmd;
+    close($fh);
+}
+
+sub wait_reload {
+    my $w = shift;
+    like(<$w>, qr/ts=(\S+) gid=\d+ type=proxy_conf status=start/, "reload started");
+    like(<$w>, qr/ts=(\S+) gid=\d+ type=proxy_conf status=done/, "reload completed");
+}
+
+my @mocksrvs = ();
+diag "making mock servers";
+for my $port (11511, 11512, 11513, 11514, 11515, 11516) {
+    my $srv = mock_server($port);
+    ok(defined $srv, "mock server created");
+    push(@mocksrvs, $srv);
+}
+
+diag "testing failure to start";
+write_modefile("invalid syntax");
+eval {
+    my $p_srv = new_memcached('-o proxy_config=./t/proxyconfig.lua -l 127.0.0.1', 11510);
+};
+ok($@ && $@ =~ m/Failed to connect/, "server successfully not started");
+
+write_modefile('return "none"');
+my $p_srv = new_memcached('-o proxy_config=./t/proxyconfig.lua -l 127.0.0.1', 11510);
+my $ps = $p_srv->sock;
+$ps->autoflush(1);
+
+# Create a watcher so we can monitor when reloads complete.
+my $watcher = $p_srv->new_sock;
+print $watcher "watch proxyevents\n";
+is(<$watcher>, "OK\r\n", "watcher enabled");
+
+{
+    # test with stubbed main routes.
+    print $ps "mg foo v\r\n";
+    is(scalar <$ps>, "SERVER_ERROR no mg route\r\n", "no mg route loaded");
+}
+
+# Load some backends
+{
+    write_modefile('return "start"');
+
+    $p_srv->reload();
+    wait_reload($watcher);
+}
+
+{
+    # set up server backend sockets.
+    my @mbe = ();
+    for my $msrv ($mocksrvs[0], $mocksrvs[1], $mocksrvs[2]) {
+        my $be = $msrv->accept();
+        $be->autoflush(1);
+        ok(defined $be, "mock backend created");
+        push(@mbe, $be);
+    }
+
+    my $s = IO::Select->new();
+
+    for my $be (@mbe) {
+        $s->add($be);
+        like(<$be>, qr/version/, "received version command");
+        print $be "VERSION 1.0.0-mock\r\n";
+    }
+
+    # Try sending something.
+    my $cmd = "mg foo v\r\n";
+    print $ps $cmd;
+    my @readable = $s->can_read(1);
+    is(scalar @readable, 1, "only one backend became readable");
+    my $be = shift @readable;
+    is(scalar <$be>, $cmd, "metaget passthrough");
+    print $be "EN\r\n";
+    is(scalar <$ps>, "EN\r\n", "miss received");
+}
+
+# TODO:
+# remove backends
+# do dead sockets close?
+# adding user stats
+# changing user stats
+# adding backends with the same label don't create more connections
+# total backend counters
+# change top level routes mid-request
+#  - send the request to backend
+#  - issue and wait for reload
+#  - read from backend and respond, should use the original code still.
+#  - could also read from backend and then do reload/etc.
+
+done_testing();
+
+END {
+    unlink $modefile;
+}
diff --git a/t/proxyunits.lua b/t/proxyunits.lua
new file mode 100644
index 0000000..6b492b4
--- /dev/null
+++ b/t/proxyunits.lua
@@ -0,0 +1,234 @@
+mcp.backend_read_timeout(0.5)
+
+function mcp_config_pools(oldss)
+    local srv = mcp.backend
+
+    -- Single backend for zones to ease testing.
+    -- For purposes of this config the proxy is always "zone 1" (z1)
+    local b1 = srv('b1', '127.0.0.1', 11411)
+    local b2 = srv('b2', '127.0.0.1', 11412)
+    local b3 = srv('b3', '127.0.0.1', 11413)
+
+    local b1z = {b1}
+    local b2z = {b2}
+    local b3z = {b3}
+
+    -- convert the backends to pools.
+    -- as per a normal full config see simple.lua or t/startfile.lua
+    local zones = {
+        z1 = mcp.pool(b1z),
+        z2 = mcp.pool(b2z),
+        z3 = mcp.pool(b3z),
+    }
+
+    return zones
+end
+
+-- WORKER CODE:
+
+-- Using a very simple route handler only to allow testing the three
+-- workarounds in the same configuration file.
+function prefix_factory(pattern, list, default)
+    local p = pattern
+    local l = list
+    local d = default
+    return function(r)
+        local route = l[string.match(r:key(), p)]
+        if route == nil then
+            return d(r)
+        end
+        return route(r)
+    end
+end
+
+-- just for golfing the code in mcp_config_routes()
+function toproute_factory(pfx, label)
+    local err = "SERVER_ERROR no " .. label .. " route\r\n"
+    return prefix_factory("^/(%a+)/", pfx, function(r) return err end)
+end
+
+-- Do specialized testing based on the key prefix.
+function mcp_config_routes(zones)
+    local pfx_get = {}
+    local pfx_set = {}
+    local pfx_touch = {}
+    local pfx_gets = {}
+    local pfx_gat = {}
+    local pfx_gats = {}
+    local pfx_cas = {}
+    local pfx_add = {}
+    local pfx_delete = {}
+    local pfx_incr = {}
+    local pfx_decr = {}
+    local pfx_append = {}
+    local pfx_prepend = {}
+    local pfx_mg = {}
+    local pfx_ms = {}
+    local pfx_md = {}
+    local pfx_ma = {}
+
+    local basic = function(r)
+        return zones.z1(r)
+    end
+
+    pfx_get["b"] = basic
+    pfx_set["b"] = basic
+    pfx_touch["b"] = basic
+    pfx_gets["b"] = basic
+    pfx_gat["b"] = basic
+    pfx_gats["b"] = basic
+    pfx_cas["b"] = basic
+    pfx_add["b"] = basic
+    pfx_delete["b"] = basic
+    pfx_incr["b"] = basic
+    pfx_decr["b"] = basic
+    pfx_append["b"] = basic
+    pfx_prepend["b"] = basic
+    pfx_mg["b"] = basic
+    pfx_ms["b"] = basic
+    pfx_md["b"] = basic
+    pfx_ma["b"] = basic
+
+    -- show that we fetched the key by generating our own response string.
+    pfx_get["getkey"] = function(r)
+        return "VALUE |" .. r:key() .. " 0 2\r\nts\r\nEND\r\n"
+    end
+
+    pfx_get["rtrimkey"] = function(r)
+        r:rtrimkey(4)
+        return zones.z1(r)
+    end
+
+    pfx_get["ltrimkey"] = function(r)
+        r:ltrimkey(10)
+        return zones.z1(r)
+    end
+
+    -- Basic test for routing requests to specific pools.
+    -- Not sure how this could possibly break but testing for completeness.
+    pfx_get["zonetest"] = function(r)
+        local key = r:key()
+        if key == "/zonetest/a" then
+            return zones.z1(r)
+        elseif key == "/zonetest/b" then
+            return zones.z2(r)
+        elseif key == "/zonetest/c" then
+            return zones.z3(r)
+        else
+            return "END\r\n"
+        end
+    end
+
+    pfx_get["logtest"] = function(r)
+        mcp.log("testing manual log messages")
+        return "END\r\n"
+    end
+
+    pfx_get["logreqtest"] = function(r)
+        local res = zones.z1(r)
+        mcp.log_req(r, res, "logreqtest")
+        return res
+    end
+
+    -- tell caller what we got back via a fake response
+    pfx_get["awaitbasic"] = function(r)
+        local vals = {}
+        local rtable = mcp.await(r, { zones.z1, zones.z2, zones.z3 })
+
+        for i, res in pairs(rtable) do
+            if res:hit() == true then
+                vals[i] = "hit"
+            elseif res:ok() == true then
+                vals[i] = "ok"
+            else
+                vals[i] = "err"
+            end
+        end
+
+        local val = table.concat(vals, " ")
+        local vlen = string.len(val)
+        -- convenience functions for creating responses would be nice :)
+        return "VALUE " .. r:key() .. " 0 " .. vlen .. "\r\n" .. val .. "\r\nEND\r\n"
+    end
+
+    pfx_get["awaitone"] = function(r)
+        local mode = string.sub(r:key(), -1, -1)
+        local num = 0
+        if mode == "a" then
+            num = 1
+        elseif mode == "b" then
+            num = 2
+        end
+        local rtable = mcp.await(r, { zones.z1, zones.z2, zones.z3 }, num)
+
+        local count = 0
+        for i, res in pairs(rtable) do
+            count = count + 1
+        end
+
+        local vlen = string.len(count)
+        return "VALUE " .. r:key() .. " 0 " .. vlen .. "\r\n" .. count .. "\r\nEND\r\n"
+    end
+
+    -- should be the same as awaitone
+    pfx_get["awaitgood"] = function(r)
+        local mode = string.sub(r:key(), -1, -1)
+        local num = 0
+        if mode == "a" then
+            num = 1
+        elseif mode == "b" then
+            num = 2
+        end
+        local rtable = mcp.await(r, { zones.z1, zones.z2, zones.z3 }, num, mcp.AWAIT_GOOD)
+
+        local count = 0
+        for i, res in pairs(rtable) do
+            count = count + 1
+        end
+
+        local vlen = string.len(count)
+        return "VALUE " .. r:key() .. " 0 " .. vlen .. "\r\n" .. count .. "\r\nEND\r\n"
+    end
+
+    -- not sure if anything else should be checked here? if err or not?
+    pfx_get["awaitany"] = function(r)
+        local rtable = mcp.await(r, { zones.z1, zones.z2, zones.z3 }, 2, mcp.AWAIT_ANY)
+        local count = 0
+        for i, res in pairs(rtable) do
+            count = count + 1
+        end
+
+        local vlen = string.len(count)
+        return "VALUE " .. r:key() .. " 0 " .. vlen .. "\r\n" .. count .. "\r\nEND\r\n"
+    end
+
+    pfx_get["awaitbg"] = function(r)
+        local rtable = mcp.await(r, { zones.z1, zones.z2, zones.z3 }, 1, mcp.AWAIT_BACKGROUND)
+        local count = 0
+        for i, res in pairs(rtable) do
+            count = count + 1
+        end
+
+        local vlen = string.len(count)
+        return "VALUE " .. r:key() .. " 0 " .. vlen .. "\r\n" .. count .. "\r\nEND\r\n"
+    end
+
+    mcp.attach(mcp.CMD_GET, toproute_factory(pfx_get, "get"))
+    mcp.attach(mcp.CMD_SET, toproute_factory(pfx_set, "set"))
+    mcp.attach(mcp.CMD_TOUCH, toproute_factory(pfx_touch, "touch"))
+    mcp.attach(mcp.CMD_GETS, toproute_factory(pfx_gets, "gets"))
+    mcp.attach(mcp.CMD_GAT, toproute_factory(pfx_gat, "gat"))
+    mcp.attach(mcp.CMD_GATS, toproute_factory(pfx_gats, "gats"))
+    mcp.attach(mcp.CMD_CAS, toproute_factory(pfx_cas, "cas"))
+    mcp.attach(mcp.CMD_ADD, toproute_factory(pfx_add, "add"))
+    mcp.attach(mcp.CMD_DELETE, toproute_factory(pfx_delete, "delete"))
+    mcp.attach(mcp.CMD_INCR, toproute_factory(pfx_incr, "incr"))
+    mcp.attach(mcp.CMD_DECR, toproute_factory(pfx_decr, "decr"))
+    mcp.attach(mcp.CMD_APPEND, toproute_factory(pfx_append, "append"))
+    mcp.attach(mcp.CMD_PREPEND, toproute_factory(pfx_prepend, "prepend"))
+    mcp.attach(mcp.CMD_MG, toproute_factory(pfx_mg, "mg"))
+    mcp.attach(mcp.CMD_MS, toproute_factory(pfx_ms, "ms"))
+    mcp.attach(mcp.CMD_MD, toproute_factory(pfx_md, "md"))
+    mcp.attach(mcp.CMD_MA, toproute_factory(pfx_ma, "ma"))
+
+end
diff --git a/t/proxyunits.t b/t/proxyunits.t
new file mode 100644
index 0000000..0914b98
--- /dev/null
+++ b/t/proxyunits.t
@@ -0,0 +1,516 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+use Test::More;
+use FindBin qw($Bin);
+use lib "$Bin/lib";
+use Carp qw(croak);
+use MemcachedTest;
+use IO::Socket qw(AF_INET SOCK_STREAM);
+
+if (!supports_proxy()) {
+    plan skip_all => 'proxy not enabled';
+    exit 0;
+}
+
+# Set up some server sockets.
+sub mock_server {
+    my $port = shift;
+    my $srv = IO::Socket->new(
+        Domain => AF_INET,
+        Type => SOCK_STREAM,
+        Proto => 'tcp',
+        LocalHost => '127.0.0.1',
+        LocalPort => $port,
+        ReusePort => 1,
+        Listen => 5) || die "IO::Socket: $@";
+    return $srv;
+}
+
+# Put a version command down the pipe to ensure the socket is clear.
+# client version commands skip the proxy code
+sub check_version {
+    my $ps = shift;
+    print $ps "version\r\n";
+    like(<$ps>, qr/VERSION /, "version received");
+}
+
+my @mocksrvs = ();
+diag "making mock servers";
+for my $port (11411, 11412, 11413) {
+    my $srv = mock_server($port);
+    ok(defined $srv, "mock server created");
+    push(@mocksrvs, $srv);
+}
+
+my $p_srv = new_memcached('-o proxy_config=./t/proxyunits.lua -l 127.0.0.1', 11410);
+my $ps = $p_srv->sock;
+$ps->autoflush(1);
+
+# set up server backend sockets.
+my @mbe = ();
+diag "accepting mock backends";
+for my $msrv (@mocksrvs) {
+    my $be = $msrv->accept();
+    $be->autoflush(1);
+    ok(defined $be, "mock backend created");
+    push(@mbe, $be);
+}
+
+diag "validating backends";
+for my $be (@mbe) {
+    like(<$be>, qr/version/, "received version command");
+    print $be "VERSION 1.0.0-mock\r\n";
+}
+
+diag "ready for main tests";
+# Target a single backend, validating basic syntax.
+# Should test all command types.
+# uses /b/ path for "basic"
+{
+    # Test invalid route.
+    print $ps "set /invalid/a 0 0 2\r\nhi\r\n";
+    is(scalar <$ps>, "SERVER_ERROR no set route\r\n");
+
+    # Testing against just one backend. Results should make sense despite our
+    # invalid request above.
+    my $be = $mbe[0];
+    my $cmd;
+
+    # TODO: add more tests for the varying response codes.
+
+    # Basic set.
+    $cmd = "set /b/a 0 0 2";
+    print $ps "$cmd\r\nhi\r\n";
+    is(scalar <$be>, "$cmd\r\n", "set passthrough");
+    is(scalar <$be>, "hi\r\n", "set value");
+    print $be "STORED\r\n";
+
+    is(scalar <$ps>, "STORED\r\n", "got STORED from set");
+
+    # Basic get
+    $cmd = "get /b/a\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, $cmd, "get passthrough");
+    print $be "VALUE /b/a 0 2\r\nhi\r\nEND\r\n";
+
+    is(scalar <$ps>, "VALUE /b/a 0 2\r\n", "get rline");
+    is(scalar <$ps>, "hi\r\n", "get data");
+    is(scalar <$ps>, "END\r\n", "get end");
+
+    # touch
+    $cmd = "touch /b/a 50\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, $cmd, "touch passthrough");
+    print $be "TOUCHED\r\n";
+
+    is(scalar <$ps>, "TOUCHED\r\n", "got touch response");
+
+    # gets
+    $cmd = "gets /b/a\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, $cmd, "gets passthrough");
+    print $be "VALUE /b/a 0 2 2\r\nhi\r\nEND\r\n";
+
+    is(scalar <$ps>, "VALUE /b/a 0 2 2\r\n", "gets rline");
+    is(scalar <$ps>, "hi\r\n", "gets data");
+    is(scalar <$ps>, "END\r\n", "gets end");
+
+    # gat
+    $cmd = "gat 10 /b/a\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, $cmd, "gat passthrough");
+    print $be "VALUE /b/a 0 2\r\nhi\r\nEND\r\n";
+
+    is(scalar <$ps>, "VALUE /b/a 0 2\r\n", "gat rline");
+    is(scalar <$ps>, "hi\r\n", "gat data");
+    is(scalar <$ps>, "END\r\n", "gat end");
+
+    # gats
+    $cmd = "gats 11 /b/a\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, $cmd, "gats passthrough");
+    print $be "VALUE /b/a 0 2 1\r\nhi\r\nEND\r\n";
+
+    is(scalar <$ps>, "VALUE /b/a 0 2 1\r\n", "gats rline");
+    is(scalar <$ps>, "hi\r\n", "gats data");
+    is(scalar <$ps>, "END\r\n", "gats end");
+
+    # cas
+    $cmd = "cas /b/a 0 0 2 5";
+    print $ps "$cmd\r\nhi\r\n";
+    is(scalar <$be>, "$cmd\r\n", "cas passthrough");
+    is(scalar <$be>, "hi\r\n", "cas value");
+    print $be "STORED\r\n";
+
+    is(scalar <$ps>, "STORED\r\n", "got STORED from cas");
+
+    # add
+    $cmd = "add /b/a 0 0 2";
+    print $ps "$cmd\r\nhi\r\n";
+    is(scalar <$be>, "$cmd\r\n", "add passthrough");
+    is(scalar <$be>, "hi\r\n", "add value");
+    print $be "STORED\r\n";
+
+    is(scalar <$ps>, "STORED\r\n", "got STORED from add");
+
+    # delete
+    $cmd = "delete /b/a\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, $cmd, "delete passthrough");
+    print $be "DELETED\r\n";
+
+    is(scalar <$ps>, "DELETED\r\n", "got delete response");
+
+    # incr
+    $cmd = "incr /b/a 1\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, $cmd, "incr passthrough");
+    print $be "2\r\n";
+
+    is(scalar <$ps>, "2\r\n", "got incr response");
+
+    # decr
+    $cmd = "decr /b/a 1\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, $cmd, "decr passthrough");
+    print $be "10\r\n";
+
+    is(scalar <$ps>, "10\r\n", "got decr response");
+
+    # append
+    $cmd = "append /b/a 0 0 2";
+    print $ps "$cmd\r\nhi\r\n";
+    is(scalar <$be>, "$cmd\r\n", "append passthrough");
+    is(scalar <$be>, "hi\r\n", "append value");
+    print $be "STORED\r\n";
+
+    is(scalar <$ps>, "STORED\r\n", "got STORED from append");
+
+    # prepend
+    $cmd = "prepend /b/a 0 0 2";
+    print $ps "$cmd\r\nhi\r\n";
+    is(scalar <$be>, "$cmd\r\n", "prepend passthrough");
+    is(scalar <$be>, "hi\r\n", "prepend value");
+    print $be "STORED\r\n";
+
+    is(scalar <$ps>, "STORED\r\n", "got STORED from prepend");
+
+    # [meta commands]
+    # testing the bare meta commands.
+    # TODO: add more tests for tokens and changing response codes.
+    # mg
+    $cmd = "mg /b/a\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, $cmd, "mg passthrough");
+    print $be "HD\r\n";
+
+    is(scalar <$ps>, "HD\r\n", "got mg response");
+    # ms
+    $cmd = "ms /b/a 2";
+    print $ps "$cmd\r\nhi\r\n";
+    is(scalar <$be>, "$cmd\r\n", "ms passthrough");
+    is(scalar <$be>, "hi\r\n", "ms value");
+    print $be "HD\r\n";
+
+    is(scalar <$ps>, "HD\r\n", "got HD from ms");
+
+    # md
+    $cmd = "md /b/a\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, $cmd, "md passthrough");
+    print $be "HD\r\n";
+
+    is(scalar <$ps>, "HD\r\n", "got HD from md");
+    # ma
+    $cmd = "ma /b/a\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, $cmd, "ma passthrough");
+    print $be "HD\r\n";
+
+    is(scalar <$ps>, "HD\r\n", "got HD from ma");
+    # mn?
+    # me?
+}
+
+# run a cleanser check between each set of tests.
+check_version($ps);
+
+{
+    # multiget syntax
+    # - gets broken into individual gets on backend
+    my $be = $mbe[0];
+    my $cmd = "get /b/a /b/b /b/c\r\n";
+    print $ps $cmd;
+    # NOTE: the proxy ends up reversing the keys to the backend, but returns keys in the
+    # proper order. This is undesireable but not problematic: because of how
+    # ascii multiget syntax works the server cannot start responding until all
+    # answers are resolved anyway.
+    is(scalar <$be>, "get /b/c\r\n", "multiget breakdown c");
+    is(scalar <$be>, "get /b/b\r\n", "multiget breakdown b");
+    is(scalar <$be>, "get /b/a\r\n", "multiget breakdown a");
+
+    print $be "VALUE /b/c 0 1\r\nc\r\n",
+              "END\r\n",
+              "VALUE /b/b 0 1\r\nb\r\n",
+              "END\r\n",
+              "VALUE /b/a 0 1\r\na\r\n",
+              "END\r\n";
+
+    for my $key ('a', 'b', 'c') {
+        is(scalar <$ps>, "VALUE /b/$key 0 1\r\n", "multiget res $key");
+        is(scalar <$ps>, "$key\r\n", "multiget value $key");
+    }
+    is(scalar <$ps>, "END\r\n", "final END from multiget");
+}
+
+check_version($ps);
+
+{
+    # noreply tests.
+    # - backend should receive with noreply/q stripped or mangled
+    # - backend should reply as normal
+    # - frontend should get nothing; to test issue another command and ensure
+    # it only gets that response.
+    my $be = $mbe[0];
+    my $cmd = "set /b/a 0 0 2 noreply\r\nhi\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, "set /b/a 0 0 2 noreplY\r\n", "set received with broken noreply");
+    is(scalar <$be>, "hi\r\n", "set payload received");
+
+    print $be "STORED\r\n";
+
+    # To ensure success, make another req and ensure res isn't STORED
+    $cmd = "touch /b/a 50\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, $cmd, "canary touch received");
+    print $be "TOUCHED\r\n";
+
+    is(scalar <$ps>, "TOUCHED\r\n", "got TOUCHED instread of STORED");
+
+    # TODO: meta quiet cases
+    # - q should be turned into a space on the backend
+    # - errors should still pass through to client
+}
+
+check_version($ps);
+# TODO: Test specifically responding to a get but missing the END\r\n. it
+# should time out and not leak to the client.
+
+# Test Lua request API
+{
+    my $be = $mbe[0];
+
+    # fetching the key.
+    print $ps "get /getkey/testkey\r\n";
+    # look for the key to be slightly different to ensure we hit lua.
+    is(scalar <$ps>, "VALUE |/getkey/testkey 0 2\r\n", "request:key()");
+    is(scalar <$ps>, "ts\r\n", "request:key() value");
+    is(scalar <$ps>, "END\r\n", "request:key() END");
+
+    # rtrimkey
+    # this overwrites part of the key with spaces, which should be skipped by
+    # a valid protocol parser.
+    print $ps "get /rtrimkey/onehalf\r\n";
+    is(scalar <$be>, "get /rtrimkey/one    \r\n", "request:rtrimkey()");
+    print $be "END\r\n";
+    is(scalar <$ps>, "END\r\n", "rtrimkey END");
+
+    # ltrimkey
+    print $ps "get /ltrimkey/test\r\n";
+    is(scalar <$be>, "get           test\r\n", "request:ltrimkey()");
+    print $be "END\r\n";
+    is(scalar <$ps>, "END\r\n", "ltrimkey END");
+
+    # token(n) fetch
+    # token(n, "replacement")
+    # token(n, "") removal
+    # ntokens()
+    # command() integer
+    #
+    # meta:
+    # has_flag("F")
+    # test has_flag() against non-meta command
+    # flag_token("F") with no token (bool, nil|token)
+    # flag_token("F") with token
+    # flag_token("F", "FReplacement")
+    # flag_token("F", "") removal
+    # flag_token("F", "FReplacement") -> flag_token("F") test repeated fetch
+
+    # mcp.request() - has a few modes to test
+    # - allows passing in an existing request to clone/edit
+    # - passing in value blob
+}
+
+check_version($ps);
+# Test Lua response API
+#{
+    # elapsed()
+    # ok()
+    # hit()
+    # vlen()
+    # code()
+    # line()
+#}
+
+# Test requests land in proper backend in basic scenarios
+{
+    # TODO: maybe should send values to ensure the right response?
+    # I don't think this test is very useful though; probably better to try
+    # harder when testing error conditions.
+    for my $tu (['a', $mbe[0]], ['b', $mbe[1]], ['c', $mbe[2]]) {
+        my $be = $tu->[1];
+        my $cmd = "get /zonetest/" . $tu->[0] . "\r\n";
+        print $ps $cmd;
+        is(scalar <$be>, $cmd, "routed proper zone: " . $tu->[0]);
+        print $be "END\r\n";
+        is(scalar <$ps>, "END\r\n", "end from zone fetch");
+    }
+    my $cmd = "get /zonetest/invalid\r\n";
+    print $ps $cmd;
+    is(scalar <$ps>, "END\r\n", "END from invalid route");
+}
+
+check_version($ps);
+# Test re-requests in lua.
+# - fetch zones.z1() then fetch zones.z2()
+# - return z1 or z2 or netiher
+# - fetch all three zones
+# - hit the same zone multiple times
+
+# Test out of spec commands from client
+# - wrong # of tokens
+# - bad key size
+# - etc
+
+# Test errors/garbage from server
+# - certain errors pass through to the client, most close the backend.
+
+# Test delayed read (timeout)
+
+# Test Lua logging (see t/watcher.t)
+{
+    my $be = $mbe[0];
+    my $watcher = $p_srv->new_sock;
+    print $watcher "watch proxyuser proxyreqs\n";
+    is(<$watcher>, "OK\r\n", "watcher enabled");
+
+    # log(msg)
+    print $ps "get /logtest/a\r\n";
+    like(<$watcher>, qr/ts=(\S+) gid=\d+ type=proxy_user msg=testing manual log messages/,
+        "log a manual message");
+    is(scalar <$ps>, "END\r\n", "logtest END");
+
+    # log_req(r, res)
+    my $cmd = "get /logreqtest/a\r\n";
+    print $ps $cmd;
+    is(scalar <$be>, $cmd, "got passthru for log");
+    print $be "END\r\n";
+    is(scalar <$ps>, "END\r\n", "got END from log test");
+    like(<$watcher>, qr/ts=(\S+) gid=\d+ type=proxy_req elapsed=\d+ type=105 code=17 status=0 be=127.0.0.1:11411 detail=logreqtest req=get \/logreqtest\/a/, "found request log entry");
+
+    # test log_req with nil res (should be 0's in places)
+    # log_reqsample()
+}
+
+# Basic proxy stats validation
+
+# Test user stats
+
+check_version($ps);
+# Test await arguments (may move to own file?)
+# TODO: the results table from mcp.await() contains all of the results so far,
+# regardless of the mode.
+# need some tests that show this.
+{
+    my $cmd;
+    # await(r, p)
+    # this should hit all three backends
+    my $key = "/awaitbasic/a";
+    $cmd = "get $key\r\n";
+    print $ps $cmd;
+    for my $be (@mbe) {
+        is(scalar <$be>, $cmd, "awaitbasic backend req");
+        print $be "VALUE $key 0 2\r\nok\r\nEND\r\n";
+    }
+    is(scalar <$ps>, "VALUE $key 0 11\r\n", "response from await");
+    is(scalar <$ps>, "hit hit hit\r\n", "hit responses from await");
+    is(scalar <$ps>, "END\r\n", "end from await");
+    # repeat above test but with different combo of results
+
+    # await(r, p, 1)
+    $key = "/awaitone/a";
+    $cmd = "get $key\r\n";
+    print $ps $cmd;
+    for my $be (@mbe) {
+        is(scalar <$be>, $cmd, "awaitone backend req");
+        print $be "VALUE $key 0 2\r\nok\r\nEND\r\n";
+    }
+    is(scalar <$ps>, "VALUE $key 0 1\r\n", "response from await");
+    is(scalar <$ps>, "1\r\n", "looking for a single response");
+    is(scalar <$ps>, "END\r\n", "end from await");
+
+    # await(r, p(3+), 2)
+    $key = "/awaitone/b";
+    $cmd = "get $key\r\n";
+    print $ps $cmd;
+    for my $be (@mbe) {
+        is(scalar <$be>, $cmd, "awaitone backend req");
+        print $be "VALUE $key 0 2\r\nok\r\nEND\r\n";
+    }
+    is(scalar <$ps>, "VALUE $key 0 1\r\n", "response from await");
+    is(scalar <$ps>, "2\r\n", "looking two responses");
+    is(scalar <$ps>, "END\r\n", "end from await");
+
+    # await(r, p, 1, mcp.AWAIT_GOOD)
+    $key = "/awaitgood/a";
+    $cmd = "get $key\r\n";
+    print $ps $cmd;
+    for my $be (@mbe) {
+        is(scalar <$be>, $cmd, "awaitgood backend req");
+        print $be "VALUE $key 0 2\r\nok\r\nEND\r\n";
+    }
+    is(scalar <$ps>, "VALUE $key 0 1\r\n", "response from await");
+    is(scalar <$ps>, "1\r\n", "looking for a single response");
+    is(scalar <$ps>, "END\r\n", "end from await");
+    # should test above with first response being err, second good, third
+    # miss, and a few similar iterations.
+
+    # await(r, p, 2, mcp.AWAIT_ANY)
+    $key = "/awaitany/a";
+    $cmd = "get $key\r\n";
+    print $ps $cmd;
+    for my $be (@mbe) {
+        is(scalar <$be>, $cmd, "awaitany backend req");
+        print $be "VALUE $key 0 2\r\nok\r\nEND\r\n";
+    }
+    is(scalar <$ps>, "VALUE $key 0 1\r\n", "response from await");
+    is(scalar <$ps>, "2\r\n", "looking for a two responses");
+    is(scalar <$ps>, "END\r\n", "end from await");
+
+    # await(r, p, 2, mcp.AWAIT_OK)
+    # await(r, p, 1, mcp.AWAIT_FIRST)
+    # more AWAIT_FIRST tests? to see how much it waits on/etc.
+    # await(r, p, 2, mcp.AWAIT_FASTGOOD)
+    # - should return 1 res on good, else wait for N non-error responses
+    # - test three pools, but third returns good. should have returned already
+    # await(r, p, 1, mcp.AWAIT_BACKGROUND) - ensure res without waiting
+    $key = "/awaitbg/a";
+    $cmd = "get $key\r\n";
+    print $ps $cmd;
+    # check we can get a response _before_ the backends are consulted.
+    is(scalar <$ps>, "VALUE $key 0 1\r\n", "response from await");
+    is(scalar <$ps>, "0\r\n", "looking for zero responses");
+    is(scalar <$ps>, "END\r\n", "end from await");
+    for my $be (@mbe) {
+        is(scalar <$be>, $cmd, "awaitbg backend req");
+        print $be "VALUE $key 0 2\r\nok\r\nEND\r\n";
+    }
+
+    # test hitting a pool normally then hit mcp.await()
+    # test hitting mcp.await() then a pool normally
+}
+
+check_version($ps);
+done_testing();