Rewrite ddoc_cache to improve performance

There were a couple issues with the previous ddoc_cache implementation
that made it possible to tip over the ddoc_cache_opener process. First,
there were a lot of messages flowing through a single gen_server. And
second, the cache relied on periodically evicting entries to ensure
proper behavior in not caching an entry forever after it had changed on
disk.

The new version makes two important changes. First, entries now have an
associated process that manages the cache entry. This process will
periodically refresh the entry and if the entry has changed or no longer
exists the process will remove its entry from cache.

The second major change is that the cache entry process directly mutates
the related ets table entries so that our performance is not dependent
on the speed of ets table mutations. Using a custom entry that does no
work the cache can now sustain roughly one million operations a second
with a twenty thousand clients fighting over a cache limited to one
thousand items. In production this means that cache performance will
likely be rate limited by other factors like loading design documents
from disk.

27 files changed, 2146 insertions, 351 deletions

diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl
index b4ad257ff..78e0b8c19 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -319,7 +319,7 @@ handle_info({update_docs, Client, GroupedDocs, NonRepDocs, MergeConflicts,
                     couch_event:notify(Db2#db.name, {ddoc_updated, DDocId})
                 end, UpdatedDDocIds),
                 couch_event:notify(Db2#db.name, ddoc_updated),
-                ddoc_cache:evict(Db2#db.name, UpdatedDDocIds),
+                ddoc_cache:refresh(Db2#db.name, UpdatedDDocIds),
                 refresh_validate_doc_funs(Db2);
             false ->
                 Db2
diff --git a/src/ddoc_cache/src/ddoc_cache.app.src b/src/ddoc_cache/src/ddoc_cache.app.src
index a64b2f526..38fcedab0 100644
--- a/src/ddoc_cache/src/ddoc_cache.app.src
+++ b/src/ddoc_cache/src/ddoc_cache.app.src
@@ -13,14 +13,8 @@
 {application, ddoc_cache, [
     {description, "Design Document Cache"},
     {vsn, git},
-    {modules, [
-        ddoc_cache,
-        ddoc_cache_app,
-        ddoc_cache_opener,
-        ddoc_cache_sup,
-        ddoc_cache_util
-    ]},
     {registered, [
+        ddoc_cache_tables,
         ddoc_cache_lru,
         ddoc_cache_opener
     ]},
@@ -28,17 +22,11 @@
         kernel,
         stdlib,
         crypto,
+        config,
         couch_event,
-        ets_lru,
-        mem3,
-        fabric,
         couch_log,
-        couch_stats
+        couch_stats,
+        fabric
     ]},
-    {mod, {ddoc_cache_app, []}},
-    {env, [
-        {max_objects, unlimited},
-        {max_size, 104857600}, % 100M
-        {max_lifetime, 60000} % 1m
-    ]}
+    {mod, {ddoc_cache_app, []}}
 ]}.
diff --git a/src/ddoc_cache/src/ddoc_cache.erl b/src/ddoc_cache/src/ddoc_cache.erl
index ed9330998..50cac3039 100644
--- a/src/ddoc_cache/src/ddoc_cache.erl
+++ b/src/ddoc_cache/src/ddoc_cache.erl
@@ -12,86 +12,43 @@
 
 -module(ddoc_cache).
 
--export([
-    start/0,
-    stop/0
-]).
 
 -export([
     open_doc/2,
     open_doc/3,
     open_validation_funs/1,
-    evict/2,
+    open_custom/2,
+    refresh/2,
 
     %% deprecated
     open/2
 ]).
 
-start() ->
-    application:start(ddoc_cache).
-
-stop() ->
-    application:stop(ddoc_cache).
 
 open_doc(DbName, DocId) ->
-    Key = {DbName, DocId, '_'},
-    case ddoc_cache_opener:match_newest(Key) of
-        {ok, _} = Resp ->
-            couch_stats:increment_counter([ddoc_cache, hit]),
-            Resp;
-        missing ->
-            couch_stats:increment_counter([ddoc_cache, miss]),
-            ddoc_cache_opener:open_doc(DbName, DocId);
-        recover ->
-            couch_stats:increment_counter([ddoc_cache, recovery]),
-            ddoc_cache_opener:recover_doc(DbName, DocId)
-    end.
+    Key = {ddoc_cache_entry_ddocid, {DbName, DocId}},
+    ddoc_cache_lru:open(Key).
+
 
 open_doc(DbName, DocId, RevId) ->
-    Key = {DbName, DocId, RevId},
-    case ddoc_cache_opener:lookup(Key) of
-        {ok, _} = Resp ->
-            couch_stats:increment_counter([ddoc_cache, hit]),
-            Resp;
-        missing ->
-            couch_stats:increment_counter([ddoc_cache, miss]),
-            ddoc_cache_opener:open_doc(DbName, DocId, RevId);
-        recover ->
-            couch_stats:increment_counter([ddoc_cache, recovery]),
-            ddoc_cache_opener:recover_doc(DbName, DocId, RevId)
-    end.
+    Key = {ddoc_cache_entry_ddocid_rev, {DbName, DocId, RevId}},
+    ddoc_cache_lru:open(Key).
+
 
 open_validation_funs(DbName) ->
-    Key = {DbName, validation_funs},
-    case ddoc_cache_opener:lookup(Key) of
-        {ok, _} = Resp ->
-            couch_stats:increment_counter([ddoc_cache, hit]),
-            Resp;
-        missing ->
-            couch_stats:increment_counter([ddoc_cache, miss]),
-            ddoc_cache_opener:open_validation_funs(DbName);
-        recover ->
-            couch_stats:increment_counter([ddoc_cache, recovery]),
-            ddoc_cache_opener:recover_validation_funs(DbName)
-    end.
+    Key = {ddoc_cache_entry_validation_funs, DbName},
+    ddoc_cache_lru:open(Key).
+
 
 open_custom(DbName, Mod) ->
-    Key = {DbName, Mod},
-    case ddoc_cache_opener:lookup(Key) of
-        {ok, _} = Resp ->
-            couch_stats:increment_counter([ddoc_cache, hit]),
-            Resp;
-        missing ->
-            couch_stats:increment_counter([ddoc_cache, miss]),
-            ddoc_cache_opener:open_doc(DbName, Mod);
-        recover ->
-            couch_stats:increment_counter([ddoc_cache, recovery]),
-            Mod:recover(DbName)
-    end.
-
-evict(ShardDbName, DDocIds) ->
+    Key = {ddoc_cache_entry_custom, {DbName, Mod}},
+    ddoc_cache_lru:open(Key).
+
+
+refresh(ShardDbName, DDocIds) when is_list(DDocIds) ->
     DbName = mem3:dbname(ShardDbName),
-    ddoc_cache_opener:evict_docs(DbName, DDocIds).
+    ddoc_cache_lru:refresh(DbName, DDocIds).
+
 
 open(DbName, validation_funs) ->
     open_validation_funs(DbName);
diff --git a/src/ddoc_cache/src/ddoc_cache.hrl b/src/ddoc_cache/src/ddoc_cache.hrl
new file mode 100644
index 000000000..dba0d37b2
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache.hrl
@@ -0,0 +1,40 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-type dbname() :: iodata().
+-type docid() :: iodata().
+-type doc_hash() :: <<_:128>>.
+-type revision() :: {pos_integer(), doc_hash()}.
+
+-define(CACHE, ddoc_cache_entries).
+-define(LRU, ddoc_cache_lru).
+-define(REFRESH_TIMEOUT, 67000).
+-define(SHUTDOWN_TIMEOUT, 1000).
+
+-record(entry, {
+    key,
+    val,
+    pid
+}).
+
+-record(opener, {
+    key,
+    pid,
+    clients
+}).
+
+
+-ifdef(TEST).
+-define(EVENT(Name, Arg), ddoc_cache_ev:event(Name, Arg)).
+-else.
+-define(EVENT(Name, Arg), ignore).
+-endif.
diff --git a/src/ddoc_cache/src/ddoc_cache_entry.erl b/src/ddoc_cache/src/ddoc_cache_entry.erl
new file mode 100644
index 000000000..4dee2a11f
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache_entry.erl
@@ -0,0 +1,279 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_entry).
+
+
+-export([
+    dbname/1,
+    ddocid/1,
+    recover/1,
+
+    start_link/1,
+    shutdown/1,
+    open/2,
+    accessed/1,
+    refresh/1
+]).
+
+-export([
+    init/1,
+    terminate/2,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    code_change/3
+]).
+
+-export([
+    do_open/1
+]).
+
+
+-include("ddoc_cache.hrl").
+
+
+-record(st, {
+    key,
+    val,
+    opener,
+    waiters,
+    ts
+}).
+
+
+dbname({Mod, Arg}) ->
+    Mod:dbname(Arg).
+
+
+ddocid({Mod, Arg}) ->
+    Mod:ddocid(Arg).
+
+
+recover({Mod, Arg}) ->
+    Mod:recover(Arg).
+
+
+start_link(Key) ->
+    Pid = proc_lib:spawn_link(?MODULE, init, [Key]),
+    {ok, Pid}.
+
+
+shutdown(Pid) ->
+    ok = gen_server:call(Pid, shutdown).
+
+
+open(Pid, Key) ->
+    try
+        Resp = gen_server:call(Pid, open),
+        case Resp of
+            {open_ok, Val} ->
+                Val;
+            {open_error, {T, R, S}} ->
+                erlang:raise(T, R, S)
+        end
+    catch exit:_ ->
+        % Its possible that this process was evicted just
+        % before we tried talking to it. Just fallback
+        % to a standard recovery
+        recover(Key)
+    end.
+
+
+accessed(Pid) ->
+    gen_server:cast(Pid, accessed).
+
+
+refresh(Pid) ->
+    gen_server:cast(Pid, refresh).
+
+
+init(Key) ->
+    true = ets:update_element(?CACHE, Key, {#entry.pid, self()}),
+    St = #st{
+        key = Key,
+        opener = spawn_opener(Key),
+        waiters = []
+    },
+    ?EVENT(started, Key),
+    gen_server:enter_loop(?MODULE, [], St).
+
+
+terminate(_Reason, St) ->
+    #st{
+        key = Key,
+        opener = Pid,
+        ts = Ts
+    } = St,
+    % We may have already deleted our cache entry
+    % during shutdown
+    Pattern = #entry{key = Key, pid = self(), _ = '_'},
+    CacheMSpec = [{Pattern, [], [true]}],
+    true = ets:select_delete(?CACHE, CacheMSpec) < 2,
+    % We may have already deleted our LRU entry
+    % during shutdown
+    if Ts == undefined -> ok; true ->
+        LruMSpec = [{{{Ts, Key, self()}}, [], [true]}],
+        true = ets:select_delete(?LRU, LruMSpec) < 2
+    end,
+    % Blow away any current opener if it exists
+    if not is_pid(Pid) -> ok; true ->
+        catch exit(Pid, kill)
+    end,
+    ok.
+
+
+handle_call(open, From, #st{val = undefined} = St) ->
+    NewSt = St#st{
+        waiters = [From | St#st.waiters]
+    },
+    {noreply, NewSt};
+
+handle_call(open, _From, St) ->
+    {reply, St#st.val, St};
+
+handle_call(shutdown, _From, St) ->
+    remove_from_cache(St),
+    {stop, normal, ok, St};
+
+handle_call(Msg, _From, St) ->
+    {stop, {bad_call, Msg}, {bad_call, Msg}, St}.
+
+
+handle_cast(accessed, St) ->
+    ?EVENT(accessed, St#st.key),
+    drain_accessed(),
+    {noreply, update_lru(St)};
+
+handle_cast(refresh, #st{opener = Ref} = St) when is_reference(Ref) ->
+    #st{
+        key = Key
+    } = St,
+    erlang:cancel_timer(Ref),
+    NewSt = St#st{
+        opener = spawn_opener(Key)
+    },
+    {noreply, NewSt};
+
+handle_cast(refresh, #st{opener = Pid} = St) when is_pid(Pid) ->
+    catch exit(Pid, kill),
+    receive
+        {'DOWN', _, _, Pid, _} -> ok
+    end,
+    NewSt = St#st{
+        opener = spawn_opener(St#st.key)
+    },
+    {noreply, NewSt};
+
+handle_cast(Msg, St) ->
+    {stop, {bad_cast, Msg}, St}.
+
+
+handle_info({'DOWN', _, _, Pid, Resp}, #st{key = Key, opener = Pid} = St) ->
+    case Resp of
+        {open_ok, Key, {ok, Val}} ->
+            if not is_list(St#st.waiters) -> ok; true ->
+                respond(St#st.waiters, {open_ok, {ok, Val}})
+            end,
+            update_cache(St, Val),
+            Msg = {'$gen_cast', refresh},
+            Timer = erlang:send_after(?REFRESH_TIMEOUT, self(), Msg),
+            NewSt = St#st{
+                val = {open_ok, {ok, Val}},
+                opener = Timer
+            },
+            {noreply, update_lru(NewSt)};
+        {Status, Key, Other} ->
+            NewSt = St#st{
+                val = {Status, Other},
+                opener = undefined,
+                waiters = undefined
+            },
+            remove_from_cache(NewSt),
+            if not is_list(St#st.waiters) -> ok; true ->
+                respond(St#st.waiters, {Status, Other})
+            end,
+            {stop, normal, NewSt}
+    end;
+
+handle_info(Msg, St) ->
+    {stop, {bad_info, Msg}, St}.
+
+
+code_change(_, St, _) ->
+    {ok, St}.
+
+
+spawn_opener(Key) ->
+    {Pid, _} = erlang:spawn_monitor(?MODULE, do_open, [Key]),
+    Pid.
+
+
+do_open(Key) ->
+    try recover(Key) of
+        Resp ->
+            erlang:exit({open_ok, Key, Resp})
+    catch T:R ->
+        S = erlang:get_stacktrace(),
+        erlang:exit({open_error, Key, {T, R, S}})
+    end.
+
+
+update_lru(#st{key = Key, ts = Ts} = St) ->
+    if Ts == undefined -> ok; true ->
+        MSpec = [{{{Ts, Key, self()}}, [], [true]}],
+        1 = ets:select_delete(?LRU, MSpec)
+    end,
+    NewTs = os:timestamp(),
+    true = ets:insert(?LRU, {{NewTs, Key, self()}}),
+    St#st{ts = NewTs}.
+
+
+update_cache(#st{val = undefined} = St, Val) ->
+    true = ets:update_element(?CACHE, St#st.key, {#entry.val, Val}),
+    ?EVENT(inserted, St#st.key);
+
+update_cache(#st{val = V1} = _St, V2) when {open_ok, {ok, V2}} == V1 ->
+    ?EVENT(update_noop, _St#st.key);
+
+update_cache(St, Val) ->
+    true = ets:update_element(?CACHE, St#st.key, {#entry.val, Val}),
+    ?EVENT(updated, {St#st.key, Val}).
+
+
+remove_from_cache(St) ->
+    #st{
+        key = Key,
+        ts = Ts
+    } = St,
+    Pattern = #entry{key = Key, pid = self(), _ = '_'},
+    CacheMSpec = [{Pattern, [], [true]}],
+    1 = ets:select_delete(?CACHE, CacheMSpec),
+    if Ts == undefined -> ok; true ->
+        LruMSpec = [{{{Ts, Key, self()}}, [], [true]}],
+        1 = ets:select_delete(?LRU, LruMSpec)
+    end,
+    ?EVENT(removed, St#st.key),
+    ok.
+
+
+drain_accessed() ->
+    receive
+        {'$gen_cast', accessed} ->
+            drain_accessed()
+    after 0 ->
+        ok
+    end.
+
+
+respond(Waiters, Resp) ->
+    [gen_server:reply(W, Resp) || W <- Waiters].
diff --git a/src/ddoc_cache/src/ddoc_cache_util.erl b/src/ddoc_cache/src/ddoc_cache_entry_custom.erl
index 24c4b0d9e..d858ad6a8 100644
--- a/src/ddoc_cache/src/ddoc_cache_util.erl
+++ b/src/ddoc_cache/src/ddoc_cache_entry_custom.erl
@@ -10,25 +10,23 @@
 % License for the specific language governing permissions and limitations under
 % the License.
 
--module(ddoc_cache_util).
+-module(ddoc_cache_entry_custom).
 
 
 -export([
-    new_uuid/0
+    dbname/1,
+    ddocid/1,
+    recover/1
 ]).
 
 
-new_uuid() ->
-    to_hex(crypto:strong_rand_bytes(16), []).
+dbname({DbName, _}) ->
+    DbName.
 
 
-to_hex(<<>>, Acc) ->
-    list_to_binary(lists:reverse(Acc));
-to_hex(<<C1:4, C2:4, Rest/binary>>, Acc) ->
-    to_hex(Rest, [hexdig(C1), hexdig(C2) | Acc]).
+ddocid(_) ->
+    no_ddocid.
 
 
-hexdig(C) when C >= 0, C =< 9 ->
-    C + $0;
-hexdig(C) when C >= 10, C =< 15 ->
-    C + $A - 10.
+recover({DbName, Mod}) ->
+    Mod:recover(DbName).
diff --git a/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl b/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl
new file mode 100644
index 000000000..cac9abc21
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl
@@ -0,0 +1,35 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_entry_ddocid).
+
+
+-export([
+    dbname/1,
+    ddocid/1,
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+
+
+dbname({DbName, _}) ->
+    DbName.
+
+
+ddocid({_, DDocId}) ->
+    DDocId.
+
+
+recover({DbName, DDocId}) ->
+    fabric:open_doc(DbName, DDocId, [ejson_body, ?ADMIN_CTX]).
diff --git a/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl b/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl
new file mode 100644
index 000000000..012abab46
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl
@@ -0,0 +1,37 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_entry_ddocid_rev).
+
+
+-export([
+    dbname/1,
+    ddocid/1,
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+
+
+dbname({DbName, _, _}) ->
+    DbName.
+
+
+ddocid({_, DDocId, _}) ->
+    DDocId.
+
+
+recover({DbName, DDocId, Rev}) ->
+    Opts = [ejson_body, ?ADMIN_CTX],
+    {ok, [Resp]} = fabric:open_revs(DbName, DDocId, [Rev], Opts),
+    Resp.
diff --git a/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl b/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl
new file mode 100644
index 000000000..3d43f7a87
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl
@@ -0,0 +1,39 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_entry_validation_funs).
+
+
+-export([
+    dbname/1,
+    ddocid/1,
+    recover/1
+]).
+
+
+dbname(DbName) ->
+    DbName.
+
+
+ddocid(_) ->
+    no_ddocid.
+
+
+recover(DbName) ->
+    {ok, DDocs} = fabric:design_docs(mem3:dbname(DbName)),
+    Funs = lists:flatmap(fun(DDoc) ->
+        case couch_doc:get_validate_doc_fun(DDoc) of
+            nil -> [];
+            Fun -> [Fun]
+        end
+    end, DDocs),
+    {ok, Funs}.
diff --git a/src/ddoc_cache/src/ddoc_cache_lru.erl b/src/ddoc_cache/src/ddoc_cache_lru.erl
new file mode 100644
index 000000000..5d4763983
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache_lru.erl
@@ -0,0 +1,293 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_lru).
+-behaviour(gen_server).
+-vsn(1).
+
+
+-export([
+    start_link/0,
+    open/1,
+    refresh/2
+]).
+
+-export([
+    init/1,
+    terminate/2,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    code_change/3
+]).
+
+-export([
+    handle_db_event/3
+]).
+
+
+-include("ddoc_cache.hrl").
+
+
+-record(st, {
+    pids, % pid -> key
+    dbs, % dbname -> docid -> key -> pid
+    size,
+    evictor
+}).
+
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+
+open(Key) ->
+    try ets:lookup(?CACHE, Key) of
+        [] ->
+            lru_start(Key);
+        [#entry{pid = undefined}] ->
+            lru_start(Key);
+        [#entry{val = undefined, pid = Pid}] ->
+            couch_stats:increment_counter([ddoc_cache, miss]),
+            ddoc_cache_entry:open(Pid, Key);
+        [#entry{val = Val, pid = Pid}] ->
+            couch_stats:increment_counter([ddoc_cache, hit]),
+            ddoc_cache_entry:accessed(Pid),
+            {ok, Val}
+    catch _:_ ->
+        couch_stats:increment_counter([ddoc_cache, recovery]),
+        ddoc_cache_entry:recover(Key)
+    end.
+
+
+refresh(DbName, DDocIds) ->
+    gen_server:cast(?MODULE, {refresh, DbName, DDocIds}).
+
+
+init(_) ->
+    process_flag(trap_exit, true),
+    {ok, Pids} = khash:new(),
+    {ok, Dbs} = khash:new(),
+    {ok, Evictor} = couch_event:link_listener(
+            ?MODULE, handle_db_event, nil, [all_dbs]
+        ),
+    {ok, #st{
+        pids = Pids,
+        dbs = Dbs,
+        size = 0,
+        evictor = Evictor
+    }}.
+
+
+terminate(_Reason, St) ->
+    case is_pid(St#st.evictor) of
+        true -> catch exit(St#st.evictor, kill);
+        false -> ok
+    end,
+    ok.
+
+
+handle_call({start, Key}, _From, St) ->
+    #st{
+        pids = Pids,
+        dbs = Dbs,
+        size = CurSize
+    } = St,
+    case ets:lookup(?CACHE, Key) of
+        [] ->
+            MaxSize = config:get_integer("ddoc_cache", "max_size", 1000),
+            case trim(St, CurSize, max(0, MaxSize)) of
+                {ok, N} ->
+                    true = ets:insert_new(?CACHE, #entry{key = Key}),
+                    {ok, Pid} = ddoc_cache_entry:start_link(Key),
+                    true = ets:update_element(?CACHE, Key, {#entry.pid, Pid}),
+                    ok = khash:put(Pids, Pid, Key),
+                    store_key(Dbs, Key, Pid),
+                    {reply, {ok, Pid}, St#st{size = CurSize - N + 1}};
+                full ->
+                    ?EVENT(full, Key),
+                    {reply, full, St}
+            end;
+        [#entry{pid = Pid}] ->
+            {reply, {ok, Pid}, St}
+    end;
+
+handle_call(Msg, _From, St) ->
+    {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}.
+
+
+handle_cast({evict, DbName}, St) ->
+    gen_server:abcast(mem3:nodes(), ?MODULE, {do_evict, DbName}),
+    {noreply, St};
+
+handle_cast({refresh, DbName, DDocIds}, St) ->
+    gen_server:abcast(mem3:nodes(), ?MODULE, {do_refresh, DbName, DDocIds}),
+    {noreply, St};
+
+handle_cast({do_evict, DbName}, St) ->
+    #st{
+        dbs = Dbs
+    } = St,
+    ToRem = case khash:lookup(Dbs, DbName) of
+        {value, DDocIds} ->
+            AccOut = khash:fold(DDocIds, fun(_, Keys, Acc1) ->
+                khash:to_list(Keys) ++ Acc1
+            end, []),
+            ?EVENT(evicted, DbName),
+            AccOut;
+        not_found ->
+            ?EVENT(evict_noop, DbName),
+            []
+    end,
+    lists:foreach(fun({Key, Pid}) ->
+        remove_entry(St, Key, Pid)
+    end, ToRem),
+    khash:del(Dbs, DbName),
+    {noreply, St};
+
+handle_cast({do_refresh, DbName, DDocIdList}, St) ->
+    #st{
+        dbs = Dbs
+    } = St,
+    case khash:lookup(Dbs, DbName) of
+        {value, DDocIds} ->
+            lists:foreach(fun(DDocId) ->
+                case khash:lookup(DDocIds, DDocId) of
+                    {value, Keys} ->
+                        khash:fold(Keys, fun(_, Pid, _) ->
+                            ddoc_cache_entry:refresh(Pid)
+                        end, nil);
+                    not_found ->
+                        ok
+                end
+            end, [no_ddocid | DDocIdList]);
+        not_found ->
+            ok
+    end,
+    {noreply, St};
+
+handle_cast(Msg, St) ->
+    {stop, {invalid_cast, Msg}, St}.
+
+
+handle_info({'EXIT', Pid, _Reason}, #st{evictor = Pid} = St) ->
+    ?EVENT(evictor_died, Pid),
+    {ok, Evictor} = couch_event:link_listener(
+            ?MODULE, handle_db_event, nil, [all_dbs]
+        ),
+    {noreply, St#st{evictor=Evictor}};
+
+handle_info({'EXIT', Pid, normal}, St) ->
+    % This clause handles when an entry starts
+    % up but encounters an error or uncacheable
+    % response from its recover call.
+    #st{
+        pids = Pids
+    } = St,
+    {value, Key} = khash:lookup(Pids, Pid),
+    khash:del(Pids, Pid),
+    remove_key(St, Key),
+    {noreply, St};
+
+handle_info(Msg, St) ->
+    {stop, {invalid_info, Msg}, St}.
+
+
+code_change(_OldVsn, St, _Extra) ->
+    {ok, St}.
+
+
+handle_db_event(ShardDbName, created, St) ->
+    gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
+    {ok, St};
+
+handle_db_event(ShardDbName, deleted, St) ->
+    gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
+    {ok, St};
+
+handle_db_event(_DbName, _Event, St) ->
+    {ok, St}.
+
+
+lru_start(Key) ->
+    case gen_server:call(?MODULE, {start, Key}, infinity) of
+        {ok, Pid} ->
+            couch_stats:increment_counter([ddoc_cache, miss]),
+            ddoc_cache_entry:open(Pid, Key);
+        full ->
+            couch_stats:increment_counter([ddoc_cache, recovery]),
+            ddoc_cache_entry:recover(Key)
+    end.
+
+
+trim(_, _, 0) ->
+    full;
+
+trim(_St, CurSize, MaxSize) when CurSize < MaxSize ->
+    {ok, 0};
+
+trim(St, CurSize, MaxSize) when CurSize >= MaxSize ->
+    case ets:first(?LRU) of
+        '$end_of_table' ->
+            full;
+        {_Ts, Key, Pid} ->
+            remove_entry(St, Key, Pid),
+            {ok, 1}
+    end.
+
+
+remove_entry(St, Key, Pid) ->
+    #st{
+        pids = Pids
+    } = St,
+    unlink(Pid),
+    ddoc_cache_entry:shutdown(Pid),
+    khash:del(Pids, Pid),
+    remove_key(St, Key).
+
+
+store_key(Dbs, Key, Pid) ->
+    DbName = ddoc_cache_entry:dbname(Key),
+    DDocId = ddoc_cache_entry:ddocid(Key),
+    case khash:lookup(Dbs, DbName) of
+        {value, DDocIds} ->
+            case khash:lookup(DDocIds, DDocId) of
+                {value, Keys} ->
+                    khash:put(Keys, Key, Pid);
+                not_found ->
+                    {ok, Keys} = khash:from_list([{Key, Pid}]),
+                    khash:put(DDocIds, DDocId, Keys)
+            end;
+        not_found ->
+            {ok, Keys} = khash:from_list([{Key, Pid}]),
+            {ok, DDocIds} = khash:from_list([{DDocId, Keys}]),
+            khash:put(Dbs, DbName, DDocIds)
+    end.
+
+
+remove_key(St, Key) ->
+    #st{
+        dbs = Dbs
+    } = St,
+    DbName = ddoc_cache_entry:dbname(Key),
+    DDocId = ddoc_cache_entry:ddocid(Key),
+    {value, DDocIds} = khash:lookup(Dbs, DbName),
+    {value, Keys} = khash:lookup(DDocIds, DDocId),
+    khash:del(Keys, Key),
+    case khash:size(Keys) of
+        0 -> khash:del(DDocIds, DDocId);
+        _ -> ok
+    end,
+    case khash:size(DDocIds) of
+        0 -> khash:del(Dbs, DbName);
+        _ -> ok
+    end.
diff --git a/src/ddoc_cache/src/ddoc_cache_opener.erl b/src/ddoc_cache/src/ddoc_cache_opener.erl
index b76a228b0..7839bcb6e 100644
--- a/src/ddoc_cache/src/ddoc_cache_opener.erl
+++ b/src/ddoc_cache/src/ddoc_cache_opener.erl
@@ -14,279 +14,52 @@
 -behaviour(gen_server).
 -vsn(1).
 
--include_lib("couch/include/couch_db.hrl").
--include_lib("mem3/include/mem3.hrl").
-
 -export([
     start_link/0
 ]).
+
 -export([
     init/1,
     terminate/2,
-
     handle_call/3,
     handle_cast/2,
     handle_info/2,
-
     code_change/3
 ]).
 
--export([
-    open_doc/2,
-    open_doc/3,
-    open_validation_funs/1,
-    evict_docs/2,
-    lookup/1,
-    match_newest/1,
-    recover_doc/2,
-    recover_doc/3,
-    recover_validation_funs/1
-]).
--export([
-    handle_db_event/3
-]).
--export([
-    fetch_doc_data/1
-]).
-
--define(CACHE, ddoc_cache_lru).
--define(OPENING, ddoc_cache_opening).
-
--type dbname() :: iodata().
--type docid() :: iodata().
--type doc_hash() :: <<_:128>>.
--type revision() :: {pos_integer(), doc_hash()}.
-
--record(opener, {
-    key,
-    pid,
-    clients
-}).
-
--record(st, {
-    db_ddocs,
-    evictor
-}).
 
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 
--spec open_doc(dbname(), docid()) -> {ok, #doc{}}.
-open_doc(DbName, DocId) ->
-    Resp = gen_server:call(?MODULE, {open, {DbName, DocId}}, infinity),
-    handle_open_response(Resp).
-
--spec open_doc(dbname(), docid(), revision()) -> {ok, #doc{}}.
-open_doc(DbName, DocId, Rev) ->
-    Resp = gen_server:call(?MODULE, {open, {DbName, DocId, Rev}}, infinity),
-    handle_open_response(Resp).
-
--spec open_validation_funs(dbname()) -> {ok, [fun()]}.
-open_validation_funs(DbName) ->
-    Resp = gen_server:call(?MODULE, {open, {DbName, validation_funs}}, infinity),
-    handle_open_response(Resp).
-
--spec evict_docs(dbname(), [docid()]) -> ok.
-evict_docs(DbName, DocIds) ->
-    gen_server:cast(?MODULE, {evict, DbName, DocIds}).
-
-lookup(Key) ->
-    try ets_lru:lookup_d(?CACHE, Key) of
-        {ok, _} = Resp ->
-            Resp;
-        _ ->
-            missing
-    catch
-        error:badarg ->
-            recover
-    end.
-
-match_newest(Key) ->
-    try ets_lru:match_object(?CACHE, Key, '_') of
-        [] ->
-            missing;
-        Docs ->
-            Sorted = lists:sort(
-                fun (#doc{deleted=DelL, revs=L}, #doc{deleted=DelR, revs=R}) ->
-                    {not DelL, L} > {not DelR, R}
-                end, Docs),
-            {ok, hd(Sorted)}
-    catch
-        error:badarg ->
-            recover
-    end.
-
-recover_doc(DbName, DDocId) ->
-    fabric:open_doc(DbName, DDocId, [ejson_body, ?ADMIN_CTX]).
-
-recover_doc(DbName, DDocId, Rev) ->
-    {ok, [Resp]} = fabric:open_revs(DbName, DDocId, [Rev], [ejson_body, ?ADMIN_CTX]),
-    Resp.
-
-recover_validation_funs(DbName) ->
-    {ok, DDocs} = fabric:design_docs(mem3:dbname(DbName)),
-    Funs = lists:flatmap(fun(DDoc) ->
-        case couch_doc:get_validate_doc_fun(DDoc) of
-            nil -> [];
-            Fun -> [Fun]
-        end
-    end, DDocs),
-    {ok, Funs}.
-
-handle_db_event(ShardDbName, created, St) ->
-    gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
-    {ok, St};
-handle_db_event(ShardDbName, deleted, St) ->
-    gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
-    {ok, St};
-handle_db_event(_DbName, _Event, St) ->
-    {ok, St}.
 
 init(_) ->
-    process_flag(trap_exit, true),
-    _ = ets:new(?OPENING, [set, protected, named_table, {keypos, #opener.key}]),
-    {ok, Evictor} = couch_event:link_listener(
-            ?MODULE, handle_db_event, nil, [all_dbs]
-        ),
-    {ok, #st{
-        evictor = Evictor
-    }}.
+    {ok, nil}.
 
-terminate(_Reason, St) ->
-    case is_pid(St#st.evictor) of
-        true -> exit(St#st.evictor, kill);
-        false -> ok
-    end,
+terminate(_Reason, _St) ->
     ok.
 
-handle_call({open, OpenerKey}, From, St) ->
-    case ets:lookup(?OPENING, OpenerKey) of
-        [#opener{clients=Clients}=O] ->
-            ets:insert(?OPENING, O#opener{clients=[From | Clients]}),
-            {noreply, St};
-        [] ->
-            Pid = spawn_link(?MODULE, fetch_doc_data, [OpenerKey]),
-            ets:insert(?OPENING, #opener{key=OpenerKey, pid=Pid, clients=[From]}),
-            {noreply, St}
-    end;
 
 handle_call(Msg, _From, St) ->
     {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}.
 
 
-handle_cast({evict, DbName}, St) ->
-    gen_server:abcast(mem3:nodes(), ?MODULE, {do_evict, DbName}),
+% The do_evict clauses are upgrades while we're
+% in a rolling reboot.
+handle_cast({do_evict, _} = Msg, St) ->
+    gen_server:cast(ddoc_cache_lru, Msg),
     {noreply, St};
 
-handle_cast({evict, DbName, DDocIds}, St) ->
-    gen_server:abcast(mem3:nodes(), ?MODULE, {do_evict, DbName, DDocIds}),
-    {noreply, St};
-
-handle_cast({do_evict, DbName}, St) ->
-    DDocIds = lists:flatten(ets_lru:match(?CACHE, {DbName, '$1', '_'}, '_')),
-    handle_cast({do_evict, DbName, DDocIds}, St);
-
 handle_cast({do_evict, DbName, DDocIds}, St) ->
-    CustomKeys = lists:flatten(ets_lru:match(?CACHE, {DbName, '$1'}, '_')),
-    lists:foreach(fun(Mod) ->
-        ets_lru:remove(?CACHE, {DbName, Mod})
-    end, CustomKeys),
-    lists:foreach(fun(DDocId) ->
-        Revs = ets_lru:match(?CACHE, {DbName, DDocId, '$1'}, '_'),
-        lists:foreach(fun([Rev]) ->
-            ets_lru:remove(?CACHE, {DbName, DDocId, Rev})
-        end, Revs)
-    end, DDocIds),
+    gen_server:cast(ddoc_cache_lru, {do_refresh, DbName, DDocIds}),
     {noreply, St};
 
 handle_cast(Msg, St) ->
     {stop, {invalid_cast, Msg}, St}.
 
-handle_info({'EXIT', Pid, Reason}, #st{evictor=Pid}=St) ->
-    couch_log:error("ddoc_cache_opener evictor died ~w", [Reason]),
-    {ok, Evictor} = couch_event:link_listener(?MODULE, handle_db_event, nil, [all_dbs]),
-    {noreply, St#st{evictor=Evictor}};
-
-handle_info({'EXIT', _Pid, {open_ok, OpenerKey, Resp}}, St) ->
-    respond(OpenerKey, {open_ok, Resp}),
-    {noreply, St};
-
-handle_info({'EXIT', _Pid, {open_error, OpenerKey, Type, Error}}, St) ->
-    respond(OpenerKey, {open_error, Type, Error}),
-    {noreply, St};
-
-handle_info({'EXIT', Pid, Reason}, St) ->
-    Pattern = #opener{pid=Pid, _='_'},
-    case ets:match_object(?OPENING, Pattern) of
-        [#opener{key=OpenerKey, clients=Clients}] ->
-            _ = [gen_server:reply(C, {error, Reason}) || C <- Clients],
-            ets:delete(?OPENING, OpenerKey),
-            {noreply, St};
-        [] ->
-            {stop, {unknown_pid_died, {Pid, Reason}}, St}
-    end;
 
 handle_info(Msg, St) ->
     {stop, {invalid_info, Msg}, St}.
 
+
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
-
--spec fetch_doc_data({dbname(), validation_funs}) -> no_return();
-                    ({dbname(), atom()}) -> no_return();
-                    ({dbname(), docid()}) -> no_return();
-                    ({dbname(), docid(), revision()}) -> no_return().
-fetch_doc_data({DbName, validation_funs}=OpenerKey) ->
-    {ok, Funs} = recover_validation_funs(DbName),
-    ok = ets_lru:insert(?CACHE, OpenerKey, Funs),
-    exit({open_ok, OpenerKey, {ok, Funs}});
-fetch_doc_data({DbName, Mod}=OpenerKey) when is_atom(Mod) ->
-    % This is not actually a docid but rather a custom cache key.
-    % Treat the argument as a code module and invoke its recover function.
-    try Mod:recover(DbName) of
-        {ok, Result} ->
-            ok = ets_lru:insert(?CACHE, OpenerKey, Result),
-            exit({open_ok, OpenerKey, {ok, Result}});
-        Else ->
-            exit({open_ok, OpenerKey, Else})
-    catch
-        Type:Reason ->
-            exit({open_error, OpenerKey, Type, Reason})
-    end;
-fetch_doc_data({DbName, DocId}=OpenerKey) ->
-    try recover_doc(DbName, DocId) of
-        {ok, Doc} ->
-            {RevDepth, [RevHash| _]} = Doc#doc.revs,
-            Rev = {RevDepth, RevHash},
-            ok = ets_lru:insert(?CACHE, {DbName, DocId, Rev}, Doc),
-            exit({open_ok, OpenerKey, {ok, Doc}});
-        Else ->
-            exit({open_ok, OpenerKey, Else})
-    catch
-        Type:Reason ->
-            exit({open_error, OpenerKey, Type, Reason})
-    end;
-fetch_doc_data({DbName, DocId, Rev}=OpenerKey) ->
-    try recover_doc(DbName, DocId, Rev) of
-        {ok, Doc} ->
-            ok = ets_lru:insert(?CACHE, {DbName, DocId, Rev}, Doc),
-            exit({open_ok, OpenerKey, {ok, Doc}});
-        Else ->
-            exit({open_ok, OpenerKey, Else})
-    catch
-        Type:Reason ->
-            exit({open_error, OpenerKey, Type, Reason})
-    end.
-
-handle_open_response(Resp) ->
-    case Resp of
-        {open_ok, Value} -> Value;
-        {open_error, throw, Error} -> throw(Error);
-        {open_error, error, Error} -> erlang:error(Error);
-        {open_error, exit, Error} -> exit(Error)
-    end.
-
-respond(OpenerKey, Resp) ->
-    [#opener{clients=Clients}] = ets:lookup(?OPENING, OpenerKey),
-    _ = [gen_server:reply(C, Resp) || C <- Clients],
-    ets:delete(?OPENING, OpenerKey).
diff --git a/src/ddoc_cache/src/ddoc_cache_sup.erl b/src/ddoc_cache/src/ddoc_cache_sup.erl
index 85e90b3c5..35393e103 100644
--- a/src/ddoc_cache/src/ddoc_cache_sup.erl
+++ b/src/ddoc_cache/src/ddoc_cache_sup.erl
@@ -27,12 +27,20 @@ start_link() ->
 init([]) ->
     Children = [
         {
+            ddoc_cache_tables,
+            {ddoc_cache_tables, start_link, []},
+            permanent,
+            5000,
+            worker,
+            [ddoc_cache_tables]
+        },
+        {
             ddoc_cache_lru,
-            {ets_lru, start_link, [ddoc_cache_lru, lru_opts()]},
+            {ddoc_cache_lru, start_link, []},
             permanent,
             5000,
             worker,
-            [ets_lru]
+            [ddoc_cache_lru]
         },
         {
             ddoc_cache_opener,
@@ -43,25 +51,4 @@ init([]) ->
             [ddoc_cache_opener]
         }
     ],
-    {ok, {{one_for_one, 5, 10}, Children}}.
-
-
-lru_opts() ->
-    case application:get_env(ddoc_cache, max_objects) of
-        {ok, MxObjs} when is_integer(MxObjs), MxObjs >= 0 ->
-            [{max_objects, MxObjs}];
-        _ ->
-            []
-    end ++
-    case application:get_env(ddoc_cache, max_size) of
-        {ok, MxSize} when is_integer(MxSize), MxSize >= 0 ->
-            [{max_size, MxSize}];
-        _ ->
-            []
-    end ++
-    case application:get_env(ddoc_cache, max_lifetime) of
-        {ok, MxLT} when is_integer(MxLT), MxLT >= 0 ->
-            [{max_lifetime, MxLT}];
-        _ ->
-            []
-    end.
+    {ok, {{one_for_all, 25, 1}, Children}}.
diff --git a/src/ddoc_cache/src/ddoc_cache_tables.erl b/src/ddoc_cache/src/ddoc_cache_tables.erl
new file mode 100644
index 000000000..585677689
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache_tables.erl
@@ -0,0 +1,68 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_tables).
+-behaviour(gen_server).
+-vsn(1).
+
+
+-export([
+    start_link/0
+]).
+
+-export([
+    init/1,
+    terminate/2,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    code_change/3
+]).
+
+
+-include("ddoc_cache.hrl").
+
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+
+init(_) ->
+    BaseOpts = [public, named_table],
+    CacheOpts = [
+        set,
+        {read_concurrency, true},
+        {keypos, #entry.key}
+    ] ++ BaseOpts,
+    ets:new(?CACHE, CacheOpts),
+    ets:new(?LRU, [ordered_set, {write_concurrency, true}] ++ BaseOpts),
+    {ok, nil}.
+
+
+terminate(_Reason, _St) ->
+    ok.
+
+
+handle_call(Msg, _From, St) ->
+    {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}.
+
+
+handle_cast(Msg, St) ->
+    {stop, {invalid_cast, Msg}, St}.
+
+
+handle_info(Msg, St) ->
+    {stop, {invalid_info, Msg}, St}.
+
+
+code_change(_OldVsn, St, _Extra) ->
+    {ok, St}.
diff --git a/src/ddoc_cache/test/ddoc_cache_basic_test.erl b/src/ddoc_cache/test/ddoc_cache_basic_test.erl
new file mode 100644
index 000000000..227ac545a
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_basic_test.erl
@@ -0,0 +1,122 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_basic_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(DbName) ->
+    {ok, {DbName, totes_custom}}.
+
+
+check_basic_test_() ->
+    {
+        setup,
+        fun ddoc_cache_tutil:start_couch/0,
+        fun ddoc_cache_tutil:stop_couch/1,
+        {with, [
+            fun cache_ddoc/1,
+            fun cache_ddoc_rev/1,
+            fun cache_vdu/1,
+            fun cache_custom/1,
+            fun cache_ddoc_refresher_unchanged/1,
+            fun dont_cache_not_found/1,
+            fun deprecated_api_works/1
+        ]}
+    }.
+
+
+cache_ddoc({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    Resp1 = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    ?assertMatch({ok, #doc{id = ?FOOBAR}}, Resp1),
+    ?assertEqual(1, ets:info(?CACHE, size)),
+    Resp2 = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    ?assertEqual(Resp1, Resp2),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+
+cache_ddoc_rev({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?FOOBAR),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    Resp1 = ddoc_cache:open_doc(DbName, ?FOOBAR, Rev),
+    ?assertMatch({ok, #doc{id = ?FOOBAR}}, Resp1),
+    ?assertEqual(1, ets:info(?CACHE, size)),
+    Resp2 = ddoc_cache:open_doc(DbName, ?FOOBAR, Rev),
+    ?assertEqual(Resp1, Resp2),
+    ?assertEqual(1, ets:info(?CACHE, size)),
+
+    % Assert that the non-rev cache entry is separate
+    Resp3 = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    ?assertMatch({ok, #doc{id = ?FOOBAR}}, Resp3),
+    ?assertEqual(2, ets:info(?CACHE, size)).
+
+
+cache_vdu({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    Resp1 = ddoc_cache:open_validation_funs(DbName),
+    ?assertMatch({ok, [_]}, Resp1),
+    ?assertEqual(1, ets:info(?CACHE, size)),
+    Resp2 = ddoc_cache:open_validation_funs(DbName),
+    ?assertEqual(Resp1, Resp2),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+
+cache_custom({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    Resp1 = ddoc_cache:open_custom(DbName, ?MODULE),
+    ?assertMatch({ok, {DbName, totes_custom}}, Resp1),
+    ?assertEqual(1, ets:info(?CACHE, size)),
+    Resp2 = ddoc_cache:open_custom(DbName, ?MODULE),
+    ?assertEqual(Resp1, Resp2),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+
+cache_ddoc_refresher_unchanged({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    ddoc_cache:open_doc(DbName, ?FOOBAR),
+    [Entry1] = ets:lookup(?CACHE, ets:first(?CACHE)),
+    ddoc_cache:open_doc(DbName, ?FOOBAR),
+    [Entry2] = ets:lookup(?CACHE, ets:first(?CACHE)),
+    ?assertEqual(Entry1, Entry2).
+
+
+dont_cache_not_found({DbName, _}) ->
+    DDocId = <<"_design/not_found">>,
+    ddoc_cache_tutil:clear(),
+    Resp = ddoc_cache:open_doc(DbName, DDocId),
+    ?assertEqual({not_found, missing}, Resp),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    ?assertEqual(0, ets:info(?LRU, size)).
+
+
+deprecated_api_works({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    {ok, _} = ddoc_cache:open(DbName, ?FOOBAR),
+    {ok, _} = ddoc_cache:open(DbName, <<"foobar">>),
+    {ok, _} = ddoc_cache:open(DbName, ?MODULE),
+    {ok, _} = ddoc_cache:open(DbName, validation_funs).
+
diff --git a/src/ddoc_cache/test/ddoc_cache_coverage_test.erl b/src/ddoc_cache/test/ddoc_cache_coverage_test.erl
new file mode 100644
index 000000000..395f56048
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_coverage_test.erl
@@ -0,0 +1,86 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_coverage_test).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+coverage_test_() ->
+    {
+        setup,
+        fun ddoc_cache_tutil:start_couch/0,
+        fun ddoc_cache_tutil:stop_couch/1,
+        [
+            fun restart_lru/0,
+            fun restart_tables/0,
+            fun restart_evictor/0
+        ]
+    }.
+
+
+restart_lru() ->
+    send_bad_messages(ddoc_cache_lru),
+    ?assertEqual(ok, ddoc_cache_lru:terminate(bang, {st, a, b, c, d})),
+    ?assertEqual({ok, foo}, ddoc_cache_lru:code_change(1, foo, [])).
+
+
+restart_tables() ->
+    send_bad_messages(ddoc_cache_tables),
+    ?assertEqual(ok, ddoc_cache_tables:terminate(bang, baz)),
+    ?assertEqual({ok, foo}, ddoc_cache_tables:code_change(1, foo, [])).
+
+
+restart_evictor() ->
+    meck:new(ddoc_cache_ev, [passthrough]),
+    try
+        State = sys:get_state(ddoc_cache_lru),
+        Evictor = element(5, State),
+        Ref = erlang:monitor(process, Evictor),
+        exit(Evictor, shutdown),
+        receive
+            {'DOWN', Ref, _, _, Reason} ->
+                couch_log:error("MONITOR: ~p", [Reason]),
+                ok
+        end,
+        meck:wait(ddoc_cache_ev, event, [evictor_died, '_'], 1000),
+        NewState = sys:get_state(ddoc_cache_lru),
+        NewEvictor = element(5, NewState),
+        ?assertNotEqual(Evictor, NewEvictor)
+    after
+        meck:unload()
+    end.
+
+
+send_bad_messages(Name) ->
+    wait_for_restart(Name, fun() ->
+        ?assertEqual({invalid_call, foo}, gen_server:call(Name, foo))
+    end),
+    wait_for_restart(Name, fun() ->
+        gen_server:cast(Name, foo)
+    end),
+    wait_for_restart(Name, fun() ->
+        whereis(Name) ! foo
+    end).
+
+
+wait_for_restart(Server, Fun) ->
+    Ref = erlang:monitor(process, whereis(Server)),
+    Fun(),
+    receive
+        {'DOWN', Ref, _, _, _} ->
+            ok
+    end,
+    ?assert(is_pid(test_util:wait_process(Server))).
diff --git a/src/ddoc_cache/test/ddoc_cache_disabled_test.erl b/src/ddoc_cache/test/ddoc_cache_disabled_test.erl
new file mode 100644
index 000000000..ef7318075
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_disabled_test.erl
@@ -0,0 +1,52 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_disabled_test).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    config:set("ddoc_cache", "max_size", "0", false),
+    Ctx.
+
+
+check_disabled_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun ddoc_cache_tutil:stop_couch/1,
+        {with, [
+            fun resp_ok/1,
+            fun resp_not_found/1
+        ]}
+    }.
+
+
+resp_ok({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    Resp = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    ?assertMatch({ok, #doc{id = ?FOOBAR}}, Resp),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    ?assertEqual(0, ets:info(?LRU, size)).
+
+
+resp_not_found({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    Resp = ddoc_cache:open_doc(DbName, <<"_design/not_found">>),
+    ?assertEqual({not_found, missing}, Resp),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    ?assertEqual(0, ets:info(?LRU, size)).
diff --git a/src/ddoc_cache/test/ddoc_cache_entry_test.erl b/src/ddoc_cache/test/ddoc_cache_entry_test.erl
new file mode 100644
index 000000000..62afc7237
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_entry_test.erl
@@ -0,0 +1,120 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_entry_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(<<"foo">>) ->
+    timer:sleep(30000);
+
+recover(DbName) ->
+    {ok, {DbName, such_custom}}.
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:new(ddoc_cache_ev, [passthrough]),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_entry_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun cancel_and_replace_opener/1,
+            fun condenses_access_messages/1,
+            fun kill_opener_on_terminate/1,
+            fun open_dead_entry/1,
+            fun handles_bad_messages/1,
+            fun handles_code_change/1
+        ]}
+    }.
+
+
+cancel_and_replace_opener(_) ->
+    Key = {ddoc_cache_entry_custom, {<<"foo">>, ?MODULE}},
+    true = ets:insert_new(?CACHE, #entry{key = Key}),
+    {ok, Entry} = ddoc_cache_entry:start_link(Key),
+    Opener1 = element(4, sys:get_state(Entry)),
+    Ref1 = erlang:monitor(process, Opener1),
+    gen_server:cast(Entry, refresh),
+    receive {'DOWN', Ref1, _, _, _} -> ok end,
+    Opener2 = element(4, sys:get_state(Entry)),
+    ?assert(Opener2 /= Opener1),
+    ?assert(is_process_alive(Opener2)),
+    % Clean up after ourselves
+    unlink(Entry),
+    ddoc_cache_entry:shutdown(Entry).
+
+
+condenses_access_messages({DbName, _}) ->
+    meck:reset(ddoc_cache_ev),
+    Key = {ddoc_cache_entry_custom, {DbName, ?MODULE}},
+    true = ets:insert(?CACHE, #entry{key = Key}),
+    {ok, Entry} = ddoc_cache_entry:start_link(Key),
+    erlang:suspend_process(Entry),
+    lists:foreach(fun(_) ->
+        gen_server:cast(Entry, accessed)
+    end, lists:seq(1, 100)),
+    erlang:resume_process(Entry),
+    meck:wait(1, ddoc_cache_ev, event, [accessed, Key], 1000),
+    ?assertError(
+            timeout,
+            meck:wait(2, ddoc_cache_ev, event, [accessed, Key], 100)
+        ),
+    unlink(Entry),
+    ddoc_cache_entry:shutdown(Entry).
+
+
+kill_opener_on_terminate(_) ->
+    Pid = spawn(fun() -> receive _ -> ok end end),
+    ?assert(is_process_alive(Pid)),
+    St = {st, key, val, Pid, waiters, ts},
+    ?assertEqual(ok, ddoc_cache_entry:terminate(normal, St)),
+    ?assert(not is_process_alive(Pid)).
+
+
+open_dead_entry({DbName, _}) ->
+    Pid = spawn(fun() -> ok end),
+    Key = {ddoc_cache_entry_custom, {DbName, ?MODULE}},
+    ?assertEqual(recover(DbName), ddoc_cache_entry:open(Pid, Key)).
+
+
+handles_bad_messages(_) ->
+    CallExpect = {stop, {bad_call, foo}, {bad_call, foo}, baz},
+    CastExpect = {stop, {bad_cast, foo}, bar},
+    InfoExpect = {stop, {bad_info, foo}, bar},
+    ?assertEqual(CallExpect, ddoc_cache_entry:handle_call(foo, bar, baz)),
+    ?assertEqual(CastExpect, ddoc_cache_entry:handle_cast(foo, bar)),
+    ?assertEqual(InfoExpect, ddoc_cache_entry:handle_info(foo, bar)).
+
+
+handles_code_change(_) ->
+    CCExpect = {ok, bar},
+    ?assertEqual(CCExpect, ddoc_cache_entry:code_change(foo, bar, baz)).
diff --git a/src/ddoc_cache/test/ddoc_cache_ev.erl b/src/ddoc_cache/test/ddoc_cache_ev.erl
new file mode 100644
index 000000000..a451342cf
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_ev.erl
@@ -0,0 +1,21 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_ev).
+
+-export([
+    event/2
+]).
+
+
+event(Name, Arg) ->
+    couch_log:error("~s :: ~s :: ~p", [?MODULE, Name, Arg]).
diff --git a/src/ddoc_cache/test/ddoc_cache_eviction_test.erl b/src/ddoc_cache/test/ddoc_cache_eviction_test.erl
new file mode 100644
index 000000000..0b9f57b85
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_eviction_test.erl
@@ -0,0 +1,93 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_eviction_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(DbName) ->
+    {ok, {DbName, totes_custom}}.
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:new(ddoc_cache_ev, [passthrough]),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_eviction_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun evict_all/1,
+            fun dont_evict_all_unrelated/1,
+            fun check_upgrade_clause/1
+        ]}
+    }.
+
+
+evict_all({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?FOOBAR),
+    ShardName = element(2, hd(mem3:shards(DbName))),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR, Rev),
+    {ok, _} = ddoc_cache:open_validation_funs(DbName),
+    {ok, _} = ddoc_cache:open_custom(DbName, ?MODULE),
+    ?assertEqual(4, ets:info(?CACHE, size)),
+    {ok, _} = ddoc_cache_lru:handle_db_event(ShardName, deleted, foo),
+    meck:wait(ddoc_cache_ev, event, [evicted, DbName], 1000),
+    meck:wait(4, ddoc_cache_ev, event, [removed, '_'], 1000),
+    ?assertEqual(0, ets:info(?CACHE, size)).
+
+
+dont_evict_all_unrelated({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?FOOBAR),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR, Rev),
+    {ok, _} = ddoc_cache:open_validation_funs(DbName),
+    {ok, _} = ddoc_cache:open_custom(DbName, ?MODULE),
+    ?assertEqual(4, ets:info(?CACHE, size)),
+    ShardName = <<"shards/00000000-ffffffff/test.1384769918">>,
+    {ok, _} = ddoc_cache_lru:handle_db_event(ShardName, deleted, foo),
+    meck:wait(ddoc_cache_ev, event, [evict_noop, <<"test">>], 1000),
+    ?assertEqual(4, ets:info(?CACHE, size)).
+
+
+check_upgrade_clause({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    ?assertEqual(1, ets:info(?CACHE, size)),
+    gen_server:cast(ddoc_cache_opener, {do_evict, DbName}),
+    meck:wait(ddoc_cache_ev, event, [evicted, DbName], 1000),
+    meck:wait(ddoc_cache_ev, event, [removed, '_'], 1000),
+    ?assertEqual(0, ets:info(?CACHE, size)).
diff --git a/src/ddoc_cache/test/ddoc_cache_lru_test.erl b/src/ddoc_cache/test/ddoc_cache_lru_test.erl
new file mode 100644
index 000000000..f5cef1668
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_lru_test.erl
@@ -0,0 +1,167 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_lru_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(<<"pause", _/binary>>) ->
+    receive go -> ok end,
+    {ok, paused};
+
+recover(DbName) ->
+    {ok, DbName}.
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    config:set("ddoc_cache", "max_size", "5", false),
+    meck:new(ddoc_cache_ev, [passthrough]),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_not_started_test() ->
+    % Starting couch, but not ddoc_cache
+    Ctx = test_util:start_couch(),
+    try
+        Key = {ddoc_cache_entry_custom, {<<"dbname">>, ?MODULE}},
+        ?assertEqual({ok, <<"dbname">>}, ddoc_cache_lru:open(Key))
+    after
+        test_util:stop_couch(Ctx)
+    end.
+
+
+check_lru_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun check_multi_start/1,
+            fun check_multi_open/1,
+            fun check_capped_size/1,
+            fun check_full_cache/1
+        ]}
+    }.
+
+
+check_multi_start(_) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Key = {ddoc_cache_entry_custom, {<<"pause">>, ?MODULE}},
+    % These will all get sent through ddoc_cache_lru
+    Clients = lists:map(fun(_) ->
+        spawn_monitor(fun() ->
+            ddoc_cache_lru:open(Key)
+        end)
+    end, lists:seq(1, 10)),
+    meck:wait(ddoc_cache_ev, event, [started, Key], 1000),
+    lists:foreach(fun({Pid, _Ref}) ->
+        ?assert(is_process_alive(Pid))
+    end, Clients),
+    [#entry{pid = Pid}] = ets:tab2list(?CACHE),
+    Opener = element(4, sys:get_state(Pid)),
+    OpenerRef = erlang:monitor(process, Opener),
+    ?assert(is_process_alive(Opener)),
+    Opener ! go,
+    receive {'DOWN', OpenerRef, _, _, _} -> ok end,
+    lists:foreach(fun({CPid, Ref}) ->
+        receive
+            {'DOWN', Ref, _, _, normal} -> ok;
+            {'DOWN', Ref, _, _, Other} ->
+                io:format(standard_error, "OTHER: ~p~n", [Other]);
+            Other when not is_tuple(Other) orelse element(1, Other) /= 'DOWN' ->
+                io:format(standard_error, "MSG: ~p~n", [Other])
+        after 2000 ->
+            io:format(standard_error, "BLAH?!: ~p ~p", [CPid, process_info(CPid, current_stacktrace)])
+        end
+    end, Clients).
+
+
+check_multi_open(_) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Key = {ddoc_cache_entry_custom, {<<"pause">>, ?MODULE}},
+    % We wait after the first client so that
+    % the rest of the clients go directly to
+    % ddoc_cache_entry bypassing ddoc_cache_lru
+    Client1 = spawn_monitor(fun() ->
+        ddoc_cache_lru:open(Key)
+    end),
+    meck:wait(ddoc_cache_ev, event, [started, Key], 1000),
+    Clients = [Client1] ++ lists:map(fun(_) ->
+        spawn_monitor(fun() ->
+            ddoc_cache_lru:open(Key)
+        end)
+    end, lists:seq(1, 9)),
+    lists:foreach(fun({Pid, _Ref}) ->
+        ?assert(is_process_alive(Pid))
+    end, Clients),
+    [#entry{pid = Pid}] = ets:tab2list(?CACHE),
+    Opener = element(4, sys:get_state(Pid)),
+    OpenerRef = erlang:monitor(process, Opener),
+    ?assert(is_process_alive(Opener)),
+    Opener ! go,
+    receive {'DOWN', OpenerRef, _, _, _} -> ok end,
+    lists:foreach(fun({_, Ref}) ->
+        receive {'DOWN', Ref, _, _, normal} -> ok end
+    end, Clients).
+
+
+check_capped_size(_) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    lists:foreach(fun(I) ->
+        DbName = list_to_binary(integer_to_list(I)),
+        ddoc_cache:open_custom(DbName, ?MODULE),
+        meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000),
+        ?assertEqual(I, ets:info(?CACHE, size))
+    end, lists:seq(1, 5)),
+    lists:foreach(fun(I) ->
+        DbName = list_to_binary(integer_to_list(I)),
+        ddoc_cache:open_custom(DbName, ?MODULE),
+        meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000),
+        ?assertEqual(5, ets:info(?CACHE, size))
+    end, lists:seq(6, 20)).
+
+
+check_full_cache(_) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    lists:foreach(fun(I) ->
+        DbSuffix = list_to_binary(integer_to_list(I)),
+        DbName = <<"pause", DbSuffix/binary>>,
+        spawn(fun() -> ddoc_cache:open_custom(DbName, ?MODULE) end),
+        meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000),
+        ?assertEqual(I, ets:info(?CACHE, size))
+    end, lists:seq(1, 5)),
+    lists:foreach(fun(I) ->
+        DbSuffix = list_to_binary(integer_to_list(I)),
+        DbName = <<"pause", DbSuffix/binary>>,
+        spawn(fun() -> ddoc_cache:open_custom(DbName, ?MODULE) end),
+        meck:wait(I - 5, ddoc_cache_ev, event, [full, '_'], 1000),
+        ?assertEqual(5, ets:info(?CACHE, size))
+    end, lists:seq(6, 20)).
diff --git a/src/ddoc_cache/test/ddoc_cache_no_cache_test.erl b/src/ddoc_cache/test/ddoc_cache_no_cache_test.erl
new file mode 100644
index 000000000..a5a5751e1
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_no_cache_test.erl
@@ -0,0 +1,78 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_no_cache_test).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+
+ddoc(DDocId) ->
+    {ok, #doc{
+        id = DDocId,
+        body = {[
+            {<<"ohai">>, null}
+        ]}
+    }}.
+
+
+not_found(_DDocId) ->
+    {not_found, missing}.
+
+
+return_error(_DDocId) ->
+    {error, timeout}.
+
+
+start(Resp) ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:new(fabric),
+    meck:expect(fabric, open_doc, fun(_, DDocId, _) ->
+        Resp(DDocId)
+    end),
+    Ctx.
+
+
+stop(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+no_cache_open_ok_test() ->
+    Ctx = start(fun ddoc/1),
+    try
+        Resp = ddoc_cache:open_doc(<<"foo">>, <<"bar">>),
+        ?assertEqual(ddoc(<<"bar">>), Resp)
+    after
+        stop(Ctx)
+    end.
+
+
+no_cache_open_not_found_test() ->
+    Ctx = start(fun not_found/1),
+    try
+        Resp = ddoc_cache:open_doc(<<"foo">>, <<"bar">>),
+        ?assertEqual(not_found(<<"bar">>), Resp)
+    after
+        stop(Ctx)
+    end.
+
+
+no_cache_open_error_test() ->
+    Ctx = start(fun return_error/1),
+    try
+        Resp = ddoc_cache:open_doc(<<"foo">>, <<"bar">>),
+        ?assertEqual(return_error(<<"bar">>), Resp)
+    after
+        stop(Ctx)
+    end.
diff --git a/src/ddoc_cache/test/ddoc_cache_open_error_test.erl b/src/ddoc_cache/test/ddoc_cache_open_error_test.erl
new file mode 100644
index 000000000..0ac239087
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_open_error_test.erl
@@ -0,0 +1,46 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_open_error_test).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:expect(fabric, open_doc, fun(_, ?FOOBAR, _) ->
+        erlang:error(test_kaboom)
+    end),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_basic_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun handle_open_error/1
+        ]}
+    }.
+
+
+handle_open_error({DbName, _}) ->
+    ?assertError(test_kaboom, ddoc_cache:open_doc(DbName, ?FOOBAR)).
diff --git a/src/ddoc_cache/test/ddoc_cache_opener_test.erl b/src/ddoc_cache/test/ddoc_cache_opener_test.erl
new file mode 100644
index 000000000..c3846360c
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_opener_test.erl
@@ -0,0 +1,33 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_opener_test).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+empty_hull_test() ->
+    InitExpect = {ok, nil},
+    TermExpect = ok,
+    CallExpect = {stop, {invalid_call, foo}, {invalid_call, foo}, baz},
+    CastExpect = {stop, {invalid_cast, foo}, bar},
+    InfoExpect = {stop, {invalid_info, foo}, bar},
+    CCExpect = {ok, bar},
+    ?assertEqual(InitExpect, ddoc_cache_opener:init(foo)),
+    ?assertEqual(TermExpect, ddoc_cache_opener:terminate(foo, bar)),
+    ?assertEqual(CallExpect, ddoc_cache_opener:handle_call(foo, bar, baz)),
+    ?assertEqual(CastExpect, ddoc_cache_opener:handle_cast(foo, bar)),
+    ?assertEqual(InfoExpect, ddoc_cache_opener:handle_info(foo, bar)),
+    ?assertEqual(CCExpect, ddoc_cache_opener:code_change(foo, bar, baz)).
diff --git a/src/ddoc_cache/test/ddoc_cache_refresh_test.erl b/src/ddoc_cache/test/ddoc_cache_refresh_test.erl
new file mode 100644
index 000000000..7bc1704cb
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_refresh_test.erl
@@ -0,0 +1,167 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_refresh_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(DbName) ->
+    {ok, {DbName, rand_string()}}.
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:new(ddoc_cache_ev, [passthrough]),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_refresh_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun refresh_ddoc/1,
+            fun refresh_ddoc_rev/1,
+            fun refresh_vdu/1,
+            fun refresh_custom/1,
+            fun refresh_multiple/1,
+            fun check_upgrade_clause/1
+        ]}
+    }.
+
+
+refresh_ddoc({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    ?assertEqual(1, ets:info(?CACHE, size)),
+    [#entry{key = Key, val = DDoc}] = ets:tab2list(?CACHE),
+    NewDDoc = DDoc#doc{
+        body = {[{<<"foo">>, <<"baz">>}]}
+    },
+    {ok, {Depth, RevId}} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    Expect = NewDDoc#doc{
+        revs = {Depth, [RevId | element(2, DDoc#doc.revs)]}
+    },
+    meck:wait(ddoc_cache_ev, event, [updated, {Key, Expect}], 1000),
+    ?assertMatch({ok, Expect}, ddoc_cache:open_doc(DbName, ?FOOBAR)),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+
+refresh_ddoc_rev({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?FOOBAR),
+    {ok, RevDDoc} = ddoc_cache:open_doc(DbName, ?FOOBAR, Rev),
+    [#entry{key = Key, val = DDoc}] = ets:tab2list(?CACHE),
+    NewDDoc = DDoc#doc{
+        body = {[{<<"foo">>, <<"kazam">>}]}
+    },
+    {ok, _} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    % We pass the rev explicitly so we assert that we're
+    % getting the same original response from the cache
+    meck:wait(ddoc_cache_ev, event, [update_noop, Key], 1000),
+    ?assertMatch({ok, RevDDoc}, ddoc_cache:open_doc(DbName, ?FOOBAR, Rev)),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+
+refresh_vdu({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    {ok, [_]} = ddoc_cache:open_validation_funs(DbName),
+    [#entry{key = Key}] = ets:tab2list(?CACHE),
+    {ok, DDoc} = fabric:open_doc(DbName, ?VDU, [?ADMIN_CTX]),
+    {ok, _} = fabric:update_doc(DbName, DDoc#doc{body = {[]}}, [?ADMIN_CTX]),
+    meck:wait(ddoc_cache_ev, event, [updated, {Key, []}], 1000),
+    ?assertMatch({ok, []}, ddoc_cache:open_validation_funs(DbName)),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+
+refresh_custom({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    {ok, Resp1} = ddoc_cache:open_custom(DbName, ?MODULE),
+    {ok, DDoc} = fabric:open_doc(DbName, ?VDU, [?CUSTOM]),
+    {ok, _} = fabric:update_doc(DbName, DDoc#doc{body = {[]}}, [?ADMIN_CTX]),
+    meck:wait(ddoc_cache_ev, event, [updated, '_'], 1000),
+    ?assertNotEqual({ok, Resp1}, ddoc_cache:open_custom(DbName, ?MODULE)),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+
+refresh_multiple({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?FOOBAR),
+    {ok, DDoc} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    {ok, DDoc} = ddoc_cache:open_doc(DbName, ?FOOBAR, Rev),
+    ?assertEqual(2, ets:info(?CACHE, size)),
+    % Relying on the sort order of entry keys to make
+    % sure our entries line up for this test
+    [
+        #entry{key = NoRevKey, val = DDoc},
+        #entry{key = RevKey, val = DDoc}
+    ] = lists:sort(ets:tab2list(?CACHE)),
+    NewDDoc = DDoc#doc{
+        body = {[{<<"foo">>, <<"kalamazoo">>}]}
+    },
+    {ok, {Depth, RevId}} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    Updated = NewDDoc#doc{
+        revs = {Depth, [RevId | element(2, DDoc#doc.revs)]}
+    },
+    meck:wait(ddoc_cache_ev, event, [update_noop, RevKey], 1000),
+    meck:wait(ddoc_cache_ev, event, [updated, {NoRevKey, Updated}], 1000),
+    % We pass the rev explicitly so we assert that we're
+    % getting the same original response from the cache
+    ?assertEqual({ok, Updated}, ddoc_cache:open_doc(DbName, ?FOOBAR)),
+    ?assertEqual({ok, DDoc}, ddoc_cache:open_doc(DbName, ?FOOBAR, Rev)),
+    ?assertEqual(2, ets:info(?CACHE, size)).
+
+
+check_upgrade_clause({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    [#entry{key = Key}] = ets:tab2list(?CACHE),
+    gen_server:cast(ddoc_cache_opener, {do_evict, DbName, [?FOOBAR]}),
+    meck:wait(ddoc_cache_ev, event, [update_noop, Key], 1000).
+
+
+rand_string() ->
+    Bin = crypto:rand_bytes(8),
+    to_hex(Bin, []).
+
+
+to_hex(<<>>, Acc) ->
+    list_to_binary(lists:reverse(Acc));
+to_hex(<<C1:4, C2:4, Rest/binary>>, Acc) ->
+    to_hex(Rest, [hexdig(C1), hexdig(C2) | Acc]).
+
+
+hexdig(C) when C >= 0, C =< 9 ->
+    C + $0;
+hexdig(C) when C >= 10, C =< 15 ->
+    C + $A - 10.
\ No newline at end of file
diff --git a/src/ddoc_cache/test/ddoc_cache_remove_test.erl b/src/ddoc_cache/test/ddoc_cache_remove_test.erl
new file mode 100644
index 000000000..7596b99e8
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_remove_test.erl
@@ -0,0 +1,206 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_remove_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("mem3/include/mem3.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(DbName) ->
+    {ok, #doc{body = {Body}}} = fabric:open_doc(DbName, ?CUSTOM, [?ADMIN_CTX]),
+    case couch_util:get_value(<<"status">>, Body) of
+        <<"ok">> ->
+            {ok, yay};
+        <<"not_ok">> ->
+            {ruh, roh};
+        <<"error">> ->
+            erlang:error(thpppt)
+    end.
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:new(ddoc_cache_ev, [passthrough]),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_refresh_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun remove_ddoc/1,
+            fun remove_ddoc_rev/1,
+            fun remove_ddoc_rev_only/1,
+            fun remove_custom_not_ok/1,
+            fun remove_custom_error/1
+        ]}
+    }.
+
+
+remove_ddoc({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    ?assertEqual(1, ets:info(?CACHE, size)),
+    [#entry{key = Key, val = DDoc}] = ets:tab2list(?CACHE),
+    NewDDoc = DDoc#doc{
+        deleted = true,
+        body = {[]}
+    },
+    {ok, _} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    meck:wait(ddoc_cache_ev, event, [removed, Key], 1000),
+    ?assertMatch({not_found, deleted}, ddoc_cache:open_doc(DbName, ?FOOBAR)),
+    ?assertEqual(0, ets:info(?CACHE, size)).
+
+
+remove_ddoc_rev({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?VDU),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?VDU, Rev),
+    [#entry{key = Key, val = DDoc, pid = Pid}] = ets:tab2list(?CACHE),
+    NewDDoc = DDoc#doc{
+        body = {[{<<"an">>, <<"update">>}]}
+    },
+    {ok, _} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    meck:wait(ddoc_cache_ev, event, [update_noop, Key], 1000),
+    % Compact the database so that the old rev is removed
+    lists:foreach(fun(Shard) ->
+        do_compact(Shard#shard.name)
+    end, mem3:local_shards(DbName)),
+    % Trigger a refresh rather than wait for the timeout
+    ddoc_cache_entry:refresh(Pid),
+    meck:wait(ddoc_cache_ev, event, [removed, Key], 1000),
+    ?assertMatch(
+            {{not_found, missing}, _},
+            ddoc_cache:open_doc(DbName, ?VDU, Rev)
+        ),
+    ?assertEqual(0, ets:info(?CACHE, size)).
+
+
+remove_ddoc_rev_only({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?VDU),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?VDU),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?VDU, Rev),
+    % Relying on the sort order of keys to keep
+    % these lined up for testing
+    [
+        #entry{key = NoRevKey, val = DDoc, pid = NoRevPid},
+        #entry{key = RevKey, val = DDoc, pid = RevPid}
+    ] = lists:sort(ets:tab2list(?CACHE)),
+    NewDDoc = DDoc#doc{
+        body = {[{<<"new">>, <<"awesomeness">>}]}
+    },
+    {ok, _} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    meck:wait(ddoc_cache_ev, event, [updated, '_'], 1000),
+    meck:wait(ddoc_cache_ev, event, [update_noop, RevKey], 1000),
+    % Compact the database so that the old rev is removed
+    lists:foreach(fun(Shard) ->
+        do_compact(Shard#shard.name)
+    end, mem3:local_shards(DbName)),
+    % Trigger a refresh rather than wait for the timeout
+    ddoc_cache_entry:refresh(NoRevPid),
+    ddoc_cache_entry:refresh(RevPid),
+    meck:wait(ddoc_cache_ev, event, [update_noop, NoRevKey], 1000),
+    meck:wait(ddoc_cache_ev, event, [removed, RevKey], 1000),
+    ?assertMatch({ok, _}, ddoc_cache:open_doc(DbName, ?VDU)),
+    ?assertMatch(
+            {{not_found, missing}, _},
+            ddoc_cache:open_doc(DbName, ?VDU, Rev)
+        ),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+remove_custom_not_ok({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    init_custom_ddoc(DbName),
+    {ok, _} = ddoc_cache:open_custom(DbName, ?MODULE),
+    [#entry{key = Key}] = ets:tab2list(?CACHE),
+    {ok, DDoc} = fabric:open_doc(DbName, ?CUSTOM, [?ADMIN_CTX]),
+    NewDDoc = DDoc#doc{
+        body = {[{<<"status">>, <<"not_ok">>}]}
+    },
+    {ok, _} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    meck:wait(ddoc_cache_ev, event, [removed, Key], 1000),
+    ?assertEqual({ruh, roh}, ddoc_cache:open_custom(DbName, ?MODULE)),
+    ?assertEqual(0, ets:info(?CACHE, size)).
+
+
+remove_custom_error({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    init_custom_ddoc(DbName),
+    {ok, _} = ddoc_cache:open_custom(DbName, ?MODULE),
+    [#entry{key = Key}] = ets:tab2list(?CACHE),
+    {ok, DDoc} = fabric:open_doc(DbName, ?CUSTOM, [?ADMIN_CTX]),
+    NewDDoc = DDoc#doc{
+        body = {[{<<"status">>, <<"error">>}]}
+    },
+    {ok, _} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    meck:wait(ddoc_cache_ev, event, [removed, Key], 1000),
+    ?assertError(thpppt, ddoc_cache:open_custom(DbName, ?MODULE)),
+    ?assertEqual(0, ets:info(?CACHE, size)).
+
+
+init_custom_ddoc(DbName) ->
+    Body = {[{<<"status">>, <<"ok">>}]},
+    {ok, Doc} = fabric:open_doc(DbName, ?CUSTOM, [?ADMIN_CTX]),
+    NewDoc = Doc#doc{body = Body},
+    {ok, _} = fabric:update_doc(DbName, NewDoc, [?ADMIN_CTX]).
+
+
+do_compact(ShardName) ->
+    {ok, Db} = couch_db:open_int(ShardName, []),
+    try
+        {ok, Pid} = couch_db:start_compact(Db),
+        Ref = erlang:monitor(process, Pid),
+        receive
+            {'DOWN', Ref, _, _, _} ->
+                ok
+        end
+    after
+        couch_db:close(Db)
+    end,
+    wait_for_compaction(ShardName).
+
+
+wait_for_compaction(ShardName) ->
+    {ok, Db} = couch_db:open_int(ShardName, []),
+    CompactRunning = try
+        {ok, Info} = couch_db:get_db_info(Db),
+        couch_util:get_value(compact_running, Info)
+    after
+        couch_db:close(Db)
+    end,
+    if not CompactRunning -> ok; true ->
+        timer:sleep(100),
+        wait_for_compaction(ShardName)
+    end.
\ No newline at end of file
diff --git a/src/ddoc_cache/test/ddoc_cache_test.hrl b/src/ddoc_cache/test/ddoc_cache_test.hrl
new file mode 100644
index 000000000..73f7bc217
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_test.hrl
@@ -0,0 +1,26 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-define(CACHE, ddoc_cache_entries).
+-define(LRU, ddoc_cache_lru).
+-define(OPENERS, ddoc_cache_openers).
+
+-define(FOOBAR, <<"_design/foobar">>).
+-define(VDU, <<"_design/vdu">>).
+-define(CUSTOM, <<"_design/custom">>).
+
+-record(entry, {
+    key,
+    val,
+    pid
+}).
diff --git a/src/ddoc_cache/test/ddoc_cache_tutil.erl b/src/ddoc_cache/test/ddoc_cache_tutil.erl
new file mode 100644
index 000000000..cdd372bff
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_tutil.erl
@@ -0,0 +1,84 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_tutil).
+
+
+-compile(export_all).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+
+
+start_couch() ->
+    %purge_modules(),
+    Ctx = test_util:start_couch(?CONFIG_CHAIN, [chttpd, ddoc_cache]),
+    TmpDb = ?tempdb(),
+    ok = fabric:create_db(TmpDb, [{q, "1"}, {n, "1"}]),
+    {ok, _} = fabric:update_docs(TmpDb, ddocs(), [?ADMIN_CTX]),
+    {TmpDb, Ctx}.
+
+
+stop_couch({_TmpDb, Ctx}) ->
+    test_util:stop_couch(Ctx).
+
+
+clear() ->
+    application:stop(ddoc_cache),
+    application:start(ddoc_cache).
+
+
+get_rev(DbName, DDocId) ->
+    {_, Ref} = erlang:spawn_monitor(fun() ->
+        {ok, #doc{revs = Revs}} = fabric:open_doc(DbName, DDocId, [?ADMIN_CTX]),
+        {Depth, [RevId | _]} = Revs,
+        exit({Depth, RevId})
+    end),
+    receive
+        {'DOWN', Ref, _, _, Rev} -> Rev
+    end.
+
+
+ddocs() ->
+    FooBar = #doc{
+        id = <<"_design/foobar">>,
+        body = {[
+            {<<"foo">>, <<"bar">>}
+        ]}
+    },
+    VDU = #doc{
+        id = <<"_design/vdu">>,
+        body = {[
+            {<<"validate_doc_update">>, <<"function(doc) {return;}">>}
+        ]}
+    },
+    Custom = #doc{
+        id = <<"_design/custom">>,
+        body = {[
+            {<<"status">>, <<"ok">>},
+            {<<"custom">>, <<"hotrod">>}
+        ]}
+    },
+    [FooBar, VDU, Custom].
+
+
+purge_modules() ->
+    case application:get_key(ddoc_cache, modules) of
+        {ok, Mods} ->
+            lists:foreach(fun(Mod) ->
+                code:delete(Mod),
+                code:purge(Mod)
+            end, Mods);
+        undefined ->
+            ok
+    end.