Rewrite ddoc_cache to improve performanceoptimize-ddoc-cache

There were a couple issues with the previous ddoc_cache implementation
that made it possible to tip over the ddoc_cache_opener process. First,
there were a lot of messages flowing through a single gen_server. And
second, the cache relied on periodically evicting entries to ensure
proper behavior in not caching an entry forever after it had changed on
disk.

The new version makes two important changes. First, entries now have an
associated process that manages the cache entry. This process will
periodically refresh the entry and if the entry has changed or no longer
exists the process will remove its entry from cache.

The second major change is that the cache entry process directly mutates
the related ets table entries so that our performance is not dependent
on the speed of ets table mutations. Using a custom entry that does no
work the cache can now sustain roughly one million operations a second
with a twenty thousand clients fighting over a cache limited to one
thousand items. In production this means that cache performance will
likely be rate limited by other factors like loading design documents
from disk.

29 files changed, 2436 insertions, 356 deletions

diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index c6f4b9953..05ff6f6e6 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -194,6 +194,14 @@ credentials = false
 ; List of accepted methods
 ; methods =
 
+; Configuration for the design document cache
+;[ddoc_cache]
+; The maximum size of the cache in bytes
+;max_size = 104857600 ; 100MiB
+; The period each cache entry should wait before
+; automatically refreshing in milliseconds
+;refresh_timeout = 67000
+
 [x_frame_options]
 ; Settings same-origin will return X-Frame-Options: SAMEORIGIN.
 ; If same origin is set, it will ignore the hosts setting
diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl
index b4ad257ff..78e0b8c19 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -319,7 +319,7 @@ handle_info({update_docs, Client, GroupedDocs, NonRepDocs, MergeConflicts,
                     couch_event:notify(Db2#db.name, {ddoc_updated, DDocId})
                 end, UpdatedDDocIds),
                 couch_event:notify(Db2#db.name, ddoc_updated),
-                ddoc_cache:evict(Db2#db.name, UpdatedDDocIds),
+                ddoc_cache:refresh(Db2#db.name, UpdatedDDocIds),
                 refresh_validate_doc_funs(Db2);
             false ->
                 Db2
diff --git a/src/ddoc_cache/src/ddoc_cache.app.src b/src/ddoc_cache/src/ddoc_cache.app.src
index a64b2f526..faacdf570 100644
--- a/src/ddoc_cache/src/ddoc_cache.app.src
+++ b/src/ddoc_cache/src/ddoc_cache.app.src
@@ -13,13 +13,6 @@
 {application, ddoc_cache, [
     {description, "Design Document Cache"},
     {vsn, git},
-    {modules, [
-        ddoc_cache,
-        ddoc_cache_app,
-        ddoc_cache_opener,
-        ddoc_cache_sup,
-        ddoc_cache_util
-    ]},
     {registered, [
         ddoc_cache_lru,
         ddoc_cache_opener
@@ -28,17 +21,12 @@
         kernel,
         stdlib,
         crypto,
+        config,
         couch_event,
-        ets_lru,
-        mem3,
-        fabric,
         couch_log,
-        couch_stats
+        couch_stats,
+        mem3,
+        fabric
     ]},
-    {mod, {ddoc_cache_app, []}},
-    {env, [
-        {max_objects, unlimited},
-        {max_size, 104857600}, % 100M
-        {max_lifetime, 60000} % 1m
-    ]}
+    {mod, {ddoc_cache_app, []}}
 ]}.
diff --git a/src/ddoc_cache/src/ddoc_cache.erl b/src/ddoc_cache/src/ddoc_cache.erl
index ed9330998..50cac3039 100644
--- a/src/ddoc_cache/src/ddoc_cache.erl
+++ b/src/ddoc_cache/src/ddoc_cache.erl
@@ -12,86 +12,43 @@
 
 -module(ddoc_cache).
 
--export([
-    start/0,
-    stop/0
-]).
 
 -export([
     open_doc/2,
     open_doc/3,
     open_validation_funs/1,
-    evict/2,
+    open_custom/2,
+    refresh/2,
 
     %% deprecated
     open/2
 ]).
 
-start() ->
-    application:start(ddoc_cache).
-
-stop() ->
-    application:stop(ddoc_cache).
 
 open_doc(DbName, DocId) ->
-    Key = {DbName, DocId, '_'},
-    case ddoc_cache_opener:match_newest(Key) of
-        {ok, _} = Resp ->
-            couch_stats:increment_counter([ddoc_cache, hit]),
-            Resp;
-        missing ->
-            couch_stats:increment_counter([ddoc_cache, miss]),
-            ddoc_cache_opener:open_doc(DbName, DocId);
-        recover ->
-            couch_stats:increment_counter([ddoc_cache, recovery]),
-            ddoc_cache_opener:recover_doc(DbName, DocId)
-    end.
+    Key = {ddoc_cache_entry_ddocid, {DbName, DocId}},
+    ddoc_cache_lru:open(Key).
+
 
 open_doc(DbName, DocId, RevId) ->
-    Key = {DbName, DocId, RevId},
-    case ddoc_cache_opener:lookup(Key) of
-        {ok, _} = Resp ->
-            couch_stats:increment_counter([ddoc_cache, hit]),
-            Resp;
-        missing ->
-            couch_stats:increment_counter([ddoc_cache, miss]),
-            ddoc_cache_opener:open_doc(DbName, DocId, RevId);
-        recover ->
-            couch_stats:increment_counter([ddoc_cache, recovery]),
-            ddoc_cache_opener:recover_doc(DbName, DocId, RevId)
-    end.
+    Key = {ddoc_cache_entry_ddocid_rev, {DbName, DocId, RevId}},
+    ddoc_cache_lru:open(Key).
+
 
 open_validation_funs(DbName) ->
-    Key = {DbName, validation_funs},
-    case ddoc_cache_opener:lookup(Key) of
-        {ok, _} = Resp ->
-            couch_stats:increment_counter([ddoc_cache, hit]),
-            Resp;
-        missing ->
-            couch_stats:increment_counter([ddoc_cache, miss]),
-            ddoc_cache_opener:open_validation_funs(DbName);
-        recover ->
-            couch_stats:increment_counter([ddoc_cache, recovery]),
-            ddoc_cache_opener:recover_validation_funs(DbName)
-    end.
+    Key = {ddoc_cache_entry_validation_funs, DbName},
+    ddoc_cache_lru:open(Key).
+
 
 open_custom(DbName, Mod) ->
-    Key = {DbName, Mod},
-    case ddoc_cache_opener:lookup(Key) of
-        {ok, _} = Resp ->
-            couch_stats:increment_counter([ddoc_cache, hit]),
-            Resp;
-        missing ->
-            couch_stats:increment_counter([ddoc_cache, miss]),
-            ddoc_cache_opener:open_doc(DbName, Mod);
-        recover ->
-            couch_stats:increment_counter([ddoc_cache, recovery]),
-            Mod:recover(DbName)
-    end.
-
-evict(ShardDbName, DDocIds) ->
+    Key = {ddoc_cache_entry_custom, {DbName, Mod}},
+    ddoc_cache_lru:open(Key).
+
+
+refresh(ShardDbName, DDocIds) when is_list(DDocIds) ->
     DbName = mem3:dbname(ShardDbName),
-    ddoc_cache_opener:evict_docs(DbName, DDocIds).
+    ddoc_cache_lru:refresh(DbName, DDocIds).
+
 
 open(DbName, validation_funs) ->
     open_validation_funs(DbName);
diff --git a/src/ddoc_cache/src/ddoc_cache.hrl b/src/ddoc_cache/src/ddoc_cache.hrl
new file mode 100644
index 000000000..dba0d37b2
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache.hrl
@@ -0,0 +1,40 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-type dbname() :: iodata().
+-type docid() :: iodata().
+-type doc_hash() :: <<_:128>>.
+-type revision() :: {pos_integer(), doc_hash()}.
+
+-define(CACHE, ddoc_cache_entries).
+-define(LRU, ddoc_cache_lru).
+-define(REFRESH_TIMEOUT, 67000).
+-define(SHUTDOWN_TIMEOUT, 1000).
+
+-record(entry, {
+    key,
+    val,
+    pid
+}).
+
+-record(opener, {
+    key,
+    pid,
+    clients
+}).
+
+
+-ifdef(TEST).
+-define(EVENT(Name, Arg), ddoc_cache_ev:event(Name, Arg)).
+-else.
+-define(EVENT(Name, Arg), ignore).
+-endif.
diff --git a/src/ddoc_cache/src/ddoc_cache_entry.erl b/src/ddoc_cache/src/ddoc_cache_entry.erl
new file mode 100644
index 000000000..79f67bd67
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache_entry.erl
@@ -0,0 +1,352 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_entry).
+-behaviour(gen_server).
+-vsn(1).
+
+
+-export([
+    dbname/1,
+    ddocid/1,
+    recover/1,
+    insert/2,
+
+    start_link/2,
+    shutdown/1,
+    open/2,
+    accessed/1,
+    refresh/1
+]).
+
+-export([
+    init/1,
+    terminate/2,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    code_change/3
+]).
+
+-export([
+    do_open/1
+]).
+
+
+-include("ddoc_cache.hrl").
+
+
+-ifndef(TEST).
+-define(ENTRY_SHUTDOWN_TIMEOUT, 5000).
+-else.
+-define(ENTRY_SHUTDOWN_TIMEOUT, 500).
+-endif.
+
+
+-record(st, {
+    key,
+    val,
+    opener,
+    waiters,
+    ts,
+    accessed
+}).
+
+
+dbname({Mod, Arg}) ->
+    Mod:dbname(Arg).
+
+
+ddocid({Mod, Arg}) ->
+    Mod:ddocid(Arg).
+
+
+recover({Mod, Arg}) ->
+    Mod:recover(Arg).
+
+
+insert({Mod, Arg}, Value) ->
+    Mod:insert(Arg, Value).
+
+
+start_link(Key, Default) ->
+    Pid = proc_lib:spawn_link(?MODULE, init, [{Key, Default}]),
+    {ok, Pid}.
+
+
+shutdown(Pid) ->
+    Ref = erlang:monitor(process, Pid),
+    ok = gen_server:cast(Pid, shutdown),
+    receive
+        {'DOWN', Ref, process, Pid, normal} ->
+            ok;
+        {'DOWN', Ref, process, Pid, Reason} ->
+            erlang:exit(Reason)
+    after ?ENTRY_SHUTDOWN_TIMEOUT ->
+        erlang:demonitor(Ref, [flush]),
+        erlang:exit({timeout, {entry_shutdown, Pid}})
+    end.
+
+
+open(Pid, Key) ->
+    try
+        Resp = gen_server:call(Pid, open),
+        case Resp of
+            {open_ok, Val} ->
+                Val;
+            {open_error, {T, R, S}} ->
+                erlang:raise(T, R, S)
+        end
+    catch exit:_ ->
+        % Its possible that this process was evicted just
+        % before we tried talking to it. Just fallback
+        % to a standard recovery
+        recover(Key)
+    end.
+
+
+accessed(Pid) ->
+    gen_server:cast(Pid, accessed).
+
+
+refresh(Pid) ->
+    gen_server:cast(Pid, force_refresh).
+
+
+init({Key, undefined}) ->
+    true = ets:update_element(?CACHE, Key, {#entry.pid, self()}),
+    St = #st{
+        key = Key,
+        opener = spawn_opener(Key),
+        waiters = [],
+        accessed = 1
+    },
+    ?EVENT(started, Key),
+    gen_server:enter_loop(?MODULE, [], St);
+
+init({Key, Wrapped}) ->
+    Default = ddoc_cache_value:unwrap(Wrapped),
+    Updates = [
+        {#entry.val, Default},
+        {#entry.pid, self()}
+    ],
+    NewTs = os:timestamp(),
+    true = ets:update_element(?CACHE, Key, Updates),
+    true = ets:insert(?LRU, {{NewTs, Key, self()}}),
+    St = #st{
+        key = Key,
+        val = {open_ok, {ok, Default}},
+        opener = start_timer(),
+        waiters = [],
+        ts = NewTs,
+        accessed = 1
+    },
+    ?EVENT(default_started, Key),
+    gen_server:enter_loop(?MODULE, [], St, hibernate).
+
+
+terminate(_Reason, St) ->
+    #st{
+        key = Key,
+        opener = Pid,
+        ts = Ts
+    } = St,
+    % We may have already deleted our cache entry
+    % during shutdown
+    Pattern = #entry{key = Key, pid = self(), _ = '_'},
+    CacheMSpec = [{Pattern, [], [true]}],
+    true = ets:select_delete(?CACHE, CacheMSpec) < 2,
+    % We may have already deleted our LRU entry
+    % during shutdown
+    if Ts == undefined -> ok; true ->
+        LruMSpec = [{{{Ts, Key, self()}}, [], [true]}],
+        true = ets:select_delete(?LRU, LruMSpec) < 2
+    end,
+    % Blow away any current opener if it exists
+    if not is_pid(Pid) -> ok; true ->
+        catch exit(Pid, kill)
+    end,
+    ok.
+
+
+handle_call(open, From, #st{opener = Pid} = St) when is_pid(Pid) ->
+    NewSt = St#st{
+        waiters = [From | St#st.waiters]
+    },
+    {noreply, NewSt};
+
+handle_call(open, _From, St) ->
+    {reply, St#st.val, St};
+
+handle_call(Msg, _From, St) ->
+    {stop, {bad_call, Msg}, {bad_call, Msg}, St}.
+
+
+handle_cast(accessed, St) ->
+    ?EVENT(accessed, St#st.key),
+    drain_accessed(),
+    NewSt = St#st{
+        accessed = St#st.accessed + 1
+    },
+    {noreply, update_lru(NewSt)};
+
+handle_cast(force_refresh, St) ->
+    % If we had frequent design document updates
+    % they could end up racing accessed events and
+    % end up prematurely evicting this entry from
+    % cache. To prevent this we just make sure that
+    % accessed is set to at least 1 before we
+    % execute a refresh.
+    NewSt = if St#st.accessed > 0 -> St; true ->
+        St#st{accessed = 1}
+    end,
+    % We remove the cache entry value so that any
+    % new client comes to us for the refreshed
+    % value.
+    true = ets:update_element(?CACHE, St#st.key, {#entry.val, undefined}),
+    handle_cast(refresh, NewSt);
+
+handle_cast(refresh, #st{accessed = 0} = St) ->
+    {stop, normal, St};
+
+handle_cast(refresh, #st{opener = Ref} = St) when is_reference(Ref) ->
+    #st{
+        key = Key
+    } = St,
+    erlang:cancel_timer(Ref),
+    NewSt = St#st{
+        opener = spawn_opener(Key),
+        accessed = 0
+    },
+    {noreply, NewSt};
+
+handle_cast(refresh, #st{opener = Pid} = St) when is_pid(Pid) ->
+    catch exit(Pid, kill),
+    receive
+        {'DOWN', _, _, Pid, _} -> ok
+    end,
+    NewSt = St#st{
+        opener = spawn_opener(St#st.key),
+        accessed = 0
+    },
+    {noreply, NewSt};
+
+handle_cast(shutdown, St) ->
+    remove_from_cache(St),
+    {stop, normal, St};
+
+handle_cast(Msg, St) ->
+    {stop, {bad_cast, Msg}, St}.
+
+
+handle_info({'DOWN', _, _, Pid, Resp}, #st{key = Key, opener = Pid} = St) ->
+    case Resp of
+        {open_ok, Key, {ok, Val}} ->
+            update_cache(St, Val),
+            NewSt1 = St#st{
+                val = {open_ok, {ok, Val}},
+                opener = start_timer(),
+                waiters = []
+            },
+            NewSt2 = update_lru(NewSt1),
+            respond(St#st.waiters, {open_ok, {ok, Val}}),
+            {noreply, NewSt2};
+        {Status, Key, Other} ->
+            NewSt = St#st{
+                val = {Status, Other},
+                opener = undefined,
+                waiters = undefined
+            },
+            remove_from_cache(NewSt),
+            respond(St#st.waiters, {Status, Other}),
+            {stop, normal, NewSt}
+    end;
+
+handle_info(Msg, St) ->
+    {stop, {bad_info, Msg}, St}.
+
+
+code_change(_, St, _) ->
+    {ok, St}.
+
+
+spawn_opener(Key) ->
+    {Pid, _} = erlang:spawn_monitor(?MODULE, do_open, [Key]),
+    Pid.
+
+
+start_timer() ->
+    TimeOut = config:get_integer(
+            "ddoc_cache", "refresh_timeout", ?REFRESH_TIMEOUT),
+    erlang:send_after(TimeOut, self(), {'$gen_cast', refresh}).
+
+
+do_open(Key) ->
+    try recover(Key) of
+        Resp ->
+            erlang:exit({open_ok, Key, Resp})
+    catch T:R ->
+        S = erlang:get_stacktrace(),
+        erlang:exit({open_error, Key, {T, R, S}})
+    end.
+
+
+update_lru(#st{key = Key, ts = Ts} = St) ->
+    remove_from_lru(Ts, Key),
+    NewTs = os:timestamp(),
+    true = ets:insert(?LRU, {{NewTs, Key, self()}}),
+    St#st{ts = NewTs}.
+
+
+update_cache(#st{val = undefined} = St, Val) ->
+    true = ets:update_element(?CACHE, St#st.key, {#entry.val, Val}),
+    ?EVENT(inserted, St#st.key);
+
+update_cache(#st{val = V1} = _St, V2) when {open_ok, {ok, V2}} == V1 ->
+    ?EVENT(update_noop, _St#st.key);
+
+update_cache(St, Val) ->
+    true = ets:update_element(?CACHE, St#st.key, {#entry.val, Val}),
+    ?EVENT(updated, {St#st.key, Val}).
+
+
+remove_from_cache(St) ->
+    #st{
+        key = Key,
+        ts = Ts
+    } = St,
+    Pattern = #entry{key = Key, pid = self(), _ = '_'},
+    CacheMSpec = [{Pattern, [], [true]}],
+    1 = ets:select_delete(?CACHE, CacheMSpec),
+    remove_from_lru(Ts, Key),
+    ?EVENT(removed, St#st.key),
+    ok.
+
+
+remove_from_lru(Ts, Key) ->
+    if Ts == undefined -> ok; true ->
+        LruMSpec = [{{{Ts, Key, self()}}, [], [true]}],
+        1 = ets:select_delete(?LRU, LruMSpec)
+    end.
+
+
+drain_accessed() ->
+    receive
+        {'$gen_cast', accessed} ->
+            drain_accessed()
+    after 0 ->
+        ok
+    end.
+
+
+respond(Waiters, Resp) ->
+    [gen_server:reply(W, Resp) || W <- Waiters].
diff --git a/src/ddoc_cache/src/ddoc_cache_util.erl b/src/ddoc_cache/src/ddoc_cache_entry_custom.erl
index 24c4b0d9e..9eaf16f34 100644
--- a/src/ddoc_cache/src/ddoc_cache_util.erl
+++ b/src/ddoc_cache/src/ddoc_cache_entry_custom.erl
@@ -10,25 +10,28 @@
 % License for the specific language governing permissions and limitations under
 % the License.
 
--module(ddoc_cache_util).
+-module(ddoc_cache_entry_custom).
 
 
 -export([
-    new_uuid/0
+    dbname/1,
+    ddocid/1,
+    recover/1,
+    insert/2
 ]).
 
 
-new_uuid() ->
-    to_hex(crypto:strong_rand_bytes(16), []).
+dbname({DbName, _}) ->
+    DbName.
 
 
-to_hex(<<>>, Acc) ->
-    list_to_binary(lists:reverse(Acc));
-to_hex(<<C1:4, C2:4, Rest/binary>>, Acc) ->
-    to_hex(Rest, [hexdig(C1), hexdig(C2) | Acc]).
+ddocid(_) ->
+    no_ddocid.
 
 
-hexdig(C) when C >= 0, C =< 9 ->
-    C + $0;
-hexdig(C) when C >= 10, C =< 15 ->
-    C + $A - 10.
+recover({DbName, Mod}) ->
+    Mod:recover(DbName).
+
+
+insert(_, _) ->
+    ok.
diff --git a/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl b/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl
new file mode 100644
index 000000000..5248469fb
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache_entry_ddocid.erl
@@ -0,0 +1,46 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_entry_ddocid).
+
+
+-export([
+    dbname/1,
+    ddocid/1,
+    recover/1,
+    insert/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+
+
+dbname({DbName, _}) ->
+    DbName.
+
+
+ddocid({_, DDocId}) ->
+    DDocId.
+
+
+recover({DbName, DDocId}) ->
+    fabric:open_doc(DbName, DDocId, [ejson_body, ?ADMIN_CTX]).
+
+
+insert({DbName, DDocId}, {ok, #doc{revs = Revs} = DDoc}) ->
+    {Depth, [RevId | _]} = Revs,
+    Rev = {Depth, RevId},
+    Key = {ddoc_cache_entry_ddocid_rev, {DbName, DDocId, Rev}},
+    spawn(fun() -> ddoc_cache_lru:insert(Key, DDoc) end);
+
+insert(_, _) ->
+    ok.
diff --git a/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl b/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl
new file mode 100644
index 000000000..868fa7789
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache_entry_ddocid_rev.erl
@@ -0,0 +1,47 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_entry_ddocid_rev).
+
+
+-export([
+    dbname/1,
+    ddocid/1,
+    recover/1,
+    insert/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+
+
+dbname({DbName, _, _}) ->
+    DbName.
+
+
+ddocid({_, DDocId, _}) ->
+    DDocId.
+
+
+recover({DbName, DDocId, Rev}) ->
+    Opts = [ejson_body, ?ADMIN_CTX],
+    {ok, [Resp]} = fabric:open_revs(DbName, DDocId, [Rev], Opts),
+    Resp.
+
+
+insert({DbName, DDocId, _Rev}, {ok, #doc{} = DDoc}) ->
+    Key = {ddoc_cache_entry_ddocid, {DbName, DDocId}},
+    spawn(fun() -> ddoc_cache_lru:insert(Key, DDoc) end);
+
+insert(_, _) ->
+    ok.
+
diff --git a/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl b/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl
new file mode 100644
index 000000000..2182dead6
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache_entry_validation_funs.erl
@@ -0,0 +1,44 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_entry_validation_funs).
+
+
+-export([
+    dbname/1,
+    ddocid/1,
+    recover/1,
+    insert/2
+]).
+
+
+dbname(DbName) ->
+    DbName.
+
+
+ddocid(_) ->
+    no_ddocid.
+
+
+recover(DbName) ->
+    {ok, DDocs} = fabric:design_docs(mem3:dbname(DbName)),
+    Funs = lists:flatmap(fun(DDoc) ->
+        case couch_doc:get_validate_doc_fun(DDoc) of
+            nil -> [];
+            Fun -> [Fun]
+        end
+    end, DDocs),
+    {ok, Funs}.
+
+
+insert(_, _) ->
+    ok.
diff --git a/src/ddoc_cache/src/ddoc_cache_lru.erl b/src/ddoc_cache/src/ddoc_cache_lru.erl
new file mode 100644
index 000000000..810cbef63
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache_lru.erl
@@ -0,0 +1,329 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_lru).
+-behaviour(gen_server).
+-vsn(1).
+
+
+-export([
+    start_link/0,
+    open/1,
+    insert/2,
+    refresh/2
+]).
+
+-export([
+    init/1,
+    terminate/2,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    code_change/3
+]).
+
+-export([
+    handle_db_event/3
+]).
+
+
+-include("ddoc_cache.hrl").
+
+
+-record(st, {
+    pids, % pid -> key
+    dbs, % dbname -> docid -> key -> pid
+    evictor
+}).
+
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+
+open(Key) ->
+    try ets:lookup(?CACHE, Key) of
+        [] ->
+            lru_start(Key, true);
+        [#entry{pid = undefined}] ->
+            lru_start(Key, false);
+        [#entry{val = undefined, pid = Pid}] ->
+            couch_stats:increment_counter([ddoc_cache, miss]),
+            ddoc_cache_entry:open(Pid, Key);
+        [#entry{val = Val, pid = Pid}] ->
+            couch_stats:increment_counter([ddoc_cache, hit]),
+            ddoc_cache_entry:accessed(Pid),
+            {ok, Val}
+    catch _:_ ->
+        couch_stats:increment_counter([ddoc_cache, recovery]),
+        ddoc_cache_entry:recover(Key)
+    end.
+
+
+insert(Key, Value) ->
+    case ets:lookup(?CACHE, Key) of
+        [] ->
+            Wrapped = ddoc_cache_value:wrap(Value),
+            gen_server:call(?MODULE, {start, Key, Wrapped}, infinity);
+        [#entry{}] ->
+            ok
+    end.
+
+
+refresh(DbName, DDocIds) ->
+    gen_server:cast(?MODULE, {refresh, DbName, DDocIds}).
+
+
+init(_) ->
+    process_flag(trap_exit, true),
+    BaseOpts = [public, named_table],
+    CacheOpts = [
+        set,
+        {read_concurrency, true},
+        {keypos, #entry.key}
+    ] ++ BaseOpts,
+    ets:new(?CACHE, CacheOpts),
+    ets:new(?LRU, [ordered_set, {write_concurrency, true}] ++ BaseOpts),
+    {ok, Pids} = khash:new(),
+    {ok, Dbs} = khash:new(),
+    {ok, Evictor} = couch_event:link_listener(
+            ?MODULE, handle_db_event, nil, [all_dbs]
+        ),
+    ?EVENT(lru_init, nil),
+    {ok, #st{
+        pids = Pids,
+        dbs = Dbs,
+        evictor = Evictor
+    }}.
+
+
+terminate(_Reason, St) ->
+    case is_pid(St#st.evictor) of
+        true -> exit(St#st.evictor, kill);
+        false -> ok
+    end,
+    ok.
+
+
+handle_call({start, Key, Default}, _From, St) ->
+    #st{
+        pids = Pids,
+        dbs = Dbs
+    } = St,
+    case ets:lookup(?CACHE, Key) of
+        [] ->
+            MaxSize = config:get_integer("ddoc_cache", "max_size", 104857600),
+            case trim(St, max(0, MaxSize)) of
+                ok ->
+                    true = ets:insert_new(?CACHE, #entry{key = Key}),
+                    {ok, Pid} = ddoc_cache_entry:start_link(Key, Default),
+                    true = ets:update_element(?CACHE, Key, {#entry.pid, Pid}),
+                    ok = khash:put(Pids, Pid, Key),
+                    store_key(Dbs, Key, Pid),
+                    {reply, {ok, Pid}, St};
+                full ->
+                    ?EVENT(full, Key),
+                    {reply, full, St}
+            end;
+        [#entry{pid = Pid}] ->
+            {reply, {ok, Pid}, St}
+    end;
+
+handle_call(Msg, _From, St) ->
+    {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}.
+
+
+handle_cast({evict, DbName}, St) ->
+    gen_server:abcast(mem3:nodes(), ?MODULE, {do_evict, DbName}),
+    {noreply, St};
+
+handle_cast({refresh, DbName, DDocIds}, St) ->
+    gen_server:abcast(mem3:nodes(), ?MODULE, {do_refresh, DbName, DDocIds}),
+    {noreply, St};
+
+handle_cast({do_evict, DbName}, St) ->
+    #st{
+        dbs = Dbs
+    } = St,
+    ToRem = case khash:lookup(Dbs, DbName) of
+        {value, DDocIds} ->
+            AccOut = khash:fold(DDocIds, fun(_, Keys, Acc1) ->
+                khash:to_list(Keys) ++ Acc1
+            end, []),
+            ?EVENT(evicted, DbName),
+            AccOut;
+        not_found ->
+            ?EVENT(evict_noop, DbName),
+            []
+    end,
+    lists:foreach(fun({Key, Pid}) ->
+        remove_entry(St, Key, Pid)
+    end, ToRem),
+    khash:del(Dbs, DbName),
+    {noreply, St};
+
+handle_cast({do_refresh, DbName, DDocIdList}, St) ->
+    #st{
+        dbs = Dbs
+    } = St,
+    % We prepend no_ddocid to the DDocIdList below
+    % so that we refresh all custom and validation
+    % function entries which load data from all
+    % design documents.
+    case khash:lookup(Dbs, DbName) of
+        {value, DDocIds} ->
+            lists:foreach(fun(DDocId) ->
+                case khash:lookup(DDocIds, DDocId) of
+                    {value, Keys} ->
+                        khash:fold(Keys, fun(_, Pid, _) ->
+                            ddoc_cache_entry:refresh(Pid)
+                        end, nil);
+                    not_found ->
+                        ok
+                end
+            end, [no_ddocid | DDocIdList]);
+        not_found ->
+            ok
+    end,
+    {noreply, St};
+
+handle_cast(Msg, St) ->
+    {stop, {invalid_cast, Msg}, St}.
+
+
+handle_info({'EXIT', Pid, Reason}, #st{evictor = Pid} = St) ->
+    {stop, Reason, St};
+
+handle_info({'EXIT', Pid, normal}, St) ->
+    % This clause handles when an entry starts
+    % up but encounters an error or uncacheable
+    % response from its recover call.
+    #st{
+        pids = Pids
+    } = St,
+    {value, Key} = khash:lookup(Pids, Pid),
+    khash:del(Pids, Pid),
+    remove_key(St, Key),
+    {noreply, St};
+
+handle_info(Msg, St) ->
+    {stop, {invalid_info, Msg}, St}.
+
+
+code_change(_OldVsn, St, _Extra) ->
+    {ok, St}.
+
+
+handle_db_event(ShardDbName, created, St) ->
+    gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
+    {ok, St};
+
+handle_db_event(ShardDbName, deleted, St) ->
+    gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
+    {ok, St};
+
+handle_db_event(_DbName, _Event, St) ->
+    {ok, St}.
+
+
+lru_start(Key, DoInsert) ->
+    case gen_server:call(?MODULE, {start, Key, undefined}, infinity) of
+        {ok, Pid} ->
+            couch_stats:increment_counter([ddoc_cache, miss]),
+            Resp = ddoc_cache_entry:open(Pid, Key),
+            if not DoInsert -> ok; true ->
+                ddoc_cache_entry:insert(Key, Resp)
+            end,
+            Resp;
+        full ->
+            couch_stats:increment_counter([ddoc_cache, recovery]),
+            ddoc_cache_entry:recover(Key)
+    end.
+
+
+trim(_, 0) ->
+    full;
+
+trim(St, MaxSize) ->
+    CurSize = ets:info(?CACHE, memory) * erlang:system_info(wordsize),
+    if CurSize =< MaxSize -> ok; true ->
+        case ets:first(?LRU) of
+            {_Ts, Key, Pid} ->
+                remove_entry(St, Key, Pid),
+                trim(St, MaxSize);
+            '$end_of_table' ->
+                full
+        end
+    end.
+
+
+remove_entry(St, Key, Pid) ->
+    #st{
+        pids = Pids
+    } = St,
+    unlink_and_flush(Pid),
+    ddoc_cache_entry:shutdown(Pid),
+    khash:del(Pids, Pid),
+    remove_key(St, Key).
+
+
+store_key(Dbs, Key, Pid) ->
+    DbName = ddoc_cache_entry:dbname(Key),
+    DDocId = ddoc_cache_entry:ddocid(Key),
+    case khash:lookup(Dbs, DbName) of
+        {value, DDocIds} ->
+            case khash:lookup(DDocIds, DDocId) of
+                {value, Keys} ->
+                    khash:put(Keys, Key, Pid);
+                not_found ->
+                    {ok, Keys} = khash:from_list([{Key, Pid}]),
+                    khash:put(DDocIds, DDocId, Keys)
+            end;
+        not_found ->
+            {ok, Keys} = khash:from_list([{Key, Pid}]),
+            {ok, DDocIds} = khash:from_list([{DDocId, Keys}]),
+            khash:put(Dbs, DbName, DDocIds)
+    end.
+
+
+remove_key(St, Key) ->
+    #st{
+        dbs = Dbs
+    } = St,
+    DbName = ddoc_cache_entry:dbname(Key),
+    DDocId = ddoc_cache_entry:ddocid(Key),
+    {value, DDocIds} = khash:lookup(Dbs, DbName),
+    {value, Keys} = khash:lookup(DDocIds, DDocId),
+    khash:del(Keys, Key),
+    case khash:size(Keys) of
+        0 -> khash:del(DDocIds, DDocId);
+        _ -> ok
+    end,
+    case khash:size(DDocIds) of
+        0 -> khash:del(Dbs, DbName);
+        _ -> ok
+    end.
+
+
+unlink_and_flush(Pid) ->
+    erlang:unlink(Pid),
+    % Its possible that the entry process has already exited before
+    % we unlink it so we have to flush out a possible 'EXIT'
+    % message sitting in our message queue. Notice that we're
+    % maintaining the assertion that all entry processes only
+    % ever exit normally.
+    receive
+        {'EXIT', Pid, normal} ->
+            ok
+    after 0 ->
+        ok
+    end.
diff --git a/src/ddoc_cache/src/ddoc_cache_opener.erl b/src/ddoc_cache/src/ddoc_cache_opener.erl
index b76a228b0..52de54217 100644
--- a/src/ddoc_cache/src/ddoc_cache_opener.erl
+++ b/src/ddoc_cache/src/ddoc_cache_opener.erl
@@ -14,279 +14,53 @@
 -behaviour(gen_server).
 -vsn(1).
 
--include_lib("couch/include/couch_db.hrl").
--include_lib("mem3/include/mem3.hrl").
 
 -export([
     start_link/0
 ]).
+
 -export([
     init/1,
     terminate/2,
-
     handle_call/3,
     handle_cast/2,
     handle_info/2,
-
     code_change/3
 ]).
 
--export([
-    open_doc/2,
-    open_doc/3,
-    open_validation_funs/1,
-    evict_docs/2,
-    lookup/1,
-    match_newest/1,
-    recover_doc/2,
-    recover_doc/3,
-    recover_validation_funs/1
-]).
--export([
-    handle_db_event/3
-]).
--export([
-    fetch_doc_data/1
-]).
-
--define(CACHE, ddoc_cache_lru).
--define(OPENING, ddoc_cache_opening).
-
--type dbname() :: iodata().
--type docid() :: iodata().
--type doc_hash() :: <<_:128>>.
--type revision() :: {pos_integer(), doc_hash()}.
-
--record(opener, {
-    key,
-    pid,
-    clients
-}).
-
--record(st, {
-    db_ddocs,
-    evictor
-}).
 
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 
--spec open_doc(dbname(), docid()) -> {ok, #doc{}}.
-open_doc(DbName, DocId) ->
-    Resp = gen_server:call(?MODULE, {open, {DbName, DocId}}, infinity),
-    handle_open_response(Resp).
-
--spec open_doc(dbname(), docid(), revision()) -> {ok, #doc{}}.
-open_doc(DbName, DocId, Rev) ->
-    Resp = gen_server:call(?MODULE, {open, {DbName, DocId, Rev}}, infinity),
-    handle_open_response(Resp).
-
--spec open_validation_funs(dbname()) -> {ok, [fun()]}.
-open_validation_funs(DbName) ->
-    Resp = gen_server:call(?MODULE, {open, {DbName, validation_funs}}, infinity),
-    handle_open_response(Resp).
-
--spec evict_docs(dbname(), [docid()]) -> ok.
-evict_docs(DbName, DocIds) ->
-    gen_server:cast(?MODULE, {evict, DbName, DocIds}).
-
-lookup(Key) ->
-    try ets_lru:lookup_d(?CACHE, Key) of
-        {ok, _} = Resp ->
-            Resp;
-        _ ->
-            missing
-    catch
-        error:badarg ->
-            recover
-    end.
-
-match_newest(Key) ->
-    try ets_lru:match_object(?CACHE, Key, '_') of
-        [] ->
-            missing;
-        Docs ->
-            Sorted = lists:sort(
-                fun (#doc{deleted=DelL, revs=L}, #doc{deleted=DelR, revs=R}) ->
-                    {not DelL, L} > {not DelR, R}
-                end, Docs),
-            {ok, hd(Sorted)}
-    catch
-        error:badarg ->
-            recover
-    end.
-
-recover_doc(DbName, DDocId) ->
-    fabric:open_doc(DbName, DDocId, [ejson_body, ?ADMIN_CTX]).
-
-recover_doc(DbName, DDocId, Rev) ->
-    {ok, [Resp]} = fabric:open_revs(DbName, DDocId, [Rev], [ejson_body, ?ADMIN_CTX]),
-    Resp.
-
-recover_validation_funs(DbName) ->
-    {ok, DDocs} = fabric:design_docs(mem3:dbname(DbName)),
-    Funs = lists:flatmap(fun(DDoc) ->
-        case couch_doc:get_validate_doc_fun(DDoc) of
-            nil -> [];
-            Fun -> [Fun]
-        end
-    end, DDocs),
-    {ok, Funs}.
-
-handle_db_event(ShardDbName, created, St) ->
-    gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
-    {ok, St};
-handle_db_event(ShardDbName, deleted, St) ->
-    gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
-    {ok, St};
-handle_db_event(_DbName, _Event, St) ->
-    {ok, St}.
 
 init(_) ->
-    process_flag(trap_exit, true),
-    _ = ets:new(?OPENING, [set, protected, named_table, {keypos, #opener.key}]),
-    {ok, Evictor} = couch_event:link_listener(
-            ?MODULE, handle_db_event, nil, [all_dbs]
-        ),
-    {ok, #st{
-        evictor = Evictor
-    }}.
+    {ok, nil}.
 
-terminate(_Reason, St) ->
-    case is_pid(St#st.evictor) of
-        true -> exit(St#st.evictor, kill);
-        false -> ok
-    end,
+terminate(_Reason, _St) ->
     ok.
 
-handle_call({open, OpenerKey}, From, St) ->
-    case ets:lookup(?OPENING, OpenerKey) of
-        [#opener{clients=Clients}=O] ->
-            ets:insert(?OPENING, O#opener{clients=[From | Clients]}),
-            {noreply, St};
-        [] ->
-            Pid = spawn_link(?MODULE, fetch_doc_data, [OpenerKey]),
-            ets:insert(?OPENING, #opener{key=OpenerKey, pid=Pid, clients=[From]}),
-            {noreply, St}
-    end;
 
 handle_call(Msg, _From, St) ->
     {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}.
 
 
-handle_cast({evict, DbName}, St) ->
-    gen_server:abcast(mem3:nodes(), ?MODULE, {do_evict, DbName}),
+% The do_evict clauses are upgrades while we're
+% in a rolling reboot.
+handle_cast({do_evict, _} = Msg, St) ->
+    gen_server:cast(ddoc_cache_lru, Msg),
     {noreply, St};
 
-handle_cast({evict, DbName, DDocIds}, St) ->
-    gen_server:abcast(mem3:nodes(), ?MODULE, {do_evict, DbName, DDocIds}),
-    {noreply, St};
-
-handle_cast({do_evict, DbName}, St) ->
-    DDocIds = lists:flatten(ets_lru:match(?CACHE, {DbName, '$1', '_'}, '_')),
-    handle_cast({do_evict, DbName, DDocIds}, St);
-
 handle_cast({do_evict, DbName, DDocIds}, St) ->
-    CustomKeys = lists:flatten(ets_lru:match(?CACHE, {DbName, '$1'}, '_')),
-    lists:foreach(fun(Mod) ->
-        ets_lru:remove(?CACHE, {DbName, Mod})
-    end, CustomKeys),
-    lists:foreach(fun(DDocId) ->
-        Revs = ets_lru:match(?CACHE, {DbName, DDocId, '$1'}, '_'),
-        lists:foreach(fun([Rev]) ->
-            ets_lru:remove(?CACHE, {DbName, DDocId, Rev})
-        end, Revs)
-    end, DDocIds),
+    gen_server:cast(ddoc_cache_lru, {do_refresh, DbName, DDocIds}),
     {noreply, St};
 
 handle_cast(Msg, St) ->
     {stop, {invalid_cast, Msg}, St}.
 
-handle_info({'EXIT', Pid, Reason}, #st{evictor=Pid}=St) ->
-    couch_log:error("ddoc_cache_opener evictor died ~w", [Reason]),
-    {ok, Evictor} = couch_event:link_listener(?MODULE, handle_db_event, nil, [all_dbs]),
-    {noreply, St#st{evictor=Evictor}};
-
-handle_info({'EXIT', _Pid, {open_ok, OpenerKey, Resp}}, St) ->
-    respond(OpenerKey, {open_ok, Resp}),
-    {noreply, St};
-
-handle_info({'EXIT', _Pid, {open_error, OpenerKey, Type, Error}}, St) ->
-    respond(OpenerKey, {open_error, Type, Error}),
-    {noreply, St};
-
-handle_info({'EXIT', Pid, Reason}, St) ->
-    Pattern = #opener{pid=Pid, _='_'},
-    case ets:match_object(?OPENING, Pattern) of
-        [#opener{key=OpenerKey, clients=Clients}] ->
-            _ = [gen_server:reply(C, {error, Reason}) || C <- Clients],
-            ets:delete(?OPENING, OpenerKey),
-            {noreply, St};
-        [] ->
-            {stop, {unknown_pid_died, {Pid, Reason}}, St}
-    end;
 
 handle_info(Msg, St) ->
     {stop, {invalid_info, Msg}, St}.
 
+
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
-
--spec fetch_doc_data({dbname(), validation_funs}) -> no_return();
-                    ({dbname(), atom()}) -> no_return();
-                    ({dbname(), docid()}) -> no_return();
-                    ({dbname(), docid(), revision()}) -> no_return().
-fetch_doc_data({DbName, validation_funs}=OpenerKey) ->
-    {ok, Funs} = recover_validation_funs(DbName),
-    ok = ets_lru:insert(?CACHE, OpenerKey, Funs),
-    exit({open_ok, OpenerKey, {ok, Funs}});
-fetch_doc_data({DbName, Mod}=OpenerKey) when is_atom(Mod) ->
-    % This is not actually a docid but rather a custom cache key.
-    % Treat the argument as a code module and invoke its recover function.
-    try Mod:recover(DbName) of
-        {ok, Result} ->
-            ok = ets_lru:insert(?CACHE, OpenerKey, Result),
-            exit({open_ok, OpenerKey, {ok, Result}});
-        Else ->
-            exit({open_ok, OpenerKey, Else})
-    catch
-        Type:Reason ->
-            exit({open_error, OpenerKey, Type, Reason})
-    end;
-fetch_doc_data({DbName, DocId}=OpenerKey) ->
-    try recover_doc(DbName, DocId) of
-        {ok, Doc} ->
-            {RevDepth, [RevHash| _]} = Doc#doc.revs,
-            Rev = {RevDepth, RevHash},
-            ok = ets_lru:insert(?CACHE, {DbName, DocId, Rev}, Doc),
-            exit({open_ok, OpenerKey, {ok, Doc}});
-        Else ->
-            exit({open_ok, OpenerKey, Else})
-    catch
-        Type:Reason ->
-            exit({open_error, OpenerKey, Type, Reason})
-    end;
-fetch_doc_data({DbName, DocId, Rev}=OpenerKey) ->
-    try recover_doc(DbName, DocId, Rev) of
-        {ok, Doc} ->
-            ok = ets_lru:insert(?CACHE, {DbName, DocId, Rev}, Doc),
-            exit({open_ok, OpenerKey, {ok, Doc}});
-        Else ->
-            exit({open_ok, OpenerKey, Else})
-    catch
-        Type:Reason ->
-            exit({open_error, OpenerKey, Type, Reason})
-    end.
-
-handle_open_response(Resp) ->
-    case Resp of
-        {open_ok, Value} -> Value;
-        {open_error, throw, Error} -> throw(Error);
-        {open_error, error, Error} -> erlang:error(Error);
-        {open_error, exit, Error} -> exit(Error)
-    end.
-
-respond(OpenerKey, Resp) ->
-    [#opener{clients=Clients}] = ets:lookup(?OPENING, OpenerKey),
-    _ = [gen_server:reply(C, Resp) || C <- Clients],
-    ets:delete(?OPENING, OpenerKey).
diff --git a/src/ddoc_cache/src/ddoc_cache_sup.erl b/src/ddoc_cache/src/ddoc_cache_sup.erl
index 85e90b3c5..6fff9ef4f 100644
--- a/src/ddoc_cache/src/ddoc_cache_sup.erl
+++ b/src/ddoc_cache/src/ddoc_cache_sup.erl
@@ -28,11 +28,11 @@ init([]) ->
     Children = [
         {
             ddoc_cache_lru,
-            {ets_lru, start_link, [ddoc_cache_lru, lru_opts()]},
+            {ddoc_cache_lru, start_link, []},
             permanent,
             5000,
             worker,
-            [ets_lru]
+            [ddoc_cache_lru]
         },
         {
             ddoc_cache_opener,
@@ -43,25 +43,4 @@ init([]) ->
             [ddoc_cache_opener]
         }
     ],
-    {ok, {{one_for_one, 5, 10}, Children}}.
-
-
-lru_opts() ->
-    case application:get_env(ddoc_cache, max_objects) of
-        {ok, MxObjs} when is_integer(MxObjs), MxObjs >= 0 ->
-            [{max_objects, MxObjs}];
-        _ ->
-            []
-    end ++
-    case application:get_env(ddoc_cache, max_size) of
-        {ok, MxSize} when is_integer(MxSize), MxSize >= 0 ->
-            [{max_size, MxSize}];
-        _ ->
-            []
-    end ++
-    case application:get_env(ddoc_cache, max_lifetime) of
-        {ok, MxLT} when is_integer(MxLT), MxLT >= 0 ->
-            [{max_lifetime, MxLT}];
-        _ ->
-            []
-    end.
+    {ok, {{one_for_one, 25, 1}, Children}}.
diff --git a/src/ddoc_cache/src/ddoc_cache_value.erl b/src/ddoc_cache/src/ddoc_cache_value.erl
new file mode 100644
index 000000000..21a5bb549
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache_value.erl
@@ -0,0 +1,27 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_value).
+
+
+-export([
+    wrap/1,
+    unwrap/1
+]).
+
+
+wrap(Value) ->
+    {?MODULE, term_to_binary(Value)}.
+
+
+unwrap({?MODULE, Bin}) when is_binary(Bin) ->
+    binary_to_term(Bin).
diff --git a/src/ddoc_cache/test/ddoc_cache_basic_test.erl b/src/ddoc_cache/test/ddoc_cache_basic_test.erl
new file mode 100644
index 000000000..7f6dbc9a4
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_basic_test.erl
@@ -0,0 +1,175 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_basic_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(DbName) ->
+    {ok, {DbName, totes_custom}}.
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:new(ddoc_cache_ev, [passthrough]),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_basic_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun cache_ddoc/1,
+            fun cache_ddoc_rev/1,
+            fun cache_vdu/1,
+            fun cache_custom/1,
+            fun cache_ddoc_refresher_unchanged/1,
+            fun dont_cache_not_found/1,
+            fun deprecated_api_works/1
+        ]}
+    }.
+
+
+check_no_vdu_test_() ->
+    {
+        setup,
+        fun() -> ddoc_cache_tutil:start_couch([{write_ddocs, false}]) end,
+        fun ddoc_cache_tutil:stop_couch/1,
+        {with, [
+            fun cache_no_vdu_no_ddoc/1,
+            fun cache_no_vdu_empty_ddoc/1
+        ]}
+    }.
+
+
+cache_ddoc({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    Resp1 = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    ?assertMatch({ok, #doc{id = ?FOOBAR}}, Resp1),
+    meck:wait(ddoc_cache_ev, event, [started, '_'], 1000),
+    meck:wait(ddoc_cache_ev, event, [default_started, '_'], 1000),
+    ?assertEqual(2, ets:info(?CACHE, size)),
+    Resp2 = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    ?assertEqual(Resp1, Resp2),
+    ?assertEqual(2, ets:info(?CACHE, size)).
+
+
+cache_ddoc_rev({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?FOOBAR),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    Resp1 = ddoc_cache:open_doc(DbName, ?FOOBAR, Rev),
+    ?assertMatch({ok, #doc{id = ?FOOBAR}}, Resp1),
+    meck:wait(ddoc_cache_ev, event, [started, '_'], 1000),
+    meck:wait(ddoc_cache_ev, event, [default_started, '_'], 1000),
+    ?assertEqual(2, ets:info(?CACHE, size)),
+    Resp2 = ddoc_cache:open_doc(DbName, ?FOOBAR, Rev),
+    ?assertEqual(Resp1, Resp2),
+    ?assertEqual(2, ets:info(?CACHE, size)),
+
+    % Assert that the non-rev cache entry is separate
+    Resp3 = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    ?assertMatch({ok, #doc{id = ?FOOBAR}}, Resp3),
+    ?assertEqual(2, ets:info(?CACHE, size)).
+
+
+cache_vdu({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    Resp1 = ddoc_cache:open_validation_funs(DbName),
+    ?assertMatch({ok, [_]}, Resp1),
+    ?assertEqual(1, ets:info(?CACHE, size)),
+    Resp2 = ddoc_cache:open_validation_funs(DbName),
+    ?assertEqual(Resp1, Resp2),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+
+cache_custom({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    Resp1 = ddoc_cache:open_custom(DbName, ?MODULE),
+    ?assertMatch({ok, {DbName, totes_custom}}, Resp1),
+    ?assertEqual(1, ets:info(?CACHE, size)),
+    Resp2 = ddoc_cache:open_custom(DbName, ?MODULE),
+    ?assertEqual(Resp1, Resp2),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+
+cache_ddoc_refresher_unchanged({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    ddoc_cache:open_doc(DbName, ?FOOBAR),
+    meck:wait(ddoc_cache_ev, event, [started, '_'], 1000),
+    meck:wait(ddoc_cache_ev, event, [default_started, '_'], 1000),
+    Tab1 = [_, _] = lists:sort(ets:tab2list(?CACHE)),
+    ddoc_cache:open_doc(DbName, ?FOOBAR),
+    meck:wait(ddoc_cache_ev, event, [accessed, '_'], 1000),
+    Tab2 = lists:sort(ets:tab2list(?CACHE)),
+    ?assertEqual(Tab2, Tab1).
+
+
+dont_cache_not_found({DbName, _}) ->
+    DDocId = <<"_design/not_found">>,
+    ddoc_cache_tutil:clear(),
+    Resp = ddoc_cache:open_doc(DbName, DDocId),
+    ?assertEqual({not_found, missing}, Resp),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    ?assertEqual(0, ets:info(?LRU, size)).
+
+
+deprecated_api_works({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    {ok, _} = ddoc_cache:open(DbName, ?FOOBAR),
+    {ok, _} = ddoc_cache:open(DbName, <<"foobar">>),
+    {ok, _} = ddoc_cache:open(DbName, ?MODULE),
+    {ok, _} = ddoc_cache:open(DbName, validation_funs).
+
+
+cache_no_vdu_no_ddoc({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    Resp = ddoc_cache:open_validation_funs(DbName),
+    ?assertEqual({ok, []}, Resp),
+    ?assertEqual(1, ets:info(?CACHE, size)),
+    ?assertEqual(1, ets:info(?LRU, size)).
+
+
+cache_no_vdu_empty_ddoc({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    DDoc = #doc{
+        id = <<"_design/no_vdu">>,
+        body = {[]}
+    },
+    {ok, _} = fabric:update_docs(DbName, [DDoc], [?ADMIN_CTX]),
+    Resp = ddoc_cache:open_validation_funs(DbName),
+    ?assertEqual({ok, []}, Resp),
+    ?assertEqual(1, ets:info(?CACHE, size)),
+    ?assertEqual(1, ets:info(?LRU, size)).
diff --git a/src/ddoc_cache/test/ddoc_cache_coverage_test.erl b/src/ddoc_cache/test/ddoc_cache_coverage_test.erl
new file mode 100644
index 000000000..b1a185bdc
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_coverage_test.erl
@@ -0,0 +1,77 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_coverage_test).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+coverage_test_() ->
+    {
+        setup,
+        fun ddoc_cache_tutil:start_couch/0,
+        fun ddoc_cache_tutil:stop_couch/1,
+        [
+            fun restart_lru/0,
+            fun stop_on_evictor_death/0
+        ]
+    }.
+
+
+restart_lru() ->
+    send_bad_messages(ddoc_cache_lru),
+    ?assertEqual(ok, ddoc_cache_lru:terminate(bang, {st, a, b, c})),
+    ?assertEqual({ok, foo}, ddoc_cache_lru:code_change(1, foo, [])).
+
+
+stop_on_evictor_death() ->
+    meck:new(ddoc_cache_ev, [passthrough]),
+    try
+        Lru = whereis(ddoc_cache_lru),
+        State = sys:get_state(Lru),
+        Evictor = element(4, State),
+        Ref = erlang:monitor(process, Lru),
+        exit(Evictor, shutdown),
+        receive
+            {'DOWN', Ref, _, _, Reason} ->
+                ?assertEqual(shutdown, Reason)
+        end,
+        meck:wait(ddoc_cache_ev, event, [lru_init, '_'], 1000),
+        ?assert(whereis(ddoc_cache_lru) /= Lru)
+    after
+        meck:unload()
+    end.
+
+
+send_bad_messages(Name) ->
+    wait_for_restart(Name, fun() ->
+        ?assertEqual({invalid_call, foo}, gen_server:call(Name, foo))
+    end),
+    wait_for_restart(Name, fun() ->
+        gen_server:cast(Name, foo)
+    end),
+    wait_for_restart(Name, fun() ->
+        whereis(Name) ! foo
+    end).
+
+
+wait_for_restart(Server, Fun) ->
+    Ref = erlang:monitor(process, whereis(Server)),
+    Fun(),
+    receive
+        {'DOWN', Ref, _, _, _} ->
+            ok
+    end,
+    ?assert(is_pid(test_util:wait_process(Server))).
diff --git a/src/ddoc_cache/test/ddoc_cache_disabled_test.erl b/src/ddoc_cache/test/ddoc_cache_disabled_test.erl
new file mode 100644
index 000000000..bfc08002d
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_disabled_test.erl
@@ -0,0 +1,62 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_disabled_test).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    config:set("ddoc_cache", "max_size", "0", false),
+    Ctx.
+
+
+check_disabled_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun ddoc_cache_tutil:stop_couch/1,
+        {with, [
+            fun resp_ok/1,
+            fun resp_not_found/1,
+            fun check_effectively_disabled/1
+        ]}
+    }.
+
+
+resp_ok({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    Resp = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    ?assertMatch({ok, #doc{id = ?FOOBAR}}, Resp),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    ?assertEqual(0, ets:info(?LRU, size)).
+
+
+resp_not_found({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    Resp = ddoc_cache:open_doc(DbName, <<"_design/not_found">>),
+    ?assertEqual({not_found, missing}, Resp),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    ?assertEqual(0, ets:info(?LRU, size)).
+
+
+check_effectively_disabled({DbName, _}) ->
+    config:set("ddoc_cache", "max_size", "1", false),
+    ddoc_cache_tutil:clear(),
+    Resp = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    ?assertMatch({ok, #doc{id = ?FOOBAR}}, Resp),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    ?assertEqual(0, ets:info(?LRU, size)).
diff --git a/src/ddoc_cache/test/ddoc_cache_entry_test.erl b/src/ddoc_cache/test/ddoc_cache_entry_test.erl
new file mode 100644
index 000000000..dd7a039ec
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_entry_test.erl
@@ -0,0 +1,159 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_entry_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(<<"foo">>) ->
+    timer:sleep(30000);
+
+recover(DbName) ->
+    {ok, {DbName, such_custom}}.
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:new(ddoc_cache_ev, [passthrough]),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_entry_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun cancel_and_replace_opener/1,
+            fun condenses_access_messages/1,
+            fun kill_opener_on_terminate/1,
+            fun evict_when_not_accessed/1,
+            fun open_dead_entry/1,
+            fun handles_bad_messages/1,
+            fun handles_code_change/1
+        ]}
+    }.
+
+
+cancel_and_replace_opener(_) ->
+    Key = {ddoc_cache_entry_custom, {<<"foo">>, ?MODULE}},
+    true = ets:insert_new(?CACHE, #entry{key = Key}),
+    {ok, Entry} = ddoc_cache_entry:start_link(Key, undefined),
+    Opener1 = element(4, sys:get_state(Entry)),
+    Ref1 = erlang:monitor(process, Opener1),
+    gen_server:cast(Entry, force_refresh),
+    receive {'DOWN', Ref1, _, _, _} -> ok end,
+    Opener2 = element(4, sys:get_state(Entry)),
+    ?assert(Opener2 /= Opener1),
+    ?assert(is_process_alive(Opener2)),
+    % Clean up after ourselves
+    unlink(Entry),
+    ddoc_cache_entry:shutdown(Entry).
+
+
+condenses_access_messages({DbName, _}) ->
+    meck:reset(ddoc_cache_ev),
+    Key = {ddoc_cache_entry_custom, {DbName, ?MODULE}},
+    true = ets:insert(?CACHE, #entry{key = Key}),
+    {ok, Entry} = ddoc_cache_entry:start_link(Key, undefined),
+    erlang:suspend_process(Entry),
+    lists:foreach(fun(_) ->
+        gen_server:cast(Entry, accessed)
+    end, lists:seq(1, 100)),
+    erlang:resume_process(Entry),
+    meck:wait(1, ddoc_cache_ev, event, [accessed, Key], 1000),
+    ?assertError(
+            timeout,
+            meck:wait(2, ddoc_cache_ev, event, [accessed, Key], 100)
+        ),
+    unlink(Entry),
+    ddoc_cache_entry:shutdown(Entry).
+
+
+kill_opener_on_terminate(_) ->
+    Pid = spawn(fun() -> receive _ -> ok end end),
+    ?assert(is_process_alive(Pid)),
+    St = {st, key, val, Pid, waiters, ts, accessed},
+    ?assertEqual(ok, ddoc_cache_entry:terminate(normal, St)),
+    ?assert(not is_process_alive(Pid)).
+
+
+evict_when_not_accessed(_) ->
+    meck:reset(ddoc_cache_ev),
+    Key = {ddoc_cache_entry_custom, {<<"bar">>, ?MODULE}},
+    true = ets:insert_new(?CACHE, #entry{key = Key}),
+    {ok, Entry} = ddoc_cache_entry:start_link(Key, undefined),
+    Ref = erlang:monitor(process, Entry),
+    AccessCount1 = element(7, sys:get_state(Entry)),
+    ?assertEqual(1, AccessCount1),
+    ok = gen_server:cast(Entry, refresh),
+
+    meck:wait(ddoc_cache_ev, event, [update_noop, Key], 1000),
+
+    AccessCount2 = element(7, sys:get_state(Entry)),
+    ?assertEqual(0, AccessCount2),
+    ok = gen_server:cast(Entry, refresh),
+    receive {'DOWN', Ref, _, _, Reason} -> Reason end,
+    ?assertEqual(normal, Reason),
+    ?assertEqual(0, ets:info(?CACHE, size)).
+
+
+open_dead_entry({DbName, _}) ->
+    Pid = spawn(fun() -> ok end),
+    Key = {ddoc_cache_entry_custom, {DbName, ?MODULE}},
+    ?assertEqual(recover(DbName), ddoc_cache_entry:open(Pid, Key)).
+
+
+handles_bad_messages(_) ->
+    CallExpect = {stop, {bad_call, foo}, {bad_call, foo}, baz},
+    CastExpect = {stop, {bad_cast, foo}, bar},
+    InfoExpect = {stop, {bad_info, foo}, bar},
+    ?assertEqual(CallExpect, ddoc_cache_entry:handle_call(foo, bar, baz)),
+    ?assertEqual(CastExpect, ddoc_cache_entry:handle_cast(foo, bar)),
+    ?assertEqual(InfoExpect, ddoc_cache_entry:handle_info(foo, bar)).
+
+
+handles_code_change(_) ->
+    CCExpect = {ok, bar},
+    ?assertEqual(CCExpect, ddoc_cache_entry:code_change(foo, bar, baz)).
+
+
+handles_bad_shutdown_test_() ->
+    {timeout, 10, ?_test(begin
+        ErrorPid = spawn(fun() ->
+            receive
+               _ -> exit(bad_shutdown)
+            end
+        end),
+        ?assertExit(bad_shutdown, ddoc_cache_entry:shutdown(ErrorPid)),
+        NotDeadYetPid = spawn(fun() ->
+            timer:sleep(infinity)
+        end),
+        ?assertExit(
+                {timeout, {entry_shutdown, NotDeadYetPid}},
+                ddoc_cache_entry:shutdown(NotDeadYetPid)
+            )
+    end)}.
diff --git a/src/ddoc_cache/test/ddoc_cache_ev.erl b/src/ddoc_cache/test/ddoc_cache_ev.erl
new file mode 100644
index 000000000..a451342cf
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_ev.erl
@@ -0,0 +1,21 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_ev).
+
+-export([
+    event/2
+]).
+
+
+event(Name, Arg) ->
+    couch_log:error("~s :: ~s :: ~p", [?MODULE, Name, Arg]).
diff --git a/src/ddoc_cache/test/ddoc_cache_eviction_test.erl b/src/ddoc_cache/test/ddoc_cache_eviction_test.erl
new file mode 100644
index 000000000..5a02a5c12
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_eviction_test.erl
@@ -0,0 +1,96 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_eviction_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("mem3/include/mem3.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(DbName) ->
+    {ok, {DbName, totes_custom}}.
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:new(ddoc_cache_ev, [passthrough]),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_eviction_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun evict_all/1,
+            fun dont_evict_all_unrelated/1,
+            fun check_upgrade_clause/1
+        ]}
+    }.
+
+
+evict_all({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?FOOBAR),
+    #shard{name = ShardName} = hd(mem3:shards(DbName)),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR, Rev),
+    {ok, _} = ddoc_cache:open_validation_funs(DbName),
+    {ok, _} = ddoc_cache:open_custom(DbName, ?MODULE),
+    ?assertEqual(4, ets:info(?CACHE, size)),
+    {ok, _} = ddoc_cache_lru:handle_db_event(ShardName, deleted, foo),
+    meck:wait(ddoc_cache_ev, event, [evicted, DbName], 1000),
+    meck:wait(4, ddoc_cache_ev, event, [removed, '_'], 1000),
+    ?assertEqual(0, ets:info(?CACHE, size)).
+
+
+dont_evict_all_unrelated({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?FOOBAR),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR, Rev),
+    {ok, _} = ddoc_cache:open_validation_funs(DbName),
+    {ok, _} = ddoc_cache:open_custom(DbName, ?MODULE),
+    ?assertEqual(4, ets:info(?CACHE, size)),
+    ShardName = <<"shards/00000000-ffffffff/test.1384769918">>,
+    {ok, _} = ddoc_cache_lru:handle_db_event(ShardName, deleted, foo),
+    meck:wait(ddoc_cache_ev, event, [evict_noop, <<"test">>], 1000),
+    ?assertEqual(4, ets:info(?CACHE, size)).
+
+
+check_upgrade_clause({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    meck:wait(ddoc_cache_ev, event, [started, '_'], 1000),
+    meck:wait(ddoc_cache_ev, event, [default_started, '_'], 1000),
+    ?assertEqual(2, ets:info(?CACHE, size)),
+    gen_server:cast(ddoc_cache_opener, {do_evict, DbName}),
+    meck:wait(ddoc_cache_ev, event, [evicted, DbName], 1000),
+    meck:wait(2, ddoc_cache_ev, event, [removed, '_'], 1000),
+    ?assertEqual(0, ets:info(?CACHE, size)).
diff --git a/src/ddoc_cache/test/ddoc_cache_lru_test.erl b/src/ddoc_cache/test/ddoc_cache_lru_test.erl
new file mode 100644
index 000000000..245511563
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_lru_test.erl
@@ -0,0 +1,216 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_lru_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(<<"pause", _/binary>>) ->
+    receive go -> ok end,
+    {ok, paused};
+
+recover(<<"big", _/binary>>) ->
+    {ok, [random:uniform() || _ <- lists:seq(1, 8192)]};
+
+recover(DbName) ->
+    {ok, DbName}.
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:new(ddoc_cache_ev, [passthrough]),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_not_started_test() ->
+    % Starting couch, but not ddoc_cache
+    Ctx = test_util:start_couch(),
+    try
+        Key = {ddoc_cache_entry_custom, {<<"dbname">>, ?MODULE}},
+        ?assertEqual({ok, <<"dbname">>}, ddoc_cache_lru:open(Key))
+    after
+        test_util:stop_couch(Ctx)
+    end.
+
+
+check_lru_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun check_multi_start/1,
+            fun check_multi_open/1,
+            fun check_capped_size/1,
+            fun check_cache_refill/1,
+            fun check_evict_and_exit/1
+        ]}
+    }.
+
+
+check_multi_start(_) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Key = {ddoc_cache_entry_custom, {<<"pause">>, ?MODULE}},
+    % These will all get sent through ddoc_cache_lru
+    Clients = lists:map(fun(_) ->
+        spawn_monitor(fun() ->
+            ddoc_cache_lru:open(Key)
+        end)
+    end, lists:seq(1, 10)),
+    meck:wait(ddoc_cache_ev, event, [started, Key], 1000),
+    lists:foreach(fun({Pid, _Ref}) ->
+        ?assert(is_process_alive(Pid))
+    end, Clients),
+    [#entry{pid = Pid}] = ets:tab2list(?CACHE),
+    Opener = element(4, sys:get_state(Pid)),
+    OpenerRef = erlang:monitor(process, Opener),
+    ?assert(is_process_alive(Opener)),
+    Opener ! go,
+    receive {'DOWN', OpenerRef, _, _, _} -> ok end,
+    lists:foreach(fun({_, Ref}) ->
+        receive
+            {'DOWN', Ref, _, _, normal} -> ok
+        end
+    end, Clients).
+
+
+check_multi_open(_) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Key = {ddoc_cache_entry_custom, {<<"pause">>, ?MODULE}},
+    % We wait after the first client so that
+    % the rest of the clients go directly to
+    % ddoc_cache_entry bypassing ddoc_cache_lru
+    Client1 = spawn_monitor(fun() ->
+        ddoc_cache_lru:open(Key)
+    end),
+    meck:wait(ddoc_cache_ev, event, [started, Key], 1000),
+    Clients = [Client1] ++ lists:map(fun(_) ->
+        spawn_monitor(fun() ->
+            ddoc_cache_lru:open(Key)
+        end)
+    end, lists:seq(1, 9)),
+    lists:foreach(fun({Pid, _Ref}) ->
+        ?assert(is_process_alive(Pid))
+    end, Clients),
+    [#entry{pid = Pid}] = ets:tab2list(?CACHE),
+    Opener = element(4, sys:get_state(Pid)),
+    OpenerRef = erlang:monitor(process, Opener),
+    ?assert(is_process_alive(Opener)),
+    Opener ! go,
+    receive {'DOWN', OpenerRef, _, _, _} -> ok end,
+    lists:foreach(fun({_, Ref}) ->
+        receive {'DOWN', Ref, _, _, normal} -> ok end
+    end, Clients).
+
+
+check_capped_size(_) ->
+    % The extra factor of two in the size checks is
+    % a fudge factor. We don't reject entries from
+    % the cache if they would put us over the limit
+    % as we don't have the size information a
+    % priori.
+    config:set("ddoc_cache", "max_size", "1048576", false),
+    MaxSize = 1048576,
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    lists:foreach(fun(I) ->
+        DbName = list_to_binary("big_" ++ integer_to_list(I)),
+        ddoc_cache:open_custom(DbName, ?MODULE),
+        meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000),
+        ?assert(cache_size() < MaxSize * 2)
+    end, lists:seq(1, 25)),
+    lists:foreach(fun(I) ->
+        DbName = list_to_binary("big_" ++ integer_to_list(I)),
+        ddoc_cache:open_custom(DbName, ?MODULE),
+        meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000),
+        ?assert(cache_size() < MaxSize * 2)
+    end, lists:seq(26, 100)).
+
+
+check_cache_refill({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+
+    InitDDoc = fun(I) ->
+        NumBin = list_to_binary(integer_to_list(I)),
+        DDocId = <<"_design/", NumBin/binary>>,
+        Doc = #doc{id = DDocId, body = {[]}},
+        {ok, _} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]),
+        {ok, _} = ddoc_cache:open_doc(DbName, DDocId),
+        {ddoc_cache_entry_ddocid, {DbName, DDocId}}
+    end,
+
+    lists:foreach(fun(I) ->
+        Key = InitDDoc(I),
+        meck:wait(ddoc_cache_ev, event, [started, Key], 1000)
+    end, lists:seq(1, 5)),
+
+    ShardName = mem3:name(hd(mem3:shards(DbName))),
+    {ok, _} = ddoc_cache_lru:handle_db_event(ShardName, deleted, foo),
+    meck:wait(ddoc_cache_ev, event, [evicted, DbName], 1000),
+    meck:wait(10, ddoc_cache_ev, event, [removed, '_'], 1000),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+
+    lists:foreach(fun(I) ->
+        Key = InitDDoc(I),
+        meck:wait(ddoc_cache_ev, event, [started, Key], 1000)
+    end, lists:seq(6, 10)).
+
+
+check_evict_and_exit(_) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+
+    Key = {ddoc_cache_entry_custom, {<<"dbname">>, ?MODULE}},
+    ?assertEqual({ok, <<"dbname">>}, ddoc_cache_lru:open(Key)),
+    [#entry{key = Key, pid = Pid}] = ets:tab2list(?CACHE),
+
+    erlang:monitor(process, whereis(ddoc_cache_lru)),
+
+    % Pause the LRU so we can queue multiple messages
+    erlang:suspend_process(whereis(ddoc_cache_lru)),
+
+    gen_server:cast(ddoc_cache_lru, {do_evict, <<"dbname">>}),
+    whereis(ddoc_cache_lru) ! {'EXIT', Pid, normal},
+
+    % Resume the LRU and ensure that it doesn't die
+    erlang:resume_process(whereis(ddoc_cache_lru)),
+
+    meck:wait(ddoc_cache_ev, event, [evicted, <<"dbname">>], 1000),
+
+    % Make sure it can handle another message
+    OtherKey = {ddoc_cache_entry_custom, {<<"otherdb">>, ?MODULE}},
+    ?assertEqual({ok, <<"otherdb">>}, ddoc_cache_lru:open(OtherKey)),
+
+    % And verify our monitor doesn't fire
+    timer:sleep(500),
+    ?assertEqual({messages, []}, process_info(self(), messages)).
+
+
+cache_size() ->
+    ets:info(?CACHE, memory) * erlang:system_info(wordsize).
diff --git a/src/ddoc_cache/test/ddoc_cache_no_cache_test.erl b/src/ddoc_cache/test/ddoc_cache_no_cache_test.erl
new file mode 100644
index 000000000..637a6e872
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_no_cache_test.erl
@@ -0,0 +1,79 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_no_cache_test).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+
+ddoc(DDocId) ->
+    {ok, #doc{
+        id = DDocId,
+        revs = {1, [<<"deadbeefdeadbeef">>]},
+        body = {[
+            {<<"ohai">>, null}
+        ]}
+    }}.
+
+
+not_found(_DDocId) ->
+    {not_found, missing}.
+
+
+return_error(_DDocId) ->
+    {error, timeout}.
+
+
+start(Resp) ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:new(fabric),
+    meck:expect(fabric, open_doc, fun(_, DDocId, _) ->
+        Resp(DDocId)
+    end),
+    Ctx.
+
+
+stop(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+no_cache_open_ok_test() ->
+    Ctx = start(fun ddoc/1),
+    try
+        Resp = ddoc_cache:open_doc(<<"foo">>, <<"bar">>),
+        ?assertEqual(ddoc(<<"bar">>), Resp)
+    after
+        stop(Ctx)
+    end.
+
+
+no_cache_open_not_found_test() ->
+    Ctx = start(fun not_found/1),
+    try
+        Resp = ddoc_cache:open_doc(<<"foo">>, <<"bar">>),
+        ?assertEqual(not_found(<<"bar">>), Resp)
+    after
+        stop(Ctx)
+    end.
+
+
+no_cache_open_error_test() ->
+    Ctx = start(fun return_error/1),
+    try
+        Resp = ddoc_cache:open_doc(<<"foo">>, <<"bar">>),
+        ?assertEqual(return_error(<<"bar">>), Resp)
+    after
+        stop(Ctx)
+    end.
diff --git a/src/ddoc_cache/test/ddoc_cache_open_error_test.erl b/src/ddoc_cache/test/ddoc_cache_open_error_test.erl
new file mode 100644
index 000000000..f3a9b10f4
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_open_error_test.erl
@@ -0,0 +1,46 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_open_error_test).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:expect(fabric, open_doc, fun(_, ?FOOBAR, _) ->
+        erlang:error(test_kaboom)
+    end),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_open_error_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun handle_open_error/1
+        ]}
+    }.
+
+
+handle_open_error({DbName, _}) ->
+    ?assertError(test_kaboom, ddoc_cache:open_doc(DbName, ?FOOBAR)).
diff --git a/src/ddoc_cache/test/ddoc_cache_opener_test.erl b/src/ddoc_cache/test/ddoc_cache_opener_test.erl
new file mode 100644
index 000000000..c3846360c
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_opener_test.erl
@@ -0,0 +1,33 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_opener_test).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+empty_hull_test() ->
+    InitExpect = {ok, nil},
+    TermExpect = ok,
+    CallExpect = {stop, {invalid_call, foo}, {invalid_call, foo}, baz},
+    CastExpect = {stop, {invalid_cast, foo}, bar},
+    InfoExpect = {stop, {invalid_info, foo}, bar},
+    CCExpect = {ok, bar},
+    ?assertEqual(InitExpect, ddoc_cache_opener:init(foo)),
+    ?assertEqual(TermExpect, ddoc_cache_opener:terminate(foo, bar)),
+    ?assertEqual(CallExpect, ddoc_cache_opener:handle_call(foo, bar, baz)),
+    ?assertEqual(CastExpect, ddoc_cache_opener:handle_cast(foo, bar)),
+    ?assertEqual(InfoExpect, ddoc_cache_opener:handle_info(foo, bar)),
+    ?assertEqual(CCExpect, ddoc_cache_opener:code_change(foo, bar, baz)).
diff --git a/src/ddoc_cache/test/ddoc_cache_refresh_test.erl b/src/ddoc_cache/test/ddoc_cache_refresh_test.erl
new file mode 100644
index 000000000..f1459870d
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_refresh_test.erl
@@ -0,0 +1,174 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_refresh_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(DbName) ->
+    {ok, {DbName, rand_string()}}.
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:new(ddoc_cache_ev, [passthrough]),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_refresh_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun refresh_ddoc/1,
+            fun refresh_ddoc_rev/1,
+            fun refresh_vdu/1,
+            fun refresh_custom/1,
+            fun refresh_multiple/1,
+            fun check_upgrade_clause/1
+        ]}
+    }.
+
+
+refresh_ddoc({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    meck:wait(ddoc_cache_ev, event, [started, '_'], 1000),
+    meck:wait(ddoc_cache_ev, event, [default_started, '_'], 1000),
+
+    ?assertEqual(2, ets:info(?CACHE, size)),
+    [#entry{key = Key, val = DDoc}, _] = lists:sort(ets:tab2list(?CACHE)),
+    NewDDoc = DDoc#doc{
+        body = {[{<<"foo">>, <<"baz">>}]}
+    },
+    {ok, {Depth, RevId}} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    Expect = NewDDoc#doc{
+        revs = {Depth, [RevId | element(2, DDoc#doc.revs)]}
+    },
+    meck:wait(ddoc_cache_ev, event, [updated, {Key, Expect}], 1000),
+    ?assertMatch({ok, Expect}, ddoc_cache:open_doc(DbName, ?FOOBAR)),
+    ?assertEqual(2, ets:info(?CACHE, size)).
+
+
+refresh_ddoc_rev({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?FOOBAR),
+    {ok, RevDDoc} = ddoc_cache:open_doc(DbName, ?FOOBAR, Rev),
+
+    meck:wait(ddoc_cache_ev, event, [started, '_'], 1000),
+    meck:wait(ddoc_cache_ev, event, [default_started, '_'], 1000),
+
+    [_, #entry{key = Key, val = DDoc}] = lists:sort(ets:tab2list(?CACHE)),
+    NewDDoc = DDoc#doc{
+        body = {[{<<"foo">>, <<"kazam">>}]}
+    },
+    {ok, _} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    % We pass the rev explicitly so we assert that we're
+    % getting the same original response from the cache
+    meck:wait(ddoc_cache_ev, event, [update_noop, Key], 1000),
+    ?assertMatch({ok, RevDDoc}, ddoc_cache:open_doc(DbName, ?FOOBAR, Rev)),
+    ?assertEqual(2, ets:info(?CACHE, size)).
+
+
+refresh_vdu({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    {ok, [_]} = ddoc_cache:open_validation_funs(DbName),
+    [#entry{key = Key}] = ets:tab2list(?CACHE),
+    {ok, DDoc} = fabric:open_doc(DbName, ?VDU, [?ADMIN_CTX]),
+    {ok, _} = fabric:update_doc(DbName, DDoc#doc{body = {[]}}, [?ADMIN_CTX]),
+    meck:wait(ddoc_cache_ev, event, [updated, {Key, []}], 1000),
+    ?assertMatch({ok, []}, ddoc_cache:open_validation_funs(DbName)),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+
+refresh_custom({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    {ok, Resp1} = ddoc_cache:open_custom(DbName, ?MODULE),
+    {ok, DDoc} = fabric:open_doc(DbName, ?VDU, [?CUSTOM]),
+    {ok, _} = fabric:update_doc(DbName, DDoc#doc{body = {[]}}, [?ADMIN_CTX]),
+    meck:wait(ddoc_cache_ev, event, [updated, '_'], 1000),
+    ?assertNotEqual({ok, Resp1}, ddoc_cache:open_custom(DbName, ?MODULE)),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+
+refresh_multiple({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?FOOBAR),
+    {ok, DDoc} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    {ok, DDoc} = ddoc_cache:open_doc(DbName, ?FOOBAR, Rev),
+    ?assertEqual(2, ets:info(?CACHE, size)),
+    % Relying on the sort order of entry keys to make
+    % sure our entries line up for this test
+    [
+        #entry{key = NoRevKey, val = DDoc},
+        #entry{key = RevKey, val = DDoc}
+    ] = lists:sort(ets:tab2list(?CACHE)),
+    NewDDoc = DDoc#doc{
+        body = {[{<<"foo">>, <<"kalamazoo">>}]}
+    },
+    {ok, {Depth, RevId}} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    Updated = NewDDoc#doc{
+        revs = {Depth, [RevId | element(2, DDoc#doc.revs)]}
+    },
+    meck:wait(ddoc_cache_ev, event, [update_noop, RevKey], 1000),
+    meck:wait(ddoc_cache_ev, event, [updated, {NoRevKey, Updated}], 1000),
+    % We pass the rev explicitly so we assert that we're
+    % getting the same original response from the cache
+    ?assertEqual({ok, Updated}, ddoc_cache:open_doc(DbName, ?FOOBAR)),
+    ?assertEqual({ok, DDoc}, ddoc_cache:open_doc(DbName, ?FOOBAR, Rev)),
+    ?assertEqual(2, ets:info(?CACHE, size)).
+
+
+check_upgrade_clause({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+    [#entry{key = Key}] = ets:tab2list(?CACHE),
+    gen_server:cast(ddoc_cache_opener, {do_evict, DbName, [?FOOBAR]}),
+    meck:wait(ddoc_cache_ev, event, [update_noop, Key], 1000).
+
+
+rand_string() ->
+    Bin = crypto:rand_bytes(8),
+    to_hex(Bin, []).
+
+
+to_hex(<<>>, Acc) ->
+    list_to_binary(lists:reverse(Acc));
+to_hex(<<C1:4, C2:4, Rest/binary>>, Acc) ->
+    to_hex(Rest, [hexdig(C1), hexdig(C2) | Acc]).
+
+
+hexdig(C) when C >= 0, C =< 9 ->
+    C + $0;
+hexdig(C) when C >= 10, C =< 15 ->
+    C + $A - 10.
\ No newline at end of file
diff --git a/src/ddoc_cache/test/ddoc_cache_remove_test.erl b/src/ddoc_cache/test/ddoc_cache_remove_test.erl
new file mode 100644
index 000000000..8787482e9
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_remove_test.erl
@@ -0,0 +1,224 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_remove_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("mem3/include/mem3.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(DbName) ->
+    {ok, #doc{body = {Body}}} = fabric:open_doc(DbName, ?CUSTOM, [?ADMIN_CTX]),
+    case couch_util:get_value(<<"status">>, Body) of
+        <<"ok">> ->
+            {ok, yay};
+        <<"not_ok">> ->
+            {ruh, roh};
+        <<"error">> ->
+            erlang:error(thpppt)
+    end.
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:new(ddoc_cache_ev, [passthrough]),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_refresh_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun remove_ddoc/1,
+            fun remove_ddoc_rev/1,
+            fun remove_ddoc_rev_only/1,
+            fun remove_custom_not_ok/1,
+            fun remove_custom_error/1
+        ]}
+    }.
+
+
+remove_ddoc({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    ?assertEqual(0, ets:info(?CACHE, size)),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+
+    meck:wait(ddoc_cache_ev, event, [started, '_'], 1000),
+    meck:wait(ddoc_cache_ev, event, [default_started, '_'], 1000),
+
+    [#entry{val = DDoc}, #entry{val = DDoc}] = ets:tab2list(?CACHE),
+    {Depth, [RevId | _]} = DDoc#doc.revs,
+    NewDDoc = DDoc#doc{
+        deleted = true,
+        body = {[]}
+    },
+    {ok, _} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+
+    DDocIdKey = {ddoc_cache_entry_ddocid, {DbName, ?FOOBAR}},
+    Rev = {Depth, RevId},
+    DDocIdRevKey = {ddoc_cache_entry_ddocid_rev, {DbName, ?FOOBAR, Rev}},
+    meck:wait(ddoc_cache_ev, event, [removed, DDocIdKey], 1000),
+    meck:wait(ddoc_cache_ev, event, [update_noop, DDocIdRevKey], 1000),
+
+    ?assertMatch({not_found, deleted}, ddoc_cache:open_doc(DbName, ?FOOBAR)),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+
+remove_ddoc_rev({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?VDU),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?VDU, Rev),
+
+    meck:wait(ddoc_cache_ev, event, [started, '_'], 1000),
+    meck:wait(ddoc_cache_ev, event, [default_started, '_'], 1000),
+
+    % Notice the sort so that we know we're getting the
+    % revid version second.
+    [_, #entry{key = Key, val = DDoc, pid = Pid}]
+            = lists:sort(ets:tab2list(?CACHE)),
+
+    NewDDoc = DDoc#doc{
+        body = {[{<<"an">>, <<"update">>}]}
+    },
+    {ok, _} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    meck:wait(ddoc_cache_ev, event, [update_noop, Key], 1000),
+    % Compact the database so that the old rev is removed
+    lists:foreach(fun(Shard) ->
+        do_compact(Shard#shard.name)
+    end, mem3:local_shards(DbName)),
+    % Trigger a refresh rather than wait for the timeout
+    ddoc_cache_entry:refresh(Pid),
+    meck:wait(ddoc_cache_ev, event, [removed, Key], 1000),
+    ?assertMatch(
+            {{not_found, missing}, _},
+            ddoc_cache:open_doc(DbName, ?VDU, Rev)
+        ),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+
+remove_ddoc_rev_only({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Rev = ddoc_cache_tutil:get_rev(DbName, ?VDU),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?VDU),
+    {ok, _} = ddoc_cache:open_doc(DbName, ?VDU, Rev),
+    % Relying on the sort order of keys to keep
+    % these lined up for testing
+    [
+        #entry{key = NoRevKey, val = DDoc, pid = NoRevPid},
+        #entry{key = RevKey, val = DDoc, pid = RevPid}
+    ] = lists:sort(ets:tab2list(?CACHE)),
+    NewDDoc = DDoc#doc{
+        body = {[{<<"new">>, <<"awesomeness">>}]}
+    },
+    {ok, _} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    meck:wait(ddoc_cache_ev, event, [updated, '_'], 1000),
+    meck:wait(ddoc_cache_ev, event, [update_noop, RevKey], 1000),
+    % Compact the database so that the old rev is removed
+    lists:foreach(fun(Shard) ->
+        do_compact(Shard#shard.name)
+    end, mem3:local_shards(DbName)),
+    % Trigger a refresh rather than wait for the timeout
+    ddoc_cache_entry:refresh(NoRevPid),
+    ddoc_cache_entry:refresh(RevPid),
+    meck:wait(ddoc_cache_ev, event, [update_noop, NoRevKey], 1000),
+    meck:wait(ddoc_cache_ev, event, [removed, RevKey], 1000),
+    ?assertMatch({ok, _}, ddoc_cache:open_doc(DbName, ?VDU)),
+    ?assertMatch(
+            {{not_found, missing}, _},
+            ddoc_cache:open_doc(DbName, ?VDU, Rev)
+        ),
+    ?assertEqual(1, ets:info(?CACHE, size)).
+
+remove_custom_not_ok({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    init_custom_ddoc(DbName),
+    {ok, _} = ddoc_cache:open_custom(DbName, ?MODULE),
+    [#entry{key = Key}] = ets:tab2list(?CACHE),
+    {ok, DDoc} = fabric:open_doc(DbName, ?CUSTOM, [?ADMIN_CTX]),
+    NewDDoc = DDoc#doc{
+        body = {[{<<"status">>, <<"not_ok">>}]}
+    },
+    {ok, _} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    meck:wait(ddoc_cache_ev, event, [removed, Key], 1000),
+    ?assertEqual({ruh, roh}, ddoc_cache:open_custom(DbName, ?MODULE)),
+    ?assertEqual(0, ets:info(?CACHE, size)).
+
+
+remove_custom_error({DbName, _}) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    init_custom_ddoc(DbName),
+    {ok, _} = ddoc_cache:open_custom(DbName, ?MODULE),
+    [#entry{key = Key}] = ets:tab2list(?CACHE),
+    {ok, DDoc} = fabric:open_doc(DbName, ?CUSTOM, [?ADMIN_CTX]),
+    NewDDoc = DDoc#doc{
+        body = {[{<<"status">>, <<"error">>}]}
+    },
+    {ok, _} = fabric:update_doc(DbName, NewDDoc, [?ADMIN_CTX]),
+    meck:wait(ddoc_cache_ev, event, [removed, Key], 1000),
+    ?assertError(thpppt, ddoc_cache:open_custom(DbName, ?MODULE)),
+    ?assertEqual(0, ets:info(?CACHE, size)).
+
+
+init_custom_ddoc(DbName) ->
+    Body = {[{<<"status">>, <<"ok">>}]},
+    {ok, Doc} = fabric:open_doc(DbName, ?CUSTOM, [?ADMIN_CTX]),
+    NewDoc = Doc#doc{body = Body},
+    {ok, _} = fabric:update_doc(DbName, NewDoc, [?ADMIN_CTX]).
+
+
+do_compact(ShardName) ->
+    {ok, Db} = couch_db:open_int(ShardName, []),
+    try
+        {ok, Pid} = couch_db:start_compact(Db),
+        Ref = erlang:monitor(process, Pid),
+        receive
+            {'DOWN', Ref, _, _, _} ->
+                ok
+        end
+    after
+        couch_db:close(Db)
+    end,
+    wait_for_compaction(ShardName).
+
+
+wait_for_compaction(ShardName) ->
+    {ok, Db} = couch_db:open_int(ShardName, []),
+    CompactRunning = try
+        {ok, Info} = couch_db:get_db_info(Db),
+        couch_util:get_value(compact_running, Info)
+    after
+        couch_db:close(Db)
+    end,
+    if not CompactRunning -> ok; true ->
+        timer:sleep(100),
+        wait_for_compaction(ShardName)
+    end.
\ No newline at end of file
diff --git a/src/ddoc_cache/test/ddoc_cache_test.hrl b/src/ddoc_cache/test/ddoc_cache_test.hrl
new file mode 100644
index 000000000..73f7bc217
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_test.hrl
@@ -0,0 +1,26 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-define(CACHE, ddoc_cache_entries).
+-define(LRU, ddoc_cache_lru).
+-define(OPENERS, ddoc_cache_openers).
+
+-define(FOOBAR, <<"_design/foobar">>).
+-define(VDU, <<"_design/vdu">>).
+-define(CUSTOM, <<"_design/custom">>).
+
+-record(entry, {
+    key,
+    val,
+    pid
+}).
diff --git a/src/ddoc_cache/test/ddoc_cache_tutil.erl b/src/ddoc_cache/test/ddoc_cache_tutil.erl
new file mode 100644
index 000000000..6463b38d1
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_tutil.erl
@@ -0,0 +1,96 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_tutil).
+
+
+-compile(export_all).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+
+
+start_couch() ->
+    start_couch([{write_ddocs, true}]).
+
+
+start_couch(Options) ->
+    WriteDDocs = couch_util:get_value(write_ddocs, Options, true),
+    purge_modules(),
+    Ctx = test_util:start_couch(?CONFIG_CHAIN, [chttpd, ddoc_cache]),
+    TmpDb = ?tempdb(),
+    ok = fabric:create_db(TmpDb, [{q, "1"}, {n, "1"}]),
+    if not WriteDDocs -> ok; true ->
+        {ok, _} = fabric:update_docs(TmpDb, ddocs(), [?ADMIN_CTX])
+    end,
+    {TmpDb, Ctx}.
+
+
+stop_couch({_TmpDb, Ctx}) ->
+    test_util:stop_couch(Ctx).
+
+
+clear() ->
+    application:stop(ddoc_cache),
+    application:start(ddoc_cache).
+
+
+get_rev(DbName, DDocId) ->
+    {_, Ref} = erlang:spawn_monitor(fun() ->
+        {ok, #doc{revs = Revs}} = fabric:open_doc(DbName, DDocId, [?ADMIN_CTX]),
+        {Depth, [RevId | _]} = Revs,
+        exit({Depth, RevId})
+    end),
+    receive
+        {'DOWN', Ref, _, _, Rev} -> Rev
+    end.
+
+
+ddocs() ->
+    FooBar = #doc{
+        id = <<"_design/foobar">>,
+        body = {[
+            {<<"foo">>, <<"bar">>}
+        ]}
+    },
+    VDU = #doc{
+        id = <<"_design/vdu">>,
+        body = {[
+            {<<"validate_doc_update">>, <<"function(doc) {return;}">>}
+        ]}
+    },
+    Custom = #doc{
+        id = <<"_design/custom">>,
+        body = {[
+            {<<"status">>, <<"ok">>},
+            {<<"custom">>, <<"hotrod">>}
+        ]}
+    },
+    [FooBar, VDU, Custom].
+
+
+purge_modules() ->
+    case application:get_key(ddoc_cache, modules) of
+        {ok, Mods} ->
+            lists:foreach(fun(Mod) ->
+                case code:which(Mod) of
+                    cover_compiled ->
+                        ok;
+                    _ ->
+                        code:delete(Mod),
+                        code:purge(Mod)
+                end
+            end, Mods);
+        undefined ->
+            ok
+    end.
diff --git a/test/javascript/tests/rewrite.js b/test/javascript/tests/rewrite.js
index 8ff32295b..a984936d1 100644
--- a/test/javascript/tests/rewrite.js
+++ b/test/javascript/tests/rewrite.js
@@ -416,13 +416,15 @@ couchTests.rewrite = function(debug) {
         T(typeof(result['_revs_info']) === "object");
 
         // test path relative to server
-        designDoc.rewrites.push({
-          "from": "uuids",
+        T(db.save({
+          _id: "_design/test2",
+          rewrites: [{
+            "from": "uuids",
           "to": "../../../_uuids"
-        });
-        T(db.save(designDoc).ok);
+          }]
+        }).ok);
         
-        var xhr = CouchDB.request("GET", "/"+dbName+"/_design/test/_rewrite/uuids");
+        var xhr = CouchDB.request("GET", "/"+dbName+"/_design/test2/_rewrite/uuids");
         T(xhr.status == 500);
         var result = JSON.parse(xhr.responseText);
         T(result.error == "insecure_rewrite_rule");
@@ -432,7 +434,7 @@ couchTests.rewrite = function(debug) {
             key: "secure_rewrites",
             value: "false"}],
           function() {
-            var xhr = CouchDB.request("GET", "/"+dbName+"/_design/test/_rewrite/uuids?cache=bust");
+            var xhr = CouchDB.request("GET", "/"+dbName+"/_design/test2/_rewrite/uuids?cache=bust");
             T(xhr.status == 200);
             var result = JSON.parse(xhr.responseText);
             T(result.uuids.length == 1);