Handle libicu upgrades in views

Previously, libicu collator versions were not tracked, by the views.
So during major OS version upgrades, it was possible to experience
data loss due to collation order changes between libicu library
versions. The view order inconsistency would last until the view was
compacted.

To fix it, introduce a view info map in the header which records
the ranges of unicode collator versions used by that view. The
versions range is updated every time a view is opened. In case the
number of collator versions is greater than 2, smoosh would enqueue
the view for recompaction into the upgrade channel.

The new view info map is re-using a previously removed view header
field from 2.x views. Since there is already 2.x -> 3.x upgrade code
which ignores this field, this allows for simple downgrading to 3.2.1
and then upgrading back to 3.2.1+.

Also, the view info map may be re-used in the future to record other
metadata about the view without the need to expand the view record and
so, allow for easy downgrades.

4 files changed, 62 insertions, 14 deletions

diff --git a/src/couch_mrview/include/couch_mrview.hrl b/src/couch_mrview/include/couch_mrview.hrl
index bb0ab0b46..b31463c53 100644
--- a/src/couch_mrview/include/couch_mrview.hrl
+++ b/src/couch_mrview/include/couch_mrview.hrl
@@ -29,7 +29,8 @@
     doc_acc,
     doc_queue,
     write_queue,
-    qserver=nil
+    qserver=nil,
+    view_info=#{}
 }).
 
 
@@ -49,6 +50,7 @@
     seq=0,
     purge_seq=0,
     id_btree_state=nil,
+    view_info=#{}, % replaces log btree in versions < 3.x
     view_states=nil
 }).
 
diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl
index a024d35c8..6113ee031 100644
--- a/src/couch_mrview/src/couch_mrview_index.erl
+++ b/src/couch_mrview/src/couch_mrview_index.erl
@@ -63,7 +63,8 @@ get(info, State) ->
         language = Lang,
         update_seq = UpdateSeq,
         purge_seq = PurgeSeq,
-        views = Views
+        views = Views,
+        view_info = ViewInfo
     } = State,
     {ok, FileSize} = couch_file:bytes(Fd),
     {ok, ExternalSize} = couch_mrview_util:calculate_external_size(Views),
@@ -72,7 +73,7 @@ get(info, State) ->
 
     UpdateOptions0 = get(update_options, State),
     UpdateOptions = [atom_to_binary(O, latin1) || O <- UpdateOptions0],
-
+    CollatorVersions = couch_mrview_util:get_collator_versions(ViewInfo),
     {ok, [
         {signature, list_to_binary(couch_index_util:hexsig(Sig))},
         {language, Lang},
@@ -84,7 +85,8 @@ get(info, State) ->
             ]}},
         {update_seq, UpdateSeq},
         {purge_seq, PurgeSeq},
-        {update_options, UpdateOptions}
+        {update_options, UpdateOptions},
+        {collator_versions, CollatorVersions}
     ]};
 get(Other, _) ->
     throw({unknown_index_property, Other}).
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index b7220f71f..61dd48eaa 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -32,6 +32,7 @@
 -export([get_view_keys/1, get_view_queries/1]).
 -export([set_view_type/3]).
 -export([set_extra/3, get_extra/2, get_extra/3]).
+-export([get_collator_versions/1]).
 
 -define(MOD, couch_mrview_index).
 -define(GET_VIEW_RETRY_COUNT, 1).
@@ -285,6 +286,7 @@ init_state(Db, Fd, #mrst{views = Views} = State, nil) ->
         seq = 0,
         purge_seq = PurgeSeq,
         id_btree_state = nil,
+        view_info = update_collator_versions(#{}),
         view_states = [make_view_state(#mrview{}) || _ <- Views]
     },
     init_state(Db, Fd, State, Header);
@@ -297,6 +299,7 @@ init_state(Db, Fd, State, Header) ->
         seq = Seq,
         purge_seq = PurgeSeq,
         id_btree_state = IdBtreeState,
+        view_info = ViewInfo,
         view_states = ViewStates
     } = maybe_update_header(Header),
 
@@ -314,7 +317,8 @@ init_state(Db, Fd, State, Header) ->
         update_seq = Seq,
         purge_seq = PurgeSeq,
         id_btree = IdBtree,
-        views = Views2
+        views = Views2,
+        view_info = ViewInfo
     }.
 
 open_view(_Db, Fd, Lang, ViewState, View) ->
@@ -764,13 +768,15 @@ make_header(State) ->
         update_seq = Seq,
         purge_seq = PurgeSeq,
         id_btree = IdBtree,
-        views = Views
+        views = Views,
+        view_info = ViewInfo
     } = State,
 
     #mrheader{
         seq = Seq,
         purge_seq = PurgeSeq,
         id_btree_state = get_btree_state(IdBtree),
+        view_info = ViewInfo,
         view_states = [make_view_state(V) || V <- Views]
     }.
 
@@ -819,7 +825,8 @@ reset_state(State) ->
         qserver = nil,
         update_seq = 0,
         id_btree = nil,
-        views = [View#mrview{btree = nil} || View <- State#mrst.views]
+        views = [View#mrview{btree = nil} || View <- State#mrst.views],
+        view_info = #{}
     }.
 
 all_docs_key_opts(#mrargs{extra = Extra} = Args) ->
@@ -1070,14 +1077,30 @@ old_view_format(View, SI, KSI) ->
         View#mrview.options
     }.
 
-maybe_update_header(#mrheader{} = Header) ->
-    Header;
-maybe_update_header(Header) when tuple_size(Header) == 6 ->
+maybe_update_header(#mrheader{view_info = Info} = Header) when is_map(Info) ->
+    % Latest (3.2.1+) version. The size of the record is the same as
+    % the <2.3.1 version. The main difference is that the LogBt field
+    % is now a map. This trick allows for easy downgrading back to
+    % version 3.2.1 and then upgrading back to 3.2.1+ if needed.
+    Header#mrheader{view_info = update_collator_versions(Info)};
+maybe_update_header({mrheader, Seq, PSeq, IDBt, ViewStates}) ->
+    % Versions >2.3.1 and =<3.2.1 (no view info map)
+    % ViewStates were the same as they are now.
     #mrheader{
-        seq = element(2, Header),
-        purge_seq = element(3, Header),
-        id_btree_state = element(4, Header),
-        view_states = [make_view_state(S) || S <- element(6, Header)]
+        seq = Seq,
+        purge_seq = PSeq,
+        id_btree_state = IDBt,
+        view_info = update_collator_versions(#{}),
+        view_states = ViewStates
+    };
+maybe_update_header({mrheader, Seq, PSeq, IDBt, _LogBt, ViewStates}) ->
+    % Versions <2.3.1.
+    #mrheader{
+        seq = Seq,
+        purge_seq = PSeq,
+        id_btree_state = IDBt,
+        view_info = update_collator_versions(#{}),
+        view_states = [make_view_state(S) || S <- ViewStates]
     }.
 
 %% End of <= 2.x upgrade code.
@@ -1216,3 +1239,13 @@ kv_external_size(KVList, Reduction) ->
         ?term_size(Reduction),
         KVList
     ).
+
+update_collator_versions(#{} = ViewInfo) ->
+    Versions = maps:get(ucol_vs, ViewInfo, []),
+    Ver = tuple_to_list(couch_ejson_compare:get_collator_version()),
+    ViewInfo#{ucol_vs => lists:usort([Ver | Versions])}.
+
+get_collator_versions(#{ucol_vs := Versions}) when is_list(Versions) ->
+    Versions;
+get_collator_versions(#{}) ->
+    [].
diff --git a/src/smoosh/src/smoosh_server.erl b/src/smoosh/src/smoosh_server.erl
index 0526625ff..b5c2b56bc 100644
--- a/src/smoosh/src/smoosh_server.erl
+++ b/src/smoosh/src/smoosh_server.erl
@@ -480,6 +480,9 @@ get_priority(Channel) ->
     smoosh_utils:get(Channel, "priority", "ratio").
 
 needs_upgrade(Props) ->
+    db_needs_upgrade(Props) orelse view_needs_upgrade(Props).
+
+db_needs_upgrade(Props) ->
     DiskVersion = couch_util:get_value(disk_format_version, Props),
     case couch_util:get_value(engine, Props) of
         couch_bt_engine ->
@@ -488,6 +491,14 @@ needs_upgrade(Props) ->
             false
     end.
 
+view_needs_upgrade(Props) ->
+    case couch_util:get_value(collator_versions, Props) of
+        undefined ->
+            false;
+        Versions when is_list(Versions) ->
+            length(Versions) >= 2
+    end.
+
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").