COUCHDB-1654: Transparently update view signatures from <= 1.2.x.

Updates 1.2.x or earlier view files to 1.3.x or later view files
transparently, the first time the 1.2.x view file is opened by
1.3.x or later.

Here's how it works:

Before opening a view index,
If no matching index file is found in the new location:
 calculate the <= 1.2.x view signature
 if a file with that signature lives in the old location
   copy it to the new location with the new signature in the name.
Then proceed to open the view index as usual.
After opening, read its header.

If the header matches the <= 1.2.x style #index_header record:
  upgrade the header to the new #mrheader record
The next time the view is used, the new header is used.

If we crash after the rename, but before the header upgrade,
  the header upgrade is done on the next view opening.

If we crash between upgrading to the new header and writing
  that header to disk, we start with the old header again,
  do the upgrade and write to disk.

Includes etap tests in 250*.t.

5 files changed, 287 insertions, 0 deletions

diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl
index 6bcb63f0e..7466dbc98 100644
--- a/src/couch_mrview/src/couch_mrview_index.erl
+++ b/src/couch_mrview/src/couch_mrview_index.erl
@@ -75,9 +75,30 @@ open(Db, State) ->
         sig=Sig
     } = State,
     IndexFName = couch_mrview_util:index_file(DbName, Sig),
+
+    % If we are upgrading from <=1.2.x, we upgrade the view
+    % index file on the fly, avoiding an index reset.
+    %
+    % OldSig is `ok` if no upgrade happened.
+    %
+    % To remove suppport for 1.2.x auto-upgrades in the
+    % future, just remove the next line and the code
+    % between "upgrade code for <= 1.2.x" and
+    % "end upgrade code for <= 1.2.x" and the corresponding
+    % code in couch_mrview_util
+
+    OldSig = couch_mrview_util:maybe_update_index_file(State),
+
     case couch_mrview_util:open_file(IndexFName) of
         {ok, Fd} ->
             case (catch couch_file:read_header(Fd)) of
+                % upgrade code for <= 1.2.x
+                {ok, {OldSig, Header}} ->
+                    % Matching view signatures.
+                    NewSt = couch_mrview_util:init_state(Db, Fd, State, Header),
+                    {ok, RefCounter} = couch_ref_counter:start([Fd]),
+                    {ok, NewSt#mrst{refc=RefCounter}};
+                % end of upgrade code for <= 1.2.x
                 {ok, {Sig, Header}} ->
                     % Matching view signatures.
                     NewSt = couch_mrview_util:init_state(Db, Fd, State, Header),
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 8e6e4dc84..092ae3d5e 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -24,6 +24,7 @@
 -export([calculate_data_size/2]).
 -export([validate_args/1]).
 -export([maybe_load_doc/3, maybe_load_doc/4]).
+-export([maybe_update_index_file/1]).
 
 -define(MOD, couch_mrview_index).
 
@@ -168,6 +169,19 @@ init_state(Db, Fd, #mrst{views=Views}=State, nil) ->
         view_states=[{nil, 0, 0} || _ <- Views]
     },
     init_state(Db, Fd, State, Header);
+% read <= 1.2.x header record and transpile it to >=1.3.x
+% header record
+init_state(Db, Fd, State, #index_header{
+    seq=Seq,
+    purge_seq=PurgeSeq,
+    id_btree_state=IdBtreeState,
+    view_states=ViewStates}) ->
+    init_state(Db, Fd, State, #mrheader{
+        seq=Seq,
+        purge_seq=PurgeSeq,
+        id_btree_state=IdBtreeState,
+        view_states=ViewStates
+        });
 init_state(Db, Fd, State, Header) ->
     #mrst{language=Lang, views=Views} = State,
     #mrheader{
@@ -704,3 +718,90 @@ index_of(Key, [_ | Rest], Idx) ->
 
 mrverror(Mesg) ->
     throw({query_parse_error, Mesg}).
+
+
+%% Updates 1.2.x or earlier view files to 1.3.x or later view files
+%% transparently, the first time the 1.2.x view file is opened by
+%% 1.3.x or later.
+%%
+%% Here's how it works:
+%%
+%% Before opening a view index,
+%% If no matching index file is found in the new location:
+%%  calculate the <= 1.2.x view signature
+%%  if a file with that signature lives in the old location
+%%    rename it to the new location with the new signature in the name.
+%% Then proceed to open the view index as usual.
+%% After opening, read its header.
+%%
+%% If the header matches the <= 1.2.x style #index_header record:
+%%   upgrade the header to the new #mrheader record
+%% The next time the view is used, the new header is used.
+%%
+%% If we crash after the rename, but before the header upgrade,
+%%   the header upgrade is done on the next view opening.
+%%
+%% If we crash between upgrading to the new header and writing
+%%   that header to disk, we start with the old header again,
+%%   do the upgrade and write to disk.
+
+maybe_update_index_file(State) ->
+    DbName = State#mrst.db_name,
+    NewIndexFile = index_file(DbName, State#mrst.sig),
+    % open in read-only mode so we don't create
+    % the file if it doesn't exist.
+    case file:open(NewIndexFile, [read, raw]) of
+    {ok, Fd_Read} ->
+        % the new index file exists, there is nothing to do here.
+        file:close(Fd_Read);
+    _Error ->
+        update_index_file(State)
+    end.
+
+update_index_file(State) ->
+    Sig = sig_vsn_12x(State),
+    DbName = State#mrst.db_name,
+    FileName = couch_index_util:hexsig(Sig) ++ ".view",
+    IndexFile = couch_index_util:index_file("", DbName, FileName),
+
+    % If we have an old index, rename it to the new position.
+    case file:read_file_info(IndexFile) of
+    {ok, _FileInfo} ->
+        % Crash if the rename fails for any reason.
+        % If the target exists, e.g. the next request will find the
+        % new file and we are good. We might need to catch this
+        % further up to avoid a full server crash.
+        ?LOG_INFO("Attempting to update legacy view index file.", []),
+        NewIndexFile = index_file(DbName, State#mrst.sig),
+        ok = filelib:ensure_dir(NewIndexFile),
+        ok = file:rename(IndexFile, NewIndexFile),
+        ?LOG_INFO("Successfully updated legacy view index file.", []),
+        Sig;
+    _ ->
+        % Ignore missing index file
+        ok
+    end.
+
+sig_vsn_12x(State) ->
+    ViewInfo = [old_view_format(V) || V <- State#mrst.views],
+    SigData = case State#mrst.lib of
+    {[]} ->
+        {ViewInfo, State#mrst.language, State#mrst.design_opts};
+    _ ->
+        {ViewInfo, State#mrst.language, State#mrst.design_opts,
+            couch_index_util:sort_lib(State#mrst.lib)}
+    end,
+    couch_util:md5(term_to_binary(SigData)).
+
+old_view_format(View) ->
+{
+    view,
+    View#mrview.id_num,
+    View#mrview.map_names,
+    View#mrview.def,
+    View#mrview.btree,
+    View#mrview.reduce_funs,
+    View#mrview.options
+}.
+
+%% End of <= 1.2.x upgrade code.
diff --git a/test/etap/250-upgrade-legacy-view-files.t b/test/etap/250-upgrade-legacy-view-files.t
new file mode 100644
index 000000000..998264d6a
--- /dev/null
+++ b/test/etap/250-upgrade-legacy-view-files.t
@@ -0,0 +1,165 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+    test_util:init_code_path(),
+
+    etap:plan(8),
+    case (catch test()) of
+        ok ->
+            etap:end_tests();
+        Other ->
+            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+            etap:bail(Other)
+    end,
+    ok.
+
+
+test() ->
+    couch_server_sup:start_link(test_util:config_files()),
+
+    % commit sofort
+    ok = couch_config:set("query_server_config", "commit_freq", "0"),
+
+    test_upgrade(),
+
+    couch_server_sup:stop(),
+    ok.
+
+fixture_path() ->
+    test_util:srcdir() ++ "/test/etap/fixtures".
+
+old_db() ->
+    fixture_path() ++ "/" ++ old_db_name().
+
+old_db_name() ->
+    "test.couch".
+
+old_view() ->
+    fixture_path() ++ "/" ++ old_view_name().
+
+old_view_name() ->
+    "3b835456c235b1827e012e25666152f3.view".
+new_view_name() ->
+    "a1c5929f912aca32f13446122cc6ce50.view".
+
+couch_url() ->
+    "http://" ++ addr() ++ ":" ++ port().
+
+addr() ->
+    couch_config:get("httpd", "bind_address", "127.0.0.1").
+
+port() ->
+    integer_to_list(mochiweb_socket_server:get(couch_httpd, port)).
+
+
+% <= 1.2.x
+-record(index_header,
+    {seq=0,
+    purge_seq=0,
+    id_btree_state=nil,
+    view_states=nil
+    }).
+
+% >= 1.3.x
+-record(mrheader, {
+    seq=0,
+    purge_seq=0,
+    id_btree_state=nil,
+    view_states=nil
+}).
+
+ensure_header(File, MatchFun, Msg) ->
+    {ok, Fd} = couch_file:open(File),
+    {ok, {_Sig, Header}} = couch_file:read_header(Fd),
+    couch_file:close(Fd),
+    etap:fun_is(MatchFun, Header, "ensure " ++ Msg ++ " header for file: " ++ File).
+
+file_exists(File) ->
+    % open without creating
+    case file:open(File, [read, raw]) of
+    {ok, Fd_Read} ->
+        file:close(Fd_Read),
+        true;
+    _Error ->
+        false
+    end.
+
+cleanup() ->
+    DbDir = couch_config:get("couchdb", "database_dir"),
+    Files = [
+        DbDir ++ "/test.couch",
+        DbDir ++ "/.test_design/" ++ old_view_name(),
+        DbDir ++ "/.test_design/mrview/" ++ new_view_name()
+    ],
+    lists:foreach(fun(File) -> file:delete(File) end, Files),
+    etap:ok(true, "cleanup").
+
+test_upgrade() ->
+
+    cleanup(),
+
+    % copy old db file into db dir
+    DbDir = couch_config:get("couchdb", "database_dir"),
+    DbTarget = DbDir ++ "/" ++ old_db_name(),
+    filelib:ensure_dir(DbTarget),
+    file:copy(old_db(), DbTarget),
+
+    % copy old view file into view dir
+    ViewDir = couch_config:get("couchdb", "index_dir"),
+    ViewTarget = ViewDir ++ "/.test_design/" ++ old_view_name(),
+    filelib:ensure_dir(ViewTarget),
+    file:copy(old_view(), ViewTarget),
+
+    % ensure old header
+    ensure_header(ViewTarget, fun(#index_header{}) -> true; (_) -> false end, "old"),
+
+    % query view
+    ViewUrl = couch_url() ++ "/test/_design/test/_view/test",
+    {ok, Code, _Headers, Body}  = test_util:request(ViewUrl, [], get),
+
+    % expect results
+    etap:is(Code, 200, "valid view result http status code"),
+    ExpectBody = <<"{\"total_rows\":2,\"offset\":0,\"rows\":[\r\n{\"id\":\"193f2f9c596ddc7ad326f7da470009ec\",\"key\":1,\"value\":null},\r\n{\"id\":\"193f2f9c596ddc7ad326f7da470012b6\",\"key\":2,\"value\":null}\r\n]}\n">>,
+    etap:is(Body, ExpectBody, "valid view result"),
+
+    % ensure old file gone.
+    etap:is(file_exists(ViewTarget), false, "ensure old file is gone"),
+
+    % ensure new header
+    NewViewFile = ViewDir ++ "/.test_design/mrview/" ++ new_view_name(),
+
+    % add doc(s)
+    test_util:request(
+        couch_url() ++ "/test/boo",
+        [{"Content-Type", "application/json"}],
+        put,
+        <<"{\"a\":3}">>),
+
+    % query again
+    {ok, Code2, _Headers2, Body2} = test_util:request(ViewUrl, [], get),
+
+    % expect results
+    etap:is(Code2, 200, "valid view result http status code"),
+    ExpectBody2 = <<"{\"total_rows\":3,\"offset\":0,\"rows\":[\r\n{\"id\":\"193f2f9c596ddc7ad326f7da470009ec\",\"key\":1,\"value\":null},\r\n{\"id\":\"193f2f9c596ddc7ad326f7da470012b6\",\"key\":2,\"value\":null},\r\n{\"id\":\"boo\",\"key\":3,\"value\":null}\r\n]}\n">>,
+    etap:is(Body2, ExpectBody2, "valid view result after doc add"),
+
+    % ensure no rebuild
+    % TBD no idea how to actually test this.
+
+    % ensure new header.
+    timer:sleep(1000),
+    ensure_header(NewViewFile, fun(#mrheader{}) -> true; (_) -> false end, "new"),
+
+    ok.
diff --git a/test/etap/fixtures/3b835456c235b1827e012e25666152f3.view b/test/etap/fixtures/3b835456c235b1827e012e25666152f3.view
new file mode 100644
index 000000000..9c67648be
--- /dev/null
+++ b/test/etap/fixtures/3b835456c235b1827e012e25666152f3.view
diff --git a/test/etap/fixtures/test.couch b/test/etap/fixtures/test.couch
new file mode 100644
index 000000000..32c79af32
--- /dev/null
+++ b/test/etap/fixtures/test.couch