deal with schema integrity check failure on startup

by moving the db to a backup location, re-creating the schema, and
writing a warning to the logs

1 files changed, 51 insertions, 35 deletions

diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl
index 57dd9256..d19c37cb 100644
--- a/src/rabbit_mnesia.erl
+++ b/src/rabbit_mnesia.erl
@@ -36,9 +36,6 @@
 
 -export([table_names/0]).
 
-%% Called by rabbitmq-mnesia-current script
--export([schema_current/0]).
-
 %% create_tables/0 exported for helping embed RabbitMQ in or alongside
 %% other mnesia-using Erlang applications, such as ejabberd
 -export([create_tables/0]).
@@ -57,7 +54,6 @@
 -spec(reset/0 :: () -> 'ok').
 -spec(force_reset/0 :: () -> 'ok').
 -spec(create_tables/0 :: () -> 'ok').
--spec(schema_current/0 :: () -> bool()).
 
 -endif.
 
@@ -101,20 +97,6 @@ cluster(ClusterNodes) ->
 reset()       -> reset(false).
 force_reset() -> reset(true).
 
-%% This is invoked by rabbitmq-mnesia-current.
-schema_current() ->
-    application:start(mnesia),
-    ok = ensure_mnesia_running(),
-    ok = ensure_mnesia_dir(),
-    ok = init_db(read_cluster_nodes_config()),
-    try 
-        ensure_schema_integrity(),
-        true
-    catch
-        {error, {schema_integrity_check_failed, _Reason}} ->
-            false
-    end.
-
 %%--------------------------------------------------------------------
 
 table_definitions() ->
@@ -169,17 +151,12 @@ ensure_mnesia_not_running() ->
         yes -> throw({error, mnesia_unexpectedly_running})
     end.
 
-ensure_schema_integrity() ->
-    case catch lists:foreach(fun (Tab) ->
-                                     mnesia:table_info(Tab, version)
-                             end,
-                             table_names()) of
-        {'EXIT', Reason} -> throw({error, {schema_integrity_check_failed,
-                                           Reason}});
-        _ -> ok
-    end,
+check_schema_integrity() ->
     %%TODO: more thorough checks
-    ok.
+    case catch [mnesia:table_info(Tab, version) || Tab <- table_names()] of
+        {'EXIT', Reason} -> {error, Reason};
+        _ -> ok
+    end.
 
 %% The cluster node config file contains some or all of the disk nodes
 %% that are members of the cluster this node is / should be a part of.
@@ -259,7 +236,20 @@ init_db(ClusterNodes) ->
     case mnesia:change_config(extra_db_nodes, ClusterNodes -- [node()]) of
         {ok, []} ->
             if WasDiskNode and IsDiskNode ->
-                    ok;
+                    case check_schema_integrity() of
+                        ok ->
+                            ok;
+                        {error, Reason} ->
+                            %% NB: we cannot use rabbit_log here since
+                            %% it may not have been started yet
+                            error_logger:warning_msg(
+                              "schema integrity check failed: ~p~n" ++
+                              "moving database to backup location " ++
+                              "and recreating schema from scratch~n",
+                              [Reason]),
+                            ok = move_db(),
+                            ok = create_schema()
+                    end;
                WasDiskNode ->
                     throw({error, {cannot_convert_disk_node_to_ram_node,
                                    ClusterNodes}});
@@ -292,6 +282,28 @@ create_schema() ->
                           cannot_start_mnesia),
     create_tables().
 
+move_db() ->
+    mnesia:stop(),
+    MnesiaDir = filename:dirname(mnesia:system_info(directory) ++ "/"),
+    {{Year, Month, Day}, {Hour, Minute, Second}} = erlang:universaltime(),
+    BackupDir = lists:flatten(
+                  io_lib:format("~s_~w~2..0w~2..0w~2..0w~2..0w~2..0w",
+                                [MnesiaDir,
+                                 Year, Month, Day, Hour, Minute, Second])),
+    case file:rename(MnesiaDir, BackupDir) of
+        ok ->
+            %% NB: we cannot use rabbit_log here since it may not have
+            %% been started yet
+            error_logger:warning_msg("moved database from ~s to ~s~n",
+                                     [MnesiaDir, BackupDir]),
+            ok;
+        {error, Reason} -> throw({error, {cannot_backup_mnesia,
+                                          MnesiaDir, BackupDir, Reason}})
+    end,
+    ok = ensure_mnesia_dir(),
+    rabbit_misc:ensure_ok(mnesia:start(), cannot_start_mnesia),
+    ok.
+
 create_tables() ->
     lists:foreach(fun ({Tab, TabArgs}) ->
                           case mnesia:create_table(Tab, TabArgs) of
@@ -353,13 +365,17 @@ create_local_table_copy(Tab, Type) ->
     ok.
 
 wait_for_tables() -> 
-    ok = ensure_schema_integrity(),
-    case mnesia:wait_for_tables(table_names(), 30000) of
-        ok -> ok;
-        {timeout, BadTabs} ->
-            throw({error, {timeout_waiting_for_tables, BadTabs}});
+    case check_schema_integrity() of
+        ok ->
+            case mnesia:wait_for_tables(table_names(), 30000) of
+                ok -> ok;
+                {timeout, BadTabs} ->
+                    throw({error, {timeout_waiting_for_tables, BadTabs}});
+                {error, Reason} ->
+                    throw({error, {failed_waiting_for_tables, Reason}})
+            end;
         {error, Reason} ->
-            throw({error, {failed_waiting_for_tables, Reason}})
+            throw({error, {schema_integrity_check_failed, Reason}})
     end.
 
 reset(Force) ->