Experiment to integrate ramnesia into RabbitMQ.

Can run multiple nodes using predefined config.
Runs from the virgin node state..
Can declare a queue.
Logs are written to stdout to be visible on startup.

14 files changed, 202 insertions, 82 deletions

diff --git a/Makefile b/Makefile
index 4e61332513..b38afeb60b 100644
--- a/Makefile
+++ b/Makefile
@@ -133,13 +133,16 @@ define PROJECT_ENV
 	  ]
 endef
 
+
 LOCAL_DEPS = sasl mnesia os_mon inets compiler public_key crypto ssl syntax_tools
 BUILD_DEPS = rabbitmq_cli syslog
-DEPS = ranch lager rabbit_common ra sysmon_handler stdout_formatter
+DEPS = ranch lager rabbit_common ra sysmon_handler mnevis stdout_formatter
 TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client meck proper
 
 dep_syslog = git https://github.com/schlagert/syslog 3.4.5
 
+dep_mnevis = git https://github.com/rabbitmq/mnevis master
+
 define usage_xml_to_erl
 $(subst __,_,$(patsubst $(DOCS_DIR)/rabbitmq%.1.xml, src/rabbit_%_usage.erl, $(subst -,_,$(1))))
 endef
diff --git a/rabbit.config b/rabbit.config
new file mode 100644
index 0000000000..030e3dbdbc
--- /dev/null
+++ b/rabbit.config
@@ -0,0 +1,14 @@
+[
+{rabbit, [
+           {cluster_nodes, {['rabbit@localhost'
+           ,
+                             'rabbit1@localhost'
+                             ], disc}}
+          ]},
+{ramnesia, [
+            {initial_nodes, ['rabbit@localhost'
+            ,
+                             'rabbit1@localhost'
+                             ]}]},
+{ra, [{data_dir, "/tmp/ramnesia"}]}
+].
diff --git a/src/gm.erl b/src/gm.erl
index bc8ed76db2..faf5e6b1e0 100644
--- a/src/gm.erl
+++ b/src/gm.erl
@@ -488,7 +488,7 @@ create_tables() ->
 create_tables([]) ->
     ok;
 create_tables([{Table, Attributes} | Tables]) ->
-    case mnesia:create_table(Table, Attributes) of
+    case ramnesia:create_table(Table, Attributes) of
         {atomic, ok}                       -> create_tables(Tables);
         {aborted, {already_exists, Table}} -> create_tables(Tables);
         Err                                -> Err
@@ -535,7 +535,7 @@ validate_members(Server, Members) ->
 -spec forget_group(group_name()) -> 'ok'.
 
 forget_group(GroupName) ->
-    {atomic, ok} = mnesia:sync_transaction(
+    {atomic, ok} = ramnesia:transaction(
                      fun () ->
                              mnesia:delete({?GROUP_TABLE, GroupName})
                      end),
diff --git a/src/rabbit.erl b/src/rabbit.erl
index 4a974fd682..77f3d23ec6 100644
--- a/src/rabbit.erl
+++ b/src/rabbit.erl
@@ -175,13 +175,13 @@
                    [{description, "core initialized"},
                     {requires,    kernel_ready}]}).
 
--rabbit_boot_step({upgrade_queues,
-                   [{description, "per-vhost message store migration"},
-                    {mfa,         {rabbit_upgrade,
-                                   maybe_migrate_queues_to_per_vhost_storage,
-                                   []}},
-                    {requires,    [core_initialized]},
-                    {enables,     recovery}]}).
+% -rabbit_boot_step({upgrade_queues,
+%                    [{description, "per-vhost message store migration"},
+%                     {mfa,         {rabbit_upgrade,
+%                                    maybe_migrate_queues_to_per_vhost_storage,
+%                                    []}},
+%                     {requires,    [core_initialized]},
+%                     {enables,     recovery}]}).
 
 -rabbit_boot_step({recovery,
                    [{description, "exchange, queue and binding recovery"},
@@ -292,7 +292,7 @@ start() ->
                      Apps = load_all_apps(),
                      rabbit_feature_flags:initialize_registry(),
                      rabbit_node_monitor:prepare_cluster_status_files(),
-                     rabbit_mnesia:check_cluster_consistency(),
+                     % rabbit_mnesia:check_cluster_consistency(),
                      broker_start(Apps)
              end).
 
@@ -307,13 +307,18 @@ boot() ->
                      rabbit_hipe:log_hipe_result(HipeResult),
                      Apps = load_all_apps(),
                      rabbit_feature_flags:initialize_registry(),
-                     rabbit_node_monitor:prepare_cluster_status_files(),
-                     ok = rabbit_upgrade:maybe_upgrade_mnesia(),
-                     %% It's important that the consistency check happens after
-                     %% the upgrade, since if we are a secondary node the
-                     %% primary node will have forgotten us
-                     rabbit_mnesia:check_cluster_consistency(),
-                     broker_start(Apps)
+
+                     %% TODO: disable node monitor and mnesia upgrades
+
+                     % rabbit_node_monitor:prepare_cluster_status_files(),
+                     % ok = rabbit_upgrade:maybe_upgrade_mnesia(),
+                     % %% It's important that the consistency check happens after
+                     % %% the upgrade, since if we are a secondary node the
+                     % %% primary node will have forgotten us
+                     % rabbit_mnesia:check_cluster_consistency(),
+                     % broker_start(Apps)
+
+                     broker_start()
              end).
 
 ensure_config() ->
@@ -1061,7 +1066,9 @@ boot_delegate() ->
 -spec recover() -> 'ok'.
 
 recover() ->
+    io:format("Recover policy ~n"),
     ok = rabbit_policy:recover(),
+    io:format("Recover vhost ~n"),
     ok = rabbit_vhost:recover(),
     ok = lager_exchange_backend:maybe_init_exchange().
 
@@ -1069,7 +1076,13 @@ recover() ->
 
 maybe_insert_default_data() ->
     case rabbit_table:needs_default_data() of
-        true  -> insert_default_data();
+        true  ->
+            {ok, _, {_, Leader}} = ra:members(ramnesia_node:node_id()),
+            case Leader == node() of
+                true ->
+                    insert_default_data();
+                false -> ok
+            end;
         false -> ok
     end.
 
diff --git a/src/rabbit_amqqueue.erl b/src/rabbit_amqqueue.erl
index 9e94dd8f27..fc115d160c 100644
--- a/src/rabbit_amqqueue.erl
+++ b/src/rabbit_amqqueue.erl
@@ -1509,7 +1509,7 @@ forget_all_durable(Node) ->
     %% Note rabbit is not running so we avoid e.g. the worker pool. Also why
     %% we don't invoke the return from rabbit_binding:process_deletions/1.
     {atomic, ok} =
-        mnesia:sync_transaction(
+        ramnesia:transaction(
           fun () ->
                   Qs = mnesia:match_object(rabbit_durable_queue,
                                            amqqueue:pattern_match_all(), write),
@@ -1674,10 +1674,12 @@ delete_queues_on_node_down(Node) ->
     lists:unzip(lists:flatten([
         rabbit_misc:execute_mnesia_transaction(
           fun () -> [{Queue, delete_queue(Queue)} || Queue <- Queues] end
-        ) || Queues <- partition_queues(queues_to_delete_when_node_down(Node))
+        )
+        || Queues <- partition_queues(queues_to_delete_when_node_down(Node))
     ])).
 
 delete_queue(QueueName) ->
+io:format("Delete queue  ~p~n", [QueueName]),
     ok = mnesia:delete({rabbit_queue, QueueName}),
     rabbit_binding:remove_transient_for_destination(QueueName).
 
@@ -1696,13 +1698,27 @@ partition_queues(T) ->
 
 queues_to_delete_when_node_down(NodeDown) ->
     rabbit_misc:execute_mnesia_transaction(fun () ->
-        qlc:e(qlc:q([amqqueue:get_name(Q) ||
-            Q <- mnesia:table(rabbit_queue),
-                amqqueue:qnode(Q) == NodeDown andalso
-                not rabbit_mnesia:is_process_alive(amqqueue:get_pid(Q)) andalso
-                (not rabbit_amqqueue:is_replicated(Q) orelse
-                rabbit_amqqueue:is_dead_exclusive(Q))]
-        ))
+        % qlc:e(qlc:q([amqqueue:get_name(Q) ||
+        %     Q <- mnesia:table(rabbit_queue),
+        %         amqqueue:qnode(Q) == NodeDown andalso
+        %         not rabbit_mnesia:is_process_alive(amqqueue:get_pid(Q)) andalso
+        %         (not rabbit_amqqueue:is_replicated(Q) orelse
+        %         rabbit_amqqueue:is_dead_exclusive(Q))]
+        % ))
+        mnesia:foldl(fun(Q, Acc) ->
+            case amqqueue:qnode(Q) == NodeDown andalso
+                    not rabbit_mnesia:is_process_alive(amqqueue:get_pid(Q)) andalso
+                    (not rabbit_amqqueue:is_replicated(Q) orelse
+                    rabbit_amqqueue:is_dead_exclusive(Q)) of
+                true ->
+                    [amqqueue:get_name(Q) | Acc];
+                false ->
+                    Acc
+            end
+        end,
+        [],
+        rabbit_queue)
+
     end).
 
 notify_queue_binding_deletions(QueueDeletions) ->
diff --git a/src/rabbit_binding.erl b/src/rabbit_binding.erl
index 94b8870192..e50d82ddcc 100644
--- a/src/rabbit_binding.erl
+++ b/src/rabbit_binding.erl
@@ -149,10 +149,10 @@ add(Binding, ActingUser) -> add(Binding, fun (_Src, _Dst) -> ok end, ActingUser)
 -spec add(rabbit_types:binding(), inner_fun(), rabbit_types:username()) -> bind_res().
 
 add(Binding, InnerFun, ActingUser) ->
-    binding_action(
+    Res = binding_action(
       Binding,
       fun (Src, Dst, B) ->
-              case rabbit_exchange:validate_binding(Src, B) of
+              Res1 = case rabbit_exchange:validate_binding(Src, B) of
                   ok ->
                       lock_resource(Src, read),
                       lock_resource(Dst, read),
@@ -170,8 +170,12 @@ add(Binding, InnerFun, ActingUser) ->
                       end;
                   {error, _} = Err ->
                       rabbit_misc:const(Err)
-              end
-      end, fun not_found_or_absent_errs/1).
+              end,
+              io:format("Inner action result ~p~n", [Res1]),
+              Res1
+      end, fun not_found_or_absent_errs/1),
+    io:format("Binding action result ~p~n", [Res]),
+    Res.
 
 add(Src, Dst, B, ActingUser) ->
     [SrcDurable, DstDurable] = [durable(E) || E <- [Src, Dst]],
diff --git a/src/rabbit_connection_tracking.erl b/src/rabbit_connection_tracking.erl
index aa5eb15464..11e0de622d 100644
--- a/src/rabbit_connection_tracking.erl
+++ b/src/rabbit_connection_tracking.erl
@@ -184,7 +184,7 @@ ensure_per_vhost_tracked_connections_table_for_this_node() ->
 
 ensure_tracked_connections_table_for_node(Node) ->
     TableName = tracked_connection_table_name_for(Node),
-    case mnesia:create_table(TableName, [{record_name, tracked_connection},
+    case ramnesia:create_table(TableName, [{record_name, tracked_connection},
                                          {attributes, record_info(fields, tracked_connection)}]) of
         {atomic, ok}                   -> ok;
         {aborted, {already_exists, _}} -> ok;
@@ -198,7 +198,7 @@ ensure_tracked_connections_table_for_node(Node) ->
 
 ensure_per_vhost_tracked_connections_table_for_node(Node) ->
     TableName = tracked_connection_per_vhost_table_name_for(Node),
-    case mnesia:create_table(TableName, [{record_name, tracked_connection_per_vhost},
+    case ramnesia:create_table(TableName, [{record_name, tracked_connection_per_vhost},
                                          {attributes, record_info(fields, tracked_connection_per_vhost)}]) of
         {atomic, ok}                   -> ok;
         {aborted, {already_exists, _}} -> ok;
@@ -225,7 +225,7 @@ clear_tracked_connection_tables_for_this_node() ->
 
 delete_tracked_connections_table_for_node(Node) ->
     TableName = tracked_connection_table_name_for(Node),
-    case mnesia:delete_table(TableName) of
+    case ramnesia:delete_table(TableName) of
         {atomic, ok}              -> ok;
         {aborted, {no_exists, _}} -> ok;
         {aborted, Error} ->
@@ -238,7 +238,7 @@ delete_tracked_connections_table_for_node(Node) ->
 
 delete_per_vhost_tracked_connections_table_for_node(Node) ->
     TableName = tracked_connection_per_vhost_table_name_for(Node),
-    case mnesia:delete_table(TableName) of
+    case ramnesia:delete_table(TableName) of
         {atomic, ok}              -> ok;
         {aborted, {no_exists, _}} -> ok;
         {aborted, Error} ->
@@ -344,9 +344,15 @@ count_connections_in(VirtualHost) ->
     lists:foldl(fun (Node, Acc) ->
                         Tab = tracked_connection_per_vhost_table_name_for(Node),
                         try
-                            N = case mnesia:dirty_read(Tab, VirtualHost) of
-                                    []    -> 0;
-                                    [Val] -> Val#tracked_connection_per_vhost.connection_count
+                            N = case ramnesia:transaction(
+                                       fun() ->
+                                               case mnesia:dirty_read({Tab, VirtualHost}) of
+                                                   []    -> 0;
+                                                   [Val] -> Val#tracked_connection_per_vhost.connection_count
+                                               end
+                                       end) of
+                                    {atomic,  Val}     -> Val;
+                                    {aborted, _Reason} -> 0
                                 end,
                             Acc + N
                         catch _:Err  ->
diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl
index 162badc33d..44a941b86d 100644
--- a/src/rabbit_mnesia.erl
+++ b/src/rabbit_mnesia.erl
@@ -51,6 +51,8 @@
 %% Used internally in rpc calls
 -export([node_info/0, remove_node_if_mnesia_running/1]).
 
+-compile(nowarn_unused_function).
+
 -ifdef(TEST).
 -compile(export_all).
 -export([init_with_lock/3]).
@@ -74,27 +76,68 @@
 init() ->
     ensure_mnesia_running(),
     ensure_mnesia_dir(),
-    case is_virgin_node() of
-        true  ->
-            rabbit_log:info("Node database directory at ~s is empty. "
-                            "Assuming we need to join an existing cluster or initialise from scratch...~n",
-                            [dir()]),
-            rabbit_peer_discovery:log_configured_backend(),
-            rabbit_peer_discovery:maybe_init(),
-            init_with_lock();
-        false ->
-            NodeType = node_type(),
-            init_db_and_upgrade(cluster_nodes(all), NodeType,
-                                NodeType =:= ram, _Retry = true),
-            rabbit_peer_discovery:maybe_init(),
-            rabbit_peer_discovery:maybe_register()
-    end,
+
+    %% Create schema on all nodes
+    ok = create_schema(),
+
+    ramnesia_node:start(),
+    ramnesia_node:trigger_election(),
+
+    {ok, _, _} = ra:members(ramnesia_node:node_id()),
+
+    io:format("~nDb nodes ~p~n", [ramnesia:db_nodes()]),
+
+    io:format("~nRunning Db nodes ~p~n", [ramnesia:running_db_nodes()]),
+
+    io:format("Get cluster status ~n"),
+    {ok, Status} = cluster_status_from_mnesia(),
+    io:format("Cluster status ~p~n", [Status]),
+
+    rabbit_node_monitor:write_cluster_status(Status),
+
+    % rabbit_node_monitor:update_cluster_status(),
+
+    % case is_virgin_node() of
+    %     true  ->
+
+    %         rabbit_log:info("Node database directory at ~s is empty. "
+    %                         "Assuming we need to join an existing cluster or initialise from scratch...~n",
+    %                         [dir()]),
+    %         rabbit_peer_discovery:log_configured_backend(),
+    %         rabbit_peer_discovery:maybe_init(),
+    %         init_with_lock();
+    %     false ->
+
+    %         NodeType = node_type(),
+    %         init_db_and_upgrade(cluster_nodes(all), NodeType,
+    %                             NodeType =:= ram, _Retry = true),
+    %         rabbit_peer_discovery:maybe_init(),
+    %         rabbit_peer_discovery:maybe_register()
+    % end,
     %% We intuitively expect the global name server to be synced when
     %% Mnesia is up. In fact that's not guaranteed to be the case -
     %% let's make it so.
     ok = rabbit_node_monitor:global_sync(),
     ok.
 
+wait_for_tables() ->
+    Tables = [T || {T, _D} <- rabbit_table:definitions()],
+    lists:foreach(fun(Table) ->
+        wait_for_table(Table)
+    end,
+    Tables),
+    ok.
+
+wait_for_table(Table) ->
+    rabbit_log:error("Waiting for table ~p~n", [Table]),
+    case lists:member(Table, mnesia:system_info(tables)) of
+        true -> ok;
+        false ->
+            timer:sleep(1000),
+            wait_for_table(Table)
+    end.
+
+
 init_with_lock() ->
     {Retries, Timeout} = rabbit_peer_discovery:retry_timeout(),
     init_with_lock(Retries, Timeout, fun init_from_config/0).
@@ -362,7 +405,7 @@ remove_node_offline_node(Node) ->
     %% want - we need to know the running nodes *now*.  If the current node is a
     %% RAM node it will return bogus results, but we don't care since we only do
     %% this operation from disc nodes.
-    case {mnesia:system_info(running_db_nodes) -- [Node], node_type()} of
+    case {ramnesia:running_db_nodes() -- [Node], node_type()} of
         {[], disc} ->
             start_mnesia(),
             try
@@ -460,23 +503,26 @@ cluster_status_from_mnesia() ->
             %% `init_db/3' hasn't been run yet. In other words, either
             %% we are a virgin node or a restarted RAM node. In both
             %% cases we're not interested in what mnesia has to say.
-            NodeType = case mnesia:system_info(use_dir) of
-                           true  -> disc;
-                           false -> ram
-                       end,
-            case rabbit_table:is_present() of
-                true  -> AllNodes = mnesia:system_info(db_nodes),
-                         DiscCopies = mnesia:table_info(schema, disc_copies),
-                         DiscNodes = case NodeType of
-                                         disc -> nodes_incl_me(DiscCopies);
-                                         ram  -> DiscCopies
-                                     end,
+            % NodeType = case mnesia:system_info(use_dir) of
+            %                true  -> disc;
+            %                false -> ram
+            %            end,
+            % case rabbit_table:is_present() of
+                % true  ->
+                AllNodes = ramnesia:db_nodes(),
+
+                         %% Ignoring disk node setting
+                         % DiscCopies = mnesia:table_info(schema, disc_copies),
+                         % DiscNodes = case NodeType of
+                                         % disc -> nodes_incl_me(DiscCopies);
+                                         % ram  -> DiscCopies
+                                     % end,
                          %% `mnesia:system_info(running_db_nodes)' is safe since
                          %% we know that mnesia is running
-                         RunningNodes = mnesia:system_info(running_db_nodes),
-                         {ok, {AllNodes, DiscNodes, RunningNodes}};
-                false -> {error, tables_not_present}
-            end
+                         RunningNodes = ramnesia:running_db_nodes(),
+                         {ok, {AllNodes, AllNodes, RunningNodes}}
+                % false -> {error, tables_not_present}
+            % end
     end.
 
 cluster_status(WhichNodes) ->
@@ -669,7 +715,9 @@ check_cluster_consistency() ->
     case lists:foldl(
            fun (Node,  {error, _})    -> check_cluster_consistency(Node, true);
                (_Node, {ok, Status})  -> {ok, Status}
-           end, {error, not_found}, nodes_excl_me(cluster_nodes(all)))
+           end,
+           {error, not_found},
+           nodes_excl_me(cluster_nodes(all)))
     of
         {ok, Status = {RemoteAllNodes, _, _}} ->
             case ordsets:is_subset(ordsets:from_list(cluster_nodes(all)),
@@ -677,6 +725,8 @@ check_cluster_consistency() ->
                 true  ->
                     ok;
                 false ->
+                    error({should_not_get_here, cluster_inconsistency}),
+
                     %% We delete the schema here since we think we are
                     %% clustered with nodes that are no longer in the
                     %% cluster and there is no other way to remove
@@ -790,11 +840,16 @@ schema_ok_or_move() ->
 %% We only care about disc nodes since ram nodes are supposed to catch
 %% up only
 create_schema() ->
+    io:format("Create schema ~n"),
     stop_mnesia(),
     rabbit_misc:ensure_ok(mnesia:create_schema([node()]), cannot_create_schema),
     start_mnesia(),
+
+    io:format("Create tables ~n"),
     ok = rabbit_table:create(),
+    io:format("Check integrity ~n"),
     ensure_schema_integrity(),
+    io:format("Record version ~n"),
     ok = rabbit_version:record_desired().
 
 move_db() ->
@@ -858,11 +913,11 @@ wait_for(Condition) ->
     rabbit_log:info("Waiting for ~p...~n", [Condition]),
     timer:sleep(1000).
 
-start_mnesia(CheckConsistency) ->
-    case CheckConsistency of
-        true  -> check_cluster_consistency();
-        false -> ok
-    end,
+start_mnesia(_CheckConsistency) ->
+    % case CheckConsistency of
+        % true  -> check_cluster_consistency();
+        % false -> ok
+    % end,
     rabbit_misc:ensure_ok(mnesia:start(), cannot_start_mnesia),
     ensure_mnesia_running().
 
diff --git a/src/rabbit_mnesia_rename.erl b/src/rabbit_mnesia_rename.erl
index 0fc0d3364f..f5c5697498 100644
--- a/src/rabbit_mnesia_rename.erl
+++ b/src/rabbit_mnesia_rename.erl
@@ -254,7 +254,7 @@ rename_in_running_mnesia(FromNode, ToNode) ->
     ok.
 
 transform_table(Table, Map) ->
-    mnesia:sync_transaction(
+    ramnesia:transaction(
       fun () ->
               mnesia:lock({table, Table}, write),
               transform_table(Table, Map, mnesia:first(Table))
diff --git a/src/rabbit_node_monitor.erl b/src/rabbit_node_monitor.erl
index 9a47c4030e..14083c0b79 100644
--- a/src/rabbit_node_monitor.erl
+++ b/src/rabbit_node_monitor.erl
@@ -129,6 +129,7 @@ write_cluster_status({All, Disc, Running}) ->
               E1 = {error, _} ->
                   {ClusterStatusFN, E1}
           end,
+    io:format("Written cluster status ~p~n", [{cluster_status_filename(), Res}]),
     case Res of
         {_, ok}           -> ok;
         {FN, {error, E2}} -> throw({error, {could_not_write_file, FN, E2}})
@@ -137,6 +138,9 @@ write_cluster_status({All, Disc, Running}) ->
 -spec read_cluster_status() -> rabbit_mnesia:cluster_status().
 
 read_cluster_status() ->
+io:format("Read cluster status ~p~n", [cluster_status_filename()]),
+io:format("Running nodes status ~p~n", [running_nodes_filename()]),
+
     case {try_read_file(cluster_status_filename()),
           try_read_file(running_nodes_filename())} of
         {{ok, [{All, Disc}]}, {ok, [Running]}} when is_list(Running) ->
@@ -825,7 +829,7 @@ legacy_cluster_nodes(Nodes) ->
     %% We get all the info that we can, including the nodes from
     %% mnesia, which will be there if the node is a disc node (empty
     %% list otherwise)
-    lists:usort(Nodes ++ mnesia:system_info(db_nodes)).
+    lists:usort(Nodes ++ ramnesia:db_nodes()).
 
 legacy_disc_nodes(AllNodes) ->
     case AllNodes == [] orelse lists:member(node(), AllNodes) of
diff --git a/src/rabbit_table.erl b/src/rabbit_table.erl
index 7dfc5fb573..1f6d25cd64 100644
--- a/src/rabbit_table.erl
+++ b/src/rabbit_table.erl
@@ -54,6 +54,7 @@ ensure_secondary_indexes() ->
   ensure_secondary_index(rabbit_queue, vhost),
   ok.
 
+%% TODO ramnesia add table index
 ensure_secondary_index(Table, Field) ->
   case mnesia:add_table_index(Table, Field) of
     {atomic, ok}                          -> ok;
diff --git a/src/rabbit_upgrade_functions.erl b/src/rabbit_upgrade_functions.erl
index d3cc363cbc..e6db8df485 100644
--- a/src/rabbit_upgrade_functions.erl
+++ b/src/rabbit_upgrade_functions.erl
@@ -418,7 +418,7 @@ internal_system_x() ->
 -spec cluster_name() -> 'ok'.
 
 cluster_name() ->
-    {atomic, ok} = mnesia:transaction(fun cluster_name_tx/0),
+    {atomic, ok} = ramnesia:transaction(fun cluster_name_tx/0),
     ok.
 
 cluster_name_tx() ->
@@ -656,7 +656,7 @@ transform(TableName, Fun, FieldList, NewRecordName) ->
     ok.
 
 create(Tab, TabDef) ->
-    {atomic, ok} = mnesia:create_table(Tab, TabDef),
+    {atomic, ok} = ramnesia:create_table(Tab, TabDef),
     ok.
 
 %% Dumb replacement for rabbit_exchange:declare that does not require
diff --git a/src/rabbit_vhost.erl b/src/rabbit_vhost.erl
index 8ab5a09725..3c157f2d34 100644
--- a/src/rabbit_vhost.erl
+++ b/src/rabbit_vhost.erl
@@ -31,15 +31,19 @@
 recover() ->
     %% Clear out remnants of old incarnation, in case we restarted
     %% faster than other nodes handled DOWN messages from us.
+    io:format("amqq on node down ~n"),
     rabbit_amqqueue:on_node_down(node()),
 
+    io:format("amqq warn limit ~n"),
     rabbit_amqqueue:warn_file_limit(),
 
     %% Prepare rabbit_semi_durable_route table
+    io:format("binding recover ~n"),
     rabbit_binding:recover(),
 
     %% rabbit_vhost_sup_sup will start the actual recovery.
     %% So recovery will be run every time a vhost supervisor is restarted.
+    io:format("vhost sup sup start ~n"),
     ok = rabbit_vhost_sup_sup:start(),
 
     [ok = rabbit_vhost_sup_sup:init_vhost(VHost) || VHost <- rabbit_vhost:list()],
diff --git a/test/mirrored_supervisor_SUITE.erl b/test/mirrored_supervisor_SUITE.erl
index d4bf6883da..55deee8b25 100644
--- a/test/mirrored_supervisor_SUITE.erl
+++ b/test/mirrored_supervisor_SUITE.erl
@@ -48,7 +48,7 @@ init_per_suite(Config) ->
     lists:foreach(
       fun ({Tab, TabDef}) ->
               TabDef1 = proplists:delete(match, TabDef),
-              case mnesia:create_table(Tab, TabDef1) of
+              case ramnesia:create_table(Tab, TabDef1) of
                   {atomic, ok} ->
                       ok;
                   {aborted, Reason} ->
@@ -277,7 +277,7 @@ pid_of(Id) ->
     Pid.
 
 tx_fun(Fun) ->
-    case mnesia:sync_transaction(Fun) of
+    case ramnesia:transaction(Fun) of
         {atomic,  Result}         -> Result;
         {aborted, Reason}         -> throw({error, Reason})
     end.