%% The contents of this file are subject to the Mozilla Public License %% Version 1.1 (the "License"); you may not use this file except in %% compliance with the License. You may obtain a copy of the License %% at https://www.mozilla.org/MPL/ %% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and %% limitations under the License. %% %% The Original Code is RabbitMQ. %% %% The Initial Developer of the Original Code is GoPivotal, Inc. %% Copyright (c) 2018-2020 Pivotal Software, Inc. All rights reserved. %% %% @author The RabbitMQ team %% @copyright 2018-2019 Pivotal Software, Inc. %% %% @doc %% This module offers a framework to declare capabilities a RabbitMQ node %% supports and therefore a way to determine if multiple RabbitMQ nodes in %% a cluster are compatible and can work together. %% %% == What a feature flag is == %% %% A feature flag is a name and several properties given %% to a change in RabbitMQ which impacts its communication with other %% RabbitMQ nodes. This kind of change can be: %% %% %% A feature flag is qualified by: %% %% %% == How to declare a feature flag == %% %% To define a new feature flag, you need to use the %% `rabbitmq_feature_flag()' module attribute: %% %% ``` %% -rabitmq_feature_flag(FeatureFlag). %% ''' %% %% `FeatureFlag' is a {@type feature_flag_modattr()}. %% %% == How to enable a feature flag == %% %% To enable a supported feature flag, you have the following solutions: %% %% %% %% == How to disable a feature flag == %% %% Once enabled, there is currently no way to disable a %% feature flag. -module(rabbit_feature_flags). -export([list/0, list/1, list/2, enable/1, enable_all/0, disable/1, disable_all/0, is_supported/1, is_supported/2, is_supported_locally/1, is_supported_remotely/1, is_supported_remotely/2, is_supported_remotely/3, is_enabled/1, is_enabled/2, is_disabled/1, is_disabled/2, info/0, info/1, init/0, get_state/1, get_stability/1, check_node_compatibility/1, check_node_compatibility/2, is_node_compatible/1, is_node_compatible/2, sync_feature_flags_with_cluster/2, sync_feature_flags_with_cluster/3, refresh_feature_flags_after_app_load/1, enabled_feature_flags_list_file/0 ]). %% RabbitMQ internal use only. -export([initialize_registry/0, initialize_registry/1, mark_as_enabled_locally/2, remote_nodes/0, running_remote_nodes/0, does_node_support/3, merge_feature_flags_from_unknown_apps/1, do_sync_feature_flags_with_node/1]). -ifdef(TEST). -export([initialize_registry/3, query_supported_feature_flags/0, mark_as_enabled_remotely/2, mark_as_enabled_remotely/4]). -endif. %% Default timeout for operations on remote nodes. -define(TIMEOUT, 60000). -define(FF_REGISTRY_LOADING_LOCK, {feature_flags_registry_loading, self()}). -define(FF_STATE_CHANGE_LOCK, {feature_flags_state_change, self()}). -type feature_flag_modattr() :: {feature_name(), feature_props()}. %% The value of a `-rabbitmq_feature_flag()' module attribute used to %% declare a new feature flag. -type feature_name() :: atom(). %% The feature flag's name. It is used in many places to identify a %% specific feature flag. In particular, this is how an end-user (or %% the CLI) can enable a feature flag. This is also the only bit which %% is persisted so a node remember which feature flags are enabled. -type feature_props() :: #{desc => string(), doc_url => string(), stability => stability(), depends_on => [feature_name()], migration_fun => migration_fun_name()}. %% The feature flag properties. %% %% All properties are optional. %% %% The properties are: %% %% %% Note that the `migration_fun' is a {@type migration_fun_name()}, %% not a {@type migration_fun()}. However, the function signature %% must conform to the {@type migration_fun()} signature. The reason %% is that we must be able to represent it as an Erlang term when %% we regenerate the registry module source code (using {@link %% erl_syntax:abstract/1}). -type feature_flags() :: #{feature_name() => feature_props_extended()}. %% The feature flags map as returned or accepted by several functions in %% this module. In particular, this what the {@link list/0} function %% returns. -type feature_props_extended() :: #{desc => string(), doc_url => string(), stability => stability(), migration_fun => migration_fun_name(), depends_on => [feature_name()], provided_by => atom()}. %% The feature flag properties, once expanded by this module when feature %% flags are discovered. %% %% The new properties compared to {@type feature_props()} are: %% -type feature_state() :: boolean() | state_changing. %% The state of the feature flag: enabled if `true', disabled if `false' %% or `state_changing'. -type feature_states() :: #{feature_name() => feature_state()}. -type stability() :: stable | experimental. %% The level of stability of a feature flag. Currently, only informational. -type migration_fun_name() :: {Module :: atom(), Function :: atom()}. %% The name of the module and function to call when changing the state of %% the feature flag. -type migration_fun() :: fun((feature_name(), feature_props_extended(), migration_fun_context()) -> ok | {error, any()} | % context = enable boolean() | undefined). % context = is_enabled %% The migration function signature. %% %% It is called with context `enable' when a feature flag is being enabled. %% The function is responsible for this feature-flag-specific verification %% and data conversion. It returns `ok' if RabbitMQ can mark the feature %% flag as enabled an continue with the next one, if any. Otherwise, it %% returns `{error, any()}' if there is an error and the feature flag should %% remain disabled. The function must be idempotent: if the feature flag is %% already enabled on another node and the local node is running this function %% again because it is syncing its feature flags state, it should succeed. %% %% It is called with the context `is_enabled' to check if a feature flag %% is actually enabled. It is useful on RabbitMQ startup, just in case %% the previous instance failed to write the feature flags list file. -type migration_fun_context() :: enable | is_enabled. -export_type([feature_flag_modattr/0, feature_props/0, feature_name/0, feature_flags/0, feature_props_extended/0, feature_state/0, feature_states/0, stability/0, migration_fun_name/0, migration_fun/0, migration_fun_context/0]). -on_load(on_load/0). -spec list() -> feature_flags(). %% @doc %% Lists all supported feature flags. %% %% @returns A map of all supported feature flags. list() -> list(all). -spec list(Which :: all | enabled | disabled) -> feature_flags(). %% @doc %% Lists all, enabled or disabled feature flags, depending on the argument. %% %% @param Which The group of feature flags to return: `all', `enabled' or %% `disabled'. %% @returns A map of selected feature flags. list(all) -> rabbit_ff_registry:list(all); list(enabled) -> rabbit_ff_registry:list(enabled); list(disabled) -> maps:filter( fun(FeatureName, _) -> is_disabled(FeatureName) end, list(all)). -spec list(all | enabled | disabled, stability()) -> feature_flags(). %% @doc %% Lists all, enabled or disabled feature flags, depending on the first %% argument, only keeping those having the specified stability. %% %% @param Which The group of feature flags to return: `all', `enabled' or %% `disabled'. %% @param Stability The level of stability used to filter the map of feature %% flags. %% @returns A map of selected feature flags. list(Which, Stability) when Stability =:= stable orelse Stability =:= experimental -> maps:filter(fun(_, FeatureProps) -> Stability =:= get_stability(FeatureProps) end, list(Which)). -spec enable(feature_name() | [feature_name()]) -> ok | {error, Reason :: any()}. %% @doc %% Enables the specified feature flag or set of feature flags. %% %% @param FeatureName The name or the list of names of feature flags to %% enable. %% @returns `ok' if the feature flags (and all the feature flags they %% depend on) were successfully enabled, or `{error, Reason}' if one %% feature flag could not be enabled (subsequent feature flags in the %% dependency tree are left unchanged). enable(FeatureName) when is_atom(FeatureName) -> rabbit_log_feature_flags:debug( "Feature flag `~s`: REQUEST TO ENABLE", [FeatureName]), case is_enabled(FeatureName) of true -> rabbit_log_feature_flags:debug( "Feature flag `~s`: already enabled", [FeatureName]), ok; false -> rabbit_log_feature_flags:debug( "Feature flag `~s`: not enabled, check if supported by cluster", [FeatureName]), %% The feature flag must be supported locally and remotely %% (i.e. by all members of the cluster). case is_supported(FeatureName) of true -> rabbit_log_feature_flags:info( "Feature flag `~s`: supported, attempt to enable...", [FeatureName]), do_enable(FeatureName); false -> rabbit_log_feature_flags:error( "Feature flag `~s`: not supported", [FeatureName]), {error, unsupported} end end; enable(FeatureNames) when is_list(FeatureNames) -> with_feature_flags(FeatureNames, fun enable/1). -spec enable_all() -> ok | {error, any()}. %% @doc %% Enables all supported feature flags. %% %% @returns `ok' if the feature flags were successfully enabled, %% or `{error, Reason}' if one feature flag could not be enabled %% (subsequent feature flags in the dependency tree are left %% unchanged). enable_all() -> with_feature_flags(maps:keys(list(all)), fun enable/1). -spec disable(feature_name() | [feature_name()]) -> ok | {error, any()}. %% @doc %% Disables the specified feature flag or set of feature flags. %% %% @param FeatureName The name or the list of names of feature flags to %% disable. %% @returns `ok' if the feature flags (and all the feature flags they %% depend on) were successfully disabled, or `{error, Reason}' if one %% feature flag could not be disabled (subsequent feature flags in the %% dependency tree are left unchanged). disable(FeatureName) when is_atom(FeatureName) -> {error, unsupported}; disable(FeatureNames) when is_list(FeatureNames) -> with_feature_flags(FeatureNames, fun disable/1). -spec disable_all() -> ok | {error, any()}. %% @doc %% Disables all supported feature flags. %% %% @returns `ok' if the feature flags were successfully disabled, %% or `{error, Reason}' if one feature flag could not be disabled %% (subsequent feature flags in the dependency tree are left %% unchanged). disable_all() -> with_feature_flags(maps:keys(list(all)), fun disable/1). -spec with_feature_flags([feature_name()], fun((feature_name()) -> ok | {error, any()})) -> ok | {error, any()}. %% @private with_feature_flags([FeatureName | Rest], Fun) -> case Fun(FeatureName) of ok -> with_feature_flags(Rest, Fun); Error -> Error end; with_feature_flags([], _) -> ok. -spec is_supported(feature_name() | [feature_name()]) -> boolean(). %% @doc %% Returns if a single feature flag or a set of feature flags is %% supported by the entire cluster. %% %% This is the same as calling both {@link is_supported_locally/1} and %% {@link is_supported_remotely/1} with a logical AND. %% %% @param FeatureNames The name or a list of names of the feature flag(s) %% to be checked. %% @returns `true' if the set of feature flags is entirely supported, or %% `false' if one of them is not or the RPC timed out. is_supported(FeatureNames) -> is_supported_locally(FeatureNames) andalso is_supported_remotely(FeatureNames). -spec is_supported(feature_name() | [feature_name()], timeout()) -> boolean(). %% @doc %% Returns if a single feature flag or a set of feature flags is %% supported by the entire cluster. %% %% This is the same as calling both {@link is_supported_locally/1} and %% {@link is_supported_remotely/2} with a logical AND. %% %% @param FeatureNames The name or a list of names of the feature flag(s) %% to be checked. %% @param Timeout Time in milliseconds after which the RPC gives up. %% @returns `true' if the set of feature flags is entirely supported, or %% `false' if one of them is not or the RPC timed out. is_supported(FeatureNames, Timeout) -> is_supported_locally(FeatureNames) andalso is_supported_remotely(FeatureNames, Timeout). -spec is_supported_locally(feature_name() | [feature_name()]) -> boolean(). %% @doc %% Returns if a single feature flag or a set of feature flags is %% supported by the local node. %% %% @param FeatureNames The name or a list of names of the feature flag(s) %% to be checked. %% @returns `true' if the set of feature flags is entirely supported, or %% `false' if one of them is not. is_supported_locally(FeatureName) when is_atom(FeatureName) -> rabbit_ff_registry:is_supported(FeatureName); is_supported_locally(FeatureNames) when is_list(FeatureNames) -> lists:all(fun(F) -> rabbit_ff_registry:is_supported(F) end, FeatureNames). -spec is_supported_remotely(feature_name() | [feature_name()]) -> boolean(). %% @doc %% Returns if a single feature flag or a set of feature flags is %% supported by all remote nodes. %% %% @param FeatureNames The name or a list of names of the feature flag(s) %% to be checked. %% @returns `true' if the set of feature flags is entirely supported, or %% `false' if one of them is not or the RPC timed out. is_supported_remotely(FeatureNames) -> is_supported_remotely(FeatureNames, ?TIMEOUT). -spec is_supported_remotely(feature_name() | [feature_name()], timeout()) -> boolean(). %% @doc %% Returns if a single feature flag or a set of feature flags is %% supported by all remote nodes. %% %% @param FeatureNames The name or a list of names of the feature flag(s) %% to be checked. %% @param Timeout Time in milliseconds after which the RPC gives up. %% @returns `true' if the set of feature flags is entirely supported, or %% `false' if one of them is not or the RPC timed out. is_supported_remotely(FeatureName, Timeout) when is_atom(FeatureName) -> is_supported_remotely([FeatureName], Timeout); is_supported_remotely([], _) -> rabbit_log_feature_flags:debug( "Feature flags: skipping query for feature flags support as the " "given list is empty"), true; is_supported_remotely(FeatureNames, Timeout) when is_list(FeatureNames) -> case running_remote_nodes() of [] -> rabbit_log_feature_flags:debug( "Feature flags: isolated node; skipping remote node query " "=> consider `~p` supported", [FeatureNames]), true; RemoteNodes -> rabbit_log_feature_flags:debug( "Feature flags: about to query these remote nodes about " "support for `~p`: ~p", [FeatureNames, RemoteNodes]), is_supported_remotely(RemoteNodes, FeatureNames, Timeout) end. -spec is_supported_remotely([node()], feature_name() | [feature_name()], timeout()) -> boolean(). %% @doc %% Returns if a single feature flag or a set of feature flags is %% supported by specified remote nodes. %% %% @param RemoteNodes The list of remote nodes to query. %% @param FeatureNames The name or a list of names of the feature flag(s) %% to be checked. %% @param Timeout Time in milliseconds after which the RPC gives up. %% @returns `true' if the set of feature flags is entirely supported by %% all nodes, or `false' if one of them is not or the RPC timed out. is_supported_remotely(_, [], _) -> rabbit_log_feature_flags:debug( "Feature flags: skipping query for feature flags support as the " "given list is empty"), true; is_supported_remotely([Node | Rest], FeatureNames, Timeout) -> case does_node_support(Node, FeatureNames, Timeout) of true -> is_supported_remotely(Rest, FeatureNames, Timeout); false -> rabbit_log_feature_flags:debug( "Feature flags: stopping query for support for `~p` here", [FeatureNames]), false end; is_supported_remotely([], FeatureNames, _) -> rabbit_log_feature_flags:debug( "Feature flags: all running remote nodes support `~p`", [FeatureNames]), true. -spec is_enabled(feature_name() | [feature_name()]) -> boolean(). %% @doc %% Returns if a single feature flag or a set of feature flags is %% enabled. %% %% This is the same as calling {@link is_enabled/2} as a `blocking' %% call. %% %% @param FeatureNames The name or a list of names of the feature flag(s) %% to be checked. %% @returns `true' if the set of feature flags is enabled, or %% `false' if one of them is not. is_enabled(FeatureNames) -> is_enabled(FeatureNames, blocking). -spec is_enabled (feature_name() | [feature_name()], blocking) -> boolean(); (feature_name() | [feature_name()], non_blocking) -> feature_state(). %% @doc %% Returns if a single feature flag or a set of feature flags is %% enabled. %% %% When `blocking' is passed, the function waits (blocks) for the %% state of a feature flag being disabled or enabled stabilizes before %% returning its final state. %% %% When `non_blocking' is passed, the function returns immediately with %% the state of the feature flag (`true' if enabled, `false' otherwise) %% or `state_changing' is the state is being changed at the time of the %% call. %% %% @param FeatureNames The name or a list of names of the feature flag(s) %% to be checked. %% @returns `true' if the set of feature flags is enabled, %% `false' if one of them is not, or `state_changing' if one of them %% is being worked on. Note that `state_changing' has precedence over %% `false', so if one is `false' and another one is `state_changing', %% `state_changing' is returned. is_enabled(FeatureNames, non_blocking) -> is_enabled_nb(FeatureNames); is_enabled(FeatureNames, blocking) -> case is_enabled_nb(FeatureNames) of state_changing -> global:set_lock(?FF_STATE_CHANGE_LOCK), global:del_lock(?FF_STATE_CHANGE_LOCK), is_enabled(FeatureNames, blocking); IsEnabled -> IsEnabled end. is_enabled_nb(FeatureName) when is_atom(FeatureName) -> rabbit_ff_registry:is_enabled(FeatureName); is_enabled_nb(FeatureNames) when is_list(FeatureNames) -> lists:foldl( fun (_F, state_changing = Acc) -> Acc; (F, false = Acc) -> case rabbit_ff_registry:is_enabled(F) of state_changing -> state_changing; _ -> Acc end; (F, _) -> rabbit_ff_registry:is_enabled(F) end, true, FeatureNames). -spec is_disabled(feature_name() | [feature_name()]) -> boolean(). %% @doc %% Returns if a single feature flag or one feature flag in a set of %% feature flags is disabled. %% %% This is the same as negating the result of {@link is_enabled/1}. %% %% @param FeatureNames The name or a list of names of the feature flag(s) %% to be checked. %% @returns `true' if one of the feature flags is disabled, or %% `false' if they are all enabled. is_disabled(FeatureNames) -> is_disabled(FeatureNames, blocking). -spec is_disabled (feature_name() | [feature_name()], blocking) -> boolean(); (feature_name() | [feature_name()], non_blocking) -> feature_state(). %% @doc %% Returns if a single feature flag or one feature flag in a set of %% feature flags is disabled. %% %% This is the same as negating the result of {@link is_enabled/2}, %% except that `state_changing' is returned as is. %% %% See {@link is_enabled/2} for a description of the `blocking' and %% `non_blocking' modes. %% %% @param FeatureNames The name or a list of names of the feature flag(s) %% to be checked. %% @returns `true' if one feature flag in the set of feature flags is %% disabled, `false' if they are all enabled, or `state_changing' if %% one of them is being worked on. Note that `state_changing' has %% precedence over `true', so if one is `true' (i.e. disabled) and %% another one is `state_changing', `state_changing' is returned. %% %% @see is_enabled/2 is_disabled(FeatureName, Blocking) -> case is_enabled(FeatureName, Blocking) of state_changing -> state_changing; IsEnabled -> not IsEnabled end. -spec info() -> ok. %% @doc %% Displays a table on stdout summing up the supported feature flags, %% their state and various informations about them. info() -> info(#{}). -spec info(#{color => boolean(), lines => boolean(), verbose => non_neg_integer()}) -> ok. %% @doc %% Displays a table on stdout summing up the supported feature flags, %% their state and various informations about them. %% %% Supported options are: %% %% %% @param Options A map of various options to tune the displayed table. info(Options) when is_map(Options) -> rabbit_ff_extra:info(Options). -spec get_state(feature_name()) -> enabled | disabled | unavailable. %% @doc %% Returns the state of a feature flag. %% %% The possible states are: %% %% %% @param FeatureName The name of the feature flag to check. %% @returns `enabled', `disabled' or `unavailable'. get_state(FeatureName) when is_atom(FeatureName) -> IsEnabled = is_enabled(FeatureName), IsSupported = is_supported(FeatureName), case IsEnabled of true -> enabled; false -> case IsSupported of true -> disabled; false -> unavailable end end. -spec get_stability(feature_name() | feature_props_extended()) -> stability(). %% @doc %% Returns the stability of a feature flag. %% %% The possible stability levels are: %% %% %% @param FeatureName The name of the feature flag to check. %% @returns `stable' or `experimental'. get_stability(FeatureName) when is_atom(FeatureName) -> case rabbit_ff_registry:get(FeatureName) of undefined -> undefined; FeatureProps -> get_stability(FeatureProps) end; get_stability(FeatureProps) when is_map(FeatureProps) -> maps:get(stability, FeatureProps, stable). %% ------------------------------------------------------------------- %% Feature flags registry. %% ------------------------------------------------------------------- -spec init() -> ok | no_return(). %% @private init() -> %% We want to make sure the `feature_flags` file exists once %% RabbitMQ was started at least once. This is not required by %% this module (it works fine if the file is missing) but it helps %% external tools. _ = ensure_enabled_feature_flags_list_file_exists(), %% We also "list" supported feature flags. We are not interested in %% that list, however, it triggers the first initialization of the %% registry. _ = list(all), ok. -spec initialize_registry() -> ok | {error, any()} | no_return(). %% @private %% @doc %% Initializes or reinitializes the registry. %% %% The registry is an Erlang module recompiled at runtime to hold the %% state of all supported feature flags. %% %% That Erlang module is called {@link rabbit_ff_registry}. The initial %% source code of this module simply calls this function so it is %% replaced by a proper registry. %% %% Once replaced, the registry contains the map of all supported feature %% flags and their state. This is makes it very efficient to query a %% feature flag state or property. %% %% The registry is local to all RabbitMQ nodes. initialize_registry() -> initialize_registry(#{}). -spec initialize_registry(feature_flags()) -> ok | {error, any()} | no_return(). %% @private %% @doc %% Initializes or reinitializes the registry. %% %% See {@link initialize_registry/0} for a description of the registry. %% %% This function takes a map of new supported feature flags (so their %% name and extended properties) to add to the existing known feature %% flags. initialize_registry(NewSupportedFeatureFlags) -> %% The first step is to get the feature flag states: if this is the %% first time we initialize it, we read the list from disk (the %% `feature_flags` file). Otherwise we query the existing registry %% before it is replaced. RegistryInitialized = rabbit_ff_registry:is_registry_initialized(), FeatureStates = case RegistryInitialized of true -> rabbit_ff_registry:states(); false -> EnabledFeatureNames = read_enabled_feature_flags_list(), list_of_enabled_feature_flags_to_feature_states( EnabledFeatureNames) end, %% We also record if the feature flags state was correctly written %% to disk. Currently we don't use this information, but in the %% future, we might want to retry the write if it failed so far. %% %% TODO: Retry to write the feature flags state if the first try %% failed. WrittenToDisk = case RegistryInitialized of true -> rabbit_ff_registry:is_registry_written_to_disk(); false -> true end, initialize_registry(NewSupportedFeatureFlags, FeatureStates, WrittenToDisk). -spec list_of_enabled_feature_flags_to_feature_states([feature_name()]) -> feature_states(). list_of_enabled_feature_flags_to_feature_states(FeatureNames) -> maps:from_list([{FeatureName, true} || FeatureName <- FeatureNames]). -spec initialize_registry(feature_flags(), feature_states(), boolean()) -> ok | {error, any()} | no_return(). %% @private %% @doc %% Initializes or reinitializes the registry. %% %% See {@link initialize_registry/0} for a description of the registry. %% %% This function takes a map of new supported feature flags (so their %% name and extended properties) to add to the existing known feature %% flags, a map of the new feature flag states (whether they are %% enabled, disabled or `state_changing'), and a flag to indicate if the %% feature flag states was recorded to disk. %% %% The latter is used to block callers asking if a feature flag is %% enabled or disabled while its state is changing. initialize_registry(NewSupportedFeatureFlags, NewFeatureStates, WrittenToDisk) -> %% We take the feature flags already registered. RegistryInitialized = rabbit_ff_registry:is_registry_initialized(), KnownFeatureFlags1 = case RegistryInitialized of true -> rabbit_ff_registry:list(all); false -> #{} end, %% Query the list (it's a map to be exact) of known %% supported feature flags. That list comes from the %% `-rabbitmq_feature_flag().` module attributes exposed by all %% currently loaded Erlang modules. KnownFeatureFlags2 = query_supported_feature_flags(), %% We merge the feature flags we already knew about %% (KnownFeatureFlags1), those found in the loaded applications %% (KnownFeatureFlags2) and those specified in arguments %% (NewSupportedFeatureFlags). The latter come from remote nodes %% usually: for example, they can come from plugins loaded on remote %% node but the plugins are missing locally. In this case, we %% consider those feature flags supported because there is no code %% locally which would cause issues. %% %% It means that the list of feature flags only grows. we don't try %% to clean it at some point because we want to remember about the %% feature flags we saw (and their state). It should be fine because %% that list should remain small. KnownFeatureFlags = maps:merge(KnownFeatureFlags1, KnownFeatureFlags2), AllFeatureFlags = maps:merge(KnownFeatureFlags, NewSupportedFeatureFlags), %% Next we want to update the feature states, based on the new %% states passed as arguments. FeatureStates0 = case RegistryInitialized of true -> maps:merge(rabbit_ff_registry:states(), NewFeatureStates); false -> NewFeatureStates end, FeatureStates = maps:filter( fun(_, true) -> true; (_, state_changing) -> true; (_, false) -> false end, FeatureStates0), Proceed = does_registry_need_refresh(AllFeatureFlags, FeatureStates, WrittenToDisk), case Proceed of true -> rabbit_log_feature_flags:debug( "Feature flags: (re)initialize registry"), do_initialize_registry(AllFeatureFlags, FeatureStates, WrittenToDisk); false -> rabbit_log_feature_flags:debug( "Feature flags: registry already up-to-date, skipping init"), ok end. -spec does_registry_need_refresh(feature_flags(), feature_states(), boolean()) -> boolean(). does_registry_need_refresh(AllFeatureFlags, FeatureStates, WrittenToDisk) -> case rabbit_ff_registry:is_registry_initialized() of true -> %% Before proceeding with the actual %% (re)initialization, let's see if there are any %% changes. CurrentAllFeatureFlags = rabbit_ff_registry:list(all), CurrentFeatureStates = rabbit_ff_registry:states(), CurrentWrittenToDisk = rabbit_ff_registry:is_registry_written_to_disk(), AllFeatureFlags =/= CurrentAllFeatureFlags orelse FeatureStates =/= CurrentFeatureStates orelse WrittenToDisk =/= CurrentWrittenToDisk; false -> true end. -spec do_initialize_registry(feature_flags(), feature_states(), boolean()) -> ok | {error, any()} | no_return(). %% @private do_initialize_registry(AllFeatureFlags, FeatureStates, WrittenToDisk) -> %% We log the state of those feature flags. rabbit_log_feature_flags:info( "Feature flags: list of feature flags found:"), lists:foreach( fun(FeatureName) -> rabbit_log_feature_flags:info( "Feature flags: [~s] ~s", [case maps:is_key(FeatureName, FeatureStates) of true -> case maps:get(FeatureName, FeatureStates) of true -> "x"; state_changing -> "~" end; false -> " " end, FeatureName]) end, lists:sort(maps:keys(AllFeatureFlags))), rabbit_log_feature_flags:info( "Feature flags: feature flag states written to disk: ~s", [case WrittenToDisk of true -> "yes"; false -> "no" end]), %% We request the registry to be regenerated and reloaded with the %% new state. T0 = erlang:timestamp(), Ret = regen_registry_mod(AllFeatureFlags, FeatureStates, WrittenToDisk), T1 = erlang:timestamp(), rabbit_log_feature_flags:debug( "Feature flags: time to regen registry: ~p µs", [timer:now_diff(T1, T0)]), Ret. -spec query_supported_feature_flags() -> feature_flags(). %% @private -ifdef(TEST). module_attributes_from_testsuite() -> try throw(force_exception) catch throw:force_exception:Stacktrace -> Modules = lists:filter( fun({Mod, _, _, _}) -> ModS = atom_to_list(Mod), re:run(ModS, "_SUITE$", [{capture, none}]) =:= match end, Stacktrace), case Modules of [{Module, _, _, _} | _] -> ModInfo = Module:module_info(attributes), Attrs = lists:append( [Attr || {Name, Attr} <- ModInfo, Name =:= rabbit_feature_flag]), case Attrs of [] -> []; _ -> [{Module, Module, Attrs}] end; _ -> [] end end. query_supported_feature_flags() -> rabbit_log_feature_flags:debug( "Feature flags: query feature flags in loaded applications + test " "module"), T0 = erlang:timestamp(), AttributesPerApp = rabbit_misc:rabbitmq_related_module_attributes( rabbit_feature_flag), AttributesFromTestsuite = module_attributes_from_testsuite(), T1 = erlang:timestamp(), rabbit_log_feature_flags:debug( "Feature flags: time to find supported feature flags: ~p µs", [timer:now_diff(T1, T0)]), AllAttributes = AttributesPerApp ++ AttributesFromTestsuite, prepare_queried_feature_flags(AllAttributes, #{}). -else. query_supported_feature_flags() -> rabbit_log_feature_flags:debug( "Feature flags: query feature flags in loaded applications"), T0 = erlang:timestamp(), AttributesPerApp = rabbit_misc:rabbitmq_related_module_attributes( rabbit_feature_flag), T1 = erlang:timestamp(), rabbit_log_feature_flags:debug( "Feature flags: time to find supported feature flags: ~p µs", [timer:now_diff(T1, T0)]), prepare_queried_feature_flags(AttributesPerApp, #{}). -endif. prepare_queried_feature_flags([{App, _Module, Attributes} | Rest], AllFeatureFlags) -> rabbit_log_feature_flags:debug( "Feature flags: application `~s` has ~b feature flags", [App, length(Attributes)]), AllFeatureFlags1 = lists:foldl( fun({FeatureName, FeatureProps}, AllFF) -> merge_new_feature_flags(AllFF, App, FeatureName, FeatureProps) end, AllFeatureFlags, Attributes), prepare_queried_feature_flags(Rest, AllFeatureFlags1); prepare_queried_feature_flags([], AllFeatureFlags) -> AllFeatureFlags. -spec merge_new_feature_flags(feature_flags(), atom(), feature_name(), feature_props()) -> feature_flags(). %% @private merge_new_feature_flags(AllFeatureFlags, App, FeatureName, FeatureProps) when is_atom(FeatureName) andalso is_map(FeatureProps) -> %% We expand the feature flag properties map with: %% - the name of the application providing it: only informational %% for now, but can be handy to understand that a feature flag %% comes from a plugin. FeatureProps1 = maps:put(provided_by, App, FeatureProps), maps:merge(AllFeatureFlags, #{FeatureName => FeatureProps1}). -spec regen_registry_mod(feature_flags(), feature_states(), boolean()) -> ok | {error, any()} | no_return(). %% @private regen_registry_mod(AllFeatureFlags, FeatureStates, WrittenToDisk) -> %% Here, we recreate the source code of the `rabbit_ff_registry` %% module from scratch. %% %% IMPORTANT: We want both modules to have the exact same public %% API in order to simplify the life of developers and their tools %% (Dialyzer, completion, and so on). %% -module(rabbit_ff_registry). ModuleAttr = erl_syntax:attribute( erl_syntax:atom(module), [erl_syntax:atom(rabbit_ff_registry)]), ModuleForm = erl_syntax:revert(ModuleAttr), %% -export([...]). ExportAttr = erl_syntax:attribute( erl_syntax:atom(export), [erl_syntax:list( [erl_syntax:arity_qualifier( erl_syntax:atom(F), erl_syntax:integer(A)) || {F, A} <- [{get, 1}, {list, 1}, {states, 0}, {is_supported, 1}, {is_enabled, 1}, {is_registry_initialized, 0}, {is_registry_written_to_disk, 0}]] ) ] ), ExportForm = erl_syntax:revert(ExportAttr), %% get(_) -> ... GetClauses = [erl_syntax:clause( [erl_syntax:atom(FeatureName)], [], [erl_syntax:abstract(maps:get(FeatureName, AllFeatureFlags))]) || FeatureName <- maps:keys(AllFeatureFlags) ], GetUnknownClause = erl_syntax:clause( [erl_syntax:variable("_")], [], [erl_syntax:atom(undefined)]), GetFun = erl_syntax:function( erl_syntax:atom(get), GetClauses ++ [GetUnknownClause]), GetFunForm = erl_syntax:revert(GetFun), %% list(_) -> ... ListAllBody = erl_syntax:abstract(AllFeatureFlags), ListAllClause = erl_syntax:clause([erl_syntax:atom(all)], [], [ListAllBody]), EnabledFeatureFlags = maps:filter( fun(FeatureName, _) -> maps:is_key(FeatureName, FeatureStates) andalso maps:get(FeatureName, FeatureStates) =:= true end, AllFeatureFlags), ListEnabledBody = erl_syntax:abstract(EnabledFeatureFlags), ListEnabledClause = erl_syntax:clause( [erl_syntax:atom(enabled)], [], [ListEnabledBody]), DisabledFeatureFlags = maps:filter( fun(FeatureName, _) -> not maps:is_key(FeatureName, FeatureStates) end, AllFeatureFlags), ListDisabledBody = erl_syntax:abstract(DisabledFeatureFlags), ListDisabledClause = erl_syntax:clause( [erl_syntax:atom(disabled)], [], [ListDisabledBody]), StateChangingFeatureFlags = maps:filter( fun(FeatureName, _) -> maps:is_key(FeatureName, FeatureStates) andalso maps:get(FeatureName, FeatureStates) =:= state_changing end, AllFeatureFlags), ListStateChangingBody = erl_syntax:abstract(StateChangingFeatureFlags), ListStateChangingClause = erl_syntax:clause( [erl_syntax:atom(state_changing)], [], [ListStateChangingBody]), ListFun = erl_syntax:function( erl_syntax:atom(list), [ListAllClause, ListEnabledClause, ListDisabledClause, ListStateChangingClause]), ListFunForm = erl_syntax:revert(ListFun), %% states() -> ... StatesBody = erl_syntax:abstract(FeatureStates), StatesClause = erl_syntax:clause([], [], [StatesBody]), StatesFun = erl_syntax:function( erl_syntax:atom(states), [StatesClause]), StatesFunForm = erl_syntax:revert(StatesFun), %% is_supported(_) -> ... IsSupportedClauses = [erl_syntax:clause( [erl_syntax:atom(FeatureName)], [], [erl_syntax:atom(true)]) || FeatureName <- maps:keys(AllFeatureFlags) ], NotSupportedClause = erl_syntax:clause( [erl_syntax:variable("_")], [], [erl_syntax:atom(false)]), IsSupportedFun = erl_syntax:function( erl_syntax:atom(is_supported), IsSupportedClauses ++ [NotSupportedClause]), IsSupportedFunForm = erl_syntax:revert(IsSupportedFun), %% is_enabled(_) -> ... IsEnabledClauses = [erl_syntax:clause( [erl_syntax:atom(FeatureName)], [], [case maps:is_key(FeatureName, FeatureStates) of true -> erl_syntax:atom( maps:get(FeatureName, FeatureStates)); false -> erl_syntax:atom(false) end]) || FeatureName <- maps:keys(AllFeatureFlags) ], NotEnabledClause = erl_syntax:clause( [erl_syntax:variable("_")], [], [erl_syntax:atom(false)]), IsEnabledFun = erl_syntax:function( erl_syntax:atom(is_enabled), IsEnabledClauses ++ [NotEnabledClause]), IsEnabledFunForm = erl_syntax:revert(IsEnabledFun), %% is_registry_initialized() -> ... IsInitializedClauses = [erl_syntax:clause( [], [], [erl_syntax:atom(true)]) ], IsInitializedFun = erl_syntax:function( erl_syntax:atom(is_registry_initialized), IsInitializedClauses), IsInitializedFunForm = erl_syntax:revert(IsInitializedFun), %% is_registry_written_to_disk() -> ... IsWrittenToDiskClauses = [erl_syntax:clause( [], [], [erl_syntax:atom(WrittenToDisk)]) ], IsWrittenToDiskFun = erl_syntax:function( erl_syntax:atom(is_registry_written_to_disk), IsWrittenToDiskClauses), IsWrittenToDiskFunForm = erl_syntax:revert(IsWrittenToDiskFun), %% Compilation! Forms = [ModuleForm, ExportForm, GetFunForm, ListFunForm, StatesFunForm, IsSupportedFunForm, IsEnabledFunForm, IsInitializedFunForm, IsWrittenToDiskFunForm], maybe_log_registry_source_code(Forms), CompileOpts = [return_errors, return_warnings], case compile:forms(Forms, CompileOpts) of {ok, Mod, Bin, _} -> load_registry_mod(Mod, Bin); {error, Errors, Warnings} -> rabbit_log_feature_flags:error( "Feature flags: registry compilation:~n" "Errors: ~p~n" "Warnings: ~p", [Errors, Warnings]), {error, {compilation_failure, Errors, Warnings}} end. maybe_log_registry_source_code(Forms) -> case rabbit_prelaunch:get_context() of #{log_feature_flags_registry := true} -> rabbit_log_feature_flags:debug( "== FEATURE FLAGS REGISTRY ==~n" "~s~n" "== END ==~n", [erl_prettypr:format(erl_syntax:form_list(Forms))]); _ -> ok end. -spec load_registry_mod(atom(), binary()) -> ok | {error, any()} | no_return(). %% @private load_registry_mod(Mod, Bin) -> rabbit_log_feature_flags:debug( "Feature flags: registry module ready, loading it..."), FakeFilename = "Compiled and loaded by " ++ ?MODULE_STRING, %% Time to load the new registry, replacing the old one. We use a %% lock here to synchronize concurrent reloads. global:set_lock(?FF_REGISTRY_LOADING_LOCK, [node()]), _ = code:soft_purge(Mod), _ = code:delete(Mod), Ret = code:load_binary(Mod, FakeFilename, Bin), global:del_lock(?FF_REGISTRY_LOADING_LOCK, [node()]), case Ret of {module, _} -> rabbit_log_feature_flags:debug( "Feature flags: registry module loaded"), ok; {error, Reason} -> rabbit_log_feature_flags:error( "Feature flags: failed to load registry module: ~p", [Reason]), throw({feature_flag_registry_reload_failure, Reason}) end. %% ------------------------------------------------------------------- %% Feature flags state storage. %% ------------------------------------------------------------------- -spec ensure_enabled_feature_flags_list_file_exists() -> ok | {error, any()}. %% @private ensure_enabled_feature_flags_list_file_exists() -> File = enabled_feature_flags_list_file(), case filelib:is_regular(File) of true -> ok; false -> write_enabled_feature_flags_list([]) end. -spec read_enabled_feature_flags_list() -> [feature_name()] | no_return(). %% @private read_enabled_feature_flags_list() -> case try_to_read_enabled_feature_flags_list() of {error, Reason} -> File = enabled_feature_flags_list_file(), throw({feature_flags_file_read_error, File, Reason}); Ret -> Ret end. -spec try_to_read_enabled_feature_flags_list() -> [feature_name()] | {error, any()}. %% @private try_to_read_enabled_feature_flags_list() -> File = enabled_feature_flags_list_file(), case file:consult(File) of {ok, [List]} -> List; {error, enoent} -> %% If the file is missing, we consider the list of enabled %% feature flags to be empty. []; {error, Reason} = Error -> rabbit_log_feature_flags:error( "Feature flags: failed to read the `feature_flags` " "file at `~s`: ~s", [File, file:format_error(Reason)]), Error end. -spec write_enabled_feature_flags_list([feature_name()]) -> ok | no_return(). %% @private write_enabled_feature_flags_list(FeatureNames) -> case try_to_write_enabled_feature_flags_list(FeatureNames) of {error, Reason} -> File = enabled_feature_flags_list_file(), throw({feature_flags_file_write_error, File, Reason}); Ret -> Ret end. -spec try_to_write_enabled_feature_flags_list([feature_name()]) -> ok | {error, any()}. %% @private try_to_write_enabled_feature_flags_list(FeatureNames) -> %% Before writing the new file, we read the existing one. If there %% are unknown feature flags in that file, we want to keep their %% state, even though they are unsupported at this time. It could be %% that a plugin was disabled in the meantime. PreviouslyEnabled = case try_to_read_enabled_feature_flags_list() of {error, _} -> []; List -> List end, FeatureNames1 = lists:foldl( fun(Name, Acc) -> case is_supported_locally(Name) of true -> Acc; false -> [Name | Acc] end end, FeatureNames, PreviouslyEnabled), FeatureNames2 = lists:sort(FeatureNames1), File = enabled_feature_flags_list_file(), Content = io_lib:format("~p.~n", [FeatureNames2]), %% TODO: If we fail to write the the file, we should spawn a process %% to retry the operation. case file:write_file(File, Content) of ok -> ok; {error, Reason} = Error -> rabbit_log_feature_flags:error( "Feature flags: failed to write the `feature_flags` " "file at `~s`: ~s", [File, file:format_error(Reason)]), Error end. -spec enabled_feature_flags_list_file() -> file:filename(). %% @doc %% Returns the path to the file where the state of feature flags is stored. %% %% @returns the path to the file. enabled_feature_flags_list_file() -> case application:get_env(rabbit, feature_flags_file) of {ok, Val} -> Val; undefined -> throw(feature_flags_file_not_set) end. %% ------------------------------------------------------------------- %% Feature flags management: enabling. %% ------------------------------------------------------------------- -spec do_enable(feature_name()) -> ok | {error, any()} | no_return(). %% @private do_enable(FeatureName) -> %% We mark this feature flag as "state changing" before doing the %% actual state change. We also take a global lock: this permits %% to block callers asking about a feature flag changing state. global:set_lock(?FF_STATE_CHANGE_LOCK), Ret = case mark_as_enabled(FeatureName, state_changing) of ok -> case enable_dependencies(FeatureName, true) of ok -> case run_migration_fun(FeatureName, enable) of ok -> mark_as_enabled(FeatureName, true); {error, no_migration_fun} -> mark_as_enabled(FeatureName, true); Error -> Error end; Error -> Error end; Error -> Error end, case Ret of ok -> ok; _ -> mark_as_enabled(FeatureName, false) end, global:del_lock(?FF_STATE_CHANGE_LOCK), Ret. -spec enable_locally(feature_name()) -> ok | {error, any()} | no_return(). %% @private enable_locally(FeatureName) when is_atom(FeatureName) -> case is_enabled(FeatureName) of true -> ok; false -> rabbit_log_feature_flags:debug( "Feature flag `~s`: enable locally (as part of feature " "flag states synchronization)", [FeatureName]), do_enable_locally(FeatureName) end. -spec do_enable_locally(feature_name()) -> ok | {error, any()} | no_return(). %% @private do_enable_locally(FeatureName) -> case enable_dependencies(FeatureName, false) of ok -> case run_migration_fun(FeatureName, enable) of ok -> mark_as_enabled_locally(FeatureName, true); {error, no_migration_fun} -> mark_as_enabled_locally(FeatureName, true); Error -> Error end; Error -> Error end. -spec enable_dependencies(feature_name(), boolean()) -> ok | {error, any()} | no_return(). %% @private enable_dependencies(FeatureName, Everywhere) -> FeatureProps = rabbit_ff_registry:get(FeatureName), DependsOn = maps:get(depends_on, FeatureProps, []), rabbit_log_feature_flags:debug( "Feature flag `~s`: enable dependencies: ~p", [FeatureName, DependsOn]), enable_dependencies(FeatureName, DependsOn, Everywhere). -spec enable_dependencies(feature_name(), [feature_name()], boolean()) -> ok | {error, any()} | no_return(). %% @private enable_dependencies(TopLevelFeatureName, [FeatureName | Rest], Everywhere) -> Ret = case Everywhere of true -> enable(FeatureName); false -> enable_locally(FeatureName) end, case Ret of ok -> enable_dependencies(TopLevelFeatureName, Rest, Everywhere); Error -> Error end; enable_dependencies(_, [], _) -> ok. -spec run_migration_fun(feature_name(), any()) -> any() | {error, any()}. %% @private run_migration_fun(FeatureName, Arg) -> FeatureProps = rabbit_ff_registry:get(FeatureName), run_migration_fun(FeatureName, FeatureProps, Arg). run_migration_fun(FeatureName, FeatureProps, Arg) -> case maps:get(migration_fun, FeatureProps, none) of {MigrationMod, MigrationFun} when is_atom(MigrationMod) andalso is_atom(MigrationFun) -> rabbit_log_feature_flags:debug( "Feature flag `~s`: run migration function ~p with arg: ~p", [FeatureName, MigrationFun, Arg]), try erlang:apply(MigrationMod, MigrationFun, [FeatureName, FeatureProps, Arg]) catch _:Reason:Stacktrace -> rabbit_log_feature_flags:error( "Feature flag `~s`: migration function crashed: ~p~n~p", [FeatureName, Reason, Stacktrace]), {error, {migration_fun_crash, Reason, Stacktrace}} end; none -> {error, no_migration_fun}; Invalid -> rabbit_log_feature_flags:error( "Feature flag `~s`: invalid migration function: ~p", [FeatureName, Invalid]), {error, {invalid_migration_fun, Invalid}} end. -spec mark_as_enabled(feature_name(), feature_state()) -> any() | {error, any()} | no_return(). %% @private mark_as_enabled(FeatureName, IsEnabled) -> case mark_as_enabled_locally(FeatureName, IsEnabled) of ok -> mark_as_enabled_remotely(FeatureName, IsEnabled); Error -> Error end. -spec mark_as_enabled_locally(feature_name(), feature_state()) -> any() | {error, any()} | no_return(). %% @private mark_as_enabled_locally(FeatureName, IsEnabled) -> rabbit_log_feature_flags:info( "Feature flag `~s`: mark as enabled=~p", [FeatureName, IsEnabled]), EnabledFeatureNames = maps:keys(list(enabled)), NewEnabledFeatureNames = case IsEnabled of true -> [FeatureName | EnabledFeatureNames]; false -> EnabledFeatureNames -- [FeatureName]; state_changing -> EnabledFeatureNames end, WrittenToDisk = case NewEnabledFeatureNames of EnabledFeatureNames -> rabbit_ff_registry:is_registry_written_to_disk(); _ -> ok =:= try_to_write_enabled_feature_flags_list( NewEnabledFeatureNames) end, initialize_registry(#{}, #{FeatureName => IsEnabled}, WrittenToDisk). -spec mark_as_enabled_remotely(feature_name(), feature_state()) -> any() | {error, any()} | no_return(). %% @private mark_as_enabled_remotely(FeatureName, IsEnabled) -> Nodes = running_remote_nodes(), mark_as_enabled_remotely(Nodes, FeatureName, IsEnabled, ?TIMEOUT). -spec mark_as_enabled_remotely([node()], feature_name(), feature_state(), timeout()) -> any() | {error, any()} | no_return(). %% @private mark_as_enabled_remotely([], _FeatureName, _IsEnabled, _Timeout) -> ok; mark_as_enabled_remotely(Nodes, FeatureName, IsEnabled, Timeout) -> T0 = erlang:timestamp(), Rets = [{Node, rpc:call(Node, ?MODULE, mark_as_enabled_locally, [FeatureName, IsEnabled], Timeout)} || Node <- Nodes], FailedNodes = [Node || {Node, Ret} <- Rets, Ret =/= ok], case FailedNodes of [] -> rabbit_log_feature_flags:debug( "Feature flags: `~s` successfully marked as enabled=~p on all " "nodes", [FeatureName, IsEnabled]), ok; _ -> T1 = erlang:timestamp(), rabbit_log_feature_flags:error( "Feature flags: failed to mark feature flag `~s` as enabled=~p " "on the following nodes:", [FeatureName, IsEnabled]), [rabbit_log_feature_flags:error( "Feature flags: - ~s: ~p", [Node, Ret]) || {Node, Ret} <- Rets, Ret =/= ok], NewTimeout = Timeout - (timer:now_diff(T1, T0) div 1000), if NewTimeout > 0 -> rabbit_log_feature_flags:debug( "Feature flags: retrying with a timeout of ~b " "milliseconds", [NewTimeout]), mark_as_enabled_remotely(FailedNodes, FeatureName, IsEnabled, NewTimeout); true -> rabbit_log_feature_flags:debug( "Feature flags: not retrying; RPC went over the " "~b milliseconds timeout", [Timeout]), %% FIXME: Is crashing the process the best solution here? throw( {failed_to_mark_feature_flag_as_enabled_on_remote_nodes, FeatureName, IsEnabled, FailedNodes}) end end. %% ------------------------------------------------------------------- %% Coordination with remote nodes. %% ------------------------------------------------------------------- -spec remote_nodes() -> [node()]. %% @private remote_nodes() -> mnesia:system_info(db_nodes) -- [node()]. -spec running_remote_nodes() -> [node()]. %% @private running_remote_nodes() -> mnesia:system_info(running_db_nodes) -- [node()]. query_running_remote_nodes(Node, Timeout) -> case rpc:call(Node, mnesia, system_info, [running_db_nodes], Timeout) of {badrpc, _} = Error -> Error; Nodes -> Nodes -- [node()] end. -spec does_node_support(node(), [feature_name()], timeout()) -> boolean(). %% @private does_node_support(Node, FeatureNames, Timeout) -> rabbit_log_feature_flags:debug( "Feature flags: querying `~p` support on node ~s...", [FeatureNames, Node]), Ret = case node() of Node -> is_supported_locally(FeatureNames); _ -> run_feature_flags_mod_on_remote_node( Node, is_supported_locally, [FeatureNames], Timeout) end, case Ret of {error, pre_feature_flags_rabbitmq} -> %% See run_feature_flags_mod_on_remote_node/4 for %% an explanation why we consider this node a 3.7.x %% pre-feature-flags node. rabbit_log_feature_flags:debug( "Feature flags: no feature flags support on node `~s`, " "consider the feature flags unsupported: ~p", [Node, FeatureNames]), false; {error, Reason} -> rabbit_log_feature_flags:error( "Feature flags: error while querying `~p` support on " "node ~s: ~p", [FeatureNames, Node, Reason]), false; true -> rabbit_log_feature_flags:debug( "Feature flags: node `~s` supports `~p`", [Node, FeatureNames]), true; false -> rabbit_log_feature_flags:debug( "Feature flags: node `~s` does not support `~p`; " "stopping query here", [Node, FeatureNames]), false end. -spec check_node_compatibility(node()) -> ok | {error, any()}. %% @doc %% Checks if a node is compatible with the local node. %% %% To be compatible, the following two conditions must be met: %%
    %%
  1. feature flags enabled on the local node must be supported by the %% remote node
    2. %%
  2. feature flags enabled on the remote node must be supported by the %% local node
    3. %%
%% %% @param Node the name of the remote node to test. %% @returns `ok' if they are compatible, `{error, Reason}' if they are not. check_node_compatibility(Node) -> check_node_compatibility(Node, ?TIMEOUT). -spec check_node_compatibility(node(), timeout()) -> ok | {error, any()}. %% @doc %% Checks if a node is compatible with the local node. %% %% See {@link check_node_compatibility/1} for the conditions required to %% consider two nodes compatible. %% %% @param Node the name of the remote node to test. %% @param Timeout Time in milliseconds after which the RPC gives up. %% @returns `ok' if they are compatible, `{error, Reason}' if they are not. %% %% @see check_node_compatibility/1 check_node_compatibility(Node, Timeout) -> %% Before checking compatibility, we exchange feature flags from %% unknown Erlang applications. So we fetch remote feature flags %% from applications which are not loaded locally, and the opposite. %% %% The goal is that such feature flags are not blocking the %% communication between nodes because the code (which would %% break) is missing on those nodes. Therefore they should not be %% considered when determinig compatibility. exchange_feature_flags_from_unknown_apps(Node, Timeout), %% FIXME FIXME FIXME %% Quand on tente de mettre deux nœuds en cluster, on a : %% Feature flags: starting an unclustered node: all feature flags %% will be enabled by default %% Ça ne devrait sans doute pas être le cas... %% We can now proceed with the actual compatibility check. rabbit_log_feature_flags:debug( "Feature flags: node `~s` compatibility check, part 1/2", [Node]), Part1 = local_enabled_feature_flags_is_supported_remotely(Node, Timeout), rabbit_log_feature_flags:debug( "Feature flags: node `~s` compatibility check, part 2/2", [Node]), Part2 = remote_enabled_feature_flags_is_supported_locally(Node, Timeout), case {Part1, Part2} of {true, true} -> rabbit_log_feature_flags:debug( "Feature flags: node `~s` is compatible", [Node]), ok; {false, _} -> rabbit_log_feature_flags:error( "Feature flags: node `~s` is INCOMPATIBLE: " "feature flags enabled locally are not supported remotely", [Node]), {error, incompatible_feature_flags}; {_, false} -> rabbit_log_feature_flags:error( "Feature flags: node `~s` is INCOMPATIBLE: " "feature flags enabled remotely are not supported locally", [Node]), {error, incompatible_feature_flags} end. -spec is_node_compatible(node()) -> boolean(). %% @doc %% Returns if a node is compatible with the local node. %% %% This function calls {@link check_node_compatibility/2} and returns %% `true' the latter returns `ok'. Therefore this is the same code, %% except that this function returns a boolean, but not the reason of %% the incompatibility if any. %% %% @param Node the name of the remote node to test. %% @returns `true' if they are compatible, `false' otherwise. is_node_compatible(Node) -> is_node_compatible(Node, ?TIMEOUT). -spec is_node_compatible(node(), timeout()) -> boolean(). %% @doc %% Returns if a node is compatible with the local node. %% %% This function calls {@link check_node_compatibility/2} and returns %% `true' the latter returns `ok'. Therefore this is the same code, %% except that this function returns a boolean, but not the reason %% of the incompatibility if any. If the RPC times out, nodes are %% considered incompatible. %% %% @param Node the name of the remote node to test. %% @param Timeout Time in milliseconds after which the RPC gives up. %% @returns `true' if they are compatible, `false' otherwise. is_node_compatible(Node, Timeout) -> check_node_compatibility(Node, Timeout) =:= ok. -spec local_enabled_feature_flags_is_supported_remotely(node(), timeout()) -> boolean(). %% @private local_enabled_feature_flags_is_supported_remotely(Node, Timeout) -> LocalEnabledFeatureNames = maps:keys(list(enabled)), is_supported_remotely([Node], LocalEnabledFeatureNames, Timeout). -spec remote_enabled_feature_flags_is_supported_locally(node(), timeout()) -> boolean(). %% @private remote_enabled_feature_flags_is_supported_locally(Node, Timeout) -> case query_remote_feature_flags(Node, enabled, Timeout) of {error, _} -> false; RemoteEnabledFeatureFlags when is_map(RemoteEnabledFeatureFlags) -> RemoteEnabledFeatureNames = maps:keys(RemoteEnabledFeatureFlags), is_supported_locally(RemoteEnabledFeatureNames) end. -spec run_feature_flags_mod_on_remote_node(node(), atom(), [term()], timeout()) -> term() | {error, term()}. %% @private run_feature_flags_mod_on_remote_node(Node, Function, Args, Timeout) -> case rpc:call(Node, ?MODULE, Function, Args, Timeout) of {badrpc, {'EXIT', {undef, [{?MODULE, Function, Args, []} | _]}}} -> %% If rabbit_feature_flags:Function() is undefined %% on the remote node, we consider it to be a 3.7.x %% pre-feature-flags node. %% %% Theoretically, it could be an older version (3.6.x and %% older). But the RabbitMQ version consistency check %% (rabbit_misc:version_minor_equivalent/2) called from %% rabbit_mnesia:check_rabbit_consistency/2 already blocked %% this situation from happening before we reach this point. rabbit_log_feature_flags:debug( "Feature flags: ~s:~s~p unavailable on node `~s`: " "assuming it is a RabbitMQ 3.7.x pre-feature-flags node", [?MODULE, Function, Args, Node]), {error, pre_feature_flags_rabbitmq}; {badrpc, Reason} = Error -> rabbit_log_feature_flags:error( "Feature flags: error while running ~s:~s~p " "on node `~s`: ~p", [?MODULE, Function, Args, Node, Reason]), {error, Error}; Ret -> Ret end. -spec query_remote_feature_flags(node(), Which :: all | enabled | disabled, timeout()) -> feature_flags() | {error, any()}. %% @private query_remote_feature_flags(Node, Which, Timeout) -> rabbit_log_feature_flags:debug( "Feature flags: querying ~s feature flags on node `~s`...", [Which, Node]), case run_feature_flags_mod_on_remote_node(Node, list, [Which], Timeout) of {error, pre_feature_flags_rabbitmq} -> %% See run_feature_flags_mod_on_remote_node/4 for %% an explanation why we consider this node a 3.7.x %% pre-feature-flags node. rabbit_log_feature_flags:debug( "Feature flags: no feature flags support on node `~s`, " "consider the list of feature flags empty", [Node]), #{}; {error, Reason} = Error -> rabbit_log_feature_flags:error( "Feature flags: error while querying ~s feature flags " "on node `~s`: ~p", [Which, Node, Reason]), Error; RemoteFeatureFlags when is_map(RemoteFeatureFlags) -> RemoteFeatureNames = maps:keys(RemoteFeatureFlags), rabbit_log_feature_flags:debug( "Feature flags: querying ~s feature flags on node `~s` " "done; ~s features: ~p", [Which, Node, Which, RemoteFeatureNames]), RemoteFeatureFlags end. -spec merge_feature_flags_from_unknown_apps(feature_flags()) -> ok | {error, any()}. %% @private merge_feature_flags_from_unknown_apps(FeatureFlags) when is_map(FeatureFlags) -> LoadedApps = [App || {App, _, _} <- application:loaded_applications()], FeatureFlagsFromUnknownApps = maps:fold( fun(FeatureName, FeatureProps, UnknownFF) -> case is_supported_locally(FeatureName) of true -> UnknownFF; false -> FeatureProvider = maps:get(provided_by, FeatureProps), case lists:member(FeatureProvider, LoadedApps) of true -> UnknownFF; false -> maps:put(FeatureName, FeatureProps, UnknownFF) end end end, #{}, FeatureFlags), rabbit_log_feature_flags:debug( "Feature flags: register feature flags provided by applications " "unknown locally: ~p", [maps:keys(FeatureFlagsFromUnknownApps)]), initialize_registry(FeatureFlagsFromUnknownApps). exchange_feature_flags_from_unknown_apps(Node, Timeout) -> %% The first step is to fetch feature flags from Erlang applications %% we don't know locally (they are loaded remotely, but not %% locally). fetch_remote_feature_flags_from_apps_unknown_locally(Node, Timeout), %% The next step is to do the opposite: push feature flags to remote %% nodes so they can register those from applications they don't %% know. push_local_feature_flags_from_apps_unknown_remotely(Node, Timeout). fetch_remote_feature_flags_from_apps_unknown_locally(Node, Timeout) -> RemoteFeatureFlags = query_remote_feature_flags(Node, all, Timeout), merge_feature_flags_from_unknown_apps(RemoteFeatureFlags). push_local_feature_flags_from_apps_unknown_remotely(Node, Timeout) -> LocalFeatureFlags = list(all), push_local_feature_flags_from_apps_unknown_remotely( Node, LocalFeatureFlags, Timeout). push_local_feature_flags_from_apps_unknown_remotely( Node, FeatureFlags, Timeout) when map_size(FeatureFlags) > 0 -> case query_running_remote_nodes(Node, Timeout) of {badrpc, Reason} -> {error, Reason}; Nodes -> lists:foreach( fun(N) -> run_feature_flags_mod_on_remote_node( N, merge_feature_flags_from_unknown_apps, [FeatureFlags], Timeout) end, Nodes) end; push_local_feature_flags_from_apps_unknown_remotely(_, _, _) -> ok. -spec sync_feature_flags_with_cluster([node()], boolean()) -> ok | {error, any()} | no_return(). %% @private sync_feature_flags_with_cluster(Nodes, NodeIsVirgin) -> sync_feature_flags_with_cluster(Nodes, NodeIsVirgin, ?TIMEOUT). -spec sync_feature_flags_with_cluster([node()], boolean(), timeout()) -> ok | {error, any()} | no_return(). %% @private sync_feature_flags_with_cluster([], NodeIsVirgin, _) -> verify_which_feature_flags_are_actually_enabled(), case NodeIsVirgin of true -> FeatureNames = get_forced_feature_flag_names(), case remote_nodes() of [] when FeatureNames =:= undefined -> rabbit_log_feature_flags:debug( "Feature flags: starting an unclustered node " "for the first time: all feature flags will be " "enabled by default"), enable_all(); [] -> case FeatureNames of [] -> rabbit_log_feature_flags:debug( "Feature flags: starting an unclustered " "node for the first time: all feature " "flags are forcibly left disabled from " "the $RABBITMQ_FEATURE_FLAGS environment " "variable"), ok; _ -> rabbit_log_feature_flags:debug( "Feature flags: starting an unclustered " "node for the first time: only the " "following feature flags specified in " "the $RABBITMQ_FEATURE_FLAGS environment " "variable will be enabled: ~p", [FeatureNames]), enable(FeatureNames) end; _ -> ok end; false -> rabbit_log_feature_flags:debug( "Feature flags: starting an unclustered node which is " "already initialized: all feature flags left in their " "current state"), ok end; sync_feature_flags_with_cluster(Nodes, _, Timeout) -> verify_which_feature_flags_are_actually_enabled(), RemoteNodes = Nodes -- [node()], sync_feature_flags_with_cluster1(RemoteNodes, Timeout). sync_feature_flags_with_cluster1([], _) -> ok; sync_feature_flags_with_cluster1(RemoteNodes, Timeout) -> RandomRemoteNode = pick_one_node(RemoteNodes), rabbit_log_feature_flags:debug( "Feature flags: SYNCING FEATURE FLAGS with node `~s`...", [RandomRemoteNode]), case query_remote_feature_flags(RandomRemoteNode, enabled, Timeout) of {error, _} = Error -> Error; RemoteFeatureFlags -> RemoteFeatureNames = maps:keys(RemoteFeatureFlags), rabbit_log_feature_flags:debug( "Feature flags: enabling locally feature flags already " "enabled on node `~s`...", [RandomRemoteNode]), case do_sync_feature_flags_with_node(RemoteFeatureNames) of ok -> sync_feature_flags_with_cluster2( RandomRemoteNode, Timeout); Error -> Error end end. sync_feature_flags_with_cluster2(RandomRemoteNode, Timeout) -> LocalFeatureNames = maps:keys(list(enabled)), rabbit_log_feature_flags:debug( "Feature flags: enabling on node `~s` feature flags already " "enabled locally...", [RandomRemoteNode]), Ret = run_feature_flags_mod_on_remote_node( RandomRemoteNode, do_sync_feature_flags_with_node, [LocalFeatureNames], Timeout), case Ret of {error, pre_feature_flags_rabbitmq} -> ok; _ -> Ret end. pick_one_node(Nodes) -> RandomIndex = rand:uniform(length(Nodes)), lists:nth(RandomIndex, Nodes). do_sync_feature_flags_with_node([FeatureFlag | Rest]) -> case enable_locally(FeatureFlag) of ok -> do_sync_feature_flags_with_node(Rest); Error -> Error end; do_sync_feature_flags_with_node([]) -> ok. -spec get_forced_feature_flag_names() -> [feature_name()] | undefined. %% @private %% @doc %% Returns the (possibly empty) list of feature flags the user want %% to enable out-of-the-box when starting a node for the first time. %% %% Without this, the default is to enable all the supported feature %% flags. %% %% There are two ways to specify that list: %%
    %%
  1. Using the `$RABBITMQ_FEATURE_FLAGS' environment variable; for %% instance `RABBITMQ_FEATURE_FLAGS=quorum_queue,mnevis'.
    2. %%
  2. Using the `forced_feature_flags_on_init' configuration parameter; %% for instance %% `{rabbit, [{forced_feature_flags_on_init, [quorum_queue, mnevis]}]}'.
    3. %%
%% %% The environment variable has precedence over the configuration %% parameter. get_forced_feature_flag_names() -> Ret = case get_forced_feature_flag_names_from_env() of undefined -> get_forced_feature_flag_names_from_config(); List -> List end, case Ret of undefined -> ok; [] -> rabbit_log_feature_flags:info( "Feature flags: automatic enablement of feature " "flags disabled (i.e. none will be enabled " "automatically)"); _ -> rabbit_log_feature_flags:info( "Feature flags: automatic enablement of feature " "flags limited to the following list: ~p", [Ret]) end, Ret. -spec get_forced_feature_flag_names_from_env() -> [feature_name()] | undefined. %% @private get_forced_feature_flag_names_from_env() -> case rabbit_prelaunch:get_context() of #{forced_feature_flags_on_init := ForcedFFs} when is_list(ForcedFFs) -> ForcedFFs; _ -> undefined end. -spec get_forced_feature_flag_names_from_config() -> [feature_name()] | undefined. %% @private get_forced_feature_flag_names_from_config() -> Value = application:get_env(rabbit, forced_feature_flags_on_init, undefined), case Value of undefined -> Value; _ when is_list(Value) -> case lists:all(fun is_atom/1, Value) of true -> Value; false -> undefined end; _ -> undefined end. -spec verify_which_feature_flags_are_actually_enabled() -> ok | {error, any()} | no_return(). %% @private verify_which_feature_flags_are_actually_enabled() -> AllFeatureFlags = list(all), EnabledFeatureNames = read_enabled_feature_flags_list(), rabbit_log_feature_flags:debug( "Feature flags: double-checking feature flag states..."), %% In case the previous instance of the node failed to write the %% feature flags list file, we want to double-check the list of %% enabled feature flags read from disk. For each feature flag, %% we call the migration function to query if the feature flag is %% actually enabled. %% %% If a feature flag doesn't provide a migration function (or if the %% function fails), we keep the current state of the feature flag. List1 = maps:fold( fun(Name, Props, Acc) -> Ret = run_migration_fun(Name, Props, is_enabled), case Ret of true -> [Name | Acc]; false -> Acc; _ -> MarkedAsEnabled = is_enabled(Name), case MarkedAsEnabled of true -> [Name | Acc]; false -> Acc end end end, [], AllFeatureFlags), RepairedEnabledFeatureNames = lists:sort(List1), %% We log the list of feature flags for which the state changes %% after the check above. WereEnabled = RepairedEnabledFeatureNames -- EnabledFeatureNames, WereDisabled = EnabledFeatureNames -- RepairedEnabledFeatureNames, case {WereEnabled, WereDisabled} of {[], []} -> ok; _ -> rabbit_log_feature_flags:warning( "Feature flags: the previous instance of this node " "must have failed to write the `feature_flags` " "file at `~s`:", [enabled_feature_flags_list_file()]) end, case WereEnabled of [] -> ok; _ -> rabbit_log_feature_flags:warning( "Feature flags: - list of previously enabled " "feature flags now marked as such: ~p", [WereEnabled]) end, case WereDisabled of [] -> ok; _ -> rabbit_log_feature_flags:warning( "Feature flags: - list of previously disabled " "feature flags now marked as such: ~p", [WereDisabled]) end, %% Finally, if the new list of enabled feature flags is different %% than the one on disk, we write the new list and re-initialize the %% registry. case RepairedEnabledFeatureNames of EnabledFeatureNames -> ok; _ -> rabbit_log_feature_flags:debug( "Feature flags: write the repaired list of enabled feature " "flags"), WrittenToDisk = ok =:= try_to_write_enabled_feature_flags_list( RepairedEnabledFeatureNames), initialize_registry( #{}, list_of_enabled_feature_flags_to_feature_states( RepairedEnabledFeatureNames), WrittenToDisk) end. -spec refresh_feature_flags_after_app_load([atom()]) -> ok | {error, any()} | no_return(). refresh_feature_flags_after_app_load([]) -> ok; refresh_feature_flags_after_app_load(Apps) -> rabbit_log_feature_flags:debug( "Feature flags: new apps loaded: ~p -> refreshing feature flags", [Apps]), FeatureFlags0 = list(all), FeatureFlags1 = query_supported_feature_flags(), %% The following list contains all the feature flags this node %% learned about only because remote nodes have them. Now, the %% applications providing them are loaded locally as well. %% Therefore, we may run their migration function in case the state %% of this node needs it. AlreadySupportedFeatureNames = maps:keys( maps:filter( fun(_, #{provided_by := App}) -> lists:member(App, Apps) end, FeatureFlags0)), case AlreadySupportedFeatureNames of [] -> ok; _ -> rabbit_log_feature_flags:debug( "Feature flags: new apps loaded: feature flags already " "supported: ~p", [lists:sort(AlreadySupportedFeatureNames)]) end, %% The following list contains all the feature flags no nodes in the %% cluster knew about before: this is the first time we see them in %% this instance of the cluster. We need to register them on all %% nodes. NewSupportedFeatureFlags = maps:filter( fun(FeatureName, _) -> not maps:is_key(FeatureName, FeatureFlags0) end, FeatureFlags1), case maps:keys(NewSupportedFeatureFlags) of [] -> ok; NewSupportedFeatureNames -> rabbit_log_feature_flags:debug( "Feature flags: new apps loaded: new feature flags (unseen so " "far): ~p ", [lists:sort(NewSupportedFeatureNames)]) end, case initialize_registry() of ok -> Ret = maybe_enable_locally_after_app_load( AlreadySupportedFeatureNames), case Ret of ok -> share_new_feature_flags_after_app_load( NewSupportedFeatureFlags, ?TIMEOUT); Error -> Error end; Error -> Error end. maybe_enable_locally_after_app_load([]) -> ok; maybe_enable_locally_after_app_load([FeatureName | Rest]) -> case is_enabled(FeatureName) of true -> case do_enable_locally(FeatureName) of ok -> maybe_enable_locally_after_app_load(Rest); Error -> Error end; false -> maybe_enable_locally_after_app_load(Rest) end. share_new_feature_flags_after_app_load(FeatureFlags, Timeout) -> push_local_feature_flags_from_apps_unknown_remotely( node(), FeatureFlags, Timeout). on_load() -> %% The goal of this `on_load()` code server hook is to prevent this %% module from being loaded in an already running RabbitMQ node if %% the running version does not have the feature flags subsystem. %% %% This situation happens when an upgrade overwrites RabbitMQ files %% with the node still running. This is the case with many packages: %% files are updated on disk, then a post-install step takes care of %% restarting the service. %% %% The problem is that if many nodes in a cluster are updated at the %% same time, one node running the newer version might query feature %% flags on an old node where this module is already available %% (because files were already overwritten). This causes the query %% to report an unexpected answer and the newer node to refuse to %% start. %% %% However, when the module is executed outside of RabbitMQ (for %% debugging purpose or in the context of EUnit for instance), we %% want to allow the load. That's why we first check if RabbitMQ is %% actually running. case rabbit:is_running() of true -> %% RabbitMQ is running. %% %% Now we want to differentiate a pre-feature-flags node %% from one having the subsystem. %% %% To do that, we verify if the `feature_flags_file` %% application environment variable is defined. With a %% feature-flags-enabled node, this application environment %% variable is defined by rabbitmq-server(8). case application:get_env(rabbit, feature_flags_file) of {ok, _} -> %% This is a feature-flags-enabled version. Loading %% the module is permitted. ok; _ -> %% This is a pre-feature-flags version. We deny the %% load and report why, possibly specifying the %% version of RabbitMQ. Vsn = case application:get_key(rabbit, vsn) of {ok, V} -> V; undefined -> "unknown version" end, "Refusing to load '" ?MODULE_STRING "' on this " "node. It appears to be running a pre-feature-flags " "version of RabbitMQ (" ++ Vsn ++ "). This is fine: " "a newer version of RabbitMQ was deployed on this " "node, but it was not restarted yet. This warning " "is probably caused by a remote node querying this " "node for its feature flags." end; false -> %% RabbitMQ is not running. Loading the module is permitted %% because this Erlang node will never be queried for its %% feature flags. ok end.