diff options
author | ILYA Khlopotov <iilyak@apache.org> | 2018-10-18 09:40:34 -0700 |
---|---|---|
committer | ILYA Khlopotov <iilyak@apache.org> | 2018-11-13 02:25:35 -0800 |
commit | e9abe501df4339639f15f34ec7f92528836dda2c (patch) | |
tree | f6f6e5fd538df049d26cad8d1b430eed0b138941 | |
parent | a4a2486bf127bf872327dfb57ae92f356ea18644 (diff) | |
download | couchdb-e9abe501df4339639f15f34ec7f92528836dda2c.tar.gz |
Implement efficient feature flags
-rw-r--r-- | src/couch/src/couch_db_epi.erl | 5 | ||||
-rw-r--r-- | src/couch/src/couch_flags.erl | 135 | ||||
-rw-r--r-- | src/couch/src/couch_flags_config.erl | 340 | ||||
-rw-r--r-- | src/couch/src/couch_util.erl | 3 | ||||
-rw-r--r-- | src/couch/test/couch_flags_tests.erl | 146 |
5 files changed, 626 insertions, 3 deletions
diff --git a/src/couch/src/couch_db_epi.erl b/src/couch/src/couch_db_epi.erl index 5ff8cfcd6..21879f683 100644 --- a/src/couch/src/couch_db_epi.erl +++ b/src/couch/src/couch_db_epi.erl @@ -35,14 +35,15 @@ providers() -> services() -> [ - {couch_db, couch_db_plugin} + {couch_db, couch_db_plugin}, + {feature_flags, couch_flags} ]. data_subscriptions() -> []. data_providers() -> - []. + [couch_flags_config:data_provider()]. processes() -> []. diff --git a/src/couch/src/couch_flags.erl b/src/couch/src/couch_flags.erl new file mode 100644 index 000000000..5cfe7f6d1 --- /dev/null +++ b/src/couch/src/couch_flags.erl @@ -0,0 +1,135 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +% This module serves two functions +% - provides public API to use to get value for a given feature flag and subject +% - implements {feature_flags, couch_flags} service + +% The module relies on couch_epi_data_gen which uses the data returned by +% `couch_flags_config:data()` to generate callback module `couch_epi_data_gen_flags_config`. +% The generated module shouldn't be used directly. We use following APIs +% - `couch_epi:get_handle({flags, config})` - to get handler (name of generated module) +% - `couch_epi:get_value(Handle, Key) - to do efficient matching +% +% The generated module implements clauses like the following +% - get(couch, {binary_match_rule()}) -> +% {matched_pattern(), size(matched_pattern()), [flag()]} | undefined +% For example +% - get(couch, {<<"/shards/test/exact">>}) -> +% {<<"/shards/test/exact">>,18,[baz,flag_bar,flag_foo]}; +% - get(couch, {<<"/shards/test", _/binary>>}) -> +% {<<"/shards/test*">>,13,[baz,flag_bar,flag_foo]}; +% - get(couch, {<<"/shards/exact">>}) -> +% {<<"/shards/exact">>,13,[flag_bar,flag_foo]}; +% - get(couch, {<<"/shards/blacklist", _/binary>>}) -> +% {<<"/shards/blacklist*">>,18,[]}; +% - get(couch, {<<"/", _/binary>>}) -> +% {<<"/*">>,2,[flag_foo]}; +% - get(_, _) -> undefined. +% +% The `couch_epi:get/2` uses the Handler module to implement efficient matching. + +% In order to distinguish between shards and clustered db the following +% convention is used. +% - it is a shard if pattern starts with `/` + +-module(couch_flags). + +%% Public API +-export([ + enabled/1, + is_enabled/2 +]). + +%% For internal use +-export([ + rules/0 +]). + +%% For use from plugin +-export([ + subject_key/1 +]). + +-include_lib("couch/include/couch_db.hrl"). +-include_lib("mem3/include/mem3.hrl"). +-include("couch_db_int.hrl"). + +-type subject() + :: #db{} + | #httpd{} + | #shard{} + | #ordered_shard{} + | string() + | binary(). + +-define(SERVICE_ID, feature_flags). + +-spec enabled(subject()) -> [atom()]. + +enabled(Subject) -> + Key = maybe_handle(subject_key, [Subject], fun subject_key/1), + Handle = couch_epi:get_handle({flags, config}), + lists:usort(enabled(Handle, {<<"/", Key/binary>>}) + ++ enabled(Handle, {couch_db:normalize_dbname(Key)})). + +-spec is_enabled(FlagId :: atom(), subject()) -> boolean(). + +is_enabled(FlagId, Subject) -> + lists:member(FlagId, enabled(Subject)). + +-spec rules() -> + [{Key :: string(), Value :: string()}]. + +rules() -> + Handle = couch_epi:get_handle(?SERVICE_ID), + lists:flatten(couch_epi:apply(Handle, ?SERVICE_ID, rules, [], [])). + +-spec enabled(Handle :: couch_epi:handle(), Key :: {binary()}) -> [atom()]. + +enabled(Handle, Key) -> + case couch_epi:get_value(Handle, couch, Key) of + {_, _, Flags} -> Flags; + undefined -> [] + end. + +-spec subject_key(subject()) -> binary(). + +subject_key(#db{name = Name}) -> + subject_key(Name); +subject_key(#httpd{path_parts=[Name | _Rest]}) -> + subject_key(Name); +subject_key(#httpd{path_parts=[]}) -> + <<>>; +subject_key(#shard{name = Name}) -> + subject_key(Name); +subject_key(#ordered_shard{name = Name}) -> + subject_key(Name); +subject_key(Name) when is_list(Name) -> + subject_key(list_to_binary(Name)); +subject_key(Name) when is_binary(Name) -> + Name. + +-spec maybe_handle( + Function :: atom(), + Args :: [term()], + Default :: fun((Args :: [term()]) -> term())) -> + term(). + +maybe_handle(Func, Args, Default) -> + Handle = couch_epi:get_handle(?SERVICE_ID), + case couch_epi:decide(Handle, ?SERVICE_ID, Func, Args, []) of + no_decision when is_function(Default) -> + apply(Default, Args); + {decided, Result} -> + Result + end. diff --git a/src/couch/src/couch_flags_config.erl b/src/couch/src/couch_flags_config.erl new file mode 100644 index 000000000..ad45add31 --- /dev/null +++ b/src/couch/src/couch_flags_config.erl @@ -0,0 +1,340 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +% This module implements {flags, config} data provider +-module(couch_flags_config). + +-export([ + enable/2, + data/0, + data/1, + data_provider/0 +]). + +-define(DATA_INTERVAL, 1000). + +-type pattern() + :: binary(). %% non empty binary which optionally can end with * + +-type flag_id() :: atom(). + +-type flags() :: list(flag_id()). + +-type parse_pattern() + :: { + binary(), %% pattern without trainig * if it is present + pattern(), + IsWildCard :: boolean(), %% true if the pattern has training * + PatternSize :: pos_integer() + }. + +-type rule() + :: { + parse_pattern(), + EnabledFlags :: flags(), + DisabledFlags :: flags() + }. + +data_provider() -> + { + {flags, config}, + {callback_module, ?MODULE}, + [{interval, ?DATA_INTERVAL}] + }. + +-spec enable(FlagId :: atom(), Pattern :: string()) -> + ok | {error, Reason :: term()}. + +enable(FlagId, Pattern) -> + Key = atom_to_list(FlagId) ++ "||" ++ Pattern, + config:set("feature_flags", Key, "true", false). + +-spec data() -> + [{{pattern()}, {pattern(), PatternSize :: pos_integer(), flags()}}]. + +data() -> + data(get_config_section("feature_flags") ++ couch_flags:rules()). + +-spec data(Rules :: [{Key :: string(), Value :: string()}]) -> + [{{pattern()}, {pattern(), PatternSize :: pos_integer(), flags()}}]. + +data(Config) -> + ByPattern = collect_rules(Config), + lists:reverse([{{P}, {P, size(P), E -- D}} || {P, {_, E, D}} <- ByPattern]). + +-spec parse_rules([{Key :: string(), Value :: string()}]) -> [rule()]. + +parse_rules(Config) -> + lists:filtermap(fun({K, V}) -> + case parse_rule(K, V) of + {error, {Format, Args}} -> + couch_log:error(Format, Args), + false; + Rule -> + {true, Rule} + end + end, Config). + +-spec parse_rule(Key :: string(), Value :: string()) -> + rule() + | {error, Reason :: term()}. + +parse_rule(Key, "true") -> + parse_flags(binary:split(list_to_binary(Key), <<"||">>), true); +parse_rule(Key, "false") -> + parse_flags(binary:split(list_to_binary(Key), <<"||">>), false); +parse_rule(Key, Value) -> + Reason = { + "Expected value for the `~p` either `true` or `false`, (got ~p)", + [Key, Value] + }, + {error, Reason}. + +-spec parse_flags([binary()], Value :: boolean()) -> + rule() | {error, Reason :: term()}. + +parse_flags([FlagsBin, PatternBin], Value) -> + case {parse_flags_term(FlagsBin), Value} of + {{error, _} = Error, _} -> + Error; + {Flags, true} -> + {parse_pattern(PatternBin), Flags, []}; + {Flags, false} -> + {parse_pattern(PatternBin), [], Flags} + end; +parse_flags(_Tokens, _) -> + couch_log:error( + "Key should be in the form of `[flags]||pattern` (got ~s)", []), + false. + +-spec parse_flags_term(Flags :: binary()) -> + [flag_id()] | {error, Reason :: term()}. + +parse_flags_term(FlagsBin) -> + case couch_util:parse_term(FlagsBin) of + {ok, Flags} when is_list(Flags) -> + lists:usort(Flags); + Term -> + {error, { + "Flags should be list of atoms (got \"~s\"): ~p", + [FlagsBin, Term] + }} + end. + +-spec parse_pattern(Pattern :: binary()) -> parse_pattern(). + +parse_pattern(PatternBin) -> + PatternSize = size(PatternBin), + case binary:last(PatternBin) of + $* -> + PrefixBin = binary:part(PatternBin, 0, PatternSize - 1), + {PrefixBin, PatternBin, true, PatternSize - 1}; + _ -> + {PatternBin, PatternBin, false, PatternSize} + end. + +-spec collect_rules([{ConfigurationKey :: string(), ConfigurationValue :: string()}]) -> + [{pattern(), rule()}]. + +collect_rules(ConfigData) -> + ByKey = by_key(parse_rules(ConfigData)), + Keys = lists:sort(fun sort_by_length/2, gb_trees:keys(ByKey)), + FuzzyKeys = lists:sort(fun sort_by_length/2, + [K || {K, {{_, _, true, _}, _, _}} <- gb_trees:to_list(ByKey)]), + Rules = collect_rules(lists:reverse(Keys), FuzzyKeys, ByKey), + gb_trees:to_list(Rules). + +-spec sort_by_length(A :: binary(), B :: binary()) -> boolean(). + +sort_by_length(A, B) -> + size(A) =< size(B). + +-spec by_key(Items :: [rule()]) -> Dictionary :: gb_trees:tree(). + +by_key(Items) -> + lists:foldl(fun({{_, K, _, _}, _, _} = Item, Acc) -> + update_element(Acc, K, Item, fun(Value) -> + update_flags(Value, Item) + end) + end, gb_trees:empty(), Items). + +-spec update_element( + Tree :: gb_trees:tree(), + Key :: pattern(), + Default :: rule(), + Fun :: fun((Item :: rule()) -> rule())) -> + gb_trees:tree(). + +update_element(Tree, Key, Default, Fun) -> + case gb_trees:lookup(Key, Tree) of + none -> + gb_trees:insert(Key, Default, Tree); + {value, Value} -> + gb_trees:update(Key, Fun(Value), Tree) + end. + +-spec collect_rules( + Keys :: [pattern()], + FuzzyKeys :: [pattern()], + ByKey :: gb_trees:tree()) -> + gb_trees:tree(). + +collect_rules([], _, Acc) -> + Acc; +collect_rules([Current | Rest], Items, Acc) -> + collect_rules(Rest, Items -- [Current], inherit_flags(Current, Items, Acc)). + +-spec inherit_flags( + Current :: pattern(), + FuzzyKeys :: [pattern()], + ByKey :: gb_trees:tree()) -> + gb_trees:tree(). + +inherit_flags(_Current, [], Acc) -> + Acc; +inherit_flags(Current, [Item | Items], Acc) -> + case match_prefix(Current, Item, Acc) of + true -> + inherit_flags(Current, Items, update_flags(Current, Item, Acc)); + false -> + inherit_flags(Current, Items, Acc) + end. + +-spec match_prefix( + AKey :: pattern(), + BKey :: pattern(), + ByKey :: gb_trees:tree()) -> + boolean(). + +match_prefix(AKey, BKey, Acc) -> + {value, A} = gb_trees:lookup(AKey, Acc), + {value, B} = gb_trees:lookup(BKey, Acc), + match_prefix(A, B). + +-spec match_prefix(A :: rule(), B :: rule()) -> boolean(). + +match_prefix({{_, _, _, _}, _, _}, {{_, _, false, _}, _, _}) -> + false; +match_prefix({{Key, _, _, _}, _, _}, {{Key, _, true, _}, _, _}) -> + true; +match_prefix({{Key0, _, _, _}, _, _}, {{Key1, _, true, S1}, _, _}) -> + case Key0 of + <<Key1:S1/binary, _/binary>> -> true; + _ -> false + end. + +-spec update_flags( + AKey :: pattern(), + BKey :: pattern(), + ByKey :: gb_trees:tree()) -> + gb_trees:tree(). + +update_flags(AKey, BKey, Acc) -> + {value, A} = gb_trees:lookup(AKey, Acc), + {value, B} = gb_trees:lookup(BKey, Acc), + gb_trees:update(AKey, update_flags(A, B), Acc). + +-spec update_flags(A :: rule(), B :: rule()) -> rule(). + +update_flags({Pattern, E0, D0}, {_, E1, D1}) -> + DisabledByParent = lists:usort(D1 -- E0), + E = lists:usort(lists:usort(E0 ++ E1) -- D0), + D = lists:usort(D0 ++ DisabledByParent), + {Pattern, E, D}. + +-spec get_config_section(Section :: string()) -> + [{Key :: string(), Value :: string()}]. + +%% When we start couch_epi the config is not started yet +% so we would get `badarg` for some time +get_config_section(Section) -> + try + config:get(Section) + catch error:badarg -> + [] + end. + +%% ------------------------------------------------------------------ +%% Tests +%% ------------------------------------------------------------------ + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +all_combinations_return_same_result_test_() -> + Config = [ + {"[foo, bar]||*", "true"}, + {"[baz, qux]||*", "false"}, + {"[baz]||shards/test*", "true"}, + {"[baz]||shards/blacklist*", "false"}, + {"[bar]||shards/test*", "false"}, + {"[bar]||shards/test/blacklist*", "true"} + ], + Expected = [ + {{<<"shards/test/blacklist*">>},{<<"shards/test/blacklist*">>,22,[bar, foo]}}, + {{<<"shards/test*">>},{<<"shards/test*">>, 12, [baz, foo]}}, + {{<<"shards/blacklist*">>},{<<"shards/blacklist*">>, 17, [bar, foo]}}, + {{<<"*">>},{<<"*">>, 1, [bar, foo]}} + ], + Combinations = couch_tests_combinatorics:permutations(Config), + [{test_id(Items), ?_assertEqual(Expected, data(Items))} + || Items <- Combinations]. + +rules_are_sorted_test() -> + Expected = [ + {{<<"shards/test/exact">>},{<<"shards/test/exact">>, 17, [baz,flag_bar,flag_foo]}}, + {{<<"shards/test/blacklist*">>},{<<"shards/test/blacklist*">>,22,[flag_foo]}}, + {{<<"shards/test*">>},{<<"shards/test*">>, 12, [baz,flag_bar,flag_foo]}}, + {{<<"shards/exact">>},{<<"shards/exact">>, 12, [flag_bar,flag_foo]}}, + {{<<"shards/blacklist*">>},{<<"shards/blacklist*">>, 17, []}}, + {{<<"*">>},{<<"*">>, 1, [flag_foo]}} + ], + ?assertEqual(Expected, data(test_config())). + +latest_overide_wins_test_() -> + Cases = [ + {[ + {"[flag]||*", "false"}, {"[flag]||a*", "true"}, + {"[flag]||ab*", "true"}, {"[flag]||abc*", "true"} + ], true}, + {[ + {"[flag]||*", "true"}, {"[flag]||a*", "false"}, + {"[flag]||ab*", "true"}, {"[flag]||abc*", "false"} + ], false} + ], + [{test_id(Rules, Expected), + ?_assertEqual(Expected, lists:member(flag, flags(hd(data(Rules)))))} + || {Rules, Expected} <- Cases]. + +flags({{_Pattern}, {_Pattern, _Size, Flags}}) -> + Flags. + +test_id(Items, ExpectedResult) -> + lists:flatten(io_lib:format("~p -> ~p", [[P || {P, _} <- Items], ExpectedResult])). + + +test_id(Items) -> + lists:flatten(io_lib:format("~p", [[P || {P, _} <- Items]])). + +test_config() -> + [ + {"[flag_foo]||*", "true"}, + {"[flag_bar]||*", "false"}, + {"[flag_bar]||shards/test*", "true"}, + {"[flag_foo]||shards/blacklist*", "false"}, + {"[baz]||shards/test*", "true"}, + {"[baz]||shards/test/blacklist*", "false"}, + {"[flag_bar]||shards/exact", "true"}, + {"[flag_bar]||shards/test/exact", "true"} + ]. + +-endif. diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl index 8f25edc9c..3efec84a9 100644 --- a/src/couch/src/couch_util.erl +++ b/src/couch/src/couch_util.erl @@ -52,7 +52,8 @@ <<"httpd_global_handlers">>, <<"native_query_servers">>, <<"os_daemons">>, - <<"query_servers">> + <<"query_servers">>, + <<"feature_flags">> ]). diff --git a/src/couch/test/couch_flags_tests.erl b/src/couch/test/couch_flags_tests.erl new file mode 100644 index 000000000..f8f27c7c0 --- /dev/null +++ b/src/couch/test/couch_flags_tests.erl @@ -0,0 +1,146 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_flags_tests). + +-include_lib("couch/include/couch_eunit.hrl"). + +%% couch_epi_plugin behaviour callbacks +-export([ + app/0, + providers/0, + services/0, + data_providers/0, + data_subscriptions/0, + processes/0, + notify/3 +]). + +-export([ + rules/0 +]). + +app() -> + test_app. + +providers() -> + [{feature_flags, ?MODULE}]. + +services() -> + []. + +data_providers() -> + []. + +data_subscriptions() -> + []. + +processes() -> + []. + +notify(_, _, _) -> + ok. + +rules() -> + test_config(). + +setup() -> + %% FIXME after we upgrade couch_epi + application:stop(couch_epi), % in case it's already running from other tests... + application:unload(couch_epi), + + application:load(couch_epi), + application:set_env(couch_epi, plugins, [couch_db_epi, ?MODULE]), + test_util:start_couch([couch_epi]). + + +teardown(Ctx) -> + test_util:stop_couch(Ctx), + ok = application:unload(couch_epi), + ok. + +couch_flags_test_() -> + { + "test couch_flags", + { + setup, fun setup/0, fun teardown/1, + enabled_flags_tests() + ++ is_enabled() +%% ++ match_performance() + } + }. + +enabled_flags_tests() -> + + [{"enabled_flags_tests", [ + {"flags_default_rule", + ?_assertEqual( + [foo], couch_flags:enabled("something"))}, + {"flags_wildcard_rule", + ?_assertEqual( + [bar, baz, foo], + couch_flags:enabled("shards/test/something"))}, + {"flags_exact_rule", + ?_assertEqual( + [bar, baz, foo], + couch_flags:enabled("shards/test/exact"))}, + {"flags_blacklist_rule", + ?_assertEqual( + [], + couch_flags:enabled("shards/blacklist/4"))} + ]}]. + +is_enabled() -> + [{"is_enabled_tests", [ + {"flags_default_rule [enabled]", + ?_assert(couch_flags:is_enabled(foo, "something"))}, + {"flags_default_rule [disabled]", + ?_assertNot(couch_flags:is_enabled(baz, "something"))}, + {"flags_default_rule [not_existent]", + ?_assertNot(couch_flags:is_enabled(non_existent, "something"))}, + + {"flags_wildcard_rule [enabled]", + ?_assert(couch_flags:is_enabled(bar, "shards/test/something"))}, + {"flags_wildcard_rule [not_existent]", + ?_assertNot(couch_flags:is_enabled(non_existent, "shards/test/something"))}, + + {"flags_exact_rule [overide_disbled]", + ?_assert(couch_flags:is_enabled(bar, "shards/test/exact"))}, + {"flags_exact_rule [not_existent]", + ?_assertNot(couch_flags:is_enabled(non_existent, "shards/test/exact"))}, + + {"flags_blacklist_rule [overide_enabled]", + ?_assertNot(couch_flags:is_enabled(foo, "shards/blacklist/4"))}, + {"flags_blacklist_rule [not_existent]", + ?_assertNot(couch_flags:is_enabled(non_existent, "shards/blacklist/4"))} + ]}]. + +match_performance() -> + [{"match_performance", [ + ?_test(begin + ?debugTime("1 million of operations took", lists:foreach(fun(_) -> + couch_flags:is_enabled(bar, "shards/test/exact") + end, lists:seq(1, 1000000))) + end) + ]}]. + + +test_config() -> + [ + {"[foo]||/*", "true"}, + {"[bar]||/*", "false"}, + {"[bar]||/shards/test*", "true"}, + {"[foo]||/shards/blacklist*", "false"}, + {"[baz]||/shards/test*", "true"}, + {"[bar]||/shards/exact", "true"}, + {"[bar]||/shards/test/exact", "true"} + ]. |