From 571274ee849201074438e2cfbbac3de4f9b2bffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Sat, 4 Jan 2020 16:46:05 +0100 Subject: Fix some typos in the bug report issue template (#2233) * Fix some typos in the bug report issue template --- .github/ISSUE_TEMPLATE/bug_report.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 6c43a139b..360d4fa62 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -27,10 +27,10 @@ assignees: '' [TIP]: # ( Include as many relevant details about your environment as possible. ) [TIP]: # ( You can paste the output of curl http://YOUR-COUCHDB:5984/ here. ) -* CouchDB Version used: +* CouchDB version used: * Browser name and version: -* Operating System and version: +* Operating system and version: -## Additional context +## Additional Context -[TIP]: # ( Add any other context about the prbolem here. ) +[TIP]: # ( Add any other context about the problem here. ) -- cgit v1.2.1 From ee47f93a0a847cc4b4c3e4e8f47462050b91c555 Mon Sep 17 00:00:00 2001 From: Juanjo Rodriguez Date: Sat, 4 Jan 2020 17:05:44 +0100 Subject: Port erlang views tests into elixir test suite (#2237) * Port erlang views tests into elixir test suite * Enable erlang views in elixir testing * Support nil or :null in response --- Makefile | 2 +- Makefile.win | 2 +- test/elixir/README.md | 2 +- test/elixir/test/erlang_views_test.exs | 117 +++++++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 test/elixir/test/erlang_views_test.exs diff --git a/Makefile b/Makefile index 9bcd389be..0d4131aec 100644 --- a/Makefile +++ b/Makefile @@ -238,7 +238,7 @@ python-black-update: .venv/bin/black elixir: export MIX_ENV=integration elixir: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1 elixir: elixir-init elixir-check-formatted elixir-credo devclean - @dev/run "$(TEST_OPTS)" -a adm:pass -n 1 --no-eval 'mix test --trace --exclude without_quorum_test --exclude with_quorum_test $(EXUNIT_OPTS)' + @dev/run "$(TEST_OPTS)" -a adm:pass -n 1 --enable-erlang-views --no-eval 'mix test --trace --exclude without_quorum_test --exclude with_quorum_test $(EXUNIT_OPTS)' .PHONY: elixir-init elixir-init: MIX_ENV=test diff --git a/Makefile.win b/Makefile.win index c74a54621..7278fec76 100644 --- a/Makefile.win +++ b/Makefile.win @@ -189,7 +189,7 @@ python-black-update: .venv/bin/black .PHONY: elixir elixir: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1 elixir: elixir-init elixir-check-formatted elixir-credo devclean - @dev\run -a adm:pass --no-eval 'mix test --trace --exclude without_quorum_test --exclude with_quorum_test $(EXUNIT_OPTS)' + @dev\run -a adm:pass --enable-erlang-views --no-eval 'mix test --trace --exclude without_quorum_test --exclude with_quorum_test $(EXUNIT_OPTS)' .PHONY: elixir-init elixir-init: diff --git a/test/elixir/README.md b/test/elixir/README.md index 7c6b10e5a..7f13d8afc 100644 --- a/test/elixir/README.md +++ b/test/elixir/README.md @@ -45,7 +45,7 @@ X means done, - means partially - [ ] Port design_docs.js - [ ] Port design_options.js - [ ] Port design_paths.js - - [ ] Port erlang_views.js + - [X] Port erlang_views.js - [ ] Port etags_head.js - [ ] Port etags_views.js - [ ] Port form_submit.js diff --git a/test/elixir/test/erlang_views_test.exs b/test/elixir/test/erlang_views_test.exs new file mode 100644 index 000000000..3346c2274 --- /dev/null +++ b/test/elixir/test/erlang_views_test.exs @@ -0,0 +1,117 @@ +defmodule ErlangViewsTest do + use CouchTestCase + + @moduletag :erlang_views + + @moduledoc """ + basic 'smoke tests' of erlang views. + This is a port of the erlang_views.js test suite. + """ + + @doc1 %{:_id => "1", :integer => 1, :string => "str1", :array => [1, 2, 3]} + + @erlang_map_fun """ + fun({Doc}) -> + K = couch_util:get_value(<<"integer">>, Doc, null), + V = couch_util:get_value(<<"string">>, Doc, null), + Emit(K, V) + end. + """ + + @erlang_reduce_fun """ + fun (_, Values, false) -> length(Values); + (_, Values, true) -> lists:sum(Values) + end. + """ + + @erlang_map_fun_2 """ + fun({Doc}) -> + Words = couch_util:get_value(<<"words">>, Doc), + lists:foreach(fun({Word}) -> + WordString = couch_util:get_value(<<"word">>, Word), + Count = couch_util:get_value(<<"count">>, Word), + Emit(WordString , Count) + end, Words) + end. + """ + + @erlang_reduce_fun_2 """ + fun(Keys, Values, RR) -> length(Values) end. + """ + + @word_list ["foo", "bar", "abc", "def", "baz", "xxyz"] + + @tag :with_db + test "Erlang map function", context do + db_name = context[:db_name] + create_doc(db_name, @doc1) + + results = + query( + db_name, + @erlang_map_fun, + nil, + nil, + nil, + "erlang" + ) + + assert results["total_rows"] == 1 + assert List.first(results["rows"])["key"] == 1 + assert List.first(results["rows"])["value"] == "str1" + end + + @tag :with_db + test "Erlang reduce function", context do + db_name = context[:db_name] + create_doc(db_name, @doc1) + doc2 = @doc1 |> Map.replace!(:_id, "2") |> Map.replace!(:string, "str2") + create_doc(db_name, doc2) + + results = + query( + db_name, + @erlang_map_fun, + @erlang_reduce_fun, + nil, + nil, + "erlang" + ) + + assert List.first(results["rows"])["value"] == 2 + end + + @tag :with_db + test "Erlang reduce function larger dataset", context do + db_name = context[:db_name] + bulk_save(db_name, create_large_dataset(250)) + + results = + query( + db_name, + @erlang_map_fun_2, + @erlang_reduce_fun_2, + nil, + nil, + "erlang" + ) + + assert Map.get(List.first(results["rows"]), "key", :null) == :null + assert List.first(results["rows"])["value"] > 0 + end + + defp create_large_dataset(size) do + doc_words = + for j <- 0..100 do + %{word: get_word(j), count: j} + end + + template_doc = %{words: doc_words} + + make_docs(0..size, template_doc) + end + + defp get_word(idx) do + Enum.at(@word_list, rem(idx, length(@word_list))) + end +end -- cgit v1.2.1 From d5568d1bbbed134091e34a1fa292f1ae2b361adf Mon Sep 17 00:00:00 2001 From: Ronny Date: Sat, 4 Jan 2020 17:29:43 +0100 Subject: Fixes #2151 - start CouchDB from symlinks (#2152) --- rel/files/couchdb.in | 12 +++++++++++- rel/overlay/bin/remsh | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/rel/files/couchdb.in b/rel/files/couchdb.in index b3c7e98e2..f64c0f860 100755 --- a/rel/files/couchdb.in +++ b/rel/files/couchdb.in @@ -12,7 +12,17 @@ # License for the specific language governing permissions and limitations under # the License. -COUCHDB_BIN_DIR=$(cd "${0%/*}" && pwd) +canonical_readlink () + { + cd $(dirname $1); + FILE=$(basename $1); + if [ -h "$FILE" ]; then + canonical_readlink $(readlink $FILE); + else + echo "$(pwd -P)"; + fi +} +COUCHDB_BIN_DIR=$(canonical_readlink $0) ERTS_BIN_DIR=$COUCHDB_BIN_DIR/../ cd "$COUCHDB_BIN_DIR/../" diff --git a/rel/overlay/bin/remsh b/rel/overlay/bin/remsh index 2ac421b07..b8946ace3 100755 --- a/rel/overlay/bin/remsh +++ b/rel/overlay/bin/remsh @@ -12,7 +12,17 @@ # License for the specific language governing permissions and limitations under # the License. -COUCHDB_BIN_DIR=$(cd "${0%/*}" && pwd) +canonical_readlink () + { + cd $(dirname $1); + FILE=$(basename $1); + if [ -h "$FILE" ]; then + canonical_readlink $(readlink $FILE); + else + echo "$(pwd -P)"; + fi +} +COUCHDB_BIN_DIR=$(canonical_readlink $0) ERTS_BIN_DIR=$COUCHDB_BIN_DIR/../ ROOTDIR=${ERTS_BIN_DIR%/*} START_ERL=$(cat "$ROOTDIR/releases/start_erl.data") -- cgit v1.2.1 From 3f1f711b451d0fb2c841cc845b1937a573224caf Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Sat, 4 Jan 2020 16:48:55 +0000 Subject: Autoupdate false test (#2311) * restore tests removed in 0a85b75ee150 * disable autoupdate when testing for stale indexes --- test/javascript/tests/design_docs.js | 32 +++++++++++++++++++++++++++++++- test/javascript/tests/view_update_seq.js | 3 ++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/test/javascript/tests/design_docs.js b/test/javascript/tests/design_docs.js index f06efc8eb..e28cb2e83 100644 --- a/test/javascript/tests/design_docs.js +++ b/test/javascript/tests/design_docs.js @@ -253,7 +253,9 @@ couchTests.design_docs = function(debug) { db.bulkSave(makeDocs(1, numDocs + 1)); T(db.ensureFullCommit().ok); - // test that we get correct design doc info back. + // test that we get correct design doc info back, + // and also that GET /db/_design/test/_info + // hasn't triggered an update of the views db.view("test/summate", {stale: "ok"}); // make sure view group's open for (var i = 0; i < 2; i++) { var dinfo = db.designInfo("_design/test"); @@ -262,6 +264,13 @@ couchTests.design_docs = function(debug) { TEquals(prev_view_size, vinfo.sizes.file, "view group disk size didn't change"); TEquals(false, vinfo.compact_running); TEquals(prev_view_sig, vinfo.signature, 'ddoc sig'); + // wait some time (there were issues where an update + // of the views had been triggered in the background) + var start = new Date().getTime(); + while (new Date().getTime() < start + 2000); + TEquals(0, db.view("test/all_docs_twice", {stale: "ok"}).total_rows, 'view info'); + TEquals(0, db.view("test/single_doc", {stale: "ok"}).total_rows, 'view info'); + TEquals(0, db.view("test/summate", {stale: "ok"}).rows.length, 'view info'); T(db.ensureFullCommit().ok); // restartServer(); }; @@ -275,6 +284,27 @@ couchTests.design_docs = function(debug) { var start = new Date().getTime(); while (new Date().getTime() < start + 2000); + // test that POST /db/_view_cleanup + // doesn't trigger an update of the views + var len1 = db.view("test/all_docs_twice", {stale: "ok"}).total_rows; + var len2 = db.view("test/single_doc", {stale: "ok"}).total_rows; + var len3 = db.view("test/summate", {stale: "ok"}).rows.length; + for (i = 0; i < 2; i++) { + T(db.viewCleanup().ok); + // wait some time (there were issues where an update + // of the views had been triggered in the background) + start = new Date().getTime(); + while (new Date().getTime() < start + 2000); + TEquals(len1, db.view("test/all_docs_twice", {stale: "ok"}).total_rows, 'view cleanup'); + TEquals(len2, db.view("test/single_doc", {stale: "ok"}).total_rows, 'view cleanup'); + TEquals(len3, db.view("test/summate", {stale: "ok"}).rows.length, 'view cleanup'); + T(db.ensureFullCommit().ok); + // restartServer(); + // we'll test whether the view group stays closed + // and the views stay uninitialized (they should!) + len1 = len2 = len3 = 0; + }; + // test commonjs in map functions resp = db.view("test/commonjs", {limit:1}); T(resp.rows[0].value == 'ok'); diff --git a/test/javascript/tests/view_update_seq.js b/test/javascript/tests/view_update_seq.js index eaba4042e..c14453f05 100644 --- a/test/javascript/tests/view_update_seq.js +++ b/test/javascript/tests/view_update_seq.js @@ -26,6 +26,7 @@ couchTests.view_update_seq = function(debug) { var designDoc = { _id:"_design/test", language: "javascript", + autoupdate: false, views: { all_docs: { map: "function(doc) { emit(doc.integer, doc.string) }" @@ -86,7 +87,7 @@ couchTests.view_update_seq = function(debug) { resp = db.view('test/all_docs', {limit: 1, stale: "update_after", update_seq: true}); T(resp.rows.length == 1); - T(seqInt(resp.update_seq) == 101 || seqInt(resp.update_seq) == 102); + TEquals(101, seqInt(resp.update_seq)); // wait 5 seconds for the next assertions to pass in very slow machines var t0 = new Date(), t1; -- cgit v1.2.1 From c30ac698333a2fb82beada8a0a656fa05457e5fe Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Sat, 4 Jan 2020 18:32:44 +0100 Subject: Multi/elixir 3 (#2394) * Add auth cache test into elixir test suite * Port cookie_auth test suite into elixir * Remove conflicting functions * remove debug log level config * Port users_db from js test suite to elixir * WIP: make test more robust * WIP: increase default timeout, revert to fail when login error * Auth cache test fixes * Cookie auth test fixes * Cookie auth test fixes * Correct test name * Correct test name * Disable all JS tests that have Elixir test counterparts. * Convert test setup/teardown logic into idiomatic ExUnit and use @moduletag config * Disable auth_cache_test * auth cache test is disabled * Update elixir test suite README.md with missing ported stuff * Port UTF8 js test suite to elixir * fix: losen assertion * disable more JS tests Co-authored-by: Juanjo Rodriguez Co-authored-by: Alessio Biancalana --- Makefile | 37 +- test/elixir/README.md | 14 +- test/elixir/lib/couch.ex | 33 +- test/elixir/lib/couch/db_test.ex | 66 +++- test/elixir/test/auth_cache_test.exs | 212 +++++++++++ test/elixir/test/cookie_auth_test.exs | 403 +++++++++++++++++++++ test/elixir/test/replication_test.exs | 2 +- test/elixir/test/users_db_test.exs | 322 ++++++++++++++++ test/elixir/test/utf8_test.exs | 65 ++++ .../tests-cluster/with-quorum/attachments.js | 1 + .../with-quorum/attachments_delete.js | 1 + .../attachments_delete_overridden_quorum.js | 1 + .../with-quorum/attachments_overridden_quorum.js | 1 + .../tests-cluster/with-quorum/db_creation.js | 1 + .../with-quorum/db_creation_overridden_quorum.js | 1 + .../tests-cluster/with-quorum/db_deletion.js | 1 + .../with-quorum/db_deletion_overridden_quorum.js | 1 + .../tests-cluster/with-quorum/doc_bulk.js | 1 + .../with-quorum/doc_bulk_overridden_quorum.js | 1 + .../tests-cluster/with-quorum/doc_copy.js | 1 + .../with-quorum/doc_copy_overridden_quorum.js | 1 + .../tests-cluster/with-quorum/doc_crud.js | 1 + .../with-quorum/doc_crud_overridden_quorum.js | 1 + .../tests-cluster/without-quorum/attachments.js | 1 + .../without-quorum/attachments_delete.js | 1 + .../attachments_delete_overridden_quorum.js | 1 + .../attachments_overridden_quorum.js | 1 + .../tests-cluster/without-quorum/db_creation.js | 1 + .../db_creation_overridden_quorum.js | 1 + .../tests-cluster/without-quorum/db_deletion.js | 1 + .../db_deletion_overridden_quorum.js | 1 + .../tests-cluster/without-quorum/doc_bulk.js | 1 + .../without-quorum/doc_bulk_overridden_quorum.js | 1 + .../tests-cluster/without-quorum/doc_copy.js | 1 + .../without-quorum/doc_copy_overridden_quorum.js | 1 + .../tests-cluster/without-quorum/doc_crud.js | 1 + .../without-quorum/doc_crud_overridden_quorum.js | 1 + test/javascript/tests/all_docs.js | 1 + test/javascript/tests/attachment_names.js | 1 + test/javascript/tests/attachment_paths.js | 1 + test/javascript/tests/attachment_ranges.js | 1 + test/javascript/tests/attachment_views.js | 1 + test/javascript/tests/attachments.js | 1 + test/javascript/tests/attachments_multipart.js | 1 + test/javascript/tests/auth_cache.js | 1 + test/javascript/tests/basics.js | 2 +- test/javascript/tests/batch_save.js | 1 + test/javascript/tests/bulk_docs.js | 1 + test/javascript/tests/coffee.js | 1 + test/javascript/tests/compact.js | 1 + test/javascript/tests/config.js | 1 + test/javascript/tests/conflicts.js | 1 + test/javascript/tests/cookie_auth.js | 1 + test/javascript/tests/copy_doc.js | 1 + test/javascript/tests/invalid_docids.js | 1 + test/javascript/tests/large_docs.js | 1 + test/javascript/tests/lots_of_docs.js | 1 + test/javascript/tests/multiple_rows.js | 1 + test/javascript/tests/reduce.js | 1 + test/javascript/tests/users_db.js | 1 + test/javascript/tests/utf8.js | 1 + test/javascript/tests/uuids.js | 1 + test/javascript/tests/view_collation.js | 1 + 63 files changed, 1142 insertions(+), 67 deletions(-) create mode 100644 test/elixir/test/auth_cache_test.exs create mode 100644 test/elixir/test/cookie_auth_test.exs create mode 100644 test/elixir/test/users_db_test.exs create mode 100644 test/elixir/test/utf8_test.exs diff --git a/Makefile b/Makefile index 0d4131aec..66b1714d3 100644 --- a/Makefile +++ b/Makefile @@ -147,8 +147,8 @@ fauxton: share/www .PHONY: check # target: check - Test everything check: all - @$(MAKE) test-cluster-with-quorum - @$(MAKE) test-cluster-without-quorum + # @$(MAKE) test-cluster-with-quorum + # @$(MAKE) test-cluster-without-quorum @$(MAKE) python-black @$(MAKE) eunit @$(MAKE) javascript @@ -286,39 +286,6 @@ endif 'test/javascript/run --suites "$(suites)" \ --ignore "$(ignore_js_suites)"' -.PHONY: test-cluster-with-quorum -test-cluster-with-quorum: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1 -test-cluster-with-quorum: devclean - @mkdir -p share/www/script/test -ifeq ($(IN_RELEASE), true) - @cp test/javascript/tests/lorem*.txt share/www/script/test/ -else - @mkdir -p src/fauxton/dist/release/test - @cp test/javascript/tests/lorem*.txt src/fauxton/dist/release/test/ -endif - @dev/run -n 3 -q --with-admin-party-please \ - --enable-erlang-views --degrade-cluster 1 \ - "$(TEST_OPTS)" \ - 'test/javascript/run --suites "$(suites)" \ - --ignore "$(ignore_js_suites)" \ - --path test/javascript/tests-cluster/with-quorum' - -.PHONY: test-cluster-without-quorum -test-cluster-without-quorum: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1 -test-cluster-without-quorum: devclean - @mkdir -p share/www/script/test -ifeq ($(IN_RELEASE), true) - @cp test/javascript/tests/lorem*.txt share/www/script/test/ -else - @mkdir -p src/fauxton/dist/release/test - @cp test/javascript/tests/lorem*.txt src/fauxton/dist/release/test/ -endif - @dev/run -n 3 -q --with-admin-party-please \ - --enable-erlang-views --degrade-cluster 2 \ - "$(TEST_OPTS)" \ - 'test/javascript/run --suites "$(suites)" \ - --ignore "$(ignore_js_suites)" \ - --path test/javascript/tests-cluster/without-quorum' .PHONY: soak-javascript soak-javascript: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1 diff --git a/test/elixir/README.md b/test/elixir/README.md index 7f13d8afc..ef95e5f61 100644 --- a/test/elixir/README.md +++ b/test/elixir/README.md @@ -31,7 +31,7 @@ X means done, - means partially - [X] Port attachments.js - [X] Port attachments_multipart.js - [X] Port attachment_views.js - - [ ] Port auth_cache.js + - [X] Port auth_cache.js - [X] Port basics.js - [X] Port batch_save.js - [X] Port bulk_docs.js @@ -40,7 +40,7 @@ X means done, - means partially - [X] Port compact.js - [X] Port config.js - [X] Port conflicts.js - - [ ] Port cookie_auth.js + - [X] Port cookie_auth.js - [X] Port copy_doc.js - [ ] Port design_docs.js - [ ] Port design_options.js @@ -54,11 +54,11 @@ X means done, - means partially - [ ] Port jsonp.js - [X] Port large_docs.js - [ ] Port list_views.js - - [ ] Port lorem_b64.txt - - [ ] Port lorem.txt + - [X] Port lorem_b64.txt + - [X] Port lorem.txt - [X] Port lots_of_docs.js - [ ] Port method_override.js - - [ ] Port multiple_rows.js + - [X] Port multiple_rows.js - [ ] Port proxyauth.js - [ ] Port purge.js - [ ] Port reader_acl.js @@ -93,9 +93,9 @@ X means done, - means partially - [ ] Port show_documents.js - [ ] Port stats.js - [ ] Port update_documents.js - - [ ] Port users_db.js + - [X] Port users_db.js - [ ] Port users_db_security.js - - [ ] Port utf8.js + - [X] Port utf8.js - [X] Port uuids.js - [X] Port view_collation.js - [ ] Port view_collation_raw.js diff --git a/test/elixir/lib/couch.ex b/test/elixir/lib/couch.ex index 6c7310d56..6a63dffb0 100644 --- a/test/elixir/lib/couch.ex +++ b/test/elixir/lib/couch.ex @@ -3,11 +3,10 @@ defmodule Couch.Session do CouchDB session helpers. """ - @enforce_keys [:cookie] - defstruct [:cookie] + defstruct [:cookie, :error] - def new(cookie) do - %Couch.Session{cookie: cookie} + def new(cookie, error \\ "") do + %Couch.Session{cookie: cookie, error: error} end def logout(sess) do @@ -20,6 +19,16 @@ defmodule Couch.Session do Couch.delete!("/_session", headers: headers) end + def info(sess) do + headers = [ + "Content-Type": "application/x-www-form-urlencoded", + "X-CouchDB-WWW-Authenticate": "Cookie", + Cookie: sess.cookie + ] + + Couch.get("/_session", headers: headers).body + end + def get(sess, url, opts \\ []), do: go(sess, :get, url, opts) def get!(sess, url, opts \\ []), do: go!(sess, :get, url, opts) def put(sess, url, opts \\ []), do: go(sess, :put, url, opts) @@ -143,12 +152,18 @@ defmodule Couch do login(user, pass) end - def login(user, pass) do + def login(user, pass, expect \\ :success) do resp = Couch.post("/_session", body: %{:username => user, :password => pass}) - true = resp.body["ok"] - cookie = resp.headers[:"set-cookie"] - [token | _] = String.split(cookie, ";") - %Couch.Session{cookie: token} + + if expect == :success do + true = resp.body["ok"] + cookie = resp.headers[:"set-cookie"] + [token | _] = String.split(cookie, ";") + %Couch.Session{cookie: token} + else + true = Map.has_key?(resp.body, "error") + %Couch.Session{error: resp.body["error"]} + end end end diff --git a/test/elixir/lib/couch/db_test.ex b/test/elixir/lib/couch/db_test.ex index 47d236ebc..b138937f2 100644 --- a/test/elixir/lib/couch/db_test.ex +++ b/test/elixir/lib/couch/db_test.ex @@ -116,16 +116,17 @@ defmodule Couch.DBTest do end) end - def create_user(user) do - required = [:name, :password, :roles] + def prepare_user_doc(user) do + required = [:name, :password] Enum.each(required, fn key -> assert Keyword.has_key?(user, key), "User missing key: #{key}" end) + id = Keyword.get(user, :id) name = Keyword.get(user, :name) password = Keyword.get(user, :password) - roles = Keyword.get(user, :roles) + roles = Keyword.get(user, :roles, []) assert is_binary(name), "User name must be a string" assert is_binary(password), "User password must be a string" @@ -135,14 +136,17 @@ defmodule Couch.DBTest do assert is_binary(role), "Roles must be a list of strings" end) - user_doc = %{ - "_id" => "org.couchdb.user:" <> name, + %{ + "_id" => id || "org.couchdb.user:" <> name, "type" => "user", "name" => name, "roles" => roles, "password" => password } + end + def create_user(user) do + user_doc = prepare_user_doc(user) resp = Couch.get("/_users/#{user_doc["_id"]}") user_doc = @@ -182,6 +186,12 @@ defmodule Couch.DBTest do {:ok, resp} end + def info(db_name) do + resp = Couch.get("/#{db_name}") + assert resp.status_code == 200 + resp.body + end + def bulk_save(db_name, docs) do resp = Couch.post( @@ -290,6 +300,27 @@ defmodule Couch.DBTest do end end + + def request_stats(path_steps, is_test) do + path = + List.foldl( + path_steps, + "/_node/_local/_stats", + fn p, acc -> + "#{acc}/#{p}" + end + ) + + path = + if is_test do + path <> "?flush=true" + else + path + end + + Couch.get(path).body + end + def retry_until(condition, sleep \\ 100, timeout \\ 30_000) do retry_until(condition, now(:ms), sleep, timeout) end @@ -349,6 +380,7 @@ defmodule Couch.DBTest do body: :jiffy.encode(setting.value) ) + assert resp.status_code == 200 Map.put(acc, node, resp.body) end) @@ -364,16 +396,22 @@ defmodule Couch.DBTest do value = elem(node_value, 1) if value == ~s(""\\n) do - Couch.delete( - "/_node/#{node}/_config/#{setting.section}/#{setting.key}", - headers: ["X-Couch-Persist": false] - ) + resp = + Couch.delete( + "/_node/#{node}/_config/#{setting.section}/#{setting.key}", + headers: ["X-Couch-Persist": false] + ) + + assert resp.status_code == 200 else - Couch.put( - "/_node/#{node}/_config/#{setting.section}/#{setting.key}", - headers: ["X-Couch-Persist": false], - body: :jiffy.encode(value) - ) + resp = + Couch.put( + "/_node/#{node}/_config/#{setting.section}/#{setting.key}", + headers: ["X-Couch-Persist": false], + body: :jiffy.encode(value) + ) + + assert resp.status_code == 200 end end) end) diff --git a/test/elixir/test/auth_cache_test.exs b/test/elixir/test/auth_cache_test.exs new file mode 100644 index 000000000..2ba396de7 --- /dev/null +++ b/test/elixir/test/auth_cache_test.exs @@ -0,0 +1,212 @@ +defmodule AuthCacheTest do + use CouchTestCase + + @moduletag :authentication + + @tag :pending + @tag :with_db + test "auth cache management", context do + db_name = context[:db_name] + + server_config = [ + %{ + :section => "chttpd_auth", + :key => "authentication_db", + :value => db_name + }, + %{ + :section => "chttpd_auth", + :key => "auth_cache_size", + :value => "3" + }, + %{ + :section => "httpd", + :key => "authentication_handlers", + :value => "{couch_httpd_auth, default_authentication_handler}" + }, + %{ + :section => "chttpd_auth", + :key => "secret", + :value => generate_secret(64) + } + ] + + run_on_modified_server(server_config, fn -> test_fun(db_name) end) + end + + defp generate_secret(len) do + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + |> String.splitter("", trim: true) + |> Enum.take_random(len) + |> Enum.join("") + end + + defp hits() do + hits = request_stats(["couchdb", "auth_cache_hits"], true) + hits["value"] || 0 + end + + defp misses() do + misses = request_stats(["couchdb", "auth_cache_misses"], true) + misses["value"] || 0 + end + + defp logout(session) do + assert Couch.Session.logout(session).body["ok"] + end + + defp login_fail(user, password) do + resp = Couch.login(user, password, :fail) + assert resp.error, "Login error is expected." + end + + defp login(user, password) do + sess = Couch.login(user, password) + assert sess.cookie, "Login correct is expected" + sess + end + + defp wait_until_compact_complete(db_name) do + retry_until( + fn -> Map.get(info(db_name), "compact_running") == false end, + 200, + 10_000 + ) + end + + defp assert_cache(event, user, password, expect \\ :expect_login_success) do + hits_before = hits() + misses_before = misses() + + session = + case expect do + :expect_login_success -> login(user, password) + :expect_login_fail -> login_fail(user, password) + _ -> assert false + end + + hits_after = hits() + misses_after = misses() + + if expect == :expect_success do + logout(session) + end + + case event do + :expect_miss -> + assert misses_after == misses_before + 1, + "Cache miss is expected for #{user} after login" + + assert hits_after == hits_before, + "No cache hit is expected for #{user} after login" + + :expect_hit -> + assert misses_after == misses_before, + "No cache miss is expected for #{user} after login" + + assert hits_after == hits_before + 1, + "Cache hit is expected for #{user} after login" + + _ -> + assert false + end + end + + defp compact(db_name) do + resp = Couch.post("/#{db_name}/_compact") + assert resp.status_code == 202 + resp.body + end + + def save_doc(db_name, body) do + resp = Couch.put("/#{db_name}/#{body["_id"]}", body: body) + assert resp.status_code in [201, 202] + assert resp.body["ok"] + Map.put(body, "_rev", resp.body["rev"]) + end + + def delete_doc(db_name, body) do + resp = Couch.delete("/#{db_name}/#{body["_id"]}", query: [rev: body["_rev"]]) + assert resp.status_code in [200, 202] + assert resp.body["ok"] + {:ok, resp} + end + + defp test_fun(db_name) do + fdmanana = + prepare_user_doc([ + {:name, "fdmanana"}, + {:password, "qwerty"}, + {:roles, ["dev"]} + ]) + + {:ok, resp} = create_doc(db_name, fdmanana) + fdmanana = Map.put(fdmanana, "_rev", resp.body["rev"]) + + chris = + prepare_user_doc([ + {:name, "chris"}, + {:password, "the_god_father"}, + {:roles, ["dev", "mafia", "white_costume"]} + ]) + + create_doc(db_name, chris) + + joe = + prepare_user_doc([ + {:name, "joe"}, + {:password, "functional"}, + {:roles, ["erlnager"]} + ]) + + create_doc(db_name, joe) + + johndoe = + prepare_user_doc([ + {:name, "johndoe"}, + {:password, "123456"}, + {:roles, ["user"]} + ]) + + create_doc(db_name, johndoe) + + assert_cache(:expect_miss, "fdmanana", "qwerty") + assert_cache(:expect_hit, "fdmanana", "qwerty") + assert_cache(:expect_miss, "chris", "the_god_father") + assert_cache(:expect_miss, "joe", "functional") + assert_cache(:expect_miss, "johndoe", "123456") + + # It's a MRU cache, joe was removed from cache to add johndoe + # BUGGED assert_cache(:expect_miss, "joe", "functional") + + assert_cache(:expect_hit, "fdmanana", "qwerty") + + fdmanana = Map.replace!(fdmanana, "password", "foobar") + fdmanana = save_doc(db_name, fdmanana) + + # Cache was refreshed + # BUGGED + # assert_cache(:expect_hit, "fdmanana", "qwerty", :expect_login_fail) + # assert_cache(:expect_hit, "fdmanana", "foobar") + + # and yet another update + fdmanana = Map.replace!(fdmanana, "password", "javascript") + fdmanana = save_doc(db_name, fdmanana) + + # Cache was refreshed + # BUGGED + # assert_cache(:expect_hit, "fdmanana", "foobar", :expect_login_fail) + # assert_cache(:expect_hit, "fdmanana", "javascript") + + delete_doc(db_name, fdmanana) + + assert_cache(:expect_hit, "fdmanana", "javascript", :expect_login_fail) + + # login, compact authentication DB, login again and verify that + # there was a cache hit + assert_cache(:expect_hit, "johndoe", "123456") + compact(db_name) + wait_until_compact_complete(db_name) + assert_cache(:expect_hit, "johndoe", "123456") + end +end diff --git a/test/elixir/test/cookie_auth_test.exs b/test/elixir/test/cookie_auth_test.exs new file mode 100644 index 000000000..ac1110be2 --- /dev/null +++ b/test/elixir/test/cookie_auth_test.exs @@ -0,0 +1,403 @@ +defmodule CookieAuthTest do + use CouchTestCase + + @moduletag :authentication + + @users_db "_users" + + @moduletag config: [ + { + "chttpd_auth", + "authentication_db", + @users_db + }, + { + "couch_httpd_auth", + "authentication_db", + @users_db + }, + { + "couch_httpd_auth", + "iterations", + "1" + }, + { + "admins", + "jan", + "apple" + } + ] + + @password "3.141592653589" + + setup do + # Create db if not exists + Couch.put("/#{@users_db}") + + resp = + Couch.get( + "/#{@users_db}/_changes", + query: [feed: "longpoll", timeout: 5000, filter: "_design"] + ) + + assert resp.body + + on_exit(&tear_down/0) + + :ok + end + + defp tear_down do + # delete users + user = URI.encode("org.couchdb.user:jchris") + user_doc = Couch.get("/#{@users_db}/#{URI.encode(user)}").body + Couch.delete("/#{@users_db}/#{user}", query: [rev: user_doc["_rev"]]) + + user = URI.encode("org.couchdb.user:Jason Davies") + user_doc = Couch.get("/#{@users_db}/#{user}").body + Couch.delete("/#{@users_db}/#{user}", query: [rev: user_doc["_rev"]]) + end + + defp login(user, password) do + sess = Couch.login(user, password) + assert sess.cookie, "Login correct is expected" + sess + end + + defp logout(session) do + assert Couch.Session.logout(session).body["ok"] + end + + defp login_as(user) do + pws = %{ + "jan" => "apple", + "Jason Davies" => @password, + "jchris" => "funnybone" + } + + user1 = Regex.replace(~r/[0-9]$/, user, "") + login(user1, pws[user]) + end + + defp create_doc_expect_error(db_name, doc, status_code, msg) do + resp = Couch.post("/#{db_name}", body: doc) + assert resp.status_code == status_code + assert resp.body["error"] == msg + resp + end + + defp open_as(db_name, doc_id, options) do + use_session = Keyword.get(options, :use_session) + user = Keyword.get(options, :user) + expect_response = Keyword.get(options, :expect_response, 200) + expect_message = Keyword.get(options, :error_message) + + session = use_session || login_as(user) + + resp = + Couch.get( + "/#{db_name}/#{URI.encode(doc_id)}", + headers: [ + Cookie: session.cookie, + "X-CouchDB-www-Authenticate": "Cookie" + ] + ) + + if use_session == nil do + logout(session) + end + + assert resp.status_code == expect_response + + if expect_message != nil do + assert resp.body["error"] == expect_message + end + + resp.body + end + + defp save_as(db_name, doc, options) do + use_session = Keyword.get(options, :use_session) + user = Keyword.get(options, :user) + expect_response = Keyword.get(options, :expect_response, [201, 202]) + expect_message = Keyword.get(options, :error_message) + + session = use_session || login_as(user) + + resp = + Couch.put( + "/#{db_name}/#{URI.encode(doc["_id"])}", + headers: [ + Cookie: session.cookie, + "X-CouchDB-www-Authenticate": "Cookie" + ], + body: doc + ) + + if use_session == nil do + logout(session) + end + + if is_list(expect_response) do + assert resp.status_code in expect_response + else + assert resp.status_code == expect_response + end + + if expect_message != nil do + assert resp.body["error"] == expect_message + end + + resp + end + + defp delete_as(db_name, doc, options) do + use_session = Keyword.get(options, :use_session) + user = Keyword.get(options, :user) + expect_response = Keyword.get(options, :expect_response, [200, 202]) + expect_message = Keyword.get(options, :error_message) + + session = use_session || login_as(user) + + resp = + Couch.delete( + "/#{db_name}/#{URI.encode(doc["_id"])}", + headers: [ + Cookie: session.cookie, + "X-CouchDB-www-Authenticate": "Cookie" + ] + ) + + if use_session == nil do + logout(session) + end + + if is_list(expect_response) do + assert resp.status_code in expect_response + else + assert resp.status_code == expect_response + end + + if expect_message != nil do + assert resp.body["error"] == expect_message + end + + resp + end + + defp test_change_admin_fun do + sess = login("jchris", "funnybone") + info = Couch.Session.info(sess) + assert info["userCtx"]["name"] == "jchris" + assert Enum.member?(info["userCtx"]["roles"], "_admin") + assert Enum.member?(info["userCtx"]["roles"], "foo") + + jchris_user_doc = + open_as( + @users_db, + "org.couchdb.user:jchris", + use_session: sess + ) + + jchris_user_doc = Map.drop(jchris_user_doc, [:salt, :password_sha]) + save_as(@users_db, jchris_user_doc, use_session: sess) + logout(sess) + sess = login("jchris", "funnybone") + info = Couch.Session.info(sess) + assert info["userCtx"]["name"] == "jchris" + assert Enum.member?(info["userCtx"]["roles"], "_admin") + assert info["info"]["authenticated"] == "cookie" + assert info["info"]["authentication_db"] == @users_db + assert Enum.member?(info["userCtx"]["roles"], "foo") + logout(sess) + end + + test "cookie auth" do + # test that the users db is born with the auth ddoc + ddoc = open_as(@users_db, "_design/_auth", user: "jan") + assert ddoc["validate_doc_update"] != nil + + jason_user_doc = + prepare_user_doc([ + {:name, "Jason Davies"}, + {:password, @password} + ]) + + create_doc(@users_db, jason_user_doc) + jason_check_doc = open_as(@users_db, jason_user_doc["_id"], user: "jan") + assert jason_check_doc["name"] == "Jason Davies" + + jchris_user_doc = + prepare_user_doc([ + {:name, "jchris"}, + {:password, "funnybone"} + ]) + + {:ok, resp} = create_doc(@users_db, jchris_user_doc) + jchris_rev = resp.body["rev"] + + duplicate_jchris_user_doc = + prepare_user_doc([ + {:name, "jchris"}, + {:password, "eh, Boo-Boo?"} + ]) + + # make sure we cant create duplicate users + create_doc_expect_error(@users_db, duplicate_jchris_user_doc, 409, "conflict") + + # we can't create _names + underscore_user_doc = + prepare_user_doc([ + {:name, "_why"}, + {:password, "copperfield"} + ]) + + create_doc_expect_error(@users_db, underscore_user_doc, 403, "forbidden") + + # we can't create malformed ids + bad_id_user_doc = + prepare_user_doc([ + {:id, "org.apache.couchdb:w00x"}, + {:name, "w00x"}, + {:password, "bar"} + ]) + + create_doc_expect_error(@users_db, bad_id_user_doc, 403, "forbidden") + + # login works + session = login_as("Jason Davies") + info = Couch.Session.info(session) + assert info["userCtx"]["name"] == "Jason Davies" + assert not Enum.member?(info["userCtx"]["roles"], "_admin") + + # update one's own credentials document + jason_user_doc = + jason_user_doc + |> Map.put("_rev", jason_check_doc["_rev"]) + |> Map.put("foo", 2) + + resp = save_as(@users_db, jason_user_doc, use_session: session) + jason_user_doc_rev = resp.body["rev"] + + # can't delete another users doc unless you are admin + + jchris_user_doc = Map.put(jchris_user_doc, "_rev", jchris_rev) + + delete_as( + @users_db, + jchris_user_doc, + use_session: session, + expect_response: 404, + error_message: "not_found" + ) + + logout(session) + + # test redirect on success + resp = + Couch.post( + "/_session", + query: [next: "/_up"], + body: %{ + :username => "Jason Davies", + :password => @password + } + ) + + assert resp.status_code == 302 + assert resp.body["ok"] + assert String.ends_with?(resp.headers["location"], "/_up") + + # test redirect on fail + resp = + Couch.post( + "/_session", + query: [fail: "/_up"], + body: %{ + :username => "Jason Davies", + :password => "foobar" + } + ) + + assert resp.status_code == 302 + assert resp.body["error"] == "unauthorized" + assert String.ends_with?(resp.headers["location"], "/_up") + + session = login("jchris", "funnybone") + info = Couch.Session.info(session) + assert info["userCtx"]["name"] == "jchris" + assert Enum.empty?(info["userCtx"]["roles"]) + + jason_user_doc = + jason_user_doc + |> Map.put("_rev", jason_user_doc_rev) + |> Map.put("foo", 3) + + save_as( + @users_db, + jason_user_doc, + use_session: session, + expect_response: 404, + error_message: "not_found" + ) + + jchris_user_doc = Map.put(jchris_user_doc, "roles", ["foo"]) + + save_as( + @users_db, + jchris_user_doc, + use_session: session, + expect_response: 403, + error_message: "forbidden" + ) + + logout(session) + + jchris_user_doc = Map.put(jchris_user_doc, "foo", ["foo"]) + + resp = + save_as( + @users_db, + jchris_user_doc, + user: "jan" + ) + + # test that you can't save system (underscore) roles even if you are admin + jchris_user_doc = + jchris_user_doc + |> Map.put("roles", ["_bar"]) + |> Map.put("_rev", resp.body["rev"]) + + save_as( + @users_db, + jchris_user_doc, + user: "jan", + expect_response: 403, + error_message: "forbidden" + ) + + session = login("jchris", "funnybone") + info = Couch.Session.info(session) + + assert not Enum.member?(info["userCtx"]["roles"], "_admin") + assert(Enum.member?(info["userCtx"]["roles"], "foo")) + + logout(session) + + login("jan", "apple") + + run_on_modified_server( + [ + %{ + :section => "admins", + :key => "jchris", + :value => "funnybone" + } + ], + &test_change_admin_fun/0 + ) + + # log in one last time so run_on_modified_server can clean up the admin account + login("jan", "apple") + end +end diff --git a/test/elixir/test/replication_test.exs b/test/elixir/test/replication_test.exs index 11687ab17..73ceca6a4 100644 --- a/test/elixir/test/replication_test.exs +++ b/test/elixir/test/replication_test.exs @@ -2,7 +2,7 @@ defmodule ReplicationTest do use CouchTestCase @moduledoc """ - Test CouchDB View Collation Behavior + Test CouchDB Replication Behavior This is a port of the view_collation.js suite """ diff --git a/test/elixir/test/users_db_test.exs b/test/elixir/test/users_db_test.exs new file mode 100644 index 000000000..71ab2f7e7 --- /dev/null +++ b/test/elixir/test/users_db_test.exs @@ -0,0 +1,322 @@ +defmodule UsersDbTest do + use CouchTestCase + + @moduletag :authentication + + @users_db_name "_users" + + @moduletag config: [ + { + "chttpd_auth", + "authentication_db", + @users_db_name + }, + { + "couch_httpd_auth", + "authentication_db", + @users_db_name + }, + { + "couch_httpd_auth", + "iterations", + "1" + }, + { + "admins", + "jan", + "apple" + } + ] + + setup do + # Create db if not exists + Couch.put("/#{@users_db_name}") + + resp = + Couch.get( + "/#{@users_db_name}/_changes", + query: [feed: "longpoll", timeout: 5000, filter: "_design"] + ) + + assert resp.body + + on_exit(&tear_down/0) + + :ok + end + + defp tear_down do + delete_db(@users_db_name) + create_db(@users_db_name) + end + + defp replicate(source, target, rep_options \\ []) do + headers = Keyword.get(rep_options, :headers, []) + body = Keyword.get(rep_options, :body, %{}) + + body = + body + |> Map.put("source", source) + |> Map.put("target", target) + + retry_until( + fn -> + resp = Couch.post("/_replicate", headers: headers, body: body, timeout: 10_000) + assert HTTPotion.Response.success?(resp) + assert resp.status_code == 200 + assert resp.body["ok"] + resp + end, + 500, + 20_000 + ) + end + + defp save_as(db_name, doc, options) do + session = Keyword.get(options, :use_session) + expect_response = Keyword.get(options, :expect_response, [201, 202]) + expect_message = Keyword.get(options, :error_message) + expect_reason = Keyword.get(options, :error_reason) + + headers = + if session != nil do + [ + Cookie: session.cookie, + "X-CouchDB-www-Authenticate": "Cookie" + ] + else + [] + end + + resp = + Couch.put( + "/#{db_name}/#{URI.encode(doc["_id"])}", + headers: headers, + body: doc + ) + + if is_list(expect_response) do + assert resp.status_code in expect_response + else + assert resp.status_code == expect_response + end + + if expect_message != nil do + assert resp.body["error"] == expect_message + end + + if expect_reason != nil do + assert resp.body["reason"] == expect_reason + end + + resp + end + + defp login(user, password) do + sess = Couch.login(user, password) + assert sess.cookie, "Login correct is expected" + sess + end + + defp logout(session) do + assert Couch.Session.logout(session).body["ok"] + end + + @tag :with_db + test "users db", context do + db_name = context[:db_name] + # test that the users db is born with the auth ddoc + ddoc = Couch.get("/#{@users_db_name}/_design/_auth") + assert ddoc.body["validate_doc_update"] != nil + + jchris_user_doc = + prepare_user_doc([ + {:name, "jchris@apache.org"}, + {:password, "funnybone"} + ]) + + {:ok, resp} = create_doc(@users_db_name, jchris_user_doc) + jchris_rev = resp.body["rev"] + + resp = + Couch.get( + "/_session", + headers: [authorization: "Basic #{:base64.encode("jchris@apache.org:funnybone")}"] + ) + + assert resp.body["userCtx"]["name"] == "jchris@apache.org" + assert resp.body["info"]["authenticated"] == "default" + assert resp.body["info"]["authentication_db"] == @users_db_name + assert resp.body["info"]["authentication_handlers"] == ["cookie", "default"] + + resp = + Couch.get( + "/_session", + headers: [authorization: "Basic Xzpf"] + ) + + assert resp.body["userCtx"]["name"] == :null + assert not Enum.member?(resp.body["info"], "authenticated") + + # ok, now create a conflicting edit on the jchris doc, and make sure there's no login. + # (use replication to create the conflict) - need 2 be admin + session = login("jan", "apple") + replicate(@users_db_name, db_name) + + jchris_user_doc = Map.put(jchris_user_doc, "_rev", jchris_rev) + + jchris_user_doc2 = Map.put(jchris_user_doc, "foo", "bar") + + save_as(@users_db_name, jchris_user_doc2, use_session: session) + save_as(@users_db_name, jchris_user_doc, use_session: session, expect_response: 409) + + # then in the other + jchris_user_doc3 = Map.put(jchris_user_doc, "foo", "barrrr") + save_as(db_name, jchris_user_doc3, use_session: session) + replicate(db_name, @users_db_name) + # now we should have a conflict + + resp = + Couch.get( + "/#{@users_db_name}/#{jchris_user_doc3["_id"]}", + query: [conflicts: true] + ) + + assert length(resp.body["_conflicts"]) == 1 + jchris_with_conflict = resp.body + + logout(session) + + # wait for auth_cache invalidation + retry_until( + fn -> + resp = + Couch.get( + "/_session", + headers: [ + authorization: "Basic #{:base64.encode("jchris@apache.org:funnybone")}" + ] + ) + + assert resp.body["error"] == "unauthorized" + assert String.contains?(resp.body["reason"], "conflict") + resp + end, + 500, + 20_000 + ) + + # You can delete a user doc + session = login("jan", "apple") + info = Couch.Session.info(session) + assert Enum.member?(info["userCtx"]["roles"], "_admin") + + resp = + Couch.delete( + "/#{@users_db_name}/#{jchris_with_conflict["_id"]}", + query: [rev: jchris_with_conflict["_rev"]], + headers: [ + Cookie: session.cookie, + "X-CouchDB-www-Authenticate": "Cookie" + ] + ) + + assert resp.body["ok"] + + # you can't change doc from type "user" + resp = + Couch.get( + "/#{@users_db_name}/#{jchris_user_doc["_id"]}", + headers: [ + Cookie: session.cookie, + "X-CouchDB-www-Authenticate": "Cookie" + ] + ) + + assert resp.status_code == 200 + + jchris_user_doc = Map.replace!(resp.body, "type", "not user") + + save_as( + @users_db_name, + jchris_user_doc, + use_session: session, + expect_response: 403, + error_message: "forbidden", + error_reason: "doc.type must be user" + ) + + # "roles" must be an array + jchris_user_doc = + jchris_user_doc + |> Map.replace!("type", "user") + |> Map.replace!("roles", "not an array") + + save_as( + @users_db_name, + jchris_user_doc, + use_session: session, + expect_response: 403, + error_message: "forbidden", + error_reason: "doc.roles must be an array" + ) + + # "roles" must be and array of strings + jchris_user_doc = Map.replace!(jchris_user_doc, "roles", [12]) + + save_as( + @users_db_name, + jchris_user_doc, + use_session: session, + expect_response: 403, + error_message: "forbidden", + error_reason: "doc.roles can only contain strings" + ) + + # "roles" must exist + jchris_user_doc = Map.drop(jchris_user_doc, ["roles"]) + + save_as( + @users_db_name, + jchris_user_doc, + use_session: session, + expect_response: 403, + error_message: "forbidden", + error_reason: "doc.roles must exist" + ) + + # character : is not allowed in usernames + joe_user_doc = + prepare_user_doc([ + {:name, "joe:erlang"}, + {:password, "querty"} + ]) + + save_as( + @users_db_name, + joe_user_doc, + use_session: session, + expect_response: 403, + error_message: "forbidden", + error_reason: "Character `:` is not allowed in usernames." + ) + + # test that you can login as a user with a password starting with : + joe_user_doc = + prepare_user_doc([ + {:name, "foo@example.org"}, + {:password, ":bar"} + ]) + + {:ok, _} = create_doc(@users_db_name, joe_user_doc) + logout(session) + + resp = + Couch.get( + "/_session", + headers: [authorization: "Basic #{:base64.encode("foo@example.org::bar")}"] + ) + + assert resp.body["userCtx"]["name"] == "foo@example.org" + end +end diff --git a/test/elixir/test/utf8_test.exs b/test/elixir/test/utf8_test.exs new file mode 100644 index 000000000..ad78080ae --- /dev/null +++ b/test/elixir/test/utf8_test.exs @@ -0,0 +1,65 @@ +defmodule UTF8Test do + use CouchTestCase + + @moduletag :utf8 + + @moduledoc """ + Test CouchDB UTF8 support + This is a port of the utf8.js test suite + """ + + @tag :with_db + test "UTF8 support", context do + db_name = context[:db_name] + texts = [ + "1. Ascii: hello", + "2. Russian: На берегу пустынных волн", + "3. Math: ∮ E⋅da = Q, n → ∞, ∑ f(i) = ∏ g(i),", + "4. Geek: STARGΛ̊TE SG-1", + "5. Braille: ⡌⠁⠧⠑ ⠼⠁⠒ ⡍⠜⠇⠑⠹⠰⠎ ⡣⠕⠌", + "6. null \u0000 byte", + ] + + texts + |> Enum.with_index() + |> Enum.each(fn {string, index} -> + status = Couch.post("/#{db_name}", query: [w: 3], body: %{"_id" => Integer.to_string(index), "text" => string}).status_code + assert status in [201, 202] + end) + + texts + |> Enum.with_index() + |> Enum.each(fn {string, index} -> + resp = Couch.get("/#{db_name}/#{index}") + %{"_id" => id, "text" => text} = resp.body + assert resp.status_code == 200 + assert Enum.at(texts, String.to_integer(id)) === text + end) + + design_doc = %{ + :_id => "_design/temp_utf8_support", + :language => "javascript", + :views => %{ + :view => %{ + :map => "function(doc) { emit(null, doc.text) }" + } + } + } + + design_resp = + Couch.put( + "/#{db_name}/_design/temp_utf8_support", + body: design_doc, + query: %{w: 3} + ) + + assert design_resp.status_code in [201, 202] + + %{"rows" => values} = Couch.get("/#{db_name}/_design/temp_utf8_support/_view/view").body + values + |> Enum.with_index() + |> Enum.each(fn {%{"value" => value}, index} -> + assert Enum.at(texts, index) === value + end) + end +end diff --git a/test/javascript/tests-cluster/with-quorum/attachments.js b/test/javascript/tests-cluster/with-quorum/attachments.js index f578f877c..8186d7574 100644 --- a/test/javascript/tests-cluster/with-quorum/attachments.js +++ b/test/javascript/tests-cluster/with-quorum/attachments.js @@ -11,6 +11,7 @@ // the License. couchTests.attachments= function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests-cluster/with-quorum/attachments_delete.js b/test/javascript/tests-cluster/with-quorum/attachments_delete.js index ed7d2db9a..1980c1124 100644 --- a/test/javascript/tests-cluster/with-quorum/attachments_delete.js +++ b/test/javascript/tests-cluster/with-quorum/attachments_delete.js @@ -11,6 +11,7 @@ // the License. couchTests.attachments_delete= function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests-cluster/with-quorum/attachments_delete_overridden_quorum.js b/test/javascript/tests-cluster/with-quorum/attachments_delete_overridden_quorum.js index 79c070e9f..48c1f34b9 100644 --- a/test/javascript/tests-cluster/with-quorum/attachments_delete_overridden_quorum.js +++ b/test/javascript/tests-cluster/with-quorum/attachments_delete_overridden_quorum.js @@ -12,6 +12,7 @@ couchTests.skip = true; couchTests.attachments_delete_overridden_quorum= function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"},{"w":3}); db.createDb(); diff --git a/test/javascript/tests-cluster/with-quorum/attachments_overridden_quorum.js b/test/javascript/tests-cluster/with-quorum/attachments_overridden_quorum.js index f9deb15c1..cbeb9858d 100644 --- a/test/javascript/tests-cluster/with-quorum/attachments_overridden_quorum.js +++ b/test/javascript/tests-cluster/with-quorum/attachments_overridden_quorum.js @@ -13,6 +13,7 @@ //Test attachments operations with an overridden quorum parameter couchTests.skip = true; couchTests.attachments_overriden_quorum= function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"},{"w":3}); db.createDb(); diff --git a/test/javascript/tests-cluster/with-quorum/db_creation.js b/test/javascript/tests-cluster/with-quorum/db_creation.js index f8efd6e68..c8a416d3e 100644 --- a/test/javascript/tests-cluster/with-quorum/db_creation.js +++ b/test/javascript/tests-cluster/with-quorum/db_creation.js @@ -12,6 +12,7 @@ // Do DB creation under cluster with quorum conditions. couchTests.db_creation = function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); if (debug) debugger; diff --git a/test/javascript/tests-cluster/with-quorum/db_creation_overridden_quorum.js b/test/javascript/tests-cluster/with-quorum/db_creation_overridden_quorum.js index 1e69cd8b4..af27f9580 100644 --- a/test/javascript/tests-cluster/with-quorum/db_creation_overridden_quorum.js +++ b/test/javascript/tests-cluster/with-quorum/db_creation_overridden_quorum.js @@ -13,6 +13,7 @@ // Do DB creation under cluster with quorum conditions but overriding write quorum. couchTests.skip = true; couchTests.db_creation_overridden_quorum = function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); if (debug) debugger; diff --git a/test/javascript/tests-cluster/with-quorum/db_deletion.js b/test/javascript/tests-cluster/with-quorum/db_deletion.js index 079fb493d..70e703411 100644 --- a/test/javascript/tests-cluster/with-quorum/db_deletion.js +++ b/test/javascript/tests-cluster/with-quorum/db_deletion.js @@ -12,6 +12,7 @@ // Do DB deletion under cluster with quorum conditions. couchTests.db_deletion = function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); if (debug) debugger; diff --git a/test/javascript/tests-cluster/with-quorum/db_deletion_overridden_quorum.js b/test/javascript/tests-cluster/with-quorum/db_deletion_overridden_quorum.js index 01417eb63..8e9c65e31 100644 --- a/test/javascript/tests-cluster/with-quorum/db_deletion_overridden_quorum.js +++ b/test/javascript/tests-cluster/with-quorum/db_deletion_overridden_quorum.js @@ -12,6 +12,7 @@ // Do DB deletion in a cluster with quorum conditions. couchTests.db_deletion_overridden_quorum = function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); if (debug) debugger; diff --git a/test/javascript/tests-cluster/with-quorum/doc_bulk.js b/test/javascript/tests-cluster/with-quorum/doc_bulk.js index 4bdd3c84b..1cb85749f 100644 --- a/test/javascript/tests-cluster/with-quorum/doc_bulk.js +++ b/test/javascript/tests-cluster/with-quorum/doc_bulk.js @@ -11,6 +11,7 @@ // the License. couchTests.doc_bulk = function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests-cluster/with-quorum/doc_bulk_overridden_quorum.js b/test/javascript/tests-cluster/with-quorum/doc_bulk_overridden_quorum.js index 0cf9a7e8c..2a3be068a 100644 --- a/test/javascript/tests-cluster/with-quorum/doc_bulk_overridden_quorum.js +++ b/test/javascript/tests-cluster/with-quorum/doc_bulk_overridden_quorum.js @@ -11,6 +11,7 @@ // the License. couchTests.doc_bulk_overridden_quorum = function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"},{"w":3}); db.createDb(); diff --git a/test/javascript/tests-cluster/with-quorum/doc_copy.js b/test/javascript/tests-cluster/with-quorum/doc_copy.js index 386ca5671..e79d38ccd 100644 --- a/test/javascript/tests-cluster/with-quorum/doc_copy.js +++ b/test/javascript/tests-cluster/with-quorum/doc_copy.js @@ -11,6 +11,7 @@ // the License. couchTests.doc_copy = function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests-cluster/with-quorum/doc_copy_overridden_quorum.js b/test/javascript/tests-cluster/with-quorum/doc_copy_overridden_quorum.js index 1ceef9743..a816817f8 100644 --- a/test/javascript/tests-cluster/with-quorum/doc_copy_overridden_quorum.js +++ b/test/javascript/tests-cluster/with-quorum/doc_copy_overridden_quorum.js @@ -12,6 +12,7 @@ couchTests.skip = true; couchTests.doc_copy_overriden_quorum = function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"},{"w":3}); db.createDb(); diff --git a/test/javascript/tests-cluster/with-quorum/doc_crud.js b/test/javascript/tests-cluster/with-quorum/doc_crud.js index f016cefdd..ab90e603e 100644 --- a/test/javascript/tests-cluster/with-quorum/doc_crud.js +++ b/test/javascript/tests-cluster/with-quorum/doc_crud.js @@ -11,6 +11,7 @@ // the License. couchTests.doc_crud = function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests-cluster/with-quorum/doc_crud_overridden_quorum.js b/test/javascript/tests-cluster/with-quorum/doc_crud_overridden_quorum.js index 41502ca5e..a3513781f 100644 --- a/test/javascript/tests-cluster/with-quorum/doc_crud_overridden_quorum.js +++ b/test/javascript/tests-cluster/with-quorum/doc_crud_overridden_quorum.js @@ -11,6 +11,7 @@ // the License. couchTests.doc_crud_overridden_quorum = function(debug) { + return console.log('done in test/elixir/test/cluster_with_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"},{"w":3}); db.createDb(); diff --git a/test/javascript/tests-cluster/without-quorum/attachments.js b/test/javascript/tests-cluster/without-quorum/attachments.js index 57563439a..349cc88d6 100644 --- a/test/javascript/tests-cluster/without-quorum/attachments.js +++ b/test/javascript/tests-cluster/without-quorum/attachments.js @@ -11,6 +11,7 @@ // the License. couchTests.attachments= function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests-cluster/without-quorum/attachments_delete.js b/test/javascript/tests-cluster/without-quorum/attachments_delete.js index 48a33d2e8..8b8a2dbcf 100644 --- a/test/javascript/tests-cluster/without-quorum/attachments_delete.js +++ b/test/javascript/tests-cluster/without-quorum/attachments_delete.js @@ -12,6 +12,7 @@ couchTests.skip = true; couchTests.attachments_delete= function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests-cluster/without-quorum/attachments_delete_overridden_quorum.js b/test/javascript/tests-cluster/without-quorum/attachments_delete_overridden_quorum.js index c3b95f865..48247e00d 100644 --- a/test/javascript/tests-cluster/without-quorum/attachments_delete_overridden_quorum.js +++ b/test/javascript/tests-cluster/without-quorum/attachments_delete_overridden_quorum.js @@ -12,6 +12,7 @@ couchTests.skip = true; couchTests.attachments_delete_overridden_quorum= function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"},{"w":1}); db.createDb(); diff --git a/test/javascript/tests-cluster/without-quorum/attachments_overridden_quorum.js b/test/javascript/tests-cluster/without-quorum/attachments_overridden_quorum.js index 434578f3a..2b8e75fd0 100644 --- a/test/javascript/tests-cluster/without-quorum/attachments_overridden_quorum.js +++ b/test/javascript/tests-cluster/without-quorum/attachments_overridden_quorum.js @@ -12,6 +12,7 @@ //Test attachments operations with an overridden quorum parameter couchTests.attachments_overriden_quorum= function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"},{"w":1}); db.createDb(); diff --git a/test/javascript/tests-cluster/without-quorum/db_creation.js b/test/javascript/tests-cluster/without-quorum/db_creation.js index a21d37746..dd9b29497 100644 --- a/test/javascript/tests-cluster/without-quorum/db_creation.js +++ b/test/javascript/tests-cluster/without-quorum/db_creation.js @@ -12,6 +12,7 @@ // Do DB creation under cluster without quorum conditions. couchTests.db_creation = function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); if (debug) debugger; diff --git a/test/javascript/tests-cluster/without-quorum/db_creation_overridden_quorum.js b/test/javascript/tests-cluster/without-quorum/db_creation_overridden_quorum.js index 7cee52ee0..8ed9b4480 100644 --- a/test/javascript/tests-cluster/without-quorum/db_creation_overridden_quorum.js +++ b/test/javascript/tests-cluster/without-quorum/db_creation_overridden_quorum.js @@ -13,6 +13,7 @@ // Do DB creation under cluster with quorum conditions but overriding write quorum. couchTests.skip = true; couchTests.db_creation_overridden_quorum = function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); if (debug) debugger; diff --git a/test/javascript/tests-cluster/without-quorum/db_deletion.js b/test/javascript/tests-cluster/without-quorum/db_deletion.js index 006345e30..f156b0e95 100644 --- a/test/javascript/tests-cluster/without-quorum/db_deletion.js +++ b/test/javascript/tests-cluster/without-quorum/db_deletion.js @@ -12,6 +12,7 @@ // Do DB creation under cluster with quorum conditions. couchTests.db_deletion = function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); if (debug) debugger; diff --git a/test/javascript/tests-cluster/without-quorum/db_deletion_overridden_quorum.js b/test/javascript/tests-cluster/without-quorum/db_deletion_overridden_quorum.js index 11b344cfb..86dea83aa 100644 --- a/test/javascript/tests-cluster/without-quorum/db_deletion_overridden_quorum.js +++ b/test/javascript/tests-cluster/without-quorum/db_deletion_overridden_quorum.js @@ -12,6 +12,7 @@ // Do DB deletion in a cluster with quorum conditions. couchTests.db_deletion_overridden_quorum = function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); if (debug) debugger; diff --git a/test/javascript/tests-cluster/without-quorum/doc_bulk.js b/test/javascript/tests-cluster/without-quorum/doc_bulk.js index 91578d88a..37f67ec6b 100644 --- a/test/javascript/tests-cluster/without-quorum/doc_bulk.js +++ b/test/javascript/tests-cluster/without-quorum/doc_bulk.js @@ -11,6 +11,7 @@ // the License. couchTests.doc_bulk = function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests-cluster/without-quorum/doc_bulk_overridden_quorum.js b/test/javascript/tests-cluster/without-quorum/doc_bulk_overridden_quorum.js index 56fb11e59..0f2f36443 100644 --- a/test/javascript/tests-cluster/without-quorum/doc_bulk_overridden_quorum.js +++ b/test/javascript/tests-cluster/without-quorum/doc_bulk_overridden_quorum.js @@ -11,6 +11,7 @@ // the License. couchTests.doc_bulk_overridden_quorum = function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"},{"w":1}); db.createDb(); diff --git a/test/javascript/tests-cluster/without-quorum/doc_copy.js b/test/javascript/tests-cluster/without-quorum/doc_copy.js index 7d7c35fcc..6e7ae45b4 100644 --- a/test/javascript/tests-cluster/without-quorum/doc_copy.js +++ b/test/javascript/tests-cluster/without-quorum/doc_copy.js @@ -11,6 +11,7 @@ // the License. couchTests.doc_copy = function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests-cluster/without-quorum/doc_copy_overridden_quorum.js b/test/javascript/tests-cluster/without-quorum/doc_copy_overridden_quorum.js index bf372cad2..301240e22 100644 --- a/test/javascript/tests-cluster/without-quorum/doc_copy_overridden_quorum.js +++ b/test/javascript/tests-cluster/without-quorum/doc_copy_overridden_quorum.js @@ -12,6 +12,7 @@ couchTests.skip = true; couchTests.doc_copy_overriden_quorum = function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"},{"w":1}); db.createDb(); diff --git a/test/javascript/tests-cluster/without-quorum/doc_crud.js b/test/javascript/tests-cluster/without-quorum/doc_crud.js index aa706976b..0a009d58a 100644 --- a/test/javascript/tests-cluster/without-quorum/doc_crud.js +++ b/test/javascript/tests-cluster/without-quorum/doc_crud.js @@ -11,6 +11,7 @@ // the License. couchTests.doc_crud = function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests-cluster/without-quorum/doc_crud_overridden_quorum.js b/test/javascript/tests-cluster/without-quorum/doc_crud_overridden_quorum.js index 44ab86ec0..9eb83bd6a 100644 --- a/test/javascript/tests-cluster/without-quorum/doc_crud_overridden_quorum.js +++ b/test/javascript/tests-cluster/without-quorum/doc_crud_overridden_quorum.js @@ -11,6 +11,7 @@ // the License. couchTests.doc_crud_overridden_quorum = function(debug) { + return console.log('done in test/elixir/test/cluster_without_quorum_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"},{"w":1}); db.createDb(); diff --git a/test/javascript/tests/all_docs.js b/test/javascript/tests/all_docs.js index a360fb9ca..0eb382fa9 100644 --- a/test/javascript/tests/all_docs.js +++ b/test/javascript/tests/all_docs.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.all_docs = function(debug) { + return console.log('done in test/elixir/test/all_docs_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}, {w: 3}); db.createDb(); diff --git a/test/javascript/tests/attachment_names.js b/test/javascript/tests/attachment_names.js index 4e9217c1a..16a23ac85 100644 --- a/test/javascript/tests/attachment_names.js +++ b/test/javascript/tests/attachment_names.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.attachment_names = function(debug) { + return console.log('done in test/elixir/test/attachment_names_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}, {w: 3}); db.createDb(); diff --git a/test/javascript/tests/attachment_paths.js b/test/javascript/tests/attachment_paths.js index 048640d0c..b8c6a794b 100644 --- a/test/javascript/tests/attachment_paths.js +++ b/test/javascript/tests/attachment_paths.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.attachment_paths = function(debug) { + return console.log('done in test/elixir/test/attachment_paths_test.exs'); if (debug) debugger; var r_db_name = get_random_db_name() var dbNames = [r_db_name, r_db_name + "/with_slashes"]; diff --git a/test/javascript/tests/attachment_ranges.js b/test/javascript/tests/attachment_ranges.js index 37700ecdf..564885cba 100644 --- a/test/javascript/tests/attachment_ranges.js +++ b/test/javascript/tests/attachment_ranges.js @@ -16,6 +16,7 @@ function cacheBust() { couchTests.elixir = true; couchTests.attachment_ranges = function(debug) { + return console.log('done in test/elixir/test/attachment_ranges_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, { "X-Couch-Full-Commit": "false" diff --git a/test/javascript/tests/attachment_views.js b/test/javascript/tests/attachment_views.js index 7be32a9c1..c6c4b1841 100644 --- a/test/javascript/tests/attachment_views.js +++ b/test/javascript/tests/attachment_views.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.attachment_views= function(debug) { + return console.log('done in test/elixir/test/attachment_views_test.exs'); var db_name = get_random_db_name() var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); diff --git a/test/javascript/tests/attachments.js b/test/javascript/tests/attachments.js index 09c6acd8a..61fe8b9b3 100644 --- a/test/javascript/tests/attachments.js +++ b/test/javascript/tests/attachments.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.attachments= function(debug) { + return console.log('done in test/elixir/test/attachment_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/attachments_multipart.js b/test/javascript/tests/attachments_multipart.js index c36083f8a..793c8c9ec 100644 --- a/test/javascript/tests/attachments_multipart.js +++ b/test/javascript/tests/attachments_multipart.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.attachments_multipart= function(debug) { + return console.log('done in test/elixir/test/attachment_multipart_test.exs'); var db_name = get_random_db_name() var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/auth_cache.js b/test/javascript/tests/auth_cache.js index 4d35d82b4..ca8f077e7 100644 --- a/test/javascript/tests/auth_cache.js +++ b/test/javascript/tests/auth_cache.js @@ -11,6 +11,7 @@ // the License. couchTests.auth_cache = function(debug) { + return console.log('done in test/elixir/test/auth_cache_test.exs'); if (debug) debugger; // Simple secret key generator diff --git a/test/javascript/tests/basics.js b/test/javascript/tests/basics.js index edf96927c..51abb4090 100644 --- a/test/javascript/tests/basics.js +++ b/test/javascript/tests/basics.js @@ -13,7 +13,7 @@ // Do some basic tests. couchTests.elixir = true; couchTests.basics = function(debug) { - + return console.log('done in test/elixir/test/basics_test.exs'); if (debug) debugger; var result = JSON.parse(CouchDB.request("GET", "/").responseText); diff --git a/test/javascript/tests/batch_save.js b/test/javascript/tests/batch_save.js index 1f85b1293..bbfb2ed9c 100644 --- a/test/javascript/tests/batch_save.js +++ b/test/javascript/tests/batch_save.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.batch_save = function(debug) { + return console.log('done in test/elixir/test/batch_save_test.exs'); var db_name = get_random_db_name() var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/bulk_docs.js b/test/javascript/tests/bulk_docs.js index 7e65ae30e..767a54367 100644 --- a/test/javascript/tests/bulk_docs.js +++ b/test/javascript/tests/bulk_docs.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.bulk_docs = function(debug) { + return console.log('done in test/elixir/test/basics_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/coffee.js b/test/javascript/tests/coffee.js index 747bacf93..42a1a68ec 100644 --- a/test/javascript/tests/coffee.js +++ b/test/javascript/tests/coffee.js @@ -13,6 +13,7 @@ // test basic coffeescript functionality couchTests.elixir = true; couchTests.coffee = function(debug) { + return console.log('done in test/elixir/test/coffee_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/compact.js b/test/javascript/tests/compact.js index 2b9dd21f0..fa05e3008 100644 --- a/test/javascript/tests/compact.js +++ b/test/javascript/tests/compact.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.compact = function(debug) { + return console.log('done in test/elixir/test/coffee_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/config.js b/test/javascript/tests/config.js index 889cbd0a6..e3cacc291 100644 --- a/test/javascript/tests/config.js +++ b/test/javascript/tests/config.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.config = function(debug) { + return console.log('done in test/elixir/test/config_test.exs'); if (debug) debugger; // test that /_config returns all the settings diff --git a/test/javascript/tests/conflicts.js b/test/javascript/tests/conflicts.js index 7b5e02093..ab25e626f 100644 --- a/test/javascript/tests/conflicts.js +++ b/test/javascript/tests/conflicts.js @@ -13,6 +13,7 @@ // Do some edit conflict detection tests couchTests.elixir = true; couchTests.conflicts = function(debug) { + return console.log('done in test/elixir/test/conflicts_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/cookie_auth.js b/test/javascript/tests/cookie_auth.js index 5c8ce8968..0dce6bdb6 100644 --- a/test/javascript/tests/cookie_auth.js +++ b/test/javascript/tests/cookie_auth.js @@ -12,6 +12,7 @@ couchTests.cookie_auth = function(debug) { // This tests cookie-based authentication. + return console.log('done in test/elixir/test/cookie_auth_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); diff --git a/test/javascript/tests/copy_doc.js b/test/javascript/tests/copy_doc.js index 708fe5360..107732c0b 100644 --- a/test/javascript/tests/copy_doc.js +++ b/test/javascript/tests/copy_doc.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.copy_doc = function(debug) { + return console.log('done in test/elixir/test/copy_doc_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/invalid_docids.js b/test/javascript/tests/invalid_docids.js index 74f0e4f9c..31c9d6cea 100644 --- a/test/javascript/tests/invalid_docids.js +++ b/test/javascript/tests/invalid_docids.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.invalid_docids = function(debug) { + return console.log('done in test/elixir/test/invalid_docids_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/large_docs.js b/test/javascript/tests/large_docs.js index bc9d22c84..aa36b6cc3 100644 --- a/test/javascript/tests/large_docs.js +++ b/test/javascript/tests/large_docs.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.large_docs = function(debug) { + return console.log('done in test/elixir/test/large_docs_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/lots_of_docs.js b/test/javascript/tests/lots_of_docs.js index dc1486aa4..453c65218 100644 --- a/test/javascript/tests/lots_of_docs.js +++ b/test/javascript/tests/lots_of_docs.js @@ -13,6 +13,7 @@ // test saving a semi-large quanitity of documents and do some view queries. couchTests.elixir = true; couchTests.lots_of_docs = function(debug) { + return console.log('done in test/elixir/test/lots_of_docs_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/multiple_rows.js b/test/javascript/tests/multiple_rows.js index 5bac8abc1..b06104460 100644 --- a/test/javascript/tests/multiple_rows.js +++ b/test/javascript/tests/multiple_rows.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.multiple_rows = function(debug) { + return console.log('done in test/elixir/test/multiple_rows_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/reduce.js b/test/javascript/tests/reduce.js index 6b8ea189c..c25ca771c 100644 --- a/test/javascript/tests/reduce.js +++ b/test/javascript/tests/reduce.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.reduce = function(debug) { + return console.log('done in test/elixir/test/reduce_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/users_db.js b/test/javascript/tests/users_db.js index 20be325ca..b13adffec 100644 --- a/test/javascript/tests/users_db.js +++ b/test/javascript/tests/users_db.js @@ -11,6 +11,7 @@ // the License. couchTests.users_db = function(debug) { + return console.log('done in test/elixir/test/users_db_test.exs'); // This tests the users db, especially validations // this should also test that you can log into the couch diff --git a/test/javascript/tests/utf8.js b/test/javascript/tests/utf8.js index a724580c0..a1092c128 100644 --- a/test/javascript/tests/utf8.js +++ b/test/javascript/tests/utf8.js @@ -11,6 +11,7 @@ // the License. couchTests.utf8 = function(debug) { + return console.log('done in test/elixir/test/utf8_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); diff --git a/test/javascript/tests/uuids.js b/test/javascript/tests/uuids.js index cbf5e8e2a..18871ecba 100644 --- a/test/javascript/tests/uuids.js +++ b/test/javascript/tests/uuids.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.uuids = function(debug) { + return console.log('done in test/elixir/test/uuids_test.exs'); var etags = []; var testHashBustingHeaders = function(xhr) { T(xhr.getResponseHeader("Cache-Control").match(/no-cache/)); diff --git a/test/javascript/tests/view_collation.js b/test/javascript/tests/view_collation.js index 7391fc85a..3ec9f8a5d 100644 --- a/test/javascript/tests/view_collation.js +++ b/test/javascript/tests/view_collation.js @@ -12,6 +12,7 @@ couchTests.elixir = true; couchTests.view_collation = function(debug) { + return console.log('done in test/elixir/test/view_collation_test.exs'); var db_name = get_random_db_name(); var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); -- cgit v1.2.1 -- cgit v1.2.1 From 3fc054d86f0844bdf851e402b05df5db08b1c230 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Sat, 4 Jan 2020 20:12:58 +0100 Subject: Feature/user db security obj readonly (#2395) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Allow to set the user db security object to readonly - Add the default config - Deny update on _security if the database is the user db and if the config is to false - Add unit test * Allow edits on _users security for the JS test Co-authored-by: Alexis Côté --- rel/overlay/etc/default.ini | 3 +++ src/chttpd/src/chttpd_db.erl | 11 +++++++++++ src/chttpd/test/eunit/chttpd_security_tests.erl | 24 +++++++++++++++++++++++- test/javascript/tests/users_db_security.js | 20 ++++++++++++++++---- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index a0c26174c..7bfbbe941 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -80,6 +80,9 @@ default_engine = couch ; document. Default is 24 hours. ;index_lag_warn_seconds = 86400 +; Allow edits on the _security object in the user db. By default, it's disabled. +users_db_security_editable = false + [couchdb_engines] ; The keys in this section are the filename extension that ; the specified engine module will use. This is important so diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl index 1787e3929..6a3df6def 100644 --- a/src/chttpd/src/chttpd_db.erl +++ b/src/chttpd/src/chttpd_db.erl @@ -781,6 +781,8 @@ db_req(#httpd{path_parts=[_,<<"_revs_diff">>]}=Req, _Db) -> db_req(#httpd{method='PUT',path_parts=[_,<<"_security">>],user_ctx=Ctx}=Req, Db) -> + DbName = ?b2l(couch_db:name(Db)), + validate_security_can_be_edited(DbName), SecObj = chttpd:json_body(Req), case fabric:set_security(Db, SecObj, [{user_ctx, Ctx}]) of ok -> @@ -1886,6 +1888,15 @@ extract_header_rev(Req, ExplicitRev) -> throw({bad_request, "Document rev and etag have different values"}) end. +validate_security_can_be_edited(DbName) -> + UserDbName = config:get("chttpd_auth", "authentication_db", "_users"), + CanEditUserSecurityObject = config:get("couchdb","users_db_security_editable","false"), + case {DbName,CanEditUserSecurityObject} of + {UserDbName,"false"} -> + Msg = "You can't edit the security object of the user database.", + throw({forbidden, Msg}); + {_,_} -> ok + end. validate_attachment_names(Doc) -> lists:foreach(fun(Att) -> diff --git a/src/chttpd/test/eunit/chttpd_security_tests.erl b/src/chttpd/test/eunit/chttpd_security_tests.erl index 955b4ff01..0bea9dbcd 100644 --- a/src/chttpd/test/eunit/chttpd_security_tests.erl +++ b/src/chttpd/test/eunit/chttpd_security_tests.erl @@ -137,7 +137,8 @@ security_object_validate_test_() -> fun should_return_ok_for_sec_obj_with_roles_and_names/1, fun should_return_error_for_sec_obj_with_incorrect_roles_and_names/1, fun should_return_error_for_sec_obj_with_incorrect_roles/1, - fun should_return_error_for_sec_obj_with_incorrect_names/1 + fun should_return_error_for_sec_obj_with_incorrect_names/1, + fun should_return_error_for_sec_obj_in_user_db/1 ] } } @@ -382,3 +383,24 @@ should_return_error_for_sec_obj_with_incorrect_names([Url,_UsersUrl]) -> {<<"reason">>,<<"no_majority">>} ]}, ResultJson) ]. + +should_return_error_for_sec_obj_in_user_db([_,_UsersUrl]) -> + SecurityUrl = lists:concat([_UsersUrl, "/_security"]), + SecurityProperties = [ + {<<"admins">>, {[{<<"names">>,[<>]}, + {<<"roles">>,[<>]}]}}, + {<<"members">>,{[{<<"names">>,[<>]}, + {<<"roles">>,[<>]}]}} + ], + + Body = jiffy:encode({SecurityProperties}), + {ok, Status, _, RespBody} = test_request:put(SecurityUrl, + [?CONTENT_JSON, ?AUTH], Body), + ResultJson = ?JSON_DECODE(RespBody), + [ + ?_assertEqual(403, Status), + ?_assertEqual({[ + {<<"error">>,<<"forbidden">>}, + {<<"reason">>,<<"You can't edit the security object of the user database.">>} + ]}, ResultJson) + ]. diff --git a/test/javascript/tests/users_db_security.js b/test/javascript/tests/users_db_security.js index 1db6c14c5..faffd8c27 100644 --- a/test/javascript/tests/users_db_security.js +++ b/test/javascript/tests/users_db_security.js @@ -374,10 +374,22 @@ couchTests.users_db_security = function(debug) { }; run_on_modified_server( - [{section: "couch_httpd_auth", - key: "iterations", value: "1"}, - {section: "admins", - key: "jan", value: "apple"}], + [ + { + section:"couchdb", + key:"users_db_security_editable", + value:"true" + }, + { + section: "couch_httpd_auth", + key: "iterations", + value: "1" + }, + { + section: "admins", + key: "jan", + value: "apple" + }], function() { try { testFun(); -- cgit v1.2.1 From 9dedf6ad35b05536e8c413b5d3bc2737d1b81cf7 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Sat, 4 Jan 2020 13:53:27 -0600 Subject: Blacklist all 21.x releases older than 21.2.3 (#2056) This basically just extends the black list to cover the `21.{0,1}` release range. This is due to a compiler bug [1] which is a duplicate of [2]. [1] https://bugs.erlang.org/browse/ERL-981 [2] https://bugs.erlang.org/browse/ERL-807 --- rebar.config.script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.script b/rebar.config.script index 39db180b5..79d3e0c32 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -86,7 +86,7 @@ case VerList of [19 | _] -> NotSupported(VerString); [20 | _] = V20 when V20 < [20, 3, 8, 11] -> BadErlang(VerString); - [21, 2, N | _] when N < 3 -> BadErlang(VerString); + [21 | _] = V21 when V21 < [21, 2, 3] -> BadErlang(VerString); [22, 0, N | _] when N < 5 -> BadErlang(VerString); _ -> ok -- cgit v1.2.1 From 43fff603bcc29f5d4138e15b764fa83cc871ce92 Mon Sep 17 00:00:00 2001 From: Nick Vatamaniuc Date: Sat, 4 Jan 2020 17:08:05 -0500 Subject: Lock shard splitting targets during the initial copy phase During the initial copy phase target shards are opened outside the couch_server. Previously, it was possible to manually (via remsh for instance) open the same targets via the couch_server by using the `couch_db:open/2` API for example. That could lead to corruption as there would be two writers for the same DB file. In order to prevent such a scenario, introduce a mechanism for the shard splitter to lock the target shards such that any regular open call would fail during the initial copy phase. The locking mechanism is generic and would allow local locking of shards for possibly other reasons in the future as well. --- src/couch/src/couch_db_split.erl | 11 +++++++- src/couch/src/couch_server.erl | 39 +++++++++++++++++++++++++-- src/couch/test/eunit/couch_db_split_tests.erl | 35 +++++++++++++++++++++--- src/couch/test/eunit/couch_db_tests.erl | 29 +++++++++++++++++++- src/mem3/test/eunit/mem3_reshard_test.erl | 35 +++++++++++++++++++++++- 5 files changed, 141 insertions(+), 8 deletions(-) diff --git a/src/couch/src/couch_db_split.erl b/src/couch/src/couch_db_split.erl index 5bf98b6fd..3a1f98d3e 100644 --- a/src/couch/src/couch_db_split.erl +++ b/src/couch/src/couch_db_split.erl @@ -132,6 +132,12 @@ split(SourceDb, Partitioned, Engine, Targets0, PickFun, {M, F, A} = HashFun) -> {error, E} -> throw({target_create_error, DbName, E, Map}) end, + case couch_server:lock(DbName, <<"shard splitting">>) of + ok -> + ok; + {error, Err} -> + throw({target_create_error, DbName, Err, Map}) + end, {ok, Filepath} = couch_server:get_engine_path(DbName, Engine), Opts = [create, ?ADMIN_CTX] ++ case Partitioned of true -> [{props, [{partitioned, true}, {hash, [M, F, A]}]}]; @@ -164,7 +170,9 @@ split(SourceDb, Partitioned, Engine, Targets0, PickFun, {M, F, A} = HashFun) -> cleanup_targets(#{} = Targets, Engine) -> maps:map(fun(_, #target{db = Db} = T) -> ok = stop_target_db(Db), - delete_target(couch_db:name(Db), Engine), + DbName = couch_db:name(Db), + delete_target(DbName, Engine), + couch_server:unlock(DbName), T end, Targets). @@ -182,6 +190,7 @@ stop_target_db(Db) -> Pid = couch_db:get_pid(Db), catch unlink(Pid), catch exit(Pid, kill), + couch_server:unlock(couch_db:name(Db)), ok. diff --git a/src/couch/src/couch_server.erl b/src/couch/src/couch_server.erl index ab0122eec..eaca3ee9b 100644 --- a/src/couch/src/couch_server.erl +++ b/src/couch/src/couch_server.erl @@ -26,6 +26,7 @@ -export([exists/1]). -export([get_engine_extensions/0]). -export([get_engine_path/2]). +-export([lock/2, unlock/1]). % config_listener api -export([handle_config_change/5, handle_config_terminate/3]). @@ -77,8 +78,15 @@ get_stats() -> sup_start_link() -> gen_server:start_link({local, couch_server}, couch_server, [], []). +open(DbName, Options) -> + case ets:lookup(couch_dbs_locks, DbName) of + [] -> + open_int(DbName, Options); + [{DbName, Reason}] -> + {error, {locked, Reason}} + end. -open(DbName, Options0) -> +open_int(DbName, Options0) -> Ctx = couch_util:get_value(user_ctx, Options0, #user_ctx{}), case ets:lookup(couch_dbs, DbName) of [#entry{db = Db0, lock = Lock} = Entry] when Lock =/= locked -> @@ -115,7 +123,15 @@ update_lru(DbName, Options) -> close_lru() -> gen_server:call(couch_server, close_lru). -create(DbName, Options0) -> +create(DbName, Options) -> + case ets:lookup(couch_dbs_locks, DbName) of + [] -> + create_int(DbName, Options); + [{DbName, Reason}] -> + {error, {locked, Reason}} + end. + +create_int(DbName, Options0) -> Options = maybe_add_sys_db_callbacks(DbName, Options0), couch_partition:validate_dbname(DbName, Options), case gen_server:call(couch_server, {create, DbName, Options}, infinity) of @@ -251,6 +267,12 @@ init([]) -> {read_concurrency, true} ]), ets:new(couch_dbs_pid_to_name, [set, protected, named_table]), + ets:new(couch_dbs_locks, [ + set, + public, + named_table, + {read_concurrency, true} + ]), process_flag(trap_exit, true), {ok, #server{root_dir=RootDir, engines = Engines, @@ -774,6 +796,19 @@ get_engine_path(DbName, Engine) when is_binary(DbName), is_atom(Engine) -> {error, {invalid_engine, Engine}} end. +lock(DbName, Reason) when is_binary(DbName), is_binary(Reason) -> + case ets:lookup(couch_dbs, DbName) of + [] -> + true = ets:insert(couch_dbs_locks, {DbName, Reason}), + ok; + [#entry{}] -> + {error, already_opened} + end. + +unlock(DbName) when is_binary(DbName) -> + true = ets:delete(couch_dbs_locks, DbName), + ok. + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/couch/test/eunit/couch_db_split_tests.erl b/src/couch/test/eunit/couch_db_split_tests.erl index 8e64c39ee..6e24c36ee 100644 --- a/src/couch/test/eunit/couch_db_split_tests.erl +++ b/src/couch/test/eunit/couch_db_split_tests.erl @@ -56,7 +56,8 @@ split_test_() -> fun should_fail_on_missing_source/1, fun should_fail_on_existing_target/1, fun should_fail_on_invalid_target_name/1, - fun should_crash_on_invalid_tmap/1 + fun should_crash_on_invalid_tmap/1, + fun should_fail_on_opened_target/1 ] } ] @@ -104,9 +105,23 @@ should_fail_on_missing_source(_DbName) -> should_fail_on_existing_target(DbName) -> Ranges = make_ranges(2), - TMap = maps:map(fun(_, _) -> DbName end, make_targets(Ranges)), + TMap = maps:map(fun(_, TName) -> + % We create the target but make sure to remove it from the cache so we + % hit the eexist error instaed of already_opened + {ok, Db} = couch_db:create(TName, [?ADMIN_CTX]), + Pid = couch_db:get_pid(Db), + ok = couch_db:close(Db), + exit(Pid, kill), + test_util:wait(fun() -> + case ets:lookup(couch_dbs, TName) of + [] -> ok; + [_ | _] -> wait + end + end), + TName + end, make_targets(Ranges)), Response = couch_db_split:split(DbName, TMap, fun fake_pickfun/3), - ?_assertMatch({error, {target_create_error, DbName, eexist}}, Response). + ?_assertMatch({error, {target_create_error, _, eexist}}, Response). should_fail_on_invalid_target_name(DbName) -> @@ -127,6 +142,20 @@ should_crash_on_invalid_tmap(DbName) -> couch_db_split:split(DbName, TMap, fun fake_pickfun/3)). +should_fail_on_opened_target(DbName) -> + Ranges = make_ranges(2), + TMap = maps:map(fun(_, TName) -> + % We create and keep the target open but delete + % its file on disk so we don't fail with eexist + {ok, Db} = couch_db:create(TName, [?ADMIN_CTX]), + FilePath = couch_db:get_filepath(Db), + ok = file:delete(FilePath), + TName + end, make_targets(Ranges)), + ?_assertMatch({error, {target_create_error, _, already_opened}}, + couch_db_split:split(DbName, TMap, fun fake_pickfun/3)). + + copy_local_docs_test_() -> Cases = [ {"Should work with no docs", 0, 2}, diff --git a/src/couch/test/eunit/couch_db_tests.erl b/src/couch/test/eunit/couch_db_tests.erl index d64f7c640..dd2cb427d 100644 --- a/src/couch/test/eunit/couch_db_tests.erl +++ b/src/couch/test/eunit/couch_db_tests.erl @@ -80,7 +80,8 @@ open_db_test_()-> fun() -> ?tempdb() end, [ fun should_create_db_if_missing/1, - fun should_open_db_if_exists/1 + fun should_open_db_if_exists/1, + fun locking_should_work/1 ] } } @@ -157,6 +158,32 @@ should_open_db_if_exists(DbName) -> ?assert(lists:member(DbName, After)) end). +locking_should_work(DbName) -> + ?_test(begin + ?assertEqual(ok, couch_server:lock(DbName, <<"x">>)), + ?assertEqual({error, {locked, <<"x">>}}, couch_db:create(DbName, [])), + ?assertEqual(ok, couch_server:unlock(DbName)), + {ok, Db} = couch_db:create(DbName, []), + ?assertEqual({error, already_opened}, + couch_server:lock(DbName, <<>>)), + + ok = couch_db:close(Db), + catch exit(couch_db:get_pid(Db), kill), + test_util:wait(fun() -> + case ets:lookup(couch_dbs, DbName) of + [] -> ok; + [_ | _] -> wait + end + end), + + ?assertEqual(ok, couch_server:lock(DbName, <<"y">>)), + ?assertEqual({error, {locked, <<"y">>}}, + couch_db:open(DbName, [])), + + couch_server:unlock(DbName), + {ok, Db1} = couch_db:open(DbName, [{create_if_missing, true}]), + ok = couch_db:close(Db1) + end). create_db(DbName) -> create_db(DbName, []). diff --git a/src/mem3/test/eunit/mem3_reshard_test.erl b/src/mem3/test/eunit/mem3_reshard_test.erl index ab6202115..1e89755a9 100644 --- a/src/mem3/test/eunit/mem3_reshard_test.erl +++ b/src/mem3/test/eunit/mem3_reshard_test.erl @@ -72,7 +72,8 @@ mem3_reshard_db_test_() -> fun couch_events_are_emitted/1, fun retries_work/1, fun target_reset_in_initial_copy/1, - fun split_an_incomplete_shard_map/1 + fun split_an_incomplete_shard_map/1, + fun target_shards_are_locked/1 ] } } @@ -479,6 +480,38 @@ split_an_incomplete_shard_map(#{db1 := Db}) -> end)}. +% Opening a db target db in initial copy phase will throw an error +target_shards_are_locked(#{db1 := Db}) -> + {timeout, ?TIMEOUT, ?_test(begin + add_test_docs(Db, #{docs => 10}), + + % Make the job stops right when it was about to copy the docs + TestPid = self(), + meck:new(couch_db, [passthrough]), + meck:expect(couch_db, start_link, fun(Engine, TName, FilePath, Opts) -> + TestPid ! {start_link, self(), TName}, + receive + continue -> + meck:passthrough([Engine, TName, FilePath, Opts]) + end + end), + + [#shard{name=Shard}] = lists:sort(mem3:local_shards(Db)), + {ok, JobId} = mem3_reshard:start_split_job(Shard), + {Target0, JobPid} = receive + {start_link, Pid, TName} -> {TName, Pid} + end, + ?assertEqual({error, {locked, <<"shard splitting">>}}, + couch_db:open_int(Target0, [])), + + % Send two continues for two targets + JobPid ! continue, + JobPid ! continue, + + wait_state(JobId, completed) + end)}. + + intercept_state(State) -> TestPid = self(), meck:new(mem3_reshard_job, [passthrough]), -- cgit v1.2.1 From 5ef18802cb82169843bd32edee838d4302d0dd97 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Sun, 5 Jan 2020 11:32:27 +0100 Subject: feat: add metric counter for unindexed mango queries. Closes #1913 (#2390) --- src/couch/priv/stats_descriptions.cfg | 4 ++++ src/mango/src/mango_cursor.erl | 1 + 2 files changed, 5 insertions(+) diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg index 0e2271350..ae203bb21 100644 --- a/src/couch/priv/stats_descriptions.cfg +++ b/src/couch/priv/stats_descriptions.cfg @@ -298,3 +298,7 @@ {type, counter}, {desc, <<"number of the attempts to read beyond set limit">>} ]}. +{[mango, unindexed_queries], [ + {type, counter}, + {desc, <<"number of mango queries that could not use an index">>} +]}. diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl index c6f21ddf8..dc2ee74c7 100644 --- a/src/mango/src/mango_cursor.erl +++ b/src/mango/src/mango_cursor.erl @@ -182,6 +182,7 @@ maybe_add_warning_int(ok, _, UserAcc) -> UserAcc; maybe_add_warning_int(Warning, UserFun, UserAcc) -> + couch_stats:increment_counter([mango, unindexed_queries]), Arg = {add_key, warning, Warning}, {_Go, UserAcc0} = UserFun(Arg, UserAcc), UserAcc0. -- cgit v1.2.1 From 3f64ef2b98796ba8e0391fb80483502fdafc6a8c Mon Sep 17 00:00:00 2001 From: Grzegorz Abramczyk Date: Sun, 5 Jan 2020 12:34:51 +0100 Subject: Fix/1977 (#2296) * Added db initialization for standalone instance * Test setup * Updated readme * Added standalone option to rel/overlay/etc/default.ini * Simplified test setup * s/standalone/single_node/ Co-authored-by: Jan Lehnardt --- dev/run | 2 +- rel/overlay/etc/default.ini | 3 +++ src/setup/README.md | 17 +++++++++++++++++ src/setup/src/setup_sup.erl | 6 ++++++ src/setup/test/t-single-node-auto-setup.sh | 24 ++++++++++++++++++++++++ 5 files changed, 51 insertions(+), 1 deletion(-) create mode 100755 src/setup/test/t-single-node-auto-setup.sh diff --git a/dev/run b/dev/run index 482a0e831..3186a1fc8 100755 --- a/dev/run +++ b/dev/run @@ -411,7 +411,7 @@ def hack_default_ini(ctx, node, contents): contents, flags=re.MULTILINE, ) - + contents = re.sub("n=3", "n=%s" % ctx["N"], contents) return contents diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index 7bfbbe941..d8476f3e1 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -68,6 +68,9 @@ default_engine = couch ; inadvertently abusing partitions resulting in hot shards. The default ; is 10GiB. A value of 0 or less will disable partition size checks. ;max_partition_size = 10737418240 +; +; Start node in single_node mode so default databases are created immediately. +;single_node = true [purge] ; Allowed maximum number of documents in one purge request diff --git a/src/setup/README.md b/src/setup/README.md index e30c40027..8a76d9dc5 100644 --- a/src/setup/README.md +++ b/src/setup/README.md @@ -141,6 +141,23 @@ b. Same as in a. _replicator and _metadata, _db_updates endpoints and whatever else is needed. // TBD: collect what else is needed. +## Single node auto setup + +Option `single_node` set to `true` in `[couchdb]` configuration executes single node configuration on startup so the node is ready for use immediately. + +### Testing single_node auto setup + +Pass `--config-overrides single_node=true` and `-n 1` to `dev/run` + + + $ dev/run --no-join -n 1 --admin a:b --config-overrides single_node=true + + +Then, in a new terminal: + + $ src/setup/test/t-single_node.sh + +The script should show that single node is enabled. ## The Setup Endpoint diff --git a/src/setup/src/setup_sup.erl b/src/setup/src/setup_sup.erl index b81aa3afb..4670a0a59 100644 --- a/src/setup/src/setup_sup.erl +++ b/src/setup/src/setup_sup.erl @@ -35,4 +35,10 @@ start_link() -> %% =================================================================== init([]) -> + case config:get_boolean("couchdb", "single_node", false) of + true -> + setup:finish_cluster([]); + false -> + ok + end, {ok, {{one_for_one, 5, 10}, couch_epi:register_service(setup_epi, [])}}. diff --git a/src/setup/test/t-single-node-auto-setup.sh b/src/setup/test/t-single-node-auto-setup.sh new file mode 100755 index 000000000..0276990f5 --- /dev/null +++ b/src/setup/test/t-single-node-auto-setup.sh @@ -0,0 +1,24 @@ +#!/bin/sh -ex +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +HEADERS="-HContent-Type:application/json" + +# Show cluster state: +curl a:b@127.0.0.1:15986/_nodes/_all_docs +curl a:b@127.0.0.1:15984/_all_dbs +curl a:b@127.0.0.1:15984/_cluster_setup + +# Change the check +curl -g 'a:b@127.0.0.1:15984/_cluster_setup?ensure_dbs_exist=["_replicator","_users"]' + +echo "YAY ALL GOOD" -- cgit v1.2.1 From c2ff7da78e3be137f6790b5d64aaf157348b1e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B0=D0=BC=D1=98=D0=B0=D0=BD=20=D0=93=D0=B5=D0=BE?= =?UTF-8?q?=D1=80=D0=B3=D0=B8=D0=B5=D0=B2=D1=81=D0=BA=D0=B8?= Date: Sun, 5 Jan 2020 13:54:55 +0100 Subject: a systemd-journald compatible log output on stderr (#1912) * a systemd-journald compatible log output on stderr based on the stderr logger but changed: - doesn't output the timestamp, the journal already has a timestamp - output the log level as where num is defined as in `sd-daemon(3)` https://www.freedesktop.org/software/systemd/man/sd-daemon.html --- rel/overlay/etc/default.ini | 17 ++++-- src/couch_log/src/couch_log_writer_journald.erl | 69 +++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 src/couch_log/src/couch_log_writer_journald.erl diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index d8476f3e1..9a8c70bc9 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -449,16 +449,25 @@ level = info ; max_message_size = 16000 ; ; -; There are three different log writers that can be configured +; There are four different log writers that can be configured ; to write log messages. The default writes to stderr of the ; Erlang VM which is useful for debugging/development as well ; as a lot of container deployments. ; -; There's also a file writer that works with logrotate and an +; There's also a file writer that works with logrotate, a ; rsyslog writer for deployments that need to have logs sent -; over the network. +; over the network, and a journald writer that's more suitable +; when using systemd journald. ; writer = stderr +; Journald Writer notes: +; +; The journald writer doesn't have any options. It still writes +; the logs to stderr, but without the timestamp prepended, since +; the journal will add it automatically, and with the log level +; formated as per +; https://www.freedesktop.org/software/systemd/man/sd-daemon.html +; ; ; File Writer Options: ; @@ -525,4 +534,4 @@ min_priority = 2.0 ; value will be rejected. If this config setting is not defined, ; CouchDB will use the value of `max_limit` instead. If neither is ; defined, the default is 2000 as stated here. -; max_limit_partitions = 2000 \ No newline at end of file +; max_limit_partitions = 2000 diff --git a/src/couch_log/src/couch_log_writer_journald.erl b/src/couch_log/src/couch_log_writer_journald.erl new file mode 100644 index 000000000..02a9c6900 --- /dev/null +++ b/src/couch_log/src/couch_log_writer_journald.erl @@ -0,0 +1,69 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_log_writer_journald). +-behaviour(couch_log_writer). + + +-export([ + init/0, + terminate/2, + write/2 +]). + + +-include("couch_log.hrl"). + + +init() -> + {ok, nil}. + + +terminate(_, _St) -> + ok. + + +write(Entry, St) -> + #log_entry{ + level = Level, + pid = Pid, + msg = Msg, + msg_id = MsgId + } = Entry, + Fmt = "<~B>~s ~p ~s ", + Args = [ + level_for_journald(Level), + node(), + Pid, + MsgId + ], + MsgSize = couch_log_config:get(max_message_size), + Data = couch_log_trunc_io:format(Fmt, Args, MsgSize), + io:format(standard_error, [Data, Msg, "\n"], []), + {ok, St}. + + +% log level mapping from sd-daemon(3) +% https://www.freedesktop.org/software/systemd/man/sd-daemon.html +-spec level_for_journald(atom()) -> integer(). +level_for_journald(Level) when is_atom(Level) -> + case Level of + debug -> 7; + info -> 6; + notice -> 5; + warning -> 4; + error -> 3; + critical -> 2; + alert -> 1; + emergency -> 0; + _ -> 3 + end. -- cgit v1.2.1 From 0449057d65d185822087c5c856ee269ee64a37c2 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Sun, 5 Jan 2020 14:28:14 +0100 Subject: Feat/1752/setup UUID and auth secret (#2398) * reafactor: generalise sync_admin to sync_config * feat: sync cluster UUID and http auth secret on setup Closes #1752 --- src/setup/src/setup.erl | 17 ++++++++++++++++- src/setup/test/t-frontend-setup.sh | 12 ++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/setup/src/setup.erl b/src/setup/src/setup.erl index 9437fbc07..12a3f4351 100644 --- a/src/setup/src/setup.erl +++ b/src/setup/src/setup.erl @@ -200,6 +200,8 @@ setup_node(NewCredentials, NewBindAddress, NodeCount, Port) -> finish_cluster(Options) -> ok = wait_connected(), ok = sync_admins(), + ok = sync_uuid(), + ok = sync_auth_secret(), Dbs = proplists:get_value(ensure_dbs_exist, Options, cluster_system_dbs()), finish_cluster_int(Dbs, has_cluster_system_dbs(Dbs)). @@ -241,8 +243,21 @@ sync_admins() -> sync_admin(User, Pass) -> + sync_config("admins", User, Pass). + + +sync_uuid() -> + Uuid = config:get("couchdb", "uuid"), + sync_config("couchdb", "uuid", Uuid). + +sync_auth_secret() -> + Secret = config:get("couch_httpd_auth", "secret"), + sync_config("couch_httpd_auth", "secret", Secret). + + +sync_config(Section, Key, Value) -> {Results, Errors} = rpc:multicall(other_nodes(), config, set, - ["admins", User, Pass]), + [Section, Key, Value]), case validate_multicall(Results, Errors) of ok -> ok; diff --git a/src/setup/test/t-frontend-setup.sh b/src/setup/test/t-frontend-setup.sh index 52056a374..e025cfba2 100755 --- a/src/setup/test/t-frontend-setup.sh +++ b/src/setup/test/t-frontend-setup.sh @@ -11,6 +11,8 @@ # License for the specific language governing permissions and limitations under # the License. +echo "To test, comment out the fake_uuid line in dev/run" + HEADERS="-HContent-Type:application/json" # show cluster state: curl a:b@127.0.0.1:15986/_nodes/_all_docs @@ -48,16 +50,22 @@ curl a:b@127.0.0.1:15984/_cluster_setup -d '{"action":"finish_cluster"}' $HEADER # Show system dbs exist on node A curl a:b@127.0.0.1:15984/_users curl a:b@127.0.0.1:15984/_replicator -curl a:b@127.0.0.1:15984/_metadata curl a:b@127.0.0.1:15984/_global_changes # Show system dbs exist on node B curl a:b@127.0.0.1:25984/_users curl a:b@127.0.0.1:25984/_replicator -curl a:b@127.0.0.1:25984/_metadata curl a:b@127.0.0.1:25984/_global_changes # Number of nodes is set to 2 curl a:b@127.0.0.1:25984/_node/node2@127.0.0.1/_config/cluster/n +# uuid and auth secret are the same +curl a:b@127.0.0.1:15984/_node/node1@127.0.0.1/_config/couchdb/uuid +curl a:b@127.0.0.1:15984/_node/node2@127.0.0.1/_config/couchdb/uuid + +curl a:b@127.0.0.1:15984/_node/node1@127.0.0.1/_config/couch_httpd_auth/secret +curl a:b@127.0.0.1:15984/_node/node2@127.0.0.1/_config/couch_httpd_auth/secret + + echo "YAY ALL GOOD" -- cgit v1.2.1 From 36cf8936f88d8e9b519bd94aab433933691ff6ae Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Wed, 19 Sep 2018 11:53:16 -0500 Subject: Simplfiy interleaved message couch_server test Previous this test relied on delete/open requests getting things into a confused state. Now that we understand the issue we can replace the more complicated logic by directly polutnig couch_server's message queue with invalid messages. --- src/couch/test/eunit/couch_server_tests.erl | 50 ++++++++++------------------- 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/src/couch/test/eunit/couch_server_tests.erl b/src/couch/test/eunit/couch_server_tests.erl index 530b7efd0..7d50700d2 100644 --- a/src/couch/test/eunit/couch_server_tests.erl +++ b/src/couch/test/eunit/couch_server_tests.erl @@ -14,6 +14,8 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). + +-include("../src/couch_db_int.hrl"). -include("../src/couch_server_int.hrl"). start() -> @@ -192,10 +194,11 @@ make_interleaved_requests({_, TestDbName}) -> t_interleaved_create_delete_open(DbName) -> - {CrtRef, DelRef, OpenRef} = {make_ref(), make_ref(), make_ref()}, + {CrtRef, OpenRef} = {make_ref(), make_ref()}, CrtMsg = {'$gen_call', {self(), CrtRef}, {create, DbName, [?ADMIN_CTX]}}, - DelMsg = {'$gen_call', {self(), DelRef}, {delete, DbName, [?ADMIN_CTX]}}, - OpenMsg = {'$gen_call', {self(), OpenRef}, {open, DbName, [?ADMIN_CTX]}}, + FakePid = spawn(fun() -> ok end), + OpenResult = {open_result, DbName, {ok, #db{main_pid = FakePid}}}, + OpenResultMsg = {'$gen_call', {self(), OpenRef}, OpenResult}, % Get the current couch_server pid so we're sure % to not end up messaging two different pids @@ -215,48 +218,29 @@ t_interleaved_create_delete_open(DbName) -> % our next requests and let the opener finish processing. erlang:suspend_process(CouchServer), - % Since couch_server is suspend, this delete request won't - % be processed until after the opener has sent its - % successful open response via gen_server:call/3 - CouchServer ! DelMsg, - - % This open request will be in the queue after the - % delete request but before the gen_server:call/3 - % message which will establish the mixed up state - % in the couch_dbs ets table - CouchServer ! OpenMsg, + % We queue a confused open_result message in front of + % the correct response from the opener. + CouchServer ! OpenResultMsg, - % First release the opener pid so it can continue - % working while we tweak meck + % Release the opener pid so it can continue Opener ! go, - % Replace our expect call to meck so that the OpenMsg - % isn't blocked on the receive - meck:expect(couch_db, start_link, fun(Engine, DbName1, Filename, Options) -> - meck:passthrough([Engine, DbName1, Filename, Options]) - end), - % Wait for the '$gen_call' message from OpenerPid to arrive % in couch_server's mailbox ok = wait_for_open_async_result(CouchServer, Opener), % Now monitor and resume the couch_server and assert that - % couch_server does not crash while processing OpenMsg + % couch_server does not crash while processing OpenResultMsg CSRef = erlang:monitor(process, CouchServer), erlang:resume_process(CouchServer), check_monitor_not_triggered(CSRef), - % The create response is expected to return not_found - % due to the delete request canceling the async opener - % pid and sending not_found to all waiters unconditionally - ?assertEqual({CrtRef, not_found}, get_next_message()), - - % Our delete request was processed normally - ?assertEqual({DelRef, ok}, get_next_message()), + % Our open_result message was processed and ignored + ?assertEqual({OpenRef, ok}, get_next_message()), - % The db was deleted thus it should be not found - % when we try and open it. - ?assertMatch({OpenRef, {not_found, no_db_file}}, get_next_message()), + % Our create request was processed normally after we + % ignored the spurious open_result + ?assertMatch({CrtRef, {ok, _}}, get_next_message()), % And finally assert that couch_server is still % alive. @@ -281,7 +265,7 @@ wait_for_open_async_result(CouchServer, Opener) -> {_, Messages} = erlang:process_info(CouchServer, messages), Found = lists:foldl(fun(Msg, Acc) -> case Msg of - {'$gen_call', {Opener, _}, {open_result, _, _, {ok, _}}} -> + {'$gen_call', {Opener, _}, {open_result, _, {ok, _}}} -> true; _ -> Acc -- cgit v1.2.1 From 009e4d5ac4a45bbbe017ba9cf4349b08e52d7335 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 18 Sep 2018 15:38:39 -0500 Subject: Avoid file_server_2 for existance tests Its not uncommon to have file_server_2 behaving poorly so we'll avoid it when there are calls that are made often. --- src/couch/src/couch_bt_engine.erl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl index e3e4d0d32..b659719f5 100644 --- a/src/couch/src/couch_bt_engine.erl +++ b/src/couch/src/couch_bt_engine.erl @@ -114,16 +114,17 @@ ]). +-include_lib("kernel/include/file.hrl"). -include_lib("couch/include/couch_db.hrl"). -include("couch_bt_engine.hrl"). exists(FilePath) -> - case filelib:is_file(FilePath) of + case is_file(FilePath) of true -> true; false -> - filelib:is_file(FilePath ++ ".compact") + is_file(FilePath ++ ".compact") end. @@ -1235,3 +1236,11 @@ finish_compaction_int(#st{} = OldSt, #st{} = NewSt1) -> {ok, NewSt2#st{ filepath = FilePath }, undefined}. + + +is_file(Path) -> + case file:read_file_info(Path, [raw]) of + {ok, #file_info{type = regular}} -> true; + {ok, #file_info{type = directory}} -> true; + _ -> false + end. -- cgit v1.2.1 From 10670fbba69ed54488350109ed7c0e250bbf17c5 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 18 Sep 2018 15:39:34 -0500 Subject: Reduce logging calls in couch_server Even when disabled these logs can take a non-trivial amount of reductions which can add extra suspensions to couch_server's main loop. --- src/couch/src/couch_server.erl | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/couch/src/couch_server.erl b/src/couch/src/couch_server.erl index eaca3ee9b..40d771751 100644 --- a/src/couch/src/couch_server.erl +++ b/src/couch/src/couch_server.erl @@ -391,7 +391,13 @@ open_async(Server, From, DbName, {Module, Filepath}, Options) -> ok end, gen_server:call(Parent, {open_result, T0, DbName, Res}, infinity), - unlink(Parent) + unlink(Parent), + case Res of + {ok, _} -> + ok; + Error -> + couch_log:info("open_result error ~p for ~s", [Error, DbName]) + end end), ReqType = case lists:member(create, Options) of true -> create; @@ -477,7 +483,6 @@ handle_call({open_result, _T0, DbName, Error}, {Opener, _}, Server) -> {reply, ok, Server}; [#entry{pid = Opener, req_type = ReqType, waiters = Waiters} = Entry] -> [gen_server:reply(Waiter, Error) || Waiter <- Waiters], - couch_log:info("open_result error ~p for ~s", [Error, DbName]), true = ets:delete(couch_dbs, DbName), true = ets:delete(couch_dbs_pid_to_name, Opener), NewServer = case ReqType of @@ -510,7 +515,8 @@ handle_call({open, DbName, Options}, From, Server) -> end; [#entry{waiters = Waiters} = Entry] when is_list(Waiters) -> true = ets:insert(couch_dbs, Entry#entry{waiters = [From | Waiters]}), - if length(Waiters) =< 10 -> ok; true -> + NumWaiters = length(Waiters), + if NumWaiters =< 10 orelse NumWaiters rem 10 /= 0 -> ok; true -> Fmt = "~b clients waiting to open db ~s", couch_log:info(Fmt, [length(Waiters), DbName]) end, @@ -650,7 +656,12 @@ handle_info({'EXIT', Pid, Reason}, Server) -> "must be built with Erlang OTP R13B04 or higher.", [DbName]), couch_log:error(Msg, []) end, - couch_log:info("db ~s died with reason ~p", [DbName, Reason]), + % We kill databases on purpose so there's no reason + % to log that fact. So we restrict logging to "interesting" + % reasons. + if Reason == normal orelse Reason == killed -> ok; true -> + couch_log:info("db ~s died with reason ~p", [DbName, Reason]) + end, if not is_list(Waiters) -> ok; true -> [gen_server:reply(Waiter, Reason) || Waiter <- Waiters] end, -- cgit v1.2.1 From 6cbe17379f9c57f0f0da84975959d17a71590854 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Wed, 19 Sep 2018 14:49:32 -0500 Subject: Track db open times in the async_open pid We can track the open latencies from the async_open process so that couch_server does not have to perform the update in its main loop. --- src/couch/src/couch_server.erl | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/couch/src/couch_server.erl b/src/couch/src/couch_server.erl index 40d771751..00f6af60a 100644 --- a/src/couch/src/couch_server.erl +++ b/src/couch/src/couch_server.erl @@ -384,18 +384,25 @@ open_async(Server, From, DbName, {Module, Filepath}, Options) -> T0 = os:timestamp(), Opener = spawn_link(fun() -> Res = couch_db:start_link(Module, DbName, Filepath, Options), - case {Res, lists:member(create, Options)} of - {{ok, _Db}, true} -> + IsSuccess = case Res of + {ok, _} -> true; + _ -> false + end, + case IsSuccess andalso lists:member(create, Options) of + true -> couch_event:notify(DbName, created); - _ -> + false -> ok end, - gen_server:call(Parent, {open_result, T0, DbName, Res}, infinity), + gen_server:call(Parent, {open_result, DbName, Res}, infinity), unlink(Parent), - case Res of - {ok, _} -> - ok; - Error -> + case IsSuccess of + true -> + % Track latency times for successful opens + Diff = timer:now_diff(os:timestamp(), T0) / 1000, + couch_stats:update_histogram([couchdb, db_open_time], Diff); + false -> + % Log unsuccessful open results couch_log:info("open_result error ~p for ~s", [Error, DbName]) end end), @@ -431,10 +438,8 @@ handle_call(reload_engines, _From, Server) -> {reply, ok, Server#server{engines = get_configured_engines()}}; handle_call(get_server, _From, Server) -> {reply, {ok, Server}, Server}; -handle_call({open_result, T0, DbName, {ok, Db}}, {Opener, _}, Server) -> +handle_call({open_result, DbName, {ok, Db}}, {Opener, _}, Server) -> true = ets:delete(couch_dbs_pid_to_name, Opener), - OpenTime = timer:now_diff(os:timestamp(), T0) / 1000, - couch_stats:update_histogram([couchdb, db_open_time], OpenTime), DbPid = couch_db:get_pid(Db), case ets:lookup(couch_dbs, DbName) of [] -> @@ -474,9 +479,9 @@ handle_call({open_result, T0, DbName, {ok, Db}}, {Opener, _}, Server) -> exit(couch_db:get_pid(Db), kill), {reply, ok, Server} end; -handle_call({open_result, T0, DbName, {error, eexist}}, From, Server) -> - handle_call({open_result, T0, DbName, file_exists}, From, Server); -handle_call({open_result, _T0, DbName, Error}, {Opener, _}, Server) -> +handle_call({open_result, DbName, {error, eexist}}, From, Server) -> + handle_call({open_result, DbName, file_exists}, From, Server); +handle_call({open_result, DbName, Error}, {Opener, _}, Server) -> case ets:lookup(couch_dbs, DbName) of [] -> % db was deleted during async open -- cgit v1.2.1 From b02599ed9f66004eac66c5129c4abd454f465217 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Wed, 19 Sep 2018 15:01:58 -0500 Subject: Move more work out of couch_server's main loop This moves the database name check and engine lookup into the open_async process. There's no reason that this work had to happen in the main loop so we can easily move the regex and engine lookups out of the loop to minimize the work per handle_call even further. --- src/couch/src/couch_server.erl | 151 +++++++++++++++++++++++++---------------- 1 file changed, 92 insertions(+), 59 deletions(-) diff --git a/src/couch/src/couch_server.erl b/src/couch/src/couch_server.erl index 00f6af60a..a72627b79 100644 --- a/src/couch/src/couch_server.erl +++ b/src/couch/src/couch_server.erl @@ -79,11 +79,11 @@ sup_start_link() -> gen_server:start_link({local, couch_server}, couch_server, [], []). open(DbName, Options) -> - case ets:lookup(couch_dbs_locks, DbName) of - [] -> - open_int(DbName, Options); - [{DbName, Reason}] -> - {error, {locked, Reason}} + try + validate_open_or_create(DbName, Options), + open_int(DbName, Options) + catch throw:{?MODULE, Error} -> + Error end. open_int(DbName, Options0) -> @@ -124,11 +124,11 @@ close_lru() -> gen_server:call(couch_server, close_lru). create(DbName, Options) -> - case ets:lookup(couch_dbs_locks, DbName) of - [] -> - create_int(DbName, Options); - [{DbName, Reason}] -> - {error, {locked, Reason}} + try + validate_open_or_create(DbName, Options), + create_int(DbName, Options) + catch throw:{?MODULE, Error} -> + Error end. create_int(DbName, Options0) -> @@ -199,7 +199,7 @@ path_ends_with(Path, Suffix) when is_binary(Suffix) -> path_ends_with(Path, Suffix) when is_list(Suffix) -> path_ends_with(Path, ?l2b(Suffix)). -check_dbname(#server{}, DbName) -> +check_dbname(DbName) -> couch_db:validate_dbname(DbName). is_admin(User, ClearPwd) -> @@ -379,11 +379,11 @@ maybe_close_lru_db(#server{lru=Lru}=Server) -> {error, all_dbs_active} end. -open_async(Server, From, DbName, {Module, Filepath}, Options) -> +open_async(Server, From, DbName, Options) -> Parent = self(), T0 = os:timestamp(), Opener = spawn_link(fun() -> - Res = couch_db:start_link(Module, DbName, Filepath, Options), + Res = open_async_int(Server, DbName, Options), IsSuccess = case Res of {ok, _} -> true; _ -> false @@ -403,7 +403,7 @@ open_async(Server, From, DbName, {Module, Filepath}, Options) -> couch_stats:update_histogram([couchdb, db_open_time], Diff); false -> % Log unsuccessful open results - couch_log:info("open_result error ~p for ~s", [Error, DbName]) + couch_log:info("open_result error ~p for ~s", [Res, DbName]) end end), ReqType = case lists:member(create, Options) of @@ -421,6 +421,20 @@ open_async(Server, From, DbName, {Module, Filepath}, Options) -> true = ets:insert(couch_dbs_pid_to_name, {Opener, DbName}), db_opened(Server, Options). +open_async_int(Server, DbName, Options) -> + DbNameList = binary_to_list(DbName), + case check_dbname(DbNameList) of + ok -> + case get_engine(Server, DbNameList, Options) of + {ok, {Module, FilePath}} -> + couch_db:start_link(Module, DbName, FilePath, Options); + Error2 -> + Error2 + end; + Error1 -> + Error1 + end. + handle_call(close_lru, _From, #server{lru=Lru} = Server) -> case couch_lru:close(Lru) of {true, NewLru} -> @@ -451,7 +465,7 @@ handle_call({open_result, DbName, {ok, Db}}, {Opener, _}, Server) -> [gen_server:reply(Waiter, {ok, Db}) || Waiter <- Waiters], % Cancel the creation request if it exists. case ReqType of - {create, DbName, _Engine, _Options, CrFrom} -> + {create, DbName, _Options, CrFrom} -> gen_server:reply(CrFrom, file_exists); _ -> ok @@ -491,8 +505,8 @@ handle_call({open_result, DbName, Error}, {Opener, _}, Server) -> true = ets:delete(couch_dbs, DbName), true = ets:delete(couch_dbs_pid_to_name, Opener), NewServer = case ReqType of - {create, DbName, Engine, Options, CrFrom} -> - open_async(Server, CrFrom, DbName, Engine, Options); + {create, DbName, Options, CrFrom} -> + open_async(Server, CrFrom, DbName, Options); _ -> Server end, @@ -505,18 +519,11 @@ handle_call({open_result, DbName, Error}, {Opener, _}, Server) -> handle_call({open, DbName, Options}, From, Server) -> case ets:lookup(couch_dbs, DbName) of [] -> - DbNameList = binary_to_list(DbName), - case check_dbname(Server, DbNameList) of - ok -> - case make_room(Server, Options) of - {ok, Server2} -> - {ok, Engine} = get_engine(Server2, DbNameList), - {noreply, open_async(Server2, From, DbName, Engine, Options)}; - CloseError -> - {reply, CloseError, Server} - end; - Error -> - {reply, Error, Server} + case make_room(Server, Options) of + {ok, Server2} -> + {noreply, open_async(Server2, From, DbName, Options)}; + CloseError -> + {reply, CloseError, Server} end; [#entry{waiters = Waiters} = Entry] when is_list(Waiters) -> true = ets:insert(couch_dbs, Entry#entry{waiters = [From | Waiters]}), @@ -530,40 +537,29 @@ handle_call({open, DbName, Options}, From, Server) -> {reply, {ok, Db}, Server} end; handle_call({create, DbName, Options}, From, Server) -> - DbNameList = binary_to_list(DbName), - case get_engine(Server, DbNameList, Options) of - {ok, Engine} -> - case check_dbname(Server, DbNameList) of - ok -> - case ets:lookup(couch_dbs, DbName) of - [] -> - case make_room(Server, Options) of - {ok, Server2} -> - {noreply, open_async(Server2, From, DbName, Engine, - [create | Options])}; - CloseError -> - {reply, CloseError, Server} - end; - [#entry{req_type = open} = Entry] -> - % We're trying to create a database while someone is in - % the middle of trying to open it. We allow one creator - % to wait while we figure out if it'll succeed. - CrOptions = [create | Options], - Req = {create, DbName, Engine, CrOptions, From}, - true = ets:insert(couch_dbs, Entry#entry{req_type = Req}), - {noreply, Server}; - [_AlreadyRunningDb] -> - {reply, file_exists, Server} - end; - Error -> - {reply, Error, Server} + case ets:lookup(couch_dbs, DbName) of + [] -> + case make_room(Server, Options) of + {ok, Server2} -> + CrOptions = [create | Options], + {noreply, open_async(Server2, From, DbName, CrOptions)}; + CloseError -> + {reply, CloseError, Server} end; - Error -> - {reply, Error, Server} + [#entry{req_type = open} = Entry] -> + % We're trying to create a database while someone is in + % the middle of trying to open it. We allow one creator + % to wait while we figure out if it'll succeed. + CrOptions = [create | Options], + Req = {create, DbName, CrOptions, From}, + true = ets:insert(couch_dbs, Entry#entry{req_type = Req}), + {noreply, Server}; + [_AlreadyRunningDb] -> + {reply, file_exists, Server} end; handle_call({delete, DbName, Options}, _From, Server) -> DbNameList = binary_to_list(DbName), - case check_dbname(Server, DbNameList) of + case check_dbname(DbNameList) of ok -> Server2 = case ets:lookup(couch_dbs, DbName) of @@ -694,6 +690,27 @@ db_closed(Server, Options) -> true -> Server end. +validate_open_or_create(DbName, Options) -> + case check_dbname(DbName) of + ok -> + ok; + DbNameError -> + throw({?MODULE, DbNameError}) + end, + + case check_engine(Options) of + ok -> + ok; + EngineError -> + throw({?MODULE, EngineError}) + end, + + case ets:lookup(couch_dbs_locks, DbName) of + [] -> + ok; + [{DbName, Reason}] -> + throw({?MODULE, {error, {locked, Reason}}}) + end. get_configured_engines() -> ConfigEntries = config:get("couchdb_engines"), @@ -803,6 +820,22 @@ get_engine_extensions() -> end. +check_engine(Options) -> + case couch_util:get_value(engine, Options) of + Ext when is_binary(Ext) -> + ExtStr = binary_to_list(Ext), + Extensions = get_engine_extensions(), + case lists:member(ExtStr, Extensions) of + true -> + ok; + false -> + {error, {invalid_engine_extension, Ext}} + end; + _ -> + ok + end. + + get_engine_path(DbName, Engine) when is_binary(DbName), is_atom(Engine) -> RootDir = config:get("couchdb", "database_dir", "."), case lists:keyfind(Engine, 2, get_configured_engines()) of -- cgit v1.2.1 From a59540132c14b3801234c8212718b3cf15f9bb44 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Wed, 19 Sep 2018 14:39:09 -0500 Subject: Set a high priority on couch_server In a VM with lots of processes couch_server can end up slowing down purely to not being scheduled which in turn will cause its message queue to backup. We've attempted this before but due to message passing have never seen it have a significant effect. However, now that we're actively avoiding synchronous message passing inside the man loop this has a significant effect by being able to process messages as soon as it has one in its mailbox. --- rel/overlay/etc/default.ini | 5 +++++ src/couch/src/couch_server.erl | 1 + src/couch/src/couch_util.erl | 11 +++++++++++ 3 files changed, 17 insertions(+) diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index 9a8c70bc9..a1df0805a 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -93,6 +93,11 @@ users_db_security_editable = false ; having to ask every configured engine. couch = couch_bt_engine +[process_priority] +; Selectively disable altering process priorities +; for modules that request it. +; couch_server = true + [cluster] q=2 n=3 diff --git a/src/couch/src/couch_server.erl b/src/couch/src/couch_server.erl index a72627b79..909e23898 100644 --- a/src/couch/src/couch_server.erl +++ b/src/couch/src/couch_server.erl @@ -235,6 +235,7 @@ close_db_if_idle(DbName) -> init([]) -> couch_util:set_mqd_off_heap(?MODULE), + couch_util:set_process_priority(?MODULE, high), % Mark pluggable storage engines as a supported feature config:enable_feature('pluggable-storage-engines'), diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl index e2885a15e..b5c93ce51 100644 --- a/src/couch/src/couch_util.erl +++ b/src/couch/src/couch_util.erl @@ -39,6 +39,7 @@ -export([check_config_blacklist/1]). -export([check_md5/2]). -export([set_mqd_off_heap/1]). +-export([set_process_priority/2]). -include_lib("couch/include/couch_db.hrl"). @@ -690,6 +691,16 @@ set_mqd_off_heap(Module) -> end. +set_process_priority(Module, Level) -> + case config:get_boolean("process_priority", atom_to_list(Module), true) of + true -> + process_flag(priority, Level), + ok; + false -> + ok + end. + + ensure_loaded(Module) when is_atom(Module) -> case code:ensure_loaded(Module) of {module, Module} -> -- cgit v1.2.1