Merge pull request #3286 from cloudant/specify-elixir-tests

Add ability to control which Elixir integration tests to run

7 files changed, 1165 insertions, 17 deletions

diff --git a/Makefile b/Makefile
index 8844b869b..e4c153e38 100644
--- a/Makefile
+++ b/Makefile
@@ -161,7 +161,7 @@ endif
 check:  all
 	@$(MAKE) emilio
 	make eunit apps=couch_eval,couch_expiring_cache,ctrace,couch_jobs,couch_views,fabric,mango,chttpd,couch_replicator
-	make elixir tests=test/elixir/test/basics_test.exs,test/elixir/test/replication_test.exs,test/elixir/test/map_test.exs,test/elixir/test/all_docs_test.exs,test/elixir/test/bulk_docs_test.exs
+	make elixir-suite
 	make exunit apps=chttpd
 	make mango-test
 
@@ -271,6 +271,17 @@ elixir-cluster-with-quorum: elixir-init elixir-check-formatted elixir-credo devc
 		--degrade-cluster 1 \
 		--no-eval 'mix test --trace --only with_quorum_test $(EXUNIT_OPTS)'
 
+.PHONY: elixir-suite
+elixir-suite: export MIX_ENV=integration
+elixir-suite: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
+elixir-suite: elixir-init elixir-check-formatted elixir-credo devclean
+	@dev/run -n 1 -q -a adm:pass \
+		--enable-erlang-views \
+		--no-join \
+		--locald-config test/elixir/test/config/test-config.ini \
+		--erlang-config rel/files/eunit.config \
+		--no-eval 'mix test --trace --include test/elixir/test/config/suite.elixir --exclude test/elixir/test/config/skip.elixir'
+
 .PHONY: elixir-check-formatted
 elixir-check-formatted: elixir-init
 	@mix format --check-formatted
diff --git a/Makefile.win b/Makefile.win
index fa676ad13..aeb7fe7b6 100644
--- a/Makefile.win
+++ b/Makefile.win
@@ -224,6 +224,17 @@ elixir-cluster-with-quorum: elixir-init elixir-check-formatted elixir-credo devc
 	    --degrade-cluster 1 \
 		--no-eval 'mix test --trace --only with_quorum_test $(EXUNIT_OPTS)'
 
+.PHONY: elixir-suite
+elixir-suite: export MIX_ENV=integration
+elixir-suite: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
+elixir-suite: elixir-init elixir-check-formatted elixir-credo devclean
+	@dev\run -n 1 -q -a adm:pass \
+		--enable-erlang-views \
+		--no-join \
+		--locald-config test/elixir/test/config/test-config.ini \
+		--erlang-config rel/files/eunit.config \
+		--no-eval 'mix test --trace --include test\elixir\test\config\suite.elixir --exclude test\elixir\test\config\skip.elixir'
+
 .PHONY: elixir-check-formatted
 elixir-check-formatted: elixir-init
 	@mix format --check-formatted
diff --git a/mix.exs b/mix.exs
index ae42af5d6..c454df331 100644
--- a/mix.exs
+++ b/mix.exs
@@ -20,6 +20,29 @@ defmodule CoverTool do
   end
 end
 
+defmodule Mix.Tasks.Suite do
+  @moduledoc """
+  Helper task to create `suites.elixir` file. It suppose to be used as follows
+  ```
+  MIX_ENV=integration mix suite > test/elixir/test/config/suite.elixir
+  ```
+  """
+  use Mix.Task
+  @shortdoc "Outputs all availabe integration tests"
+  def run(_) do
+    Path.wildcard(Path.join(Mix.Project.build_path(), "/**/ebin"))
+    |> Enum.filter(&File.dir?/1)
+    |> Enum.map(&Code.append_path/1)
+
+    tests =
+      Couch.Test.Suite.list()
+      |> Enum.sort()
+      |> Couch.Test.Suite.group_by()
+
+    IO.puts(Couch.Test.Suite.pretty_print(tests))
+  end
+end
+
 defmodule CouchDBTest.Mixfile do
   use Mix.Project
 
diff --git a/test/elixir/lib/suite.ex b/test/elixir/lib/suite.ex
new file mode 100644
index 000000000..60b7766d0
--- /dev/null
+++ b/test/elixir/lib/suite.ex
@@ -0,0 +1,213 @@
+defmodule Couch.Test.Suite do
+  @moduledoc """
+    Common code to configure ExUnit runner.
+    It replaces the usual invocation of `ExUnit.start()` in
+    `test_helper.exs` related to integration tests with:
+    ```
+    Couch.Test.Suite.start()
+    ```
+  """
+  @doc """
+  This helper function can be used to create `suite.elixir`
+  as
+  ```
+  tests =
+    Couch.Test.Suite.list()
+    |> Enum.sort()
+    |> Couch.Test.Suite.group_by()
+
+  IO.puts(Couch.Test.Suite.pretty_print(tests))
+
+  ```
+  """
+  def list() do
+    test_paths = Keyword.get(Mix.Project.config(), :test_paths, [])
+    Enum.reduce(test_paths, [], fn directory, acc ->
+      list(directory) ++ acc
+    end)
+  end
+
+  @doc """
+  This helper function can be used to create `suite.elixir`
+  as
+  ```
+  tests =
+    Couch.Test.Suite.list(["test/elixir/test"])
+    |> Enum.sort()
+    |> Couch.Test.Suite.group_by()
+
+  IO.puts(Couch.Test.Suite.pretty_print(tests))
+  ```
+  """
+  def list(directory) do
+    ensure_exunit_started()
+    Enum.reduce(test_files(directory), [], fn file_path, acc ->
+      tests_in_file(file_path) ++ acc
+    end)
+  end
+
+  @doc """
+  This helper function is used in a snippet to create `suite.elixir`
+  see list/1
+  """
+  def group_by(tests) do
+    tests |> Enum.group_by(&module_name/1, &test_name/1)
+  end
+
+  @doc """
+  This helper function is used in a snippet to create `suite.elixir`
+  see list/1
+  """
+  def pretty_print(tests) do
+    tests = Enum.join(Enum.sort(Enum.map(tests, fn {module_name, test_names} ->
+      test_names = test_names
+        |> Enum.map(fn x -> ~s("#{x}") end) |> Enum.join(",\n    ")
+      ~s(  "#{module_name}": [\n    #{test_names}\n  ])
+    end)), ",\n")
+    "%{\n#{tests}\n}"
+  end
+
+  def start(exclude \\ []) do
+    # If build number detected assume we running on Jenkins
+    # and skip certain tests that fail on jenkins.
+    default_exclude =
+      case System.get_env("BUILD_NUMBER") !== nil do
+        true -> [:pending, :skip_on_jenkins]
+        false -> [:pending]
+      end
+
+    current_exclude = Keyword.get(ExUnit.configuration(), :exclude, [])
+    {ignores, current_exclude} = from_file(current_exclude)
+
+    current_include = Keyword.get(ExUnit.configuration(), :include, [])
+    {suite, current_include} = from_file(current_include)
+
+    only_test_ids =
+      case suite -- ignores do
+        [] ->
+          nil
+
+        test_ids ->
+          to_tests(test_ids)
+      end
+
+    ExUnit.configure(
+      exclude: Enum.uniq(default_exclude ++ current_exclude ++ exclude),
+      include: current_include,
+      formatters: [JUnitFormatter, ExUnit.CLIFormatter],
+      only_test_ids: only_test_ids
+    )
+
+    ExUnit.start()
+  end
+
+  # Helpers for start/0
+
+  defp split_files(opts) do
+    {files, opts} =
+      Enum.split_with(opts, fn x ->
+        String.ends_with?(Atom.to_string(x), ".elixir")
+      end)
+
+    {Enum.map(files, &Atom.to_string/1), opts}
+  end
+
+  defp read_from_file(file_name) do
+    {map, _} = Code.eval_file(file_name)
+
+    map
+    |> Enum.reduce([], fn {module, tests}, acc ->
+      Enum.map(tests, &{module, &1}) ++ acc
+    end)
+  end
+
+  defp from_file(opts) do
+    case split_files(opts) do
+      {[], opts} ->
+        {[], opts}
+
+      {[file_name], opts} ->
+        {read_from_file(file_name), opts}
+
+      {_, _} ->
+        throw("Only one file is supported in --exclude or --include")
+    end
+  end
+
+  defp to_tests(ids) do
+    MapSet.new(
+      Enum.map(ids, fn {module_name, test_name} ->
+        {String.to_atom("Elixir.#{module_name}"), String.to_atom("test #{test_name}")}
+      end)
+    )
+  end
+
+  # Helpers for list/0
+
+  defp ensure_exunit_started() do
+    if not Process.get(EXUNIT_STARTED, false) do
+      started? =
+        Application.started_applications()
+        |> Enum.map(&Kernel.elem(&1, 0))
+        |> Enum.member?(:ex_unit)
+
+      if not started? do
+        ExUnit.start(autorun: false)
+        Process.get(EXUNIT_STARTED, true)
+      end
+    end
+  end
+
+  defp test_files(directory) do
+    files = Path.wildcard(Path.join(directory, "*_test.exs"))
+    Enum.filter(files, &File.regular?/1)
+  end
+
+  def tests_in_file(file_path) do
+    ensure_exunit_started()
+    Code.compiler_options(ignore_module_conflict: true)
+
+    tests =
+      Enum.reduce(require_file(file_path), [], fn {module_name, _}, acc ->
+        if :erlang.function_exported(module_name, :__ex_unit__, 0) do
+          module_name.__ex_unit__().tests ++ acc
+        else
+          acc
+        end
+      end)
+
+    Code.unrequire_files([file_path])
+    tests
+  end
+
+  def require_file(file_path) do
+    drop_stderr(fn ->
+      Code.require_file(file_path)
+    end)
+  end
+
+  defp drop_stderr(fun) do
+    {:ok, pid} = StringIO.open("")
+    original_pid = Process.whereis(:standard_error)
+
+    try do
+      Process.unregister(:standard_error)
+      Process.register(pid, :standard_error)
+      fun.()
+    after
+      Process.unregister(:standard_error)
+      Process.register(original_pid, :standard_error)
+      StringIO.close(pid)
+    end
+  end
+
+  defp test_name(test) do
+    String.replace_leading(Atom.to_string(test.name), "test ", "")
+  end
+
+  defp module_name(test) do
+    test.module
+    |> Atom.to_string()
+    |> String.replace_leading("Elixir.", "")
+  end
+end
diff --git a/test/elixir/test/config/skip.elixir b/test/elixir/test/config/skip.elixir
new file mode 100644
index 000000000..c63b5b3d7
--- /dev/null
+++ b/test/elixir/test/config/skip.elixir
@@ -0,0 +1,313 @@
+%{
+  "AllDocsTest": [
+  ],
+  "AttachmentMultipartTest": [
+    "manages attachments multipart requests successfully",
+    "manages compressed attachments successfully"
+  ],
+  "AttachmentNamesTest": [
+    "saves attachment names successfully"
+  ],
+  "AttachmentPathsTest": [
+    "manages attachment paths successfully - design docs"
+  ],
+  "AttachmentRangesTest": [
+  ],
+  "AttachmentViewTest": [
+    "manages attachments in views successfully"
+  ],
+  "AttachmentsTest": [
+    "COUCHDB-809 - stubs should only require the 'stub' field",
+    "empty attachments",
+    "etags for attachments",
+    "implicit doc creation allows creating docs with a reserved id. COUCHDB-565",
+    "large attachments COUCHDB-366",
+    "md5 header for attachments",
+    "saves binary",
+    "COUCHDB-497 - empty attachments",
+    "update attachment"
+  ],
+  "AuthCacheTest": [
+  ],
+  "BasicsTest": [
+  ],
+  "BatchSaveTest": [
+  ],
+  "BulkDocsTest": [
+  ],
+  "ChangesAsyncTest": [
+    "COUCHDB-1852",
+    "continuous changes",
+    "continuous filtered changes",
+    "continuous filtered changes with doc ids",
+    "eventsource changes",
+    "eventsource heartbeat",
+    "live changes",
+    "longpoll changes",
+    "longpoll filtered changes"
+  ],
+  "ChangesTest": [
+    "changes filtering on design docs"
+  ],
+  "CoffeeTest": [
+    "CoffeeScript basic functionality"
+  ],
+  "CompactTest": [
+  ],
+  "ConfigTest": [
+    "Atoms, binaries, and strings suffice as whitelist sections and keys.",
+    "Blacklist is functional",
+    "CouchDB respects configured protocols",
+    "Keys not in the whitelist may not be modified",
+    "Non-2-tuples in the whitelist are ignored",
+    "Non-list whitelist values allow further modification of the whitelist",
+    "Non-term whitelist values allow further modification of the whitelist",
+    "PORT `BUGGED` ?raw tests from config.js",
+    "Reload config",
+    "Server-side password hashing, and raw updates disabling that",
+    "Settings can be altered with undefined whitelist allowing any change",
+    "Standard config options are present"
+  ],
+  "CookieAuthTest": [
+    "cookie auth"
+  ],
+  "CopyDocTest": [
+    "Copy doc tests"
+  ],
+  "DesignDocsQueryTest": [
+  ],
+  "DesignDocsTest": [
+    "circular commonjs dependencies",
+    "commonjs require",
+    "module id values are as expected"
+  ],
+  "DesignOptionsTest": [
+    "design doc options - include_desing=false"
+  ],
+  "DesignPathTest": [
+    "design doc path",
+    "design doc path with slash in db name"
+  ],
+  "ErlangViewsTest": [
+    "Erlang map function",
+    "Erlang reduce function",
+    "Erlang reduce function larger dataset"
+  ],
+  "EtagsHeadTest": [
+  ],
+  "FormSubmitTest": [
+  ],
+  "HelperTest": [
+  ],
+  "HttpTest": [
+    "COUCHDB-708: newlines document names",
+    "location header"
+  ],
+  "InvalidDocIDsTest": [
+  ],
+  "JsonpTest": [
+  ],
+  "JwtAuthTest": [
+  ],
+  "ListViewsTest": [
+    "COUCHDB-1113",
+    "HTTP header response set after getRow() called in _list function",
+    "abort iteration with reduce",
+    "empty list",
+    "extra qs params",
+    "get with query params",
+    "handling _all_docs by _list functions. the result should be equal",
+    "multi-key fetch with GET",
+    "multi-key fetch with POST",
+    "multiple languages in design docs",
+    "no multi-key fetch allowed when group=false",
+    "reduce with 0 rows",
+    "secObj is available",
+    "standard GET",
+    "standard OPTIONS",
+    "stop iteration",
+    "the richness of the arguments",
+    "too many Get Rows",
+    "we can run lists and views from separate docs",
+    "we do multi-key requests on lists and views in separate docs",
+    "when there is a reduce present, and used",
+    "when there is a reduce present, but not used",
+    "with 0 rows",
+    "with accept headers for HTML",
+    "with include_docs and a reference to the doc"
+  ],
+  "LocalDocsTest": [
+  ],
+  "LotsOfDocsTest": [
+  ],
+  "MethodOverrideTest": [
+  ],
+  "MultipleRowsTest": [
+  ],
+  "ProxyAuthTest": [
+    "proxy auth with secret",
+    "proxy auth without secret"
+  ],
+  "PurgeTest": [
+    "COUCHDB-1065",
+    "purge documents"
+  ],
+  "ReaderACLTest": [
+    "can't set non string reader names or roles",
+    "members can query views",
+    "restricted db can be read by authorized users",
+    "unrestricted db can be read",
+    "works with readers (backwards compat with 1.0)"
+  ],
+  "RecreateDocTest": [
+  ],
+  "ReduceBuiltinTest": [
+  ],
+  "ReduceFalseTest": [
+  ],
+  "ReduceTest": [
+  ],
+  "ReplicationBadIdTest": [
+    "replication doc with bad rep id"
+  ],
+  "ReplicationTest": [
+  ],
+  "ReplicatorDBByDocIdTest": [
+    "replicatior db by doc id"
+  ],
+  "RevStemmingTest": [
+    "revs limit is kept after compaction"
+  ],
+  "RevisionTest": [
+  ],
+  "RewriteJSTest": [
+    "Test basic js rewrites on test_rewrite_suite_db",
+    "Test basic js rewrites on test_rewrite_suite_db%2Fwith_slashes",
+    "early response on test_rewrite_suite_db",
+    "early response on test_rewrite_suite_db%2Fwith_slashes",
+    "loop on test_rewrite_suite_db",
+    "loop on test_rewrite_suite_db%2Fwith_slashes",
+    "path relative to server on test_rewrite_suite_db",
+    "path relative to server on test_rewrite_suite_db%2Fwith_slashes",
+    "requests with body preserve the query string rewrite on test_rewrite_suite_db",
+    "requests with body preserve the query string rewrite on test_rewrite_suite_db%2Fwith_slashes"
+  ],
+  "RewriteTest": [
+    "Test basic rewrites on test_rewrite_suite_db",
+    "Test basic rewrites on test_rewrite_suite_db%2Fwith_slashes",
+    "loop detection on test_rewrite_suite_db",
+    "loop detection on test_rewrite_suite_db%2Fwith_slashes",
+    "path relative to server on test_rewrite_suite_db",
+    "path relative to server on test_rewrite_suite_db%2Fwith_slashes",
+    "serial execution is not spuriously counted as loop on test_rewrite_suite_db",
+    "serial execution is not spuriously counted as loop on test_rewrite_suite_db%2Fwith_slashes"
+  ],
+  "SecurityValidationTest": [
+  ],
+  "ShowDocumentsTest": [
+    "JS can't set etag",
+    "accept header switching - different mime has different etag",
+    "deleted docs",
+    "id with slash",
+    "list() compatible API",
+    "list() compatible API with provides function",
+    "missing design doc",
+    "registering types works",
+    "security object",
+    "should keep next result order: chunks + return value + provided chunks + provided return value",
+    "show error",
+    "show fail with non-existing docid",
+    "show query parameters",
+    "show with doc",
+    "show with doc - etags",
+    "show with existing doc",
+    "show with missing doc",
+    "show with non-existing docid",
+    "show without docid",
+    "the provides mime matcher",
+    "the provides mime matcher without a match"
+  ],
+  "UTF8Test": [
+  ],
+  "UUIDsTest": [
+    "sequential uuids are sequential",
+    "utc_id uuids are correct",
+    "utc_random uuids are roughly random"
+  ],
+  "UpdateDocumentsTest": [
+    "COUCHDB-1229 - allow slashes in doc ids for update handlers",
+    "COUCHDB-648 - the code in the JSON response should be honored",
+    "Insert doc with empty id",
+    "base64 response",
+    "bump counter",
+    "doc can be created",
+    "form update via application/x-www-form-urlencoded",
+    "in place update",
+    "update document"
+  ],
+  "UsersDbSecurityTest": [
+    "user db security"
+  ],
+  "UsersDbTest": [
+    "users db"
+  ],
+  "ViewCollationRawTest": [
+  ],
+  "ViewCollationTest": [
+  ],
+  "ViewCompactionTest": [
+    "view compaction"
+  ],
+  "ViewConflictsTest": [
+  ],
+  "ViewErrorsTest": [
+    "infinite loop",
+    "reduce overflow error",
+    "temporary view should give error message"
+  ],
+  "ViewIncludeDocsTest": [
+    "emitted _rev controls things"
+  ],
+  "ViewMapTest": [
+  ],
+  "ViewMultiKeyAllDocsTest": [
+  ],
+  "ViewMultiKeyDesignTest": [
+  ],
+  "ViewOffsetTest": [
+    "basic view offsets",
+    "repeated view offsets"
+  ],
+  "ViewPaginationTest": [
+    "aliases start_key and start_key_doc_id should work",
+    "basic view pagination",
+    "descending view pagination",
+    "descending=false parameter should just be ignored",
+    "endkey document id",
+    "endkey document id, but with end_key_doc_id alias"
+  ],
+  "ViewSandboxingTest": [
+  ],
+  "ViewTest": [
+  ],
+  "ViewUpdateSeqTest": [
+    "_all_docs update seq",
+    "db info update seq",
+    "view update seq"
+  ],
+  "WithQuorumTest": [
+    "Attachments overriden quorum should return 202-Acepted",
+    "Attachments should return 201-Created",
+    "Bulk docs overriden quorum should return 202-Acepted",
+    "Creating-Updating/Deleting doc with overriden quorum should return 202-Acepted/200-OK",
+    "Creating/Deleting DB should return 201-Created/202-Acepted"
+  ],
+  "WithoutQuorumTest": [
+    "Attachments overriden quorum should return 201-Created",
+    "Attachments should return 202-Acepted",
+    "Bulk docs should return 202-Acepted",
+    "Copy doc should return 202-Acepted",
+    "Creating/Deleting DB should return 202-Acepted",
+    "Creating/Updating/Deleting doc should return 202-Acepted"
+  ]
+}
diff --git a/test/elixir/test/config/suite.elixir b/test/elixir/test/config/suite.elixir
new file mode 100644
index 000000000..d4155f331
--- /dev/null
+++ b/test/elixir/test/config/suite.elixir
@@ -0,0 +1,592 @@
+%{
+  "AllDocsTest": [
+    "All Docs tests",
+    "GET with one key",
+    "POST boolean",
+    "POST edge case with colliding parameters - query takes precedence",
+    "POST with empty body",
+    "POST with keys and limit",
+    "POST with missing keys",
+    "POST with query parameter and JSON body",
+    "_local_docs POST with keys and limit",
+    "all_docs ordering"
+  ],
+  "AttachmentMultipartTest": [
+    "manages attachments multipart requests successfully",
+    "manages compressed attachments successfully"
+  ],
+  "AttachmentNamesTest": [
+    "saves attachment names successfully"
+  ],
+  "AttachmentPathsTest": [
+    "manages attachment paths successfully",
+    "manages attachment paths successfully - design docs"
+  ],
+  "AttachmentRangesTest": [
+    "manages attachment range requests successfully"
+  ],
+  "AttachmentViewTest": [
+    "manages attachments in views successfully"
+  ],
+  "AttachmentsTest": [
+    "COUCHDB-809 - stubs should only require the 'stub' field",
+    "attachment via multipart/form-data",
+    "delete attachment",
+    "empty attachments",
+    "errors for bad attachment",
+    "etags for attachments",
+    "implicit doc creation allows creating docs with a reserved id. COUCHDB-565",
+    "large attachments COUCHDB-366",
+    "md5 header for attachments",
+    "reads attachment successfully",
+    "saves attachment successfully",
+    "saves binary",
+    "COUCHDB-497 - empty attachments",
+    "update attachment"
+  ],
+  "AuthCacheTest": [
+    "auth cache management"
+  ],
+  "BasicsTest": [
+    "A document read with etag works",
+    "Can create several documents",
+    "Check _revs_limit",
+    "Check for invalid document members",
+    "Create a document and save it to the database",
+    "Created database has appropriate db info name",
+    "Creating a new DB should return location header",
+    "Creating a new DB with slashes should return Location header (COUCHDB-411)",
+    "DELETE'ing a non-existent doc should 404",
+    "Database should be in _all_dbs",
+    "Default headers are returned for doc with open_revs=all",
+    "Empty database should have zero docs",
+    "Exceeding configured DB name size limit returns an error",
+    "Make sure you can do a seq=true option",
+    "On restart, a request for creating an already existing db can not override",
+    "POST doc response has a Location header",
+    "POST doc with an _id field isn't overwritten by uuid",
+    "PUT doc has a Location header",
+    "PUT error when body not an object",
+    "PUT on existing DB should return 412 instead of 500",
+    "Ready endpoint",
+    "Regression test for COUCHDB-954",
+    "Revs info status is good",
+    "Session contains adm context",
+    "Simple map functions",
+    "Welcome endpoint",
+    "_all_docs POST error when multi-get is not a {'key': [...]} structure",
+    "_all_docs/queries works",
+    "_bulk_docs POST error when body not an object",
+    "_design_docs works",
+    "_local_docs works",
+    "oops, the doc id got lost in code nirwana"
+  ],
+  "BatchSaveTest": [
+    "batch post",
+    "batch put",
+    "batch put with identical doc ids"
+  ],
+  "BulkDocsTest": [
+    "bulk docs can create, update, & delete many docs per request",
+    "bulk docs can detect conflicts",
+    "bulk docs emits conflict error for duplicate doc `_id`s",
+    "bulk docs raises conflict error for combined update & delete",
+    "bulk docs raises error for `all_or_nothing` option",
+    "bulk docs raises error for invlaid `docs` parameter",
+    "bulk docs raises error for invlaid `new_edits` parameter",
+    "bulk docs raises error for missing `docs` parameter",
+    "bulk docs raises transaction_too_large error for transaction larger than 10MB",
+    "bulk docs supplies `id` if not provided in doc"
+  ],
+  "ChangesAsyncTest": [
+    "COUCHDB-1852",
+    "continuous changes",
+    "continuous filtered changes",
+    "continuous filtered changes with doc ids",
+    "eventsource changes",
+    "eventsource heartbeat",
+    "live changes",
+    "longpoll changes",
+    "longpoll filtered changes"
+  ],
+  "ChangesTest": [
+    "COUCHDB-1037-empty result for ?limit=1&filter=foo/bar in some cases",
+    "COUCHDB-1256",
+    "COUCHDB-1923",
+    "Changes feed negative heartbeat",
+    "Changes feed non-integer heartbeat",
+    "changes filtering on design docs",
+    "changes filtering on docids",
+    "changes limit",
+    "erlang function filtered changes",
+    "function filtered changes",
+    "map function filtered changes",
+    "non-existing desing doc and funcion for filtered changes",
+    "non-existing desing doc for filtered changes",
+    "non-existing function for filtered changes"
+  ],
+  "CoffeeTest": [
+    "CoffeeScript basic functionality"
+  ],
+  "CompactTest": [
+    "compaction reduces size of deleted docs"
+  ],
+  "ConfigTest": [
+    "Atoms, binaries, and strings suffice as whitelist sections and keys.",
+    "Blacklist is functional",
+    "CouchDB respects configured protocols",
+    "Keys not in the whitelist may not be modified",
+    "Non-2-tuples in the whitelist are ignored",
+    "Non-list whitelist values allow further modification of the whitelist",
+    "Non-term whitelist values allow further modification of the whitelist",
+    "PORT `BUGGED` ?raw tests from config.js",
+    "Reload config",
+    "Server-side password hashing, and raw updates disabling that",
+    "Settings can be altered with undefined whitelist allowing any change",
+    "Standard config options are present"
+  ],
+  "CookieAuthTest": [
+    "cookie auth"
+  ],
+  "CopyDocTest": [
+    "Copy doc tests"
+  ],
+  "DesignDocsQueryTest": [
+    "POST edge case with colliding parameters - query takes precedence",
+    "POST with empty body",
+    "POST with keys and limit",
+    "POST with query parameter and JSON body",
+    "query _design_docs (GET with no parameters)",
+    "query _design_docs descending=false",
+    "query _design_docs descending=true",
+    "query _design_docs end_key",
+    "query _design_docs end_key inclusive_end=false",
+    "query _design_docs end_key inclusive_end=false descending",
+    "query _design_docs end_key inclusive_end=true",
+    "query _design_docs end_key limit",
+    "query _design_docs end_key skip",
+    "query _design_docs endkey",
+    "query _design_docs post with keys",
+    "query _design_docs start_key",
+    "query _design_docs startkey",
+    "query _design_docs update_seq",
+    "query _design_docs with multiple key",
+    "query _design_docs with single key"
+  ],
+  "DesignDocsTest": [
+    "_all_docs view returns correctly with keys",
+    "all_docs_twice",
+    "circular commonjs dependencies",
+    "commonjs in map functions",
+    "commonjs require",
+    "consistent _rev for design docs",
+    "design doc deletion",
+    "language not specified, Javascript is implied",
+    "module id values are as expected",
+    "startkey and endkey",
+    "that we get correct design doc info back",
+    "validate doc update"
+  ],
+  "DesignOptionsTest": [
+    "design doc options - include_design default value",
+    "design doc options - include_desing=false",
+    "design doc options - include_desing=true",
+    "design doc options - local_seq=true"
+  ],
+  "DesignPathTest": [
+    "design doc path",
+    "design doc path with slash in db name"
+  ],
+  "ErlangViewsTest": [
+    "Erlang map function",
+    "Erlang reduce function",
+    "Erlang reduce function larger dataset"
+  ],
+  "EtagsHeadTest": [
+    "etag header on creation",
+    "etag header on head",
+    "etag header on retrieval",
+    "etags head"
+  ],
+  "FormSubmitTest": [
+    "form submission gives back invalid content-type"
+  ],
+  "HelperTest": [
+    "retry_until handles assertions",
+    "retry_until handles boolean conditions",
+    "retry_until times out"
+  ],
+  "HttpTest": [
+    "COUCHDB-708: newlines document names",
+    "location header",
+    "location header should include X-Forwarded-Host",
+    "location header should include custom header"
+  ],
+  "InvalidDocIDsTest": [
+    "_local-prefixed ids are illegal",
+    "a PUT request with absent _id is forbidden",
+    "accidental POST to form handling code",
+    "explicit _bulk_docks policy",
+    "invalid _prefix",
+    "using a non-string id is forbidden"
+  ],
+  "JsonpTest": [
+    "jsonp chunked callbacks",
+    "jsonp not configured callbacks",
+    "jsonp unchunked callbacks"
+  ],
+  "JwtAuthTest": [
+    "jwt auth with EC secret",
+    "jwt auth with HMAC secret",
+    "jwt auth with RSA secret",
+    "jwt auth with required iss claim",
+    "jwt auth without secret"
+  ],
+  "ListViewsTest": [
+    "COUCHDB-1113",
+    "HTTP header response set after getRow() called in _list function",
+    "abort iteration with reduce",
+    "empty list",
+    "extra qs params",
+    "get with query params",
+    "handling _all_docs by _list functions. the result should be equal",
+    "multi-key fetch with GET",
+    "multi-key fetch with POST",
+    "multiple languages in design docs",
+    "no multi-key fetch allowed when group=false",
+    "reduce with 0 rows",
+    "secObj is available",
+    "standard GET",
+    "standard OPTIONS",
+    "stop iteration",
+    "the richness of the arguments",
+    "too many Get Rows",
+    "we can run lists and views from separate docs",
+    "we do multi-key requests on lists and views in separate docs",
+    "when there is a reduce present, and used",
+    "when there is a reduce present, but not used",
+    "with 0 rows",
+    "with accept headers for HTML",
+    "with include_docs and a reference to the doc"
+  ],
+  "LocalDocsTest": [
+    "GET with multiple keys",
+    "GET with no parameters",
+    "POST edge case with colliding parameters - query takes precedence",
+    "POST with empty body",
+    "POST with keys and limit",
+    "POST with query parameter and JSON body"
+  ],
+  "LotsOfDocsTest": [
+    "lots of docs with _all_docs",
+    "lots of docs with a regular view"
+  ],
+  "MethodOverrideTest": [
+    "Method Override is ignored when original Method isn't POST",
+    "method override DELETE",
+    "method override PUT"
+  ],
+  "MultipleRowsTest": [
+    "multiple rows"
+  ],
+  "ProxyAuthTest": [
+    "proxy auth with secret",
+    "proxy auth without secret"
+  ],
+  "PurgeTest": [
+    "COUCHDB-1065",
+    "purge documents"
+  ],
+  "ReaderACLTest": [
+    "can't set non string reader names or roles",
+    "members can query views",
+    "restricted db can be read by authorized users",
+    "unrestricted db can be read",
+    "works with readers (backwards compat with 1.0)"
+  ],
+  "RecreateDocTest": [
+    "COUCHDB-1265 - changes feed after we try and break the update_seq tree",
+    "COUCHDB-292 - recreate a deleted document",
+    "Recreate a deleted document with non-exsistant rev",
+    "recreate document"
+  ],
+  "ReduceBuiltinTest": [
+    "Builtin count and sum reduce for key as array",
+    "Builtin reduce functions",
+    "Builtin reduce functions with trailings"
+  ],
+  "ReduceFalseTest": [
+    "Basic reduce functions"
+  ],
+  "ReduceTest": [
+    "Basic reduce functions",
+    "More complex array key view row testing",
+    "More complex reductions that need to use the combine option",
+    "Reduce pagination"
+  ],
+  "ReplicationBadIdTest": [
+    "replication doc with bad rep id"
+  ],
+  "ReplicationTest": [
+    "compressed attachment replication - remote-to-remote",
+    "continuous replication - remote-to-remote",
+    "create_target filter option - remote-to-remote",
+    "filtered replications - remote-to-remote",
+    "non-admin or reader user on source - remote-to-remote",
+    "non-admin user on target - remote-to-remote",
+    "replicate with since_seq - remote-to-remote",
+    "replicating attachment without conflict - COUCHDB-885",
+    "replication by doc ids - remote-to-remote",
+    "replication cancellation",
+    "replication restarts after filter change - COUCHDB-892 - remote-to-remote",
+    "simple remote-to-remote replication - remote-to-remote",
+    "source database not found with host",
+    "unauthorized replication cancellation",
+    "validate_doc_update failure replications - remote-to-remote"
+  ],
+  "ReplicatorDBByDocIdTest": [
+    "replicatior db by doc id"
+  ],
+  "RevStemmingTest": [
+    "revs limit is kept after compaction",
+    "revs limit produces replication conflict ",
+    "revs limit update"
+  ],
+  "RevisionTest": [
+    "`new_edits: false` prevents bulk updates (COUCHDB-1178)",
+    "mismatched rev in body and etag returns error",
+    "mismatched rev in body and query string returns error",
+    "multiple updates with same _rev raise conflict errors"
+  ],
+  "RewriteJSTest": [
+    "Test basic js rewrites on test_rewrite_suite_db",
+    "Test basic js rewrites on test_rewrite_suite_db%2Fwith_slashes",
+    "early response on test_rewrite_suite_db",
+    "early response on test_rewrite_suite_db%2Fwith_slashes",
+    "loop on test_rewrite_suite_db",
+    "loop on test_rewrite_suite_db%2Fwith_slashes",
+    "path relative to server on test_rewrite_suite_db",
+    "path relative to server on test_rewrite_suite_db%2Fwith_slashes",
+    "requests with body preserve the query string rewrite on test_rewrite_suite_db",
+    "requests with body preserve the query string rewrite on test_rewrite_suite_db%2Fwith_slashes"
+  ],
+  "RewriteTest": [
+    "Test basic rewrites on test_rewrite_suite_db",
+    "Test basic rewrites on test_rewrite_suite_db%2Fwith_slashes",
+    "loop detection on test_rewrite_suite_db",
+    "loop detection on test_rewrite_suite_db%2Fwith_slashes",
+    "path relative to server on test_rewrite_suite_db",
+    "path relative to server on test_rewrite_suite_db%2Fwith_slashes",
+    "serial execution is not spuriously counted as loop on test_rewrite_suite_db",
+    "serial execution is not spuriously counted as loop on test_rewrite_suite_db%2Fwith_slashes"
+  ],
+  "SecurityValidationTest": [
+    "Author presence and user security",
+    "Author presence and user security when replicated",
+    "Ddoc writes with admin and replication contexts",
+    "Force basic login",
+    "Jerry can save a document normally",
+    "Non-admin user cannot save a ddoc",
+    "Saving document using the wrong credentials",
+    "_session API",
+    "try to set a wrong value for _security"
+  ],
+  "ShowDocumentsTest": [
+    "JS can't set etag",
+    "accept header switching - different mime has different etag",
+    "deleted docs",
+    "id with slash",
+    "list() compatible API",
+    "list() compatible API with provides function",
+    "missing design doc",
+    "registering types works",
+    "security object",
+    "should keep next result order: chunks + return value + provided chunks + provided return value",
+    "show error",
+    "show fail with non-existing docid",
+    "show query parameters",
+    "show with doc",
+    "show with doc - etags",
+    "show with existing doc",
+    "show with missing doc",
+    "show with non-existing docid",
+    "show without docid",
+    "the provides mime matcher",
+    "the provides mime matcher without a match"
+  ],
+  "UTF8Test": [
+    "UTF8 support"
+  ],
+  "UUIDsTest": [
+    "Bad Request error when exceeding max UUID count",
+    "Method Not Allowed error on POST",
+    "cache busting headers are set",
+    "can return single uuid",
+    "no duplicates in 1,000 UUIDs",
+    "sequential uuids are sequential",
+    "utc_id uuids are correct",
+    "utc_random uuids are roughly random"
+  ],
+  "UpdateDocumentsTest": [
+    "COUCHDB-1229 - allow slashes in doc ids for update handlers",
+    "COUCHDB-648 - the code in the JSON response should be honored",
+    "GET is not allowed",
+    "Insert doc with empty id",
+    "Server provides UUID when POSTing without an ID in the URL",
+    "base64 response",
+    "bump counter",
+    "doc can be created",
+    "form update via application/x-www-form-urlencoded",
+    "in place update",
+    "update document",
+    "update error invalid path"
+  ],
+  "UsersDbSecurityTest": [
+    "user db security"
+  ],
+  "UsersDbTest": [
+    "users db"
+  ],
+  "ViewCollationRawTest": [
+    "ascending collation order",
+    "descending collation order",
+    "inclusive_end=false",
+    "inclusive_end=true",
+    "key query option",
+    "raw semantics in key ranges"
+  ],
+  "ViewCollationTest": [
+    "ascending collation order",
+    "descending collation order",
+    "inclusive_end=false",
+    "inclusive_end=true",
+    "key query option"
+  ],
+  "ViewCompactionTest": [
+    "view compaction"
+  ],
+  "ViewConflictsTest": [
+    "view conflict"
+  ],
+  "ViewErrorsTest": [
+    "emit undefined key results as null",
+    "emit undefined value results as null",
+    "error responses for invalid multi-get bodies",
+    "exception in map function",
+    "infinite loop",
+    "query parse error",
+    "query view with invalid params",
+    "reduce overflow error",
+    "temporary view should give error message"
+  ],
+  "ViewIncludeDocsTest": [
+    "COUCHDB-549 - include_docs=true with conflicts=true",
+    "Not an error with include_docs=false&reduce=true",
+    "Reduce support when reduce=false",
+    "emitted _rev controls things",
+    "include docs in all_docs",
+    "include docs in view",
+    "link to another doc from a value",
+    "no reduce support"
+  ],
+  "ViewMapTest": [
+    "_conflict is supported",
+    "_local_seq is supported",
+    "bad range returns error",
+    "can do design doc swap",
+    "can index design docs",
+    "can use key in query string",
+    "can use keys in query string",
+    "descending=true query with startkey_docid",
+    "inclusive = false",
+    "multiple emits in correct value order",
+    "query returns docs",
+    "supports linked documents",
+    "updated docs rebuilds index"
+  ],
+  "ViewMultiKeyAllDocsTest": [
+    "GET - get invalid rows when the key doesn't exist",
+    "POST - get invalid rows when the key doesn't exist",
+    "empty keys",
+    "keys in GET parameters",
+    "keys in GET parameters (descending)",
+    "keys in GET parameters (descending, skip, limit)",
+    "keys in GET parameters (limit)",
+    "keys in GET parameters (skip)",
+    "keys in POST body",
+    "keys in POST body (descending)",
+    "keys in POST body (descending, skip, limit)",
+    "keys in POST body (limit)",
+    "keys in POST body (skip)"
+  ],
+  "ViewMultiKeyDesignTest": [
+    "GET - invalid parameter combinations get rejected ",
+    "POST - invalid parameter combinations get rejected ",
+    "argument combinations",
+    "dir works",
+    "empty keys",
+    "keys in GET body (group)",
+    "keys in GET parameters",
+    "keys in POST body",
+    "keys in POST body (group)",
+    "limit works",
+    "offset works",
+    "that a map & reduce containing func support keys when reduce=false",
+    "that limiting by startkey_docid and endkey_docid get applied",
+    "that missing keys work too"
+  ],
+  "ViewOffsetTest": [
+    "basic view offsets",
+    "repeated view offsets"
+  ],
+  "ViewPaginationTest": [
+    "aliases start_key and start_key_doc_id should work",
+    "basic view pagination",
+    "descending view pagination",
+    "descending=false parameter should just be ignored",
+    "endkey document id",
+    "endkey document id, but with end_key_doc_id alias"
+  ],
+  "ViewSandboxingTest": [
+    "COUCHDB-925 - altering 'doc' variable in map function affects other map functions",
+    "attempting to change the document has no effect",
+    "runtime code evaluation can be prevented",
+    "view cannot access the map_funs and map_results array",
+    "view cannot invoke interpreter internals"
+  ],
+  "ViewTest": [
+    "GET with multiple keys",
+    "GET with no parameters",
+    "GET with one key",
+    "POST edge case with colliding parameters - query takes precedence",
+    "POST with boolean parameter",
+    "POST with empty body",
+    "POST with keys and limit",
+    "POST with query parameter and JSON body"
+  ],
+  "ViewUpdateSeqTest": [
+    "_all_docs update seq",
+    "db info update seq",
+    "view update seq"
+  ],
+  "WithQuorumTest": [
+    "Attachments overriden quorum should return 202-Acepted",
+    "Attachments should return 201-Created",
+    "Bulk docs overriden quorum should return 202-Acepted",
+    "Bulk docs should return 201-Created",
+    "Copy doc should return 201-Created",
+    "Creating-Updating/Deleting doc should return 201-Created/200-OK",
+    "Creating-Updating/Deleting doc with overriden quorum should return 202-Acepted/200-OK",
+    "Creating/Deleting DB should return 201-Created/202-Acepted"
+  ],
+  "WithoutQuorumTest": [
+    "Attachments overriden quorum should return 201-Created",
+    "Attachments should return 202-Acepted",
+    "Bulk docs overriden quorum should return 201-Created",
+    "Bulk docs should return 202-Acepted",
+    "Copy doc should return 202-Acepted",
+    "Creating-Updating/Deleting doc with overriden quorum should return 201-Created/200-OK",
+    "Creating/Deleting DB should return 202-Acepted",
+    "Creating/Updating/Deleting doc should return 202-Acepted"
+  ]
+}
diff --git a/test/elixir/test/test_helper.exs b/test/elixir/test/test_helper.exs
index 6311fca44..72989e7c5 100644
--- a/test/elixir/test/test_helper.exs
+++ b/test/elixir/test/test_helper.exs
@@ -1,16 +1 @@
-# If build number detected assume we running on Jenkins
-# and skip certain tests that fail on jenkins.
-exclude =
-  case System.get_env("BUILD_NUMBER") !== nil do
-    true -> [:pending, :skip_on_jenkins]
-    false -> [:pending]
-  end
-
-current_exclude = Keyword.get(ExUnit.configuration(), :exclude, [])
-
-ExUnit.configure(
-  exclude: Enum.uniq(exclude ++ current_exclude),
-  formatters: [JUnitFormatter, ExUnit.CLIFormatter]
-)
-
-ExUnit.start()
+Couch.Test.Suite.start()