1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
|
Code.require_file("../../test_helper.exs", __DIR__)
defmodule Mix.Tasks.TestTest do
use MixTest.Case
describe "ex_unit_opts/1" do
test "returns ex unit options" do
assert ex_unit_opts_from_given(unknown: "ok", seed: 13) == [seed: 13]
end
test "returns includes and excludes" do
included = [include: [:focus, key: "val"]]
assert ex_unit_opts_from_given(include: "focus", include: "key:val") == included
excluded = [exclude: [:focus, key: "val"]]
assert ex_unit_opts_from_given(exclude: "focus", exclude: "key:val") == excluded
end
test "translates :only into includes and excludes" do
assert ex_unit_opts_from_given(only: "focus") == [include: [:focus], exclude: [:test]]
only = [include: [:focus, :special], exclude: [:test]]
assert ex_unit_opts_from_given(only: "focus", include: "special") == only
end
test "translates :color into list containing an enabled key-value pair" do
assert ex_unit_opts_from_given(color: false) == [colors: [enabled: false]]
assert ex_unit_opts_from_given(color: true) == [colors: [enabled: true]]
end
test "translates :formatter into list of modules" do
assert ex_unit_opts_from_given(formatter: "A.B") == [formatters: [A.B]]
end
test "accepts custom :exit_status" do
assert {:exit_status, 5} in ex_unit_opts(exit_status: 5)
end
test "includes some default options" do
assert ex_unit_opts([]) == [
autorun: false,
exit_status: 2,
failures_manifest_file:
Path.join(Mix.Project.manifest_path(), ".mix_test_failures")
]
end
defp ex_unit_opts(opts) do
{ex_unit_opts, _allowed_files} = Mix.Tasks.Test.process_ex_unit_opts(opts)
ex_unit_opts
end
defp ex_unit_opts_from_given(passed) do
passed
|> ex_unit_opts()
|> Keyword.drop([:failures_manifest_file, :autorun, :exit_status])
end
end
describe "--stale" do
test "runs all tests for first run, then none on second" do
in_fixture("test_stale", fn ->
assert_stale_run_output("2 tests, 0 failures")
assert_stale_run_output("""
No stale tests
""")
end)
end
test "runs tests that depend on modified modules" do
in_fixture("test_stale", fn ->
assert_stale_run_output("2 tests, 0 failures")
set_all_mtimes()
force_recompilation("lib/b.ex")
assert_stale_run_output("1 test, 0 failures")
set_all_mtimes()
force_recompilation("lib/a.ex")
assert_stale_run_output("2 tests, 0 failures")
end)
end
test "doesn't write manifest when there are failures" do
in_fixture("test_stale", fn ->
assert_stale_run_output("2 tests, 0 failures")
set_all_mtimes()
File.write!("lib/b.ex", """
defmodule B do
def f, do: :error
end
""")
assert_stale_run_output("1 test, 1 failure")
assert_stale_run_output("1 test, 1 failure")
end)
end
test "runs tests that have changed" do
in_fixture("test_stale", fn ->
assert_stale_run_output("2 tests, 0 failures")
set_all_mtimes()
File.touch!("test/a_test_stale.exs")
assert_stale_run_output("1 test, 0 failures")
end)
end
test "runs tests that have changed test_helpers" do
in_fixture("test_stale", fn ->
assert_stale_run_output("2 tests, 0 failures")
set_all_mtimes()
File.touch!("test/test_helper.exs")
assert_stale_run_output("2 tests, 0 failures")
end)
end
test "runs all tests no matter what with --force" do
in_fixture("test_stale", fn ->
assert_stale_run_output("2 tests, 0 failures")
assert_stale_run_output(~w[--force], "2 tests, 0 failures")
end)
end
end
describe "--cover" do
test "reports the coverage of each app's modules in an umbrella" do
in_fixture("umbrella_test", fn ->
# This fixture by default results in coverage above the default threshold
# which should result in an exit status of 0.
assert {output, 0} = mix_code(["test", "--cover"])
assert output =~ "4 tests, 0 failures"
# For bar, we do regular --cover and also test protocols
assert output =~ """
Generating cover results ...
Percentage | Module
-----------|--------------------------
100.00% | Bar.Protocol
100.00% | Bar.Protocol.BitString
-----------|--------------------------
100.00% | Total
"""
assert output =~ "1 test, 0 failures"
# For foo, we do regular --cover and test it does not include bar
assert output =~ """
Generating cover results ...
Percentage | Module
-----------|--------------------------
100.00% | Foo
-----------|--------------------------
100.00% | Total
"""
# We skip a test in bar to force coverage below the default threshold
# which should result in an exit status of 1.
assert {output, code} = mix_code(["test", "--cover", "--exclude", "maybe_skip"])
assert output =~ """
Coverage test failed, threshold not met:
Coverage: 0.00%
Threshold: 90.00%
"""
unless windows?() do
assert code == 3
end
end)
end
test "supports unified reports by using test.coverage" do
in_fixture("umbrella_test", fn ->
assert mix(["test", "--export-coverage", "default", "--cover"]) =~
"Run \"mix test.coverage\" once all exports complete"
assert mix(["test.coverage"]) =~ """
Importing cover results: apps/bar/cover/default.coverdata
Importing cover results: apps/foo/cover/default.coverdata
Percentage | Module
-----------|--------------------------
100.00% | Bar
100.00% | Bar.Ignore
100.00% | Bar.Protocol
100.00% | Bar.Protocol.BitString
100.00% | Foo
-----------|--------------------------
100.00% | Total
"""
end)
end
end
describe "--failed" do
test "loads only files with failures and runs just the failures" do
in_fixture("test_failed", fn ->
loading_only_passing_test_msg = "loading OnlyPassingTest"
# Run `mix test` once to record failures...
output = mix(["test"])
assert output =~ loading_only_passing_test_msg
assert output =~ "4 tests, 2 failures"
# `mix test --failed` runs only failed tests and avoids loading files with no failures
output = mix(["test", "--failed"])
refute output =~ loading_only_passing_test_msg
assert output =~ "2 tests, 2 failures"
# `mix test --failed` can be applied to a directory or file
output = mix(["test", "test/passing_and_failing_test_failed.exs", "--failed"])
assert output =~ "1 test, 1 failure"
# `--failed` composes with an `--only` filter by running the intersection.
# Of the failing tests, 1 is tagged with `@tag :foo`.
# Of the passing tests, 1 is tagged with `@tag :foo`.
# But only the failing test with that tag should run.
output = mix(["test", "--failed", "--only", "foo"])
assert output =~ "2 tests, 1 failure, 1 excluded"
# Run again to give it a chance to record as passed
System.put_env("PASS_FAILING_TESTS", "true")
assert mix(["test", "--failed"]) =~ "2 tests, 0 failures"
# Nothing should get run if we try it again since everything is passing.
assert mix(["test", "--failed"]) =~ "There are no tests to run"
# `--failed` and `--stale` cannot be combined
output = mix(["test", "--failed", "--stale"])
assert output =~ "Combining --failed and --stale is not supported"
end)
after
System.delete_env("PASS_FAILING_TESTS")
end
end
describe "--listen-on-stdin" do
test "runs tests after input" do
in_fixture("test_stale", fn ->
port = mix_port(~w[test --stale --listen-on-stdin])
assert receive_until_match(port, "seed", "") =~ "2 tests"
Port.command(port, "\n")
assert receive_until_match(port, "No stale tests", "") =~ "Restarting..."
end)
end
test "does not exit on compilation failure" do
in_fixture("test_stale", fn ->
File.write!("lib/b.ex", """
defmodule B do
def f, do: error_not_a_var
end
""")
port = mix_port(~w[test --listen-on-stdin])
assert receive_until_match(port, "error", "") =~ "lib/b.ex"
File.write!("lib/b.ex", """
defmodule B do
def f, do: A.f
end
""")
Port.command(port, "\n")
assert receive_until_match(port, "seed", "") =~ "2 tests"
File.write!("test/b_test_stale.exs", """
defmodule BTest do
use ExUnit.Case
test "f" do
assert B.f() == error_not_a_var
end
end
""")
Port.command(port, "\n")
message = "undefined function error_not_a_var"
assert receive_until_match(port, message, "") =~ "test/b_test_stale.exs"
File.write!("test/b_test_stale.exs", """
defmodule BTest do
use ExUnit.Case
test "f" do
assert B.f() == :ok
end
end
""")
Port.command(port, "\n")
assert receive_until_match(port, "seed", "") =~ "2 tests"
end)
end
end
describe "--partitions" do
test "splits tests into partitions" do
in_fixture("test_stale", fn ->
assert mix(["test", "--partitions", "3", "--cover"], [{"MIX_TEST_PARTITION", "1"}]) =~
"1 test, 0 failures"
assert mix(["test", "--partitions", "3", "--cover"], [{"MIX_TEST_PARTITION", "2"}]) =~
"1 test, 0 failures"
assert mix(["test", "--partitions", "3", "--cover"], [{"MIX_TEST_PARTITION", "3"}]) =~
"There are no tests to run"
assert File.regular?("cover/1.coverdata")
assert File.regular?("cover/2.coverdata")
refute File.regular?("cover/3.coverdata")
assert mix(["test.coverage"]) == """
Importing cover results: cover/1.coverdata
Importing cover results: cover/2.coverdata
Percentage | Module
-----------|--------------------------
100.00% | A
100.00% | B
-----------|--------------------------
100.00% | Total
Generated HTML coverage results in \"cover\" directory
"""
end)
end
test "raises when no partition is given even with Mix.shell() change" do
in_fixture("test_stale", fn ->
File.write!("test/test_helper.exs", """
Mix.shell(Mix.Shell.Process)
ExUnit.start()
""")
assert_run_output(
["--partitions", "4"],
"The MIX_TEST_PARTITION environment variable must be set"
)
end)
end
test "do not raise if partitions flag is set to 1 and no partition given" do
in_fixture("test_stale", fn ->
assert mix(["test", "--partitions", "1"], []) =~
"2 tests, 0 failures"
assert mix(["test", "--partitions", "1"], [{"MIX_TEST_PARTITION", ""}]) =~
"2 tests, 0 failures"
assert mix(["test", "--partitions", "1"], [{"MIX_TEST_PARTITION", "1"}]) =~
"2 tests, 0 failures"
end)
end
test "raise if partitions is set to non-positive value" do
in_fixture("test_stale", fn ->
File.write!("test/test_helper.exs", """
Mix.shell(Mix.Shell.Process)
ExUnit.start()
""")
assert_run_output(
["--partitions", "0"],
"--partitions : expected to be positive integer, got 0"
)
assert_run_output(
["--partitions", "-1"],
"--partitions : expected to be positive integer, got -1"
)
end)
end
end
describe "logs and errors" do
test "logs test absence for a project with no test paths" do
in_fixture("test_stale", fn ->
File.rm_rf!("test")
assert_run_output("There are no tests to run")
end)
end
test "raises when no test runs even with Mix.shell() change" do
in_fixture("test_stale", fn ->
File.write!("test/test_helper.exs", """
Mix.shell(Mix.Shell.Process)
ExUnit.start()
""")
assert_run_output(
["--only", "unknown"],
"The --only option was given to \"mix test\" but no test was executed"
)
end)
end
test "raises an exception if line numbers are given with multiple files" do
in_fixture("test_stale", fn ->
assert_run_output(
[
"test/a_test_stale.exs",
"test/b_test_stale.exs:4"
],
"Line numbers can only be used when running a single test file"
)
end)
end
test "umbrella with file path" do
in_fixture("umbrella_test", fn ->
# Run false positive test first so at least the code is compiled
# and we can perform more aggressive assertions later
assert mix(["test", "apps/unknown_app/test"]) =~ """
==> bar
Paths given to "mix test" did not match any directory/file: apps/unknown_app/test
==> foo
Paths given to "mix test" did not match any directory/file: apps/unknown_app/test
"""
output = mix(["test", "apps/bar/test/bar_tests.exs"])
assert output =~ """
==> bar
....
"""
refute output =~ "==> foo"
refute output =~ "Paths given to \"mix test\" did not match any directory/file"
output = mix(["test", "apps/bar/test/bar_tests.exs:10"])
assert output =~ """
==> bar
Excluding tags: [:test]
Including tags: [line: \"10\"]
.
"""
refute output =~ "==> foo"
refute output =~ "Paths given to \"mix test\" did not match any directory/file"
end)
end
end
describe "--warnings-as-errors" do
test "fail on warning in tests" do
in_fixture("test_stale", fn ->
msg =
"Test suite aborted after successful execution due to warnings while using the --warnings-as-errors option"
refute mix(["test", "--warnings-as-errors"]) =~ msg
File.write!("lib/warning.ex", """
unused_compile_var = 1
""")
File.write!("test/warning_test_stale.exs", """
defmodule WarningTest do
use ExUnit.Case
test "warning" do
unused_test_var = 1
end
end
""")
output = mix(["test", "--warnings-as-errors", "test/warning_test_stale.exs"])
assert output =~ "variable \"unused_compile_var\" is unused"
assert output =~ "variable \"unused_test_var\" is unused"
assert output =~ msg
end)
end
test "mark failed tests" do
in_fixture("test_failed", fn ->
File.write!("test/warning_test_failed.exs", """
defmodule WarningTest do
use ExUnit.Case
test "warning" do
unused_var = 123
end
end
""")
output = mix(["test", "--warnings-as-errors"])
assert output =~ "2 failures"
refute output =~ "Test suite aborted after successful execution"
output = mix(["test", "--failed"])
assert output =~ "2 failures"
end)
end
end
describe "--exit-status" do
@describetag :unix
test "returns custom exit status" do
in_fixture("test_failed", fn ->
{output, exit_status} = mix_code(["test", "--exit-status", "5"])
assert output =~ "2 failures"
assert exit_status == 5
end)
end
end
defp receive_until_match(port, expected, acc) do
receive do
{^port, {:data, output}} ->
acc = acc <> output
if output =~ expected do
acc
else
receive_until_match(port, expected, acc)
end
end
end
defp set_all_mtimes(time \\ {{2010, 1, 1}, {0, 0, 0}}) do
Enum.each(Path.wildcard("**", match_dot: true), &File.touch!(&1, time))
end
defp assert_stale_run_output(opts \\ [], expected) do
assert_run_output(["--stale" | opts], expected)
end
defp assert_run_output(opts \\ [], expected) do
assert mix(["test" | opts]) =~ expected
end
end
|