Auto merge of #108148 - parthopdas:master, r=oli-obk

Implementing "<test_binary> --list --format json" for use by IDE test explorers / runners

Fixes #107307

PR 1 of 2 - wiring up just the new information + implement the command line changes i.e. --format json + tests

upcoming:
PR 2 of 2 - clean up "#[cfg(not(bootstrap))]" from PR 1

As per the discussions on
- MCP: https://rust-lang.zulipchat.com/#narrow/stream/233931-t-compiler.2Fmajor-changes/topic/Implementing.20.22.3Ctest_binary.3E.20--list.20--form.E2.80.A6.20compiler-team.23592/near/328747548
- preRFC: https://internals.rust-lang.org/t/pre-rfc-implementing-test-binary-list-format-json-for-use-by-ide-test-explorers-runners/18308
- FYI on Discord: https://discord.com/channels/442252698964721669/459149169546887178/1075581549409484820

23 files changed, 554 insertions, 46 deletions

diff --git a/.gitignore b/.gitignore
index 04d2597ecc6..485968d9c56 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,7 @@ Session.vim
 .project
 .favorites.json
 .settings/
+.vs/
 
 ## Tool
 .valgrindrc
diff --git a/compiler/rustc_builtin_macros/src/test.rs b/compiler/rustc_builtin_macros/src/test.rs
index e02c7e6c01b..007d14b0e74 100644
--- a/compiler/rustc_builtin_macros/src/test.rs
+++ b/compiler/rustc_builtin_macros/src/test.rs
@@ -8,7 +8,7 @@ use rustc_errors::Applicability;
 use rustc_expand::base::*;
 use rustc_session::Session;
 use rustc_span::symbol::{sym, Ident, Symbol};
-use rustc_span::Span;
+use rustc_span::{FileNameDisplayPreference, Span};
 use std::iter;
 use thin_vec::{thin_vec, ThinVec};
 
@@ -231,6 +231,8 @@ pub fn expand_test_or_bench(
         &item.ident,
     ));
 
+    let location_info = get_location_info(cx, &item);
+
     let mut test_const = cx.item(
         sp,
         Ident::new(item.ident.name, sp),
@@ -280,6 +282,16 @@ pub fn expand_test_or_bench(
                                             cx.expr_none(sp)
                                         },
                                     ),
+                                    // source_file: <relative_path_of_source_file>
+                                    field("source_file", cx.expr_str(sp, location_info.0)),
+                                    // start_line: start line of the test fn identifier.
+                                    field("start_line", cx.expr_usize(sp, location_info.1)),
+                                    // start_col: start column of the test fn identifier.
+                                    field("start_col", cx.expr_usize(sp, location_info.2)),
+                                    // end_line: end line of the test fn identifier.
+                                    field("end_line", cx.expr_usize(sp, location_info.3)),
+                                    // end_col: end column of the test fn identifier.
+                                    field("end_col", cx.expr_usize(sp, location_info.4)),
                                     // compile_fail: true | false
                                     field("compile_fail", cx.expr_bool(sp, false)),
                                     // no_run: true | false
@@ -364,6 +376,19 @@ pub fn expand_test_or_bench(
     }
 }
 
+fn get_location_info(cx: &ExtCtxt<'_>, item: &ast::Item) -> (Symbol, usize, usize, usize, usize) {
+    let span = item.ident.span;
+    let (source_file, lo_line, lo_col, hi_line, hi_col) =
+        cx.sess.source_map().span_to_location_info(span);
+
+    let file_name = match source_file {
+        Some(sf) => sf.name.display(FileNameDisplayPreference::Remapped).to_string(),
+        None => "no-location".to_string(),
+    };
+
+    (Symbol::intern(&file_name), lo_line, lo_col, hi_line, hi_col)
+}
+
 fn item_path(mod_path: &[Ident], item_ident: &Ident) -> String {
     mod_path
         .iter()
diff --git a/compiler/rustc_span/src/source_map.rs b/compiler/rustc_span/src/source_map.rs
index a1cb810a429..ee895f53eba 100644
--- a/compiler/rustc_span/src/source_map.rs
+++ b/compiler/rustc_span/src/source_map.rs
@@ -448,25 +448,36 @@ impl SourceMap {
         sp: Span,
         filename_display_pref: FileNameDisplayPreference,
     ) -> String {
-        if self.files.borrow().source_files.is_empty() || sp.is_dummy() {
-            return "no-location".to_string();
-        }
+        let (source_file, lo_line, lo_col, hi_line, hi_col) = self.span_to_location_info(sp);
+
+        let file_name = match source_file {
+            Some(sf) => sf.name.display(filename_display_pref).to_string(),
+            None => return "no-location".to_string(),
+        };
 
-        let lo = self.lookup_char_pos(sp.lo());
-        let hi = self.lookup_char_pos(sp.hi());
         format!(
-            "{}:{}:{}{}",
-            lo.file.name.display(filename_display_pref),
-            lo.line,
-            lo.col.to_usize() + 1,
+            "{file_name}:{lo_line}:{lo_col}{}",
             if let FileNameDisplayPreference::Short = filename_display_pref {
                 String::new()
             } else {
-                format!(": {}:{}", hi.line, hi.col.to_usize() + 1)
+                format!(": {hi_line}:{hi_col}")
             }
         )
     }
 
+    pub fn span_to_location_info(
+        &self,
+        sp: Span,
+    ) -> (Option<Lrc<SourceFile>>, usize, usize, usize, usize) {
+        if self.files.borrow().source_files.is_empty() || sp.is_dummy() {
+            return (None, 0, 0, 0, 0);
+        }
+
+        let lo = self.lookup_char_pos(sp.lo());
+        let hi = self.lookup_char_pos(sp.hi());
+        (Some(lo.file), lo.line, lo.col.to_usize() + 1, hi.line, hi.col.to_usize() + 1)
+    }
+
     /// Format the span location suitable for embedding in build artifacts
     pub fn span_to_embeddable_string(&self, sp: Span) -> String {
         self.span_to_string(sp, FileNameDisplayPreference::Remapped)
diff --git a/library/test/src/console.rs b/library/test/src/console.rs
index 1ee68c8540b..7eee4ca2361 100644
--- a/library/test/src/console.rs
+++ b/library/test/src/console.rs
@@ -41,6 +41,46 @@ impl<T: Write> Write for OutputLocation<T> {
     }
 }
 
+pub struct ConsoleTestDiscoveryState {
+    pub log_out: Option<File>,
+    pub tests: usize,
+    pub benchmarks: usize,
+    pub ignored: usize,
+    pub options: Options,
+}
+
+impl ConsoleTestDiscoveryState {
+    pub fn new(opts: &TestOpts) -> io::Result<ConsoleTestDiscoveryState> {
+        let log_out = match opts.logfile {
+            Some(ref path) => Some(File::create(path)?),
+            None => None,
+        };
+
+        Ok(ConsoleTestDiscoveryState {
+            log_out,
+            tests: 0,
+            benchmarks: 0,
+            ignored: 0,
+            options: opts.options,
+        })
+    }
+
+    pub fn write_log<F, S>(&mut self, msg: F) -> io::Result<()>
+    where
+        S: AsRef<str>,
+        F: FnOnce() -> S,
+    {
+        match self.log_out {
+            None => Ok(()),
+            Some(ref mut o) => {
+                let msg = msg();
+                let msg = msg.as_ref();
+                o.write_all(msg.as_bytes())
+            }
+        }
+    }
+}
+
 pub struct ConsoleTestState {
     pub log_out: Option<File>,
     pub total: usize,
@@ -138,53 +178,44 @@ impl ConsoleTestState {
 
 // List the tests to console, and optionally to logfile. Filters are honored.
 pub fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<()> {
-    let mut output = match term::stdout() {
+    let output = match term::stdout() {
         None => OutputLocation::Raw(io::stdout().lock()),
         Some(t) => OutputLocation::Pretty(t),
     };
 
-    let quiet = opts.format == OutputFormat::Terse;
-    let mut st = ConsoleTestState::new(opts)?;
-
-    let mut ntest = 0;
-    let mut nbench = 0;
+    let mut out: Box<dyn OutputFormatter> = match opts.format {
+        OutputFormat::Pretty | OutputFormat::Junit => {
+            Box::new(PrettyFormatter::new(output, false, 0, false, None))
+        }
+        OutputFormat::Terse => Box::new(TerseFormatter::new(output, false, 0, false)),
+        OutputFormat::Json => Box::new(JsonFormatter::new(output)),
+    };
+    let mut st = ConsoleTestDiscoveryState::new(opts)?;
 
+    out.write_discovery_start()?;
     for test in filter_tests(opts, tests).into_iter() {
         use crate::TestFn::*;
 
-        let TestDescAndFn { desc: TestDesc { name, .. }, testfn } = test;
+        let TestDescAndFn { desc, testfn } = test;
 
         let fntype = match testfn {
             StaticTestFn(..) | DynTestFn(..) => {
-                ntest += 1;
+                st.tests += 1;
                 "test"
             }
             StaticBenchFn(..) | DynBenchFn(..) => {
-                nbench += 1;
+                st.benchmarks += 1;
                 "benchmark"
             }
         };
 
-        writeln!(output, "{name}: {fntype}")?;
-        st.write_log(|| format!("{fntype} {name}\n"))?;
-    }
+        st.ignored += if desc.ignore { 1 } else { 0 };
 
-    fn plural(count: u32, s: &str) -> String {
-        match count {
-            1 => format!("1 {s}"),
-            n => format!("{n} {s}s"),
-        }
+        out.write_test_discovered(&desc, fntype)?;
+        st.write_log(|| format!("{fntype} {}\n", desc.name))?;
     }
 
-    if !quiet {
-        if ntest != 0 || nbench != 0 {
-            writeln!(output)?;
-        }
-
-        writeln!(output, "{}, {}", plural(ntest, "test"), plural(nbench, "benchmark"))?;
-    }
-
-    Ok(())
+    out.write_discovery_finish(&st)
 }
 
 // Updates `ConsoleTestState` depending on result of the test execution.
diff --git a/library/test/src/formatters/json.rs b/library/test/src/formatters/json.rs
index 95d2faf2506..40976ec5e1c 100644
--- a/library/test/src/formatters/json.rs
+++ b/library/test/src/formatters/json.rs
@@ -2,7 +2,7 @@ use std::{borrow::Cow, io, io::prelude::Write};
 
 use super::OutputFormatter;
 use crate::{
-    console::{ConsoleTestState, OutputLocation},
+    console::{ConsoleTestDiscoveryState, ConsoleTestState, OutputLocation},
     test_result::TestResult,
     time,
     types::TestDesc,
@@ -60,6 +60,56 @@ impl<T: Write> JsonFormatter<T> {
 }
 
 impl<T: Write> OutputFormatter for JsonFormatter<T> {
+    fn write_discovery_start(&mut self) -> io::Result<()> {
+        self.writeln_message(&format!(r#"{{ "type": "suite", "event": "discovery" }}"#))
+    }
+
+    fn write_test_discovered(&mut self, desc: &TestDesc, test_type: &str) -> io::Result<()> {
+        let TestDesc {
+            name,
+            ignore,
+            ignore_message,
+            #[cfg(not(bootstrap))]
+            source_file,
+            #[cfg(not(bootstrap))]
+            start_line,
+            #[cfg(not(bootstrap))]
+            start_col,
+            #[cfg(not(bootstrap))]
+            end_line,
+            #[cfg(not(bootstrap))]
+            end_col,
+            ..
+        } = desc;
+
+        #[cfg(bootstrap)]
+        let source_file = "";
+        #[cfg(bootstrap)]
+        let start_line = 0;
+        #[cfg(bootstrap)]
+        let start_col = 0;
+        #[cfg(bootstrap)]
+        let end_line = 0;
+        #[cfg(bootstrap)]
+        let end_col = 0;
+
+        self.writeln_message(&format!(
+            r#"{{ "type": "{test_type}", "event": "discovered", "name": "{}", "ignore": {ignore}, "ignore_message": "{}", "source_path": "{}", "start_line": {start_line}, "start_col": {start_col}, "end_line": {end_line}, "end_col": {end_col} }}"#,
+            EscapedString(name.as_slice()),
+            ignore_message.unwrap_or(""),
+            EscapedString(source_file),
+        ))
+    }
+
+    fn write_discovery_finish(&mut self, state: &ConsoleTestDiscoveryState) -> io::Result<()> {
+        let ConsoleTestDiscoveryState { tests, benchmarks, ignored, .. } = state;
+
+        let total = tests + benchmarks;
+        self.writeln_message(&format!(
+            r#"{{ "type": "suite", "event": "completed", "tests": {tests}, "benchmarks": {benchmarks}, "total": {total}, "ignored": {ignored} }}"#
+        ))
+    }
+
     fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option<u64>) -> io::Result<()> {
         let shuffle_seed_json = if let Some(shuffle_seed) = shuffle_seed {
             format!(r#", "shuffle_seed": {shuffle_seed}"#)
diff --git a/library/test/src/formatters/junit.rs b/library/test/src/formatters/junit.rs
index 7a40ce33cb7..2e07ce3c099 100644
--- a/library/test/src/formatters/junit.rs
+++ b/library/test/src/formatters/junit.rs
@@ -3,7 +3,7 @@ use std::time::Duration;
 
 use super::OutputFormatter;
 use crate::{
-    console::{ConsoleTestState, OutputLocation},
+    console::{ConsoleTestDiscoveryState, ConsoleTestState, OutputLocation},
     test_result::TestResult,
     time,
     types::{TestDesc, TestType},
@@ -27,6 +27,18 @@ impl<T: Write> JunitFormatter<T> {
 }
 
 impl<T: Write> OutputFormatter for JunitFormatter<T> {
+    fn write_discovery_start(&mut self) -> io::Result<()> {
+        Err(io::Error::new(io::ErrorKind::NotFound, "Not yet implemented!"))
+    }
+
+    fn write_test_discovered(&mut self, _desc: &TestDesc, _test_type: &str) -> io::Result<()> {
+        Err(io::Error::new(io::ErrorKind::NotFound, "Not yet implemented!"))
+    }
+
+    fn write_discovery_finish(&mut self, _state: &ConsoleTestDiscoveryState) -> io::Result<()> {
+        Err(io::Error::new(io::ErrorKind::NotFound, "Not yet implemented!"))
+    }
+
     fn write_run_start(
         &mut self,
         _test_count: usize,
diff --git a/library/test/src/formatters/mod.rs b/library/test/src/formatters/mod.rs
index cb67b6491a3..bc6ffebc1d3 100644
--- a/library/test/src/formatters/mod.rs
+++ b/library/test/src/formatters/mod.rs
@@ -1,7 +1,7 @@
 use std::{io, io::prelude::Write};
 
 use crate::{
-    console::ConsoleTestState,
+    console::{ConsoleTestDiscoveryState, ConsoleTestState},
     test_result::TestResult,
     time,
     types::{TestDesc, TestName},
@@ -18,6 +18,10 @@ pub(crate) use self::pretty::PrettyFormatter;
 pub(crate) use self::terse::TerseFormatter;
 
 pub(crate) trait OutputFormatter {
+    fn write_discovery_start(&mut self) -> io::Result<()>;
+    fn write_test_discovered(&mut self, desc: &TestDesc, test_type: &str) -> io::Result<()>;
+    fn write_discovery_finish(&mut self, state: &ConsoleTestDiscoveryState) -> io::Result<()>;
+
     fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option<u64>) -> io::Result<()>;
     fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()>;
     fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>;
diff --git a/library/test/src/formatters/pretty.rs b/library/test/src/formatters/pretty.rs
index 247778e515f..22654a3400b 100644
--- a/library/test/src/formatters/pretty.rs
+++ b/library/test/src/formatters/pretty.rs
@@ -3,7 +3,7 @@ use std::{io, io::prelude::Write};
 use super::OutputFormatter;
 use crate::{
     bench::fmt_bench_samples,
-    console::{ConsoleTestState, OutputLocation},
+    console::{ConsoleTestDiscoveryState, ConsoleTestState, OutputLocation},
     term,
     test_result::TestResult,
     time,
@@ -181,6 +181,33 @@ impl<T: Write> PrettyFormatter<T> {
 }
 
 impl<T: Write> OutputFormatter for PrettyFormatter<T> {
+    fn write_discovery_start(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+
+    fn write_test_discovered(&mut self, desc: &TestDesc, test_type: &str) -> io::Result<()> {
+        self.write_plain(format!("{}: {test_type}\n", desc.name))
+    }
+
+    fn write_discovery_finish(&mut self, state: &ConsoleTestDiscoveryState) -> io::Result<()> {
+        fn plural(count: usize, s: &str) -> String {
+            match count {
+                1 => format!("1 {s}"),
+                n => format!("{n} {s}s"),
+            }
+        }
+
+        if state.tests != 0 || state.benchmarks != 0 {
+            self.write_plain("\n")?;
+        }
+
+        self.write_plain(format!(
+            "{}, {}\n",
+            plural(state.tests, "test"),
+            plural(state.benchmarks, "benchmark")
+        ))
+    }
+
     fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option<u64>) -> io::Result<()> {
         let noun = if test_count != 1 { "tests" } else { "test" };
         let shuffle_seed_msg = if let Some(shuffle_seed) = shuffle_seed {
diff --git a/library/test/src/formatters/terse.rs b/library/test/src/formatters/terse.rs
index a431acfbc27..2931ca6ead0 100644
--- a/library/test/src/formatters/terse.rs
+++ b/library/test/src/formatters/terse.rs
@@ -3,7 +3,7 @@ use std::{io, io::prelude::Write};
 use super::OutputFormatter;
 use crate::{
     bench::fmt_bench_samples,
-    console::{ConsoleTestState, OutputLocation},
+    console::{ConsoleTestDiscoveryState, ConsoleTestState, OutputLocation},
     term,
     test_result::TestResult,
     time,
@@ -167,6 +167,18 @@ impl<T: Write> TerseFormatter<T> {
 }
 
 impl<T: Write> OutputFormatter for TerseFormatter<T> {
+    fn write_discovery_start(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+
+    fn write_test_discovered(&mut self, desc: &TestDesc, test_type: &str) -> io::Result<()> {
+        self.write_plain(format!("{}: {test_type}\n", desc.name))
+    }
+
+    fn write_discovery_finish(&mut self, _state: &ConsoleTestDiscoveryState) -> io::Result<()> {
+        Ok(())
+    }
+
     fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option<u64>) -> io::Result<()> {
         self.total_test_count = test_count;
         let noun = if test_count != 1 { "tests" } else { "test" };
diff --git a/library/test/src/tests.rs b/library/test/src/tests.rs
index 44776fb0a31..5ffdbf73fbf 100644
--- a/library/test/src/tests.rs
+++ b/library/test/src/tests.rs
@@ -63,6 +63,16 @@ fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
                 name: StaticTestName("1"),
                 ignore: true,
                 ignore_message: None,
+                #[cfg(not(bootstrap))]
+                source_file: "",
+                #[cfg(not(bootstrap))]
+                start_line: 0,
+                #[cfg(not(bootstrap))]
+                start_col: 0,
+                #[cfg(not(bootstrap))]
+                end_line: 0,
+                #[cfg(not(bootstrap))]
+                end_col: 0,
                 should_panic: ShouldPanic::No,
                 compile_fail: false,
                 no_run: false,
@@ -75,6 +85,16 @@ fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
                 name: StaticTestName("2"),
                 ignore: false,
                 ignore_message: None,
+                #[cfg(not(bootstrap))]
+                source_file: "",
+                #[cfg(not(bootstrap))]
+                start_line: 0,
+                #[cfg(not(bootstrap))]
+                start_col: 0,
+                #[cfg(not(bootstrap))]
+                end_line: 0,
+                #[cfg(not(bootstrap))]
+                end_col: 0,
                 should_panic: ShouldPanic::No,
                 compile_fail: false,
                 no_run: false,
@@ -95,6 +115,16 @@ pub fn do_not_run_ignored_tests() {
             name: StaticTestName("whatever"),
             ignore: true,
             ignore_message: None,
+            #[cfg(not(bootstrap))]
+            source_file: "",
+            #[cfg(not(bootstrap))]
+            start_line: 0,
+            #[cfg(not(bootstrap))]
+            start_col: 0,
+            #[cfg(not(bootstrap))]
+            end_line: 0,
+            #[cfg(not(bootstrap))]
+            end_col: 0,
             should_panic: ShouldPanic::No,
             compile_fail: false,
             no_run: false,
@@ -118,6 +148,16 @@ pub fn ignored_tests_result_in_ignored() {
             name: StaticTestName("whatever"),
             ignore: true,
             ignore_message: None,
+            #[cfg(not(bootstrap))]
+            source_file: "",
+            #[cfg(not(bootstrap))]
+            start_line: 0,
+            #[cfg(not(bootstrap))]
+            start_col: 0,
+            #[cfg(not(bootstrap))]
+            end_line: 0,
+            #[cfg(not(bootstrap))]
+            end_col: 0,
             should_panic: ShouldPanic::No,
             compile_fail: false,
             no_run: false,
@@ -143,6 +183,16 @@ fn test_should_panic() {
             name: StaticTestName("whatever"),
             ignore: false,
             ignore_message: None,
+            #[cfg(not(bootstrap))]
+            source_file: "",
+            #[cfg(not(bootstrap))]
+            start_line: 0,
+            #[cfg(not(bootstrap))]
+            start_col: 0,
+            #[cfg(not(bootstrap))]
+            end_line: 0,
+            #[cfg(not(bootstrap))]
+            end_col: 0,
             should_panic: ShouldPanic::Yes,
             compile_fail: false,
             no_run: false,
@@ -168,6 +218,16 @@ fn test_should_panic_good_message() {
             name: StaticTestName("whatever"),
             ignore: false,
             ignore_message: None,
+            #[cfg(not(bootstrap))]
+            source_file: "",
+            #[cfg(not(bootstrap))]
+            start_line: 0,
+            #[cfg(not(bootstrap))]
+            start_col: 0,
+            #[cfg(not(bootstrap))]
+            end_line: 0,
+            #[cfg(not(bootstrap))]
+            end_col: 0,
             should_panic: ShouldPanic::YesWithMessage("error message"),
             compile_fail: false,
             no_run: false,
@@ -198,6 +258,16 @@ fn test_should_panic_bad_message() {
             name: StaticTestName("whatever"),
             ignore: false,
             ignore_message: None,
+            #[cfg(not(bootstrap))]
+            source_file: "",
+            #[cfg(not(bootstrap))]
+            start_line: 0,
+            #[cfg(not(bootstrap))]
+            start_col: 0,
+            #[cfg(not(bootstrap))]
+            end_line: 0,
+            #[cfg(not(bootstrap))]
+            end_col: 0,
             should_panic: ShouldPanic::YesWithMessage(expected),
             compile_fail: false,
             no_run: false,
@@ -232,6 +302,16 @@ fn test_should_panic_non_string_message_type() {
             name: StaticTestName("whatever"),
             ignore: false,
             ignore_message: None,
+            #[cfg(not(bootstrap))]
+            source_file: "",
+            #[cfg(not(bootstrap))]
+            start_line: 0,
+            #[cfg(not(bootstrap))]
+            start_col: 0,
+            #[cfg(not(bootstrap))]
+            end_line: 0,
+            #[cfg(not(bootstrap))]
+            end_col: 0,
             should_panic: ShouldPanic::YesWithMessage(expected),
             compile_fail: false,
             no_run: false,
@@ -260,6 +340,16 @@ fn test_should_panic_but_succeeds() {
                 name: StaticTestName("whatever"),
                 ignore: false,
                 ignore_message: None,
+                #[cfg(not(bootstrap))]
+                source_file: "",
+                #[cfg(not(bootstrap))]
+                start_line: 0,
+                #[cfg(not(bootstrap))]
+                start_col: 0,
+                #[cfg(not(bootstrap))]
+                end_line: 0,
+                #[cfg(not(bootstrap))]
+                end_col: 0,
                 should_panic,
                 compile_fail: false,
                 no_run: false,
@@ -288,6 +378,16 @@ fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
             name: StaticTestName("whatever"),
             ignore: false,
             ignore_message: None,
+            #[cfg(not(bootstrap))]
+            source_file: "",
+            #[cfg(not(bootstrap))]
+            start_line: 0,
+            #[cfg(not(bootstrap))]
+            start_col: 0,
+            #[cfg(not(bootstrap))]
+            end_line: 0,
+            #[cfg(not(bootstrap))]
+            end_col: 0,
             should_panic: ShouldPanic::No,
             compile_fail: false,
             no_run: false,
@@ -325,6 +425,16 @@ fn time_test_failure_template(test_type: TestType) -> TestResult {
             name: StaticTestName("whatever"),
             ignore: false,
             ignore_message: None,
+            #[cfg(not(bootstrap))]
+            source_file: "",
+            #[cfg(not(bootstrap))]
+            start_line: 0,
+            #[cfg(not(bootstrap))]
+            start_col: 0,
+            #[cfg(not(bootstrap))]
+            end_line: 0,
+            #[cfg(not(bootstrap))]
+            end_col: 0,
             should_panic: ShouldPanic::No,
             compile_fail: false,
             no_run: false,
@@ -364,6 +474,16 @@ fn typed_test_desc(test_type: TestType) -> TestDesc {
         name: StaticTestName("whatever"),
         ignore: false,
         ignore_message: None,
+        #[cfg(not(bootstrap))]
+        source_file: "",
+        #[cfg(not(bootstrap))]
+        start_line: 0,
+        #[cfg(not(bootstrap))]
+        start_col: 0,
+        #[cfg(not(bootstrap))]
+        end_line: 0,
+        #[cfg(not(bootstrap))]
+        end_col: 0,
         should_panic: ShouldPanic::No,
         compile_fail: false,
         no_run: false,
@@ -476,6 +596,16 @@ pub fn exclude_should_panic_option() {
             name: StaticTestName("3"),
             ignore: false,
             ignore_message: None,
+            #[cfg(not(bootstrap))]
+            source_file: "",
+            #[cfg(not(bootstrap))]
+            start_line: 0,
+            #[cfg(not(bootstrap))]
+            start_col: 0,
+            #[cfg(not(bootstrap))]
+            end_line: 0,
+            #[cfg(not(bootstrap))]
+            end_col: 0,
             should_panic: ShouldPanic::Yes,
             compile_fail: false,
             no_run: false,
@@ -500,6 +630,16 @@ pub fn exact_filter_match() {
                     name: StaticTestName(name),
                     ignore: false,
                     ignore_message: None,
+                    #[cfg(not(bootstrap))]
+                    source_file: "",
+                    #[cfg(not(bootstrap))]
+                    start_line: 0,
+                    #[cfg(not(bootstrap))]
+                    start_col: 0,
+                    #[cfg(not(bootstrap))]
+                    end_line: 0,
+                    #[cfg(not(bootstrap))]
+                    end_col: 0,
                     should_panic: ShouldPanic::No,
                     compile_fail: false,
                     no_run: false,
@@ -591,6 +731,16 @@ fn sample_tests() -> Vec<TestDescAndFn> {
                 name: DynTestName((*name).clone()),
                 ignore: false,
                 ignore_message: None,
+                #[cfg(not(bootstrap))]
+                source_file: "",
+                #[cfg(not(bootstrap))]
+                start_line: 0,
+                #[cfg(not(bootstrap))]
+                start_col: 0,
+                #[cfg(not(bootstrap))]
+                end_line: 0,
+                #[cfg(not(bootstrap))]
+                end_col: 0,
                 should_panic: ShouldPanic::No,
                 compile_fail: false,
                 no_run: false,
@@ -720,6 +870,16 @@ pub fn test_bench_no_iter() {
         name: StaticTestName("f"),
         ignore: false,
         ignore_message: None,
+        #[cfg(not(bootstrap))]
+        source_file: "",
+        #[cfg(not(bootstrap))]
+        start_line: 0,
+        #[cfg(not(bootstrap))]
+        start_col: 0,
+        #[cfg(not(bootstrap))]
+        end_line: 0,
+        #[cfg(not(bootstrap))]
+        end_col: 0,
         should_panic: ShouldPanic::No,
         compile_fail: false,
         no_run: false,
@@ -743,6 +903,16 @@ pub fn test_bench_iter() {
         name: StaticTestName("f"),
         ignore: false,
         ignore_message: None,
+        #[cfg(not(bootstrap))]
+        source_file: "",
+        #[cfg(not(bootstrap))]
+        start_line: 0,
+        #[cfg(not(bootstrap))]
+        start_col: 0,
+        #[cfg(not(bootstrap))]
+        end_line: 0,
+        #[cfg(not(bootstrap))]
+        end_col: 0,
         should_panic: ShouldPanic::No,
         compile_fail: false,
         no_run: false,
@@ -759,6 +929,16 @@ fn should_sort_failures_before_printing_them() {
         name: StaticTestName("a"),
         ignore: false,
         ignore_message: None,
+        #[cfg(not(bootstrap))]
+        source_file: "",
+        #[cfg(not(bootstrap))]
+        start_line: 0,
+        #[cfg(not(bootstrap))]
+        start_col: 0,
+        #[cfg(not(bootstrap))]
+        end_line: 0,
+        #[cfg(not(bootstrap))]
+        end_col: 0,
         should_panic: ShouldPanic::No,
         compile_fail: false,
         no_run: false,
@@ -769,6 +949,16 @@ fn should_sort_failures_before_printing_them() {
         name: StaticTestName("b"),
         ignore: false,
         ignore_message: None,
+        #[cfg(not(bootstrap))]
+        source_file: "",
+        #[cfg(not(bootstrap))]
+        start_line: 0,
+        #[cfg(not(bootstrap))]
+        start_col: 0,
+        #[cfg(not(bootstrap))]
+        end_line: 0,
+        #[cfg(not(bootstrap))]
+        end_col: 0,
         should_panic: ShouldPanic::No,
         compile_fail: false,
         no_run: false,
@@ -816,6 +1006,16 @@ fn test_dyn_bench_returning_err_fails_when_run_as_test() {
             name: StaticTestName("whatever"),
             ignore: false,
             ignore_message: None,
+            #[cfg(not(bootstrap))]
+            source_file: "",
+            #[cfg(not(bootstrap))]
+            start_line: 0,
+            #[cfg(not(bootstrap))]
+            start_col: 0,
+            #[cfg(not(bootstrap))]
+            end_line: 0,
+            #[cfg(not(bootstrap))]
+            end_col: 0,
             should_panic: ShouldPanic::No,
             compile_fail: false,
             no_run: false,
diff --git a/library/test/src/types.rs b/library/test/src/types.rs
index 6f2e033095a..8d4e204c8ac 100644
--- a/library/test/src/types.rs
+++ b/library/test/src/types.rs
@@ -119,6 +119,16 @@ pub struct TestDesc {
     pub name: TestName,
     pub ignore: bool,
     pub ignore_message: Option<&'static str>,
+    #[cfg(not(bootstrap))]
+    pub source_file: &'static str,
+    #[cfg(not(bootstrap))]
+    pub start_line: usize,
+    #[cfg(not(bootstrap))]
+    pub start_col: usize,
+    #[cfg(not(bootstrap))]
+    pub end_line: usize,
+    #[cfg(not(bootstrap))]
+    pub end_col: usize,
     pub should_panic: options::ShouldPanic,
     pub compile_fail: bool,
     pub no_run: bool,
diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs
index 9cf84acc79f..aaa83ecce48 100644
--- a/src/librustdoc/doctest.rs
+++ b/src/librustdoc/doctest.rs
@@ -1057,6 +1057,16 @@ impl Tester for Collector {
                     Ignore::Some(ref ignores) => ignores.iter().any(|s| target_str.contains(s)),
                 },
                 ignore_message: None,
+                #[cfg(not(bootstrap))]
+                source_file: "",
+                #[cfg(not(bootstrap))]
+                start_line: 0,
+                #[cfg(not(bootstrap))]
+                start_col: 0,
+                #[cfg(not(bootstrap))]
+                end_line: 0,
+                #[cfg(not(bootstrap))]
+                end_col: 0,
                 // compiler failures are test failures
                 should_panic: test::ShouldPanic::No,
                 compile_fail: config.compile_fail,
diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs
index d9b39927ca4..22a0b1d13be 100644
--- a/src/tools/compiletest/src/header.rs
+++ b/src/tools/compiletest/src/header.rs
@@ -1047,6 +1047,16 @@ pub fn make_test_description<R: Read>(
         name,
         ignore,
         ignore_message,
+        #[cfg(not(bootstrap))]
+        source_file: "",
+        #[cfg(not(bootstrap))]
+        start_line: 0,
+        #[cfg(not(bootstrap))]
+        start_col: 0,
+        #[cfg(not(bootstrap))]
+        end_line: 0,
+        #[cfg(not(bootstrap))]
+        end_col: 0,
         should_panic,
         compile_fail: false,
         no_run: false,
diff --git a/tests/pretty/tests-are-sorted.pp b/tests/pretty/tests-are-sorted.pp
index 15dcd4ed97d..58f746f2e0e 100644
--- a/tests/pretty/tests-are-sorted.pp
+++ b/tests/pretty/tests-are-sorted.pp
@@ -4,7 +4,7 @@
 use ::std::prelude::rust_2015::*;
 #[macro_use]
 extern crate std;
-// compile-flags: --crate-type=lib --test
+// compile-flags: --crate-type=lib --test --remap-path-prefix={{src-base}}/=/the/src/ --remap-path-prefix={{src-base}}\=/the/src/
 // pretty-compare-only
 // pretty-mode:expanded
 // pp-exact:tests-are-sorted.pp
@@ -18,6 +18,11 @@ pub const m_test: test::TestDescAndFn =
             name: test::StaticTestName("m_test"),
             ignore: false,
             ignore_message: ::core::option::Option::None,
+            source_file: "/the/src/tests-are-sorted.rs",
+            start_line: 7usize,
+            start_col: 4usize,
+            end_line: 7usize,
+            end_col: 10usize,
             compile_fail: false,
             no_run: false,
             should_panic: test::ShouldPanic::No,
@@ -34,8 +39,13 @@ pub const z_test: test::TestDescAndFn =
     test::TestDescAndFn {
         desc: test::TestDesc {
             name: test::StaticTestName("z_test"),
-            ignore: false,
-            ignore_message: ::core::option::Option::None,
+            ignore: true,
+            ignore_message: ::core::option::Option::Some("not yet implemented"),
+            source_file: "/the/src/tests-are-sorted.rs",
+            start_line: 11usize,
+            start_col: 4usize,
+            end_line: 11usize,
+            end_col: 10usize,
             compile_fail: false,
             no_run: false,
             should_panic: test::ShouldPanic::No,
@@ -43,6 +53,7 @@ pub const z_test: test::TestDescAndFn =
         },
         testfn: test::StaticTestFn(|| test::assert_test_result(z_test())),
     };
+#[ignore = "not yet implemented"]
 fn z_test() {}
 
 extern crate test;
@@ -54,6 +65,11 @@ pub const a_test: test::TestDescAndFn =
             name: test::StaticTestName("a_test"),
             ignore: false,
             ignore_message: ::core::option::Option::None,
+            source_file: "/the/src/tests-are-sorted.rs",
+            start_line: 14usize,
+            start_col: 4usize,
+            end_line: 14usize,
+            end_col: 10usize,
             compile_fail: false,
             no_run: false,
             should_panic: test::ShouldPanic::No,
diff --git a/tests/pretty/tests-are-sorted.rs b/tests/pretty/tests-are-sorted.rs
index 1f737d54719..39e0922250b 100644
--- a/tests/pretty/tests-are-sorted.rs
+++ b/tests/pretty/tests-are-sorted.rs
@@ -1,4 +1,4 @@
-// compile-flags: --crate-type=lib --test
+// compile-flags: --crate-type=lib --test --remap-path-prefix={{src-base}}/=/the/src/ --remap-path-prefix={{src-base}}\=/the/src/
 // pretty-compare-only
 // pretty-mode:expanded
 // pp-exact:tests-are-sorted.pp
@@ -7,6 +7,7 @@
 fn m_test() {}
 
 #[test]
+#[ignore = "not yet implemented"]
 fn z_test() {}
 
 #[test]
diff --git a/tests/ui/test-attrs/tests-listing-format-default.rs b/tests/ui/test-attrs/tests-listing-format-default.rs
new file mode 100644
index 00000000000..d5df4b57b05
--- /dev/null
+++ b/tests/ui/test-attrs/tests-listing-format-default.rs
@@ -0,0 +1,18 @@
+// no-prefer-dynamic
+// compile-flags: --test
+// run-flags: --list
+// run-pass
+// check-run-results
+
+// Checks the listing of tests with no --format arguments.
+
+#![cfg(test)]
+#[test]
+fn m_test() {}
+
+#[test]
+#[ignore = "not yet implemented"]
+fn z_test() {}
+
+#[test]
+fn a_test() {}
diff --git a/tests/ui/test-attrs/tests-listing-format-default.run.stdout b/tests/ui/test-attrs/tests-listing-format-default.run.stdout
new file mode 100644
index 00000000000..72337daf02c
--- /dev/null
+++ b/tests/ui/test-attrs/tests-listing-format-default.run.stdout
@@ -0,0 +1,5 @@
+a_test: test
+m_test: test
+z_test: test
+
+3 tests, 0 benchmarks
diff --git a/tests/ui/test-attrs/tests-listing-format-json-without-unstableopts.rs b/tests/ui/test-attrs/tests-listing-format-json-without-unstableopts.rs
new file mode 100644
index 00000000000..5247f1f8f17
--- /dev/null
+++ b/tests/ui/test-attrs/tests-listing-format-json-without-unstableopts.rs
@@ -0,0 +1,18 @@
+// no-prefer-dynamic
+// compile-flags: --test
+// run-flags: --list --format json
+// run-fail
+// check-run-results
+
+// Checks that --format json does not work without -Zunstable-options.
+
+#![cfg(test)]
+#[test]
+fn m_test() {}
+
+#[test]
+#[ignore = "not yet implemented"]
+fn z_test() {}
+
+#[test]
+fn a_test() {}
diff --git a/tests/ui/test-attrs/tests-listing-format-json-without-unstableopts.run.stderr b/tests/ui/test-attrs/tests-listing-format-json-without-unstableopts.run.stderr
new file mode 100644
index 00000000000..9f6276300a0
--- /dev/null
+++ b/tests/ui/test-attrs/tests-listing-format-json-without-unstableopts.run.stderr
@@ -0,0 +1 @@
+error: The "json" format is only accepted on the nightly compiler
diff --git a/tests/ui/test-attrs/tests-listing-format-json.rs b/tests/ui/test-attrs/tests-listing-format-json.rs
new file mode 100644
index 00000000000..18f1521eeeb
--- /dev/null
+++ b/tests/ui/test-attrs/tests-listing-format-json.rs
@@ -0,0 +1,20 @@
+// no-prefer-dynamic
+// compile-flags: --test
+// run-flags: --list --format json -Zunstable-options
+// run-pass
+// check-run-results
+// normalize-stdout-test: "fake-test-src-base/test-attrs/" -> "$$DIR/"
+// normalize-stdout-test: "fake-test-src-base\\test-attrs\\" -> "$$DIR/"
+
+// Checks the listing of tests with --format json.
+
+#![cfg(test)]
+#[test]
+fn m_test() {}
+
+#[test]
+#[ignore = "not yet implemented"]
+fn z_test() {}
+
+#[test]
+fn a_test() {}
diff --git a/tests/ui/test-attrs/tests-listing-format-json.run.stdout b/tests/ui/test-attrs/tests-listing-format-json.run.stdout
new file mode 100644
index 00000000000..b4131e97c34
--- /dev/null
+++ b/tests/ui/test-attrs/tests-listing-format-json.run.stdout
@@ -0,0 +1,5 @@
+{ "type": "suite", "event": "discovery" }
+{ "type": "test", "event": "discovered", "name": "a_test", "ignore": false, "ignore_message": "", "source_path": "$DIR/tests-listing-format-json.rs", "start_line": 20, "start_col": 4, "end_line": 20, "end_col": 10 }
+{ "type": "test", "event": "discovered", "name": "m_test", "ignore": false, "ignore_message": "", "source_path": "$DIR/tests-listing-format-json.rs", "start_line": 13, "start_col": 4, "end_line": 13, "end_col": 10 }
+{ "type": "test", "event": "discovered", "name": "z_test", "ignore": true, "ignore_message": "not yet implemented", "source_path": "$DIR/tests-listing-format-json.rs", "start_line": 17, "start_col": 4, "end_line": 17, "end_col": 10 }
+{ "type": "suite", "event": "completed", "tests": 3, "benchmarks": 0, "total": 3, "ignored": 1 }
diff --git a/tests/ui/test-attrs/tests-listing-format-terse.rs b/tests/ui/test-attrs/tests-listing-format-terse.rs
new file mode 100644
index 00000000000..7835f71759c
--- /dev/null
+++ b/tests/ui/test-attrs/tests-listing-format-terse.rs
@@ -0,0 +1,18 @@
+// no-prefer-dynamic
+// compile-flags: --test
+// run-flags: --list --format terse
+// run-pass
+// check-run-results
+
+// Checks the listing of tests with --format terse.
+
+#![cfg(test)]
+#[test]
+fn m_test() {}
+
+#[test]
+#[ignore = "not yet implemented"]
+fn z_test() {}
+
+#[test]
+fn a_test() {}
diff --git a/tests/ui/test-attrs/tests-listing-format-terse.run.stdout b/tests/ui/test-attrs/tests-listing-format-terse.run.stdout
new file mode 100644
index 00000000000..22afe104bfb
--- /dev/null
+++ b/tests/ui/test-attrs/tests-listing-format-terse.run.stdout
@@ -0,0 +1,3 @@
+a_test: test
+m_test: test
+z_test: test