Auto merge of rust-lang#3008 - oli-obk:ui_test_progress_bars, r=RalfJung

bump ui test crate

The recommended way to run tests locally is `./miri bless -- -- --quiet`, which will show

* progress bars
* the currently running tests (allowing you to see which ones are still running towards the end of the test suite)
* the output of the currently running tests (if they are slow). This means slow running tests can output lines to `stderr` and the last line will be shown after the test name and updated every few hundred milliseconds.

As a side effect this PR also fixes rust-lang#2998 and only builds dependencies if any tests actually need them (this means that with the next ui_test update we'll be able to merge all our test suites).

Also fixes rust-lang/miri#3052.

src/tools/miri/Cargo.lock

@@ -26,6 +26,25 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "annotate-snippets"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3b9d411ecbaf79885c6df4d75fff75858d5995ff25385657a28af47e82f9c36"
+dependencies = [
+ "unicode-width",
+ "yansi-term",
+]
+
+[[package]]
+name = "ansi_term"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.71"
@@ -159,6 +178,25 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "comma"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335"
+
+[[package]]
+name = "console"
+version = "0.15.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8"
+dependencies = [
+ "encode_unicode",
+ "lazy_static",
+ "libc",
+ "unicode-width",
+ "windows-sys 0.45.0",
+]
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.8"
@@ -189,10 +227,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "diff"
-version = "0.1.13"
+name = "encode_unicode"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
+checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
 
 [[package]]
 name = "env_logger"
@@ -282,6 +320,19 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
 
+[[package]]
+name = "indicatif"
+version = "0.17.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25"
+dependencies = [
+ "console",
+ "instant",
+ "number_prefix",
+ "portable-atomic",
+ "unicode-width",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.12"
@@ -325,6 +376,12 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
+[[package]]
+name = "levenshtein"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760"
+
 [[package]]
 name = "libc"
 version = "0.2.148"
@@ -463,6 +520,12 @@ dependencies = [
  "static_assertions",
 ]
 
+[[package]]
+name = "number_prefix"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+
 [[package]]
 name = "object"
 version = "0.30.3"
@@ -484,6 +547,15 @@ version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
 
+[[package]]
+name = "pad"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2ad9b889f1b12e0b9ee24db044b5129150d5eada288edc800f789928dc8c0e3"
+dependencies = [
+ "unicode-width",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.11.2"
@@ -524,12 +596,28 @@ version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
 
+[[package]]
+name = "portable-atomic"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
+[[package]]
+name = "prettydiff"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ff1fec61082821f8236cf6c0c14e8172b62ce8a72a0eedc30d3b247bb68dc11"
+dependencies = [
+ "ansi_term",
+ "pad",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.66"
@@ -859,18 +947,23 @@ dependencies = [
 
 [[package]]
 name = "ui_test"
-version = "0.11.7"
+version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c21899b59f53717dfad29e4f46e5b21a200a1b6888ab86532a07cfc8b48dd78c"
+checksum = "accffe020b57a6dd50014d457b5842c5a2ca73cd84f07d86d0a19c460a6509ae"
 dependencies = [
+ "annotate-snippets",
+ "anyhow",
  "bstr",
  "cargo-platform",
  "cargo_metadata",
  "color-eyre",
  "colored",
+ "comma",
  "crossbeam-channel",
- "diff",
+ "indicatif",
  "lazy_static",
+ "levenshtein",
+ "prettydiff",
  "regex",
  "rustc_version",
  "rustfix",
@@ -885,6 +978,12 @@ version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
 
+[[package]]
+name = "unicode-width"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"
+
 [[package]]
 name = "valuable"
 version = "0.1.0"
@@ -1059,3 +1158,12 @@ name = "windows_x86_64_msvc"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
+
+[[package]]
+name = "yansi-term"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1"
+dependencies = [
+ "winapi",
+]

src/tools/miri/Cargo.toml

@@ -36,7 +36,7 @@ libloading = "0.7"
 
 [dev-dependencies]
 colored = "2"
-ui_test = "0.11.7"
+ui_test = "0.21.1"
 rustc_version = "0.4"
 # Features chosen to match those required by env_logger, to avoid rebuilds
 regex = { version = "1.5.5", default-features = false, features = ["perf", "std"] }

src/tools/miri/README.md

@@ -478,6 +478,8 @@ Moreover, Miri recognizes some environment variables:
 * `MIRI_TEST_TARGET` (recognized by the test suite and the `./miri` script) indicates which target
   architecture to test against.  `miri` and `cargo miri` accept the `--target` flag for the same
   purpose.
+* `MIRI_TEST_THREADS` (recognized by the test suite): set the number of threads to use for running tests.
+  By default the number of cores is used.
 * `MIRI_NO_STD` (recognized by `cargo miri` and the test suite) makes sure that the target's
   sysroot is built without libstd. This allows testing and running no_std programs.
 * `RUSTC_BLESS` (recognized by the test suite and `cargo-miri-test/run-test.py`): overwrite all

src/tools/miri/tests/compiletest.rs

@@ -1,10 +1,11 @@
 use colored::*;
 use regex::bytes::Regex;
 use std::ffi::OsString;
+use std::num::NonZeroUsize;
 use std::path::{Path, PathBuf};
 use std::{env, process::Command};
 use ui_test::{color_eyre::Result, Config, Match, Mode, OutputConflictHandling};
-use ui_test::{status_emitter, CommandBuilder};
+use ui_test::{status_emitter, CommandBuilder, Format, RustfixMode};
 
 fn miri_path() -> PathBuf {
     PathBuf::from(option_env!("MIRI").unwrap_or(env!("CARGO_BIN_EXE_miri")))
@@ -78,26 +79,18 @@ fn test_config(target: &str, path: &str, mode: Mode, with_dependencies: bool) ->
         program.args.push(flag);
     }
 
-    let bless = env::var_os("RUSTC_BLESS").is_some_and(|v| v != "0");
-    let skip_ui_checks = env::var_os("MIRI_SKIP_UI_CHECKS").is_some();
-
-    let output_conflict_handling = match (bless, skip_ui_checks) {
-        (false, false) => OutputConflictHandling::Error("./miri test --bless".into()),
-        (true, false) => OutputConflictHandling::Bless,
-        (false, true) => OutputConflictHandling::Ignore,
-        (true, true) => panic!("cannot use RUSTC_BLESS and MIRI_SKIP_UI_CHECKS at the same time"),
-    };
-
     let mut config = Config {
         target: Some(target.to_owned()),
         stderr_filters: STDERR.clone(),
         stdout_filters: STDOUT.clone(),
         mode,
         program,
-        output_conflict_handling,
         out_dir: PathBuf::from(std::env::var_os("CARGO_TARGET_DIR").unwrap()).join("ui"),
         edition: Some("2021".into()),
-        ..Config::rustc(path.into())
+        threads: std::env::var("MIRI_TEST_THREADS")
+            .ok()
+            .map(|threads| NonZeroUsize::new(threads.parse().unwrap()).unwrap()),
+        ..Config::rustc(path)
     };
 
     let use_std = env::var_os("MIRI_NO_STD").is_none();
@@ -120,51 +113,32 @@ fn test_config(target: &str, path: &str, mode: Mode, with_dependencies: bool) ->
 }
 
 fn run_tests(mode: Mode, path: &str, target: &str, with_dependencies: bool) -> Result<()> {
-    let config = test_config(target, path, mode, with_dependencies);
+    let mut config = test_config(target, path, mode, with_dependencies);
 
     // Handle command-line arguments.
-    let mut after_dashdash = false;
-    let mut quiet = false;
-    let filters = std::env::args()
-        .skip(1)
-        .filter(|arg| {
-            if after_dashdash {
-                // Just propagate everything.
-                return true;
-            }
-            match &**arg {
-                "--quiet" => {
-                    quiet = true;
-                    false
-                }
-                "--" => {
-                    after_dashdash = true;
-                    false
-                }
-                s if s.starts_with('-') => {
-                    panic!("unknown compiletest flag `{s}`");
-                }
-                _ => true,
-            }
-        })
-        .collect::<Vec<_>>();
+    let args = ui_test::Args::test()?;
+    let default_bless = env::var_os("RUSTC_BLESS").is_some_and(|v| v != "0");
+    config.with_args(&args, default_bless);
+    if let OutputConflictHandling::Error(msg) = &mut config.output_conflict_handling {
+        *msg = "./miri test --bless".into();
+    }
+    if env::var_os("MIRI_SKIP_UI_CHECKS").is_some() {
+        assert!(!default_bless, "cannot use RUSTC_BLESS and MIRI_SKIP_UI_CHECKS at the same time");
+        config.output_conflict_handling = OutputConflictHandling::Ignore;
+    }
     eprintln!("   Compiler: {}", config.program.display());
     ui_test::run_tests_generic(
-        config,
+        // Only run one test suite. In the future we can add all test suites to one `Vec` and run
+        // them all at once, making best use of systems with high parallelism.
+        vec![config],
         // The files we're actually interested in (all `.rs` files).
-        |path| {
-            path.extension().is_some_and(|ext| ext == "rs")
-                && (filters.is_empty()
-                    || filters.iter().any(|f| path.display().to_string().contains(f)))
-        },
+        ui_test::default_file_filter,
         // This could be used to overwrite the `Config` on a per-test basis.
-        |_, _| None,
+        |_, _, _| {},
         (
-            if quiet {
-                Box::<status_emitter::Quiet>::default()
-                    as Box<dyn status_emitter::StatusEmitter + Send>
-            } else {
-                Box::new(status_emitter::Text)
+            match args.format {
+                Format::Terse => status_emitter::Text::quiet(),
+                Format::Pretty => status_emitter::Text::verbose(),
             },
             status_emitter::Gha::</* GHA Actions groups*/ false> {
                 name: format!("{mode:?} {path} ({target})"),
@@ -269,11 +243,22 @@ fn main() -> Result<()> {
     ui(Mode::Pass, "tests/pass", &target, WithoutDependencies)?;
     ui(Mode::Pass, "tests/pass-dep", &target, WithDependencies)?;
     ui(Mode::Panic, "tests/panic", &target, WithDependencies)?;
-    ui(Mode::Fail { require_patterns: true }, "tests/fail", &target, WithDependencies)?;
+    ui(
+        Mode::Fail { require_patterns: true, rustfix: RustfixMode::Disabled },
+        "tests/fail",
+        &target,
+        WithoutDependencies,
+    )?;
+    ui(
+        Mode::Fail { require_patterns: true, rustfix: RustfixMode::Disabled },
+        "tests/fail-dep",
+        &target,
+        WithDependencies,
+    )?;
     if cfg!(target_os = "linux") {
         ui(Mode::Pass, "tests/extern-so/pass", &target, WithoutDependencies)?;
         ui(
-            Mode::Fail { require_patterns: true },
+            Mode::Fail { require_patterns: true, rustfix: RustfixMode::Disabled },
             "tests/extern-so/fail",
             &target,
             WithoutDependencies,
@@ -285,11 +270,17 @@ fn main() -> Result<()> {
 
 fn run_dep_mode(target: String, mut args: impl Iterator<Item = OsString>) -> Result<()> {
     let path = args.next().expect("./miri run-dep must be followed by a file name");
-    let mut config = test_config(&target, "", Mode::Yolo, /* with dependencies */ true);
+    let mut config = test_config(
+        &target,
+        "",
+        Mode::Yolo { rustfix: RustfixMode::Disabled },
+        /* with dependencies */ true,
+    );
     config.program.args.clear(); // We want to give the user full control over flags
-    config.build_dependencies_and_link_them()?;
+    let dep_args = config.build_dependencies()?;
 
     let mut cmd = config.program.build(&config.out_dir);
+    cmd.args(dep_args);
 
     cmd.arg(path);
 

src/tools/miri/tests/pass/backtrace/backtrace-api-v0.stdout

@@ -1,5 +1,5 @@
 $DIR/backtrace-api-v0.rs:24:14 (func_d)
 $DIR/backtrace-api-v0.rs:20:5 (func_c)
-$DIR/backtrace-api-v0.rs:9:5 (func_b)
+$DIR/backtrace-api-v0.rs:9:5 (func_b::<u8>)
 $DIR/backtrace-api-v0.rs:5:5 (func_a)
 $DIR/backtrace-api-v0.rs:29:18 (main)