Rollup merge of rust-lang#104046 - RalfJung:run-miri-run, r=oli-obk

bootstrap: add support for running Miri on a file

This enables:
```
./x.py run src/tools/miri --stage 0 --args src/tools/miri/tests/pass/hello.rs
```
That can be super helpful for debugging.

Also avoid sharing the Miri sysroot dir with a system-wide (rustup-managed) installation of Miri.

Fixes rust-lang#76666

Author: Manishearth

src/bootstrap/builder.rs

@@ -755,6 +755,7 @@ impl<'a> Builder<'a> {
                 run::BuildManifest,
                 run::BumpStage0,
                 run::ReplaceVersionPlaceholder,
+                run::Miri,
             ),
             // These commands either don't use paths, or they're special-cased in Build::build()
             Kind::Clean | Kind::Format | Kind::Setup => vec![],
@@ -818,7 +819,7 @@ impl<'a> Builder<'a> {
             Subcommand::Bench { ref paths, .. } => (Kind::Bench, &paths[..]),
             Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]),
             Subcommand::Install { ref paths } => (Kind::Install, &paths[..]),
-            Subcommand::Run { ref paths } => (Kind::Run, &paths[..]),
+            Subcommand::Run { ref paths, .. } => (Kind::Run, &paths[..]),
             Subcommand::Format { .. } => (Kind::Format, &[][..]),
             Subcommand::Clean { .. } | Subcommand::Setup { .. } => {
                 panic!()

src/bootstrap/flags.rs

@@ -140,6 +140,7 @@ pub enum Subcommand {
     },
     Run {
         paths: Vec<PathBuf>,
+        args: Vec<String>,
     },
     Setup {
         profile: Profile,
@@ -342,6 +343,9 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
             Kind::Format => {
                 opts.optflag("", "check", "check formatting instead of applying.");
             }
+            Kind::Run => {
+                opts.optmulti("", "args", "arguments for the tool", "ARGS");
+            }
             _ => {}
         };
 
@@ -613,7 +617,7 @@ Arguments:
                     println!("\nrun requires at least a path!\n");
                     usage(1, &opts, verbose, &subcommand_help);
                 }
-                Subcommand::Run { paths }
+                Subcommand::Run { paths, args: matches.opt_strs("args") }
             }
             Kind::Setup => {
                 let profile = if paths.len() > 1 {
@@ -721,24 +725,29 @@ impl Subcommand {
     }
 
     pub fn test_args(&self) -> Vec<&str> {
-        let mut args = vec![];
-
         match *self {
             Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => {
-                args.extend(test_args.iter().flat_map(|s| s.split_whitespace()))
+                test_args.iter().flat_map(|s| s.split_whitespace()).collect()
             }
-            _ => (),
+            _ => vec![],
         }
-
-        args
     }
 
     pub fn rustc_args(&self) -> Vec<&str> {
         match *self {
             Subcommand::Test { ref rustc_args, .. } => {
                 rustc_args.iter().flat_map(|s| s.split_whitespace()).collect()
             }
-            _ => Vec::new(),
+            _ => vec![],
+        }
+    }
+
+    pub fn args(&self) -> Vec<&str> {
+        match *self {
+            Subcommand::Run { ref args, .. } => {
+                args.iter().flat_map(|s| s.split_whitespace()).collect()
+            }
+            _ => vec![],
         }
     }
 

src/bootstrap/run.rs

@@ -1,8 +1,12 @@
+use std::process::Command;
+
 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::config::TargetSelection;
 use crate::dist::distdir;
-use crate::tool::Tool;
+use crate::test;
+use crate::tool::{self, SourceType, Tool};
 use crate::util::output;
-use std::process::Command;
+use crate::Mode;
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct ExpandYamlAnchors;
@@ -125,3 +129,63 @@ impl Step for ReplaceVersionPlaceholder {
         builder.run(&mut cmd);
     }
 }
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Miri {
+    stage: u32,
+    host: TargetSelection,
+    target: TargetSelection,
+}
+
+impl Step for Miri {
+    type Output = ();
+    const ONLY_HOSTS: bool = false;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/miri")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Miri {
+            stage: run.builder.top_stage,
+            host: run.build_triple(),
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder<'_>) {
+        let stage = self.stage;
+        let host = self.host;
+        let target = self.target;
+        let compiler = builder.compiler(stage, host);
+
+        let miri = builder
+            .ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() })
+            .expect("in-tree tool");
+        let miri_sysroot = test::Miri::build_miri_sysroot(builder, compiler, &miri, target);
+
+        // # Run miri.
+        // Running it via `cargo run` as that figures out the right dylib path.
+        // add_rustc_lib_path does not add the path that contains librustc_driver-<...>.so.
+        let mut miri = tool::prepare_tool_cargo(
+            builder,
+            compiler,
+            Mode::ToolRustc,
+            host,
+            "run",
+            "src/tools/miri",
+            SourceType::InTree,
+            &[],
+        );
+        miri.add_rustc_lib_path(builder, compiler);
+        // Forward arguments.
+        miri.arg("--").arg("--target").arg(target.rustc_target_arg());
+        miri.args(builder.config.cmd.args());
+
+        // miri tests need to know about the stage sysroot
+        miri.env("MIRI_SYSROOT", &miri_sysroot);
+
+        let mut miri = Command::from(miri);
+        builder.run(&mut miri);
+    }
+}

src/bootstrap/test.rs

@@ -465,49 +465,20 @@ pub struct Miri {
     target: TargetSelection,
 }
 
-impl Step for Miri {
-    type Output = ();
-    const ONLY_HOSTS: bool = false;
-
-    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.path("src/tools/miri")
-    }
-
-    fn make_run(run: RunConfig<'_>) {
-        run.builder.ensure(Miri {
-            stage: run.builder.top_stage,
-            host: run.build_triple(),
-            target: run.target,
-        });
-    }
-
-    /// Runs `cargo test` for miri.
-    fn run(self, builder: &Builder<'_>) {
-        let stage = self.stage;
-        let host = self.host;
-        let target = self.target;
-        let compiler = builder.compiler(stage, host);
-        // We need the stdlib for the *next* stage, as it was built with this compiler that also built Miri.
-        // Except if we are at stage 2, the bootstrap loop is complete and we can stick with our current stage.
-        let compiler_std = builder.compiler(if stage < 2 { stage + 1 } else { stage }, host);
-
-        let miri = builder
-            .ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() })
-            .expect("in-tree tool");
-        let _cargo_miri = builder
-            .ensure(tool::CargoMiri { compiler, target: self.host, extra_features: Vec::new() })
-            .expect("in-tree tool");
-        // The stdlib we need might be at a different stage. And just asking for the
-        // sysroot does not seem to populate it, so we do that first.
-        builder.ensure(compile::Std::new(compiler_std, host));
-        let sysroot = builder.sysroot(compiler_std);
-
-        // # Run `cargo miri setup` for the given target.
+impl Miri {
+    /// Run `cargo miri setup` for the given target, return where the Miri sysroot was put.
+    pub fn build_miri_sysroot(
+        builder: &Builder<'_>,
+        compiler: Compiler,
+        miri: &Path,
+        target: TargetSelection,
+    ) -> String {
+        let miri_sysroot = builder.out.join(compiler.host.triple).join("miri-sysrot");
         let mut cargo = tool::prepare_tool_cargo(
             builder,
             compiler,
             Mode::ToolRustc,
-            host,
+            compiler.host,
             "run",
             "src/tools/miri/cargo-miri",
             SourceType::InTree,
@@ -521,6 +492,8 @@ impl Step for Miri {
         cargo.env("MIRI_LIB_SRC", builder.src.join("library"));
         // Tell it where to find Miri.
         cargo.env("MIRI", &miri);
+        // Tell it where to put the sysroot.
+        cargo.env("MIRI_SYSROOT", &miri_sysroot);
         // Debug things.
         cargo.env("RUST_BACKTRACE", "1");
 
@@ -535,7 +508,7 @@ impl Step for Miri {
         cargo.arg("--print-sysroot");
 
         // FIXME: Is there a way in which we can re-use the usual `run` helpers?
-        let miri_sysroot = if builder.config.dry_run {
+        if builder.config.dry_run {
             String::new()
         } else {
             builder.verbose(&format!("running: {:?}", cargo));
@@ -548,7 +521,48 @@ impl Step for Miri {
             let sysroot = stdout.trim_end();
             builder.verbose(&format!("`cargo miri setup --print-sysroot` said: {:?}", sysroot));
             sysroot.to_owned()
-        };
+        }
+    }
+}
+
+impl Step for Miri {
+    type Output = ();
+    const ONLY_HOSTS: bool = false;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/miri")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Miri {
+            stage: run.builder.top_stage,
+            host: run.build_triple(),
+            target: run.target,
+        });
+    }
+
+    /// Runs `cargo test` for miri.
+    fn run(self, builder: &Builder<'_>) {
+        let stage = self.stage;
+        let host = self.host;
+        let target = self.target;
+        let compiler = builder.compiler(stage, host);
+        // We need the stdlib for the *next* stage, as it was built with this compiler that also built Miri.
+        // Except if we are at stage 2, the bootstrap loop is complete and we can stick with our current stage.
+        let compiler_std = builder.compiler(if stage < 2 { stage + 1 } else { stage }, host);
+
+        let miri = builder
+            .ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() })
+            .expect("in-tree tool");
+        let _cargo_miri = builder
+            .ensure(tool::CargoMiri { compiler, target: self.host, extra_features: Vec::new() })
+            .expect("in-tree tool");
+        // The stdlib we need might be at a different stage. And just asking for the
+        // sysroot does not seem to populate it, so we do that first.
+        builder.ensure(compile::Std::new(compiler_std, host));
+        let sysroot = builder.sysroot(compiler_std);
+        // We also need a Miri sysroot.
+        let miri_sysroot = Miri::build_miri_sysroot(builder, compiler, &miri, target);
 
         // # Run `cargo test`.
         let mut cargo = tool::prepare_tool_cargo(
@@ -566,7 +580,6 @@ impl Step for Miri {
         // miri tests need to know about the stage sysroot
         cargo.env("MIRI_SYSROOT", &miri_sysroot);
         cargo.env("MIRI_HOST_SYSROOT", sysroot);
-        cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
         cargo.env("MIRI", &miri);
         // propagate --bless
         if builder.config.cmd.bless() {
@@ -607,7 +620,6 @@ impl Step for Miri {
         // Tell `cargo miri` where to find things.
         cargo.env("MIRI_SYSROOT", &miri_sysroot);
         cargo.env("MIRI_HOST_SYSROOT", sysroot);
-        cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
         cargo.env("MIRI", &miri);
         // Debug things.
         cargo.env("RUST_BACKTRACE", "1");

src/tools/miri/README.md

@@ -433,8 +433,10 @@ Moreover, Miri recognizes some environment variables:
   trigger a re-build of the standard library; you have to clear the Miri build
   cache manually (on Linux, `rm -rf ~/.cache/miri`).
 * `MIRI_SYSROOT` (recognized by `cargo miri` and the Miri driver) indicates the sysroot to use. When
-  using `cargo miri`, only set this if you do not want to use the automatically created sysroot. For
-  directly invoking the Miri driver, this variable (or a `--sysroot` flag) is mandatory.
+  using `cargo miri`, this skips the automatic setup -- only set this if you do not want to use the
+  automatically created sysroot. For directly invoking the Miri driver, this variable (or a
+  `--sysroot` flag) is mandatory. When invoking `cargo miri setup`, this indicates where the sysroot
+  will be put.
 * `MIRI_TEST_TARGET` (recognized by the test suite and the `./miri` script) indicates which target
   architecture to test against.  `miri` and `cargo miri` accept the `--target` flag for the same
   purpose.

src/tools/miri/cargo-miri/src/setup.rs

@@ -17,10 +17,8 @@ pub fn setup(subcommand: &MiriCommand, target: &str, rustc_version: &VersionMeta
     let only_setup = matches!(subcommand, MiriCommand::Setup);
     let ask_user = !only_setup;
     let print_sysroot = only_setup && has_arg_flag("--print-sysroot"); // whether we just print the sysroot path
-    if std::env::var_os("MIRI_SYSROOT").is_some() {
-        if only_setup {
-            println!("WARNING: MIRI_SYSROOT already set, not doing anything.")
-        }
+    if !only_setup && std::env::var_os("MIRI_SYSROOT").is_some() {
+        // Skip setup step if MIRI_SYSROOT is explicitly set, *unless* we are `cargo miri setup`.
         return;
     }
 
@@ -61,8 +59,13 @@ pub fn setup(subcommand: &MiriCommand, target: &str, rustc_version: &VersionMeta
     }
 
     // Determine where to put the sysroot.
-    let user_dirs = directories::ProjectDirs::from("org", "rust-lang", "miri").unwrap();
-    let sysroot_dir = user_dirs.cache_dir();
+    let sysroot_dir = match std::env::var_os("MIRI_SYSROOT") {
+        Some(dir) => PathBuf::from(dir),
+        None => {
+            let user_dirs = directories::ProjectDirs::from("org", "rust-lang", "miri").unwrap();
+            user_dirs.cache_dir().to_owned()
+        }
+    };
     // Sysroot configuration and build details.
     let sysroot_config = if std::env::var_os("MIRI_NO_STD").is_some() {
         SysrootConfig::NoStd
@@ -111,7 +114,7 @@ pub fn setup(subcommand: &MiriCommand, target: &str, rustc_version: &VersionMeta
         (command, rustflags)
     };
     // Make sure all target-level Miri invocations know their sysroot.
-    std::env::set_var("MIRI_SYSROOT", sysroot_dir);
+    std::env::set_var("MIRI_SYSROOT", &sysroot_dir);
 
     // Do the build.
     if only_setup {
@@ -121,7 +124,7 @@ pub fn setup(subcommand: &MiriCommand, target: &str, rustc_version: &VersionMeta
         // We want to be quiet, but still let the user know that something is happening.
         eprint!("Preparing a sysroot for Miri (target: {target})... ");
     }
-    Sysroot::new(sysroot_dir, target)
+    Sysroot::new(&sysroot_dir, target)
         .build_from_source(&rust_src, BuildMode::Check, sysroot_config, rustc_version, cargo_cmd)
         .unwrap_or_else(|_| {
             if only_setup {