Auto merge of rust-lang#136779 - jieyouxu:rahhh, r=<try>

[DO NOT MERGE] `./x test rust-analyzer`

I somehow made `./x test rust-analyzer` work on my machine[^machine], **but at what cost?**

Not intended for merge but only as a reference. If we do want to land this, I'll need to tidy this up.

Notes:

- Unrelated tracing bits are extracted to rust-lang#137080.
- I abused a bunch of cargo features  `in-rust-tree`. It probably doesn't need to be, and simply `--cfg` might work. I was trying to get the main rust-analyzer tests to build *at all*. Anything building is already a miracle.
- I had to slap a bunch of the following to all the r-a crates to get the tests to build at all. I don't 100% understand why, but otherwise I get a whole ton of ``expected `rustc_lexer` to be available in rlib format`` build failures.

```rs
#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
#[cfg(all(feature = "in-rust-tree", test))]
extern crate rustc_driver as _;
```

- Skipped one config test that was fixed on r-a master but not synced here in r-l/r yet.

[^machine]: `x86_64-unknown-linux-gnu`, haven't bothered trying this on msvc yet.

try-job: aarch64-gnu
try-job: x86_64-apple-1
try-job: aarch64-apple
try-job: i686-mingw-1
try-job: x86_64-mingw-1
try-job: i686-msvc-1
try-job: x86_64-msvc-1

Author: bors

src/bootstrap/src/core/build_steps/compile.rs

@@ -21,7 +21,7 @@ use crate::core::config::{DryRun, TargetSelection};
 use crate::utils::cache::Cache;
 use crate::utils::exec::{BootstrapCommand, command};
 use crate::utils::helpers::{self, LldThreads, add_dylib_path, exe, libdir, linker_args, t};
-use crate::{Build, Crate};
+use crate::{Build, Crate, trace};
 
 mod cargo;
 
@@ -971,6 +971,7 @@ impl<'a> Builder<'a> {
                 test::Cargotest,
                 test::Cargo,
                 test::RustAnalyzer,
+                test::RustAnalyzerProcMacroSrv,
                 test::ErrorIndex,
                 test::Distcheck,
                 test::Nomicon,
@@ -1215,7 +1216,7 @@ impl<'a> Builder<'a> {
     /// `Compiler` since all `Compiler` instances are meant to be obtained through this function,
     /// since it ensures that they are valid (i.e., built and assembled).
     pub fn compiler(&self, stage: u32, host: TargetSelection) -> Compiler {
-        self.ensure(compile::Assemble { target_compiler: Compiler { stage, host } })
+        self.ensure(compile::Assemble { output_compiler: Compiler { stage, host } })
     }
 
     /// Similar to `compiler`, except handles the full-bootstrap option to
@@ -1331,6 +1332,8 @@ impl<'a> Builder<'a> {
             return;
         }
 
+        trace!(rustc_lib_paths = ?self.rustc_lib_paths(compiler));
+
         add_dylib_path(self.rustc_lib_paths(compiler), cmd);
     }
 

src/bootstrap/src/core/build_steps/test.rs

@@ -352,9 +352,9 @@ mod defaults {
         assert_eq!(
             first(cache.all::<compile::Assemble>()),
             &[
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } },
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } },
-                compile::Assemble { target_compiler: Compiler { host: b, stage: 1 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 0 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 1 } },
+                compile::Assemble { output_compiler: Compiler { host: b, stage: 1 } },
             ]
         );
         assert_eq!(
@@ -632,10 +632,10 @@ mod dist {
         assert_eq!(
             first(cache.all::<compile::Assemble>()),
             &[
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } },
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } },
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } },
-                compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 0 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 1 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 2 } },
+                compile::Assemble { output_compiler: Compiler { host: b, stage: 2 } },
             ]
         );
     }
@@ -713,9 +713,9 @@ mod dist {
         assert_eq!(
             first(builder.cache.all::<compile::Assemble>()),
             &[
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } },
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } },
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 0 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 1 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 2 } },
             ]
         );
         assert_eq!(

src/bootstrap/src/core/builder/mod.rs

@@ -30,5 +30,8 @@ vfs.workspace = true
 span.workspace = true
 intern.workspace = true
 
+[features]
+in-rust-tree = []
+
 [lints]
 workspace = true

src/bootstrap/src/core/builder/tests.rs

@@ -1,4 +1,9 @@
 //! base_db defines basic database traits. The concrete DB is defined by ide.
+
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 // FIXME: Rename this crate, base db is non descriptive
 mod change;
 mod input;

src/tools/rust-analyzer/crates/base-db/Cargo.toml

@@ -33,5 +33,9 @@ derive_arbitrary = "1.3.2"
 syntax-bridge.workspace = true
 syntax.workspace = true
 
+[features]
+default = []
+in-rust-tree = []
+
 [lints]
 workspace = true

src/tools/rust-analyzer/crates/base-db/src/lib.rs

@@ -1,5 +1,9 @@
 //! cfg defines conditional compiling options, `cfg` attribute parser and evaluator
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 mod cfg_expr;
 mod dnf;
 #[cfg(test)]

src/tools/rust-analyzer/crates/cfg/Cargo.toml

@@ -54,6 +54,7 @@ expect-test.workspace = true
 test-utils.workspace = true
 test-fixture.workspace = true
 syntax-bridge.workspace = true
+
 [features]
 in-rust-tree = ["hir-expand/in-rust-tree"]
 

src/tools/rust-analyzer/crates/cfg/src/lib.rs

@@ -8,6 +8,8 @@
 //! actually true.
 
 #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
 
 #[cfg(feature = "in-rust-tree")]
 extern crate rustc_parse_format;

src/tools/rust-analyzer/crates/hir-def/Cargo.toml

@@ -3,8 +3,13 @@
 //! Specifically, it implements a concept of `MacroFile` -- a file whose syntax
 //! tree originates not from the text of some `FileId`, but from some macro
 //! expansion.
+
 #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
 
+#[cfg_attr(all(feature = "in-rust-tree", test), allow(unused_extern_crates))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver;
+
 pub mod attrs;
 pub mod builtin;
 pub mod change;

src/tools/rust-analyzer/crates/hir-def/src/lib.rs

@@ -2,6 +2,8 @@
 //! information and various assists.
 
 #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
 
 #[cfg(feature = "in-rust-tree")]
 extern crate rustc_index;

src/tools/rust-analyzer/crates/hir-expand/src/lib.rs

@@ -17,9 +17,12 @@
 //! from the ide with completions, hovers, etc. It is a (soft, internal) boundary:
 //! <https://www.tedinski.com/2018/02/06/system-boundaries.html>.
 
-#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
 #![recursion_limit = "512"]
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 mod attrs;
 mod from_id;
 mod has_source;

src/tools/rust-analyzer/crates/hir-ty/src/lib.rs

@@ -59,6 +59,8 @@
 //! <https://rust-analyzer.github.io/blog/2020/09/28/how-to-make-a-light-bulb.html>
 
 #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
 
 mod assist_config;
 mod assist_context;

src/tools/rust-analyzer/crates/hir/src/lib.rs

@@ -36,5 +36,9 @@ expect-test = "1.4.0"
 test-utils.workspace = true
 test-fixture.workspace = true
 
+[features]
+default = []
+in-rust-tree = []
+
 [lints]
 workspace = true

src/tools/rust-analyzer/crates/ide-assists/src/lib.rs

@@ -1,5 +1,9 @@
 //! `completions` crate provides utilities for generating completions of user input.
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 mod completions;
 mod config;
 mod context;

src/tools/rust-analyzer/crates/ide-completion/Cargo.toml

@@ -49,5 +49,9 @@ expect-test = "1.4.0"
 test-utils.workspace = true
 test-fixture.workspace = true
 
+[features]
+default = []
+in-rust-tree = []
+
 [lints]
 workspace = true

src/tools/rust-analyzer/crates/ide-completion/src/lib.rs

@@ -2,6 +2,10 @@
 //!
 //! It is mainly a `HirDatabase` for semantic analysis, plus a `SymbolsDatabase`, for fuzzy search.
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 mod apply_change;
 
 pub mod active_parameter;

src/tools/rust-analyzer/crates/ide-db/Cargo.toml

@@ -34,5 +34,9 @@ expect-test = "1.4.0"
 test-utils.workspace = true
 test-fixture.workspace = true
 
+[features]
+default = []
+in-rust-tree = []
+
 [lints]
 workspace = true

src/tools/rust-analyzer/crates/ide-db/src/lib.rs

@@ -23,6 +23,10 @@
 //! There are also a couple of ad-hoc diagnostics implemented directly here, we
 //! don't yet have a great pattern for how to do them properly.
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 mod handlers {
     pub(crate) mod await_outside_of_async;
     pub(crate) mod break_outside_of_loop;

src/tools/rust-analyzer/crates/ide-diagnostics/Cargo.toml

@@ -32,5 +32,9 @@ expect-test = "1.4.0"
 test-utils.workspace = true
 test-fixture.workspace = true
 
+[features]
+default = []
+in-rust-tree = []
+
 [lints]
 workspace = true

src/tools/rust-analyzer/crates/ide-diagnostics/src/lib.rs

@@ -3,6 +3,10 @@
 //! Allows searching the AST for code that matches one or more patterns and then replacing that code
 //! based on a template.
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 // Feature: Structural Search and Replace
 //
 // Search and replace with named wildcards that will match any expression, type, path, pattern or item.

src/tools/rust-analyzer/crates/ide-ssr/Cargo.toml

@@ -9,9 +9,12 @@
 
 // For proving that RootDatabase is RefUnwindSafe.
 
-#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
 #![recursion_limit = "128"]
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 #[cfg(test)]
 mod fixture;
 

src/tools/rust-analyzer/crates/ide-ssr/src/lib.rs

@@ -1,5 +1,10 @@
 //! Loads a Cargo project into a static instance of analysis, without support
 //! for incorporating changes.
+
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 // Note, don't remove any public api from this. This API is consumed by external tools
 // to run rust-analyzer as a library.
 use std::{collections::hash_map::Entry, iter, mem, path::Path, sync};

src/tools/rust-analyzer/crates/ide/src/lib.rs

@@ -7,6 +7,8 @@
 //! `hir_def::macro_expansion_tests::mbe`.
 
 #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
 
 #[cfg(not(feature = "in-rust-tree"))]
 extern crate ra_ap_rustc_lexer as rustc_lexer;

src/tools/rust-analyzer/crates/load-cargo/src/lib.rs

@@ -18,7 +18,10 @@
 //! [`Parser`]: crate::parser::Parser
 
 #![allow(rustdoc::private_intra_doc_links)]
+
 #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
 
 #[cfg(not(feature = "in-rust-tree"))]
 extern crate ra_ap_rustc_lexer as rustc_lexer;

src/tools/rust-analyzer/crates/mbe/src/lib.rs

@@ -29,5 +29,9 @@ span = { path = "../span", version = "0.0.0", default-features = false}
 
 intern.workspace = true
 
+[features]
+default = []
+in-rust-tree = []
+
 [lints]
 workspace = true

src/tools/rust-analyzer/crates/parser/src/lib.rs

@@ -5,6 +5,10 @@
 //! is used to provide basic infrastructure for communication between two
 //! processes: Client (RA itself), Server (the external program)
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 pub mod legacy_protocol {
     pub mod json;
     pub mod msg;

src/tools/rust-analyzer/crates/proc-macro-api/Cargo.toml

@@ -37,5 +37,9 @@ toolchain.workspace = true
 [dev-dependencies]
 expect-test = "1.4.0"
 
+[features]
+default = []
+in-rust-tree = []
+
 [lints]
 workspace = true

src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs

@@ -15,6 +15,10 @@
 //!   procedural macros).
 //! * Lowering of concrete model to a [`base_db::CrateGraph`]
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 pub mod project_json;
 pub mod toolchain_info {
     pub mod rustc_cfg;

src/tools/rust-analyzer/crates/project-model/Cargo.toml

@@ -9,6 +9,10 @@
 //! The `cli` submodule implements some batch-processing analysis, primarily as
 //! a debugging aid.
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 pub mod cli;
 
 mod command;

src/tools/rust-analyzer/crates/project-model/src/lib.rs

@@ -10,6 +10,12 @@
 
 #![allow(clippy::disallowed_types)]
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+
+#[cfg_attr(all(feature = "in-rust-tree", test), allow(unused_extern_crates))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver;
+
 mod cli;
 mod ratoml;
 mod support;

src/tools/rust-analyzer/crates/rust-analyzer/src/lib.rs

@@ -24,6 +24,7 @@ stdx.workspace = true
 
 [features]
 default = ["ra-salsa"]
+in-rust-tree = []
 
 [lints]
 workspace = true

src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/main.rs

@@ -1,4 +1,9 @@
 //! File and span related types.
+
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 use std::fmt::{self, Write};
 
 #[cfg(feature = "ra-salsa")]

src/tools/rust-analyzer/crates/span/Cargo.toml

@@ -1,5 +1,9 @@
 //! Conversions between [`SyntaxNode`] and [`tt::TokenTree`].
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 use std::{fmt, hash::Hash};
 
 use intern::Symbol;