Unrolled build for rust-lang#128215

Rollup merge of rust-lang#128215 - ehuss:update-reference, r=Kobzol

Update the reference

This updates the reference to use the new mdbook-spec preprocessor, which is a Cargo library inside the reference submodule.

Note that this PR contains a bunch of bootstrap cleanup commits to assist with making sure the submodules are working correctly. All of the cleanup PRs should have a description in their commit. I'd be happy to move those to a separate PR if that makes review easier.

The main changes for the reference are:
- Move the `doc::Reference` bootstrap step out of the generic macro into a custom step.
    - This step needs to build rustdoc because the new mdbook-spec plugin uses rustdoc for generating links.
    - PATH is updated so that the rustdoc binary can be found.
- rustbook now includes the mdbook-spec plugin as a dependency.
- rustbook enables the mdbook-spec preprocessor.

I did a bunch of testing with the various commands and setups, such as:
- `submodules=true` and `submodules=false`
- having all submodules deinitialized
- not in a git repository

However, there are probably thousands of different permutations of different commands, settings, and environments, so there is a chance I'm missing something.

Author: rust-timer

src/bootstrap/src/core/build_steps/check.rs

@@ -9,7 +9,7 @@ use crate::core::builder::{
 };
 use crate::core::config::TargetSelection;
 use crate::{Compiler, Mode, Subcommand};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 
 pub fn cargo_subcommand(kind: Kind) -> &'static str {
     match kind {
@@ -52,7 +52,7 @@ impl Step for Std {
     }
 
     fn run(self, builder: &Builder<'_>) {
-        builder.update_submodule(&Path::new("library").join("stdarch"));
+        builder.require_submodule("library/stdarch", None);
 
         let target = self.target;
         let compiler = builder.compiler(builder.top_stage, builder.config.build);

src/bootstrap/src/core/build_steps/clippy.rs

@@ -1,7 +1,5 @@
 //! Implementation of running clippy on the compiler, standard library and various tools.
 
-use std::path::Path;
-
 use crate::builder::Builder;
 use crate::builder::ShouldRun;
 use crate::core::builder;
@@ -127,7 +125,7 @@ impl Step for Std {
     }
 
     fn run(self, builder: &Builder<'_>) {
-        builder.update_submodule(&Path::new("library").join("stdarch"));
+        builder.require_submodule("library/stdarch", None);
 
         let target = self.target;
         let compiler = builder.compiler(builder.top_stage, builder.config.build);

src/bootstrap/src/core/build_steps/compile.rs

@@ -182,11 +182,16 @@ impl Step for Std {
             return;
         }
 
-        builder.update_submodule(&Path::new("library").join("stdarch"));
+        builder.require_submodule("library/stdarch", None);
 
         // Profiler information requires LLVM's compiler-rt
         if builder.config.profiler {
-            builder.update_submodule(Path::new("src/llvm-project"));
+            builder.require_submodule(
+                "src/llvm-project",
+                Some(
+                    "The `build.profiler` config option requires `compiler-rt` sources from LLVM.",
+                ),
+            );
         }
 
         let mut target_deps = builder.ensure(StartupObjects { compiler, target });
@@ -456,13 +461,15 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car
         // That's probably ok? At least, the difference wasn't enforced before. There's a comment in
         // the compiler_builtins build script that makes me nervous, though:
         // https://github.com/rust-lang/compiler-builtins/blob/31ee4544dbe47903ce771270d6e3bea8654e9e50/build.rs#L575-L579
-        builder.update_submodule(&Path::new("src").join("llvm-project"));
+        builder.require_submodule(
+            "src/llvm-project",
+            Some(
+                "The `build.optimized-compiler-builtins` config option \
+                 requires `compiler-rt` sources from LLVM.",
+            ),
+        );
         let compiler_builtins_root = builder.src.join("src/llvm-project/compiler-rt");
-        if !compiler_builtins_root.exists() {
-            panic!(
-                "need LLVM sources available to build `compiler-rt`, but they weren't present; consider enabling `build.submodules = true` or disabling `optimized-compiler-builtins`"
-            );
-        }
+        assert!(compiler_builtins_root.exists());
         // Note that `libprofiler_builtins/build.rs` also computes this so if
         // you're changing something here please also change that.
         cargo.env("RUST_COMPILER_RT_ROOT", &compiler_builtins_root);

src/bootstrap/src/core/build_steps/dist.rs

@@ -907,7 +907,7 @@ impl Step for Src {
     /// Creates the `rust-src` installer component
     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
         if !builder.config.dry_run() {
-            builder.update_submodule(Path::new("src/llvm-project"));
+            builder.require_submodule("src/llvm-project", None);
         }
 
         let tarball = Tarball::new_targetless(builder, "rust-src");
@@ -1022,10 +1022,7 @@ impl Step for PlainSourceTarball {
             // FIXME: This code looks _very_ similar to what we have in `src/core/build_steps/vendor.rs`
             // perhaps it should be removed in favor of making `dist` perform the `vendor` step?
 
-            // Ensure we have all submodules from src and other directories checked out.
-            for submodule in build_helper::util::parse_gitmodules(&builder.src) {
-                builder.update_submodule(Path::new(submodule));
-            }
+            builder.require_and_update_all_submodules();
 
             // Vendor all Cargo dependencies
             let mut cmd = command(&builder.initial_cargo);

src/bootstrap/src/core/build_steps/doc.rs

@@ -9,14 +9,14 @@
 
 use std::io::{self, Write};
 use std::path::{Path, PathBuf};
-use std::{fs, mem};
+use std::{env, fs, mem};
 
 use crate::core::build_steps::compile;
 use crate::core::build_steps::tool::{self, prepare_tool_cargo, SourceType, Tool};
 use crate::core::builder::{self, crate_description};
 use crate::core::builder::{Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step};
 use crate::core::config::{Config, TargetSelection};
-use crate::utils::helpers::{dir_is_empty, symlink_dir, t, up_to_date};
+use crate::utils::helpers::{symlink_dir, t, up_to_date};
 use crate::Mode;
 
 macro_rules! submodule_helper {
@@ -53,15 +53,16 @@ macro_rules! book {
 
             fn run(self, builder: &Builder<'_>) {
                 $(
-                    let path = Path::new(submodule_helper!( $path, submodule $( = $submodule )? ));
-                    builder.update_submodule(&path);
+                    let path = submodule_helper!( $path, submodule $( = $submodule )? );
+                    builder.require_submodule(path, None);
                 )?
                 builder.ensure(RustbookSrc {
                     target: self.target,
                     name: $book_name.to_owned(),
                     src: builder.src.join($path),
                     parent: Some(self),
                     languages: $lang.into(),
+                    rustdoc: None,
                 })
             }
         }
@@ -80,7 +81,6 @@ book!(
     EditionGuide, "src/doc/edition-guide", "edition-guide", &[], submodule;
     EmbeddedBook, "src/doc/embedded-book", "embedded-book", &[], submodule;
     Nomicon, "src/doc/nomicon", "nomicon", &[], submodule;
-    Reference, "src/doc/reference", "reference", &[], submodule;
     RustByExample, "src/doc/rust-by-example", "rust-by-example", &["ja"], submodule;
     RustdocBook, "src/doc/rustdoc", "rustdoc", &[];
     StyleGuide, "src/doc/style-guide", "style-guide", &[];
@@ -112,6 +112,7 @@ impl Step for UnstableBook {
             src: builder.md_doc_out(self.target).join("unstable-book"),
             parent: Some(self),
             languages: vec![],
+            rustdoc: None,
         })
     }
 }
@@ -123,6 +124,7 @@ struct RustbookSrc<P: Step> {
     src: PathBuf,
     parent: Option<P>,
     languages: Vec<&'static str>,
+    rustdoc: Option<PathBuf>,
 }
 
 impl<P: Step> Step for RustbookSrc<P> {
@@ -153,13 +155,18 @@ impl<P: Step> Step for RustbookSrc<P> {
             builder.info(&format!("Rustbook ({target}) - {name}"));
             let _ = fs::remove_dir_all(&out);
 
-            builder
-                .tool_cmd(Tool::Rustbook)
-                .arg("build")
-                .arg(&src)
-                .arg("-d")
-                .arg(&out)
-                .run(builder);
+            let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook);
+            if let Some(mut rustdoc) = self.rustdoc {
+                rustdoc.pop();
+                let old_path = env::var_os("PATH").unwrap_or_default();
+                let new_path =
+                    env::join_paths(std::iter::once(rustdoc).chain(env::split_paths(&old_path)))
+                        .expect("could not add rustdoc to PATH");
+
+                rustbook_cmd.env("PATH", new_path);
+            }
+
+            rustbook_cmd.arg("build").arg(&src).arg("-d").arg(&out).run(builder);
 
             for lang in &self.languages {
                 let out = out.join(lang);
@@ -217,29 +224,22 @@ impl Step for TheBook {
     /// * Index page
     /// * Redirect pages
     fn run(self, builder: &Builder<'_>) {
-        let relative_path = Path::new("src").join("doc").join("book");
-        builder.update_submodule(&relative_path);
+        builder.require_submodule("src/doc/book", None);
 
         let compiler = self.compiler;
         let target = self.target;
 
-        let absolute_path = builder.src.join(&relative_path);
+        let absolute_path = builder.src.join("src/doc/book");
         let redirect_path = absolute_path.join("redirects");
-        if !absolute_path.exists()
-            || !redirect_path.exists()
-            || dir_is_empty(&absolute_path)
-            || dir_is_empty(&redirect_path)
-        {
-            eprintln!("Please checkout submodule: {}", relative_path.display());
-            crate::exit!(1);
-        }
+
         // build book
         builder.ensure(RustbookSrc {
             target,
             name: "book".to_owned(),
             src: absolute_path.clone(),
             parent: Some(self),
             languages: vec![],
+            rustdoc: None,
         });
 
         // building older edition redirects
@@ -252,6 +252,7 @@ impl Step for TheBook {
                 // treat the other editions as not having a parent.
                 parent: Option::<Self>::None,
                 languages: vec![],
+                rustdoc: None,
             });
         }
 
@@ -932,8 +933,8 @@ macro_rules! tool_doc {
                     let _ = source_type; // silence the "unused variable" warning
                     let source_type = SourceType::Submodule;
 
-                    let path = Path::new(submodule_helper!( $path, submodule $( = $submodule )? ));
-                    builder.update_submodule(&path);
+                    let path = submodule_helper!( $path, submodule $( = $submodule )? );
+                    builder.require_submodule(path, None);
                 )?
 
                 let stage = builder.top_stage;
@@ -1172,12 +1173,6 @@ impl Step for RustcBook {
     /// in the "md-doc" directory in the build output directory. Then
     /// "rustbook" is used to convert it to HTML.
     fn run(self, builder: &Builder<'_>) {
-        // These submodules are required to be checked out to build rustbook
-        // because they have Cargo dependencies that are needed.
-        #[allow(clippy::single_element_loop)] // This will change soon.
-        for path in ["src/doc/book"] {
-            builder.update_submodule(Path::new(path));
-        }
         let out_base = builder.md_doc_out(self.target).join("rustc");
         t!(fs::create_dir_all(&out_base));
         let out_listing = out_base.join("src/lints");
@@ -1228,6 +1223,50 @@ impl Step for RustcBook {
             src: out_base,
             parent: Some(self),
             languages: vec![],
+            rustdoc: None,
+        });
+    }
+}
+
+#[derive(Ord, PartialOrd, Debug, Clone, Hash, PartialEq, Eq)]
+pub struct Reference {
+    pub compiler: Compiler,
+    pub target: TargetSelection,
+}
+
+impl Step for Reference {
+    type Output = ();
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        let builder = run.builder;
+        run.path("src/doc/reference").default_condition(builder.config.docs)
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Reference {
+            compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+            target: run.target,
+        });
+    }
+
+    /// Builds the reference book.
+    fn run(self, builder: &Builder<'_>) {
+        builder.require_submodule("src/doc/reference", None);
+
+        // This is needed for generating links to the standard library using
+        // the mdbook-spec plugin.
+        builder.ensure(compile::Std::new(self.compiler, builder.config.build));
+        let rustdoc = builder.rustdoc(self.compiler);
+
+        // Run rustbook/mdbook to generate the HTML pages.
+        builder.ensure(RustbookSrc {
+            target: self.target,
+            name: "reference".to_owned(),
+            src: builder.src.join("src/doc/reference"),
+            parent: Some(self),
+            languages: vec![],
+            rustdoc: Some(rustdoc),
         });
     }
 }

src/bootstrap/src/core/build_steps/llvm.rs

@@ -89,7 +89,7 @@ impl LdFlags {
 /// if not).
 pub fn prebuilt_llvm_config(builder: &Builder<'_>, target: TargetSelection) -> LlvmBuildStatus {
     // If we have llvm submodule initialized already, sync it.
-    builder.update_existing_submodule(&Path::new("src").join("llvm-project"));
+    builder.update_existing_submodule("src/llvm-project");
 
     builder.config.maybe_download_ci_llvm();
 
@@ -110,7 +110,8 @@ pub fn prebuilt_llvm_config(builder: &Builder<'_>, target: TargetSelection) -> L
     }
 
     // Initialize the llvm submodule if not initialized already.
-    builder.update_submodule(&Path::new("src").join("llvm-project"));
+    // If submodules are disabled, this does nothing.
+    builder.update_submodule("src/llvm-project");
 
     let root = "src/llvm-project/llvm";
     let out_dir = builder.llvm_out(target);
@@ -1197,7 +1198,10 @@ impl Step for CrtBeginEnd {
 
     /// Build crtbegin.o/crtend.o for musl target.
     fn run(self, builder: &Builder<'_>) -> Self::Output {
-        builder.update_submodule(Path::new("src/llvm-project"));
+        builder.require_submodule(
+            "src/llvm-project",
+            Some("The LLVM sources are required for the CRT from `compiler-rt`."),
+        );
 
         let out_dir = builder.native_dir(self.target).join("crt");
 
@@ -1270,7 +1274,10 @@ impl Step for Libunwind {
 
     /// Build libunwind.a
     fn run(self, builder: &Builder<'_>) -> Self::Output {
-        builder.update_submodule(Path::new("src/llvm-project"));
+        builder.require_submodule(
+            "src/llvm-project",
+            Some("The LLVM sources are required for libunwind."),
+        );
 
         if builder.config.dry_run() {
             return PathBuf::new();