Stream cargo output instead of buffering it all at once

src/bolt/instrument.rs

@@ -8,8 +8,9 @@ use colored::Colorize;
 use crate::bolt::cli::{add_bolt_args, BoltArgs};
 use crate::bolt::env::{find_bolt_env, BoltEnv};
 use crate::bolt::{bolt_pgo_rustflags, get_binary_profile_dir};
-use crate::build::CargoCommand;
-use crate::build::{cargo_command_with_flags, get_artifact_kind, handle_metadata_message};
+use crate::build::{
+    cargo_command_with_flags, get_artifact_kind, handle_metadata_message, CargoCommand,
+};
 use crate::cli::cli_format_path;
 use crate::utils::str::capitalize;
 use crate::workspace::CargoContext;
@@ -42,9 +43,9 @@ pub fn bolt_instrument(ctx: CargoContext, args: BoltInstrumentArgs) -> anyhow::R
     );
 
     let flags = bolt_pgo_rustflags(&ctx, args.with_pgo)?;
-    let output = cargo_command_with_flags(CargoCommand::Build, &flags, args.cargo_args)?;
+    let mut cargo = cargo_command_with_flags(CargoCommand::Build, &flags, args.cargo_args)?;
 
-    for message in Message::parse_stream(output.stdout.as_slice()) {
+    for message in cargo.messages() {
         let message = message?;
         match message {
             Message::CompilerArtifact(artifact) => {
@@ -83,6 +84,8 @@ pub fn bolt_instrument(ctx: CargoContext, args: BoltInstrumentArgs) -> anyhow::R
         }
     }
 
+    cargo.check_status()?;
+
     Ok(())
 }
 

src/bolt/optimize.rs

@@ -10,8 +10,9 @@ use colored::Colorize;
 use crate::bolt::cli::{add_bolt_args, BoltArgs};
 use crate::bolt::env::{find_bolt_env, BoltEnv};
 use crate::bolt::{bolt_pgo_rustflags, get_binary_profile_dir};
-use crate::build::CargoCommand;
-use crate::build::{cargo_command_with_flags, get_artifact_kind, handle_metadata_message};
+use crate::build::{
+    cargo_command_with_flags, get_artifact_kind, handle_metadata_message, CargoCommand,
+};
 use crate::cli::cli_format_path;
 use crate::run_command;
 use crate::utils::file::gather_files_with_extension;
@@ -36,9 +37,9 @@ pub fn bolt_optimize(ctx: CargoContext, args: BoltOptimizeArgs) -> anyhow::Resul
     let bolt_env = find_bolt_env()?;
 
     let flags = bolt_pgo_rustflags(&ctx, args.with_pgo)?;
-    let output = cargo_command_with_flags(CargoCommand::Build, &flags, args.cargo_args)?;
+    let mut cargo = cargo_command_with_flags(CargoCommand::Build, &flags, args.cargo_args)?;
 
-    for message in Message::parse_stream(output.stdout.as_slice()) {
+    for message in cargo.messages() {
         let message = message?;
         match message {
             Message::CompilerArtifact(artifact) => {
@@ -85,6 +86,8 @@ The optimization will probably not be very effective.",
         }
     }
 
+    cargo.check_status()?;
+
     Ok(())
 }
 

src/build.rs

@@ -1,10 +1,9 @@
 use crate::get_default_target;
-use cargo_metadata::{Artifact, Message};
-use colored::Colorize;
+use cargo_metadata::{Artifact, Message, MessageIter};
 use std::collections::HashMap;
 use std::fmt::Write as WriteFmt;
-use std::io::Write;
-use std::process::{Command, Output};
+use std::io::{BufReader, Write};
+use std::process::{Child, ChildStdout, Command, Stdio};
 
 #[derive(Debug, Default)]
 struct CargoArgs {
@@ -17,12 +16,34 @@ enum ReleaseMode {
     NoRelease,
 }
 
-/// Run `cargo` command in release mode with the provided RUSTFLAGS and Cargo arguments.
+pub struct RunningCargo {
+    child: Child,
+    message_iter: MessageIter<BufReader<ChildStdout>>,
+}
+
+impl RunningCargo {
+    pub fn messages(&mut self) -> &mut MessageIter<BufReader<ChildStdout>> {
+        &mut self.message_iter
+    }
+
+    pub fn check_status(mut self) -> anyhow::Result<()> {
+        let status = self.child.wait()?;
+        if !status.success() {
+            return Err(anyhow::anyhow!(
+                "Cargo finished with an error ({})",
+                status.code().unwrap_or(-1),
+            ));
+        }
+        Ok(())
+    }
+}
+
+/// Start a `cargo` command in release mode with the provided RUSTFLAGS and Cargo arguments.
 pub fn cargo_command_with_flags(
     command: CargoCommand,
     flags: &str,
     cargo_args: Vec<String>,
-) -> anyhow::Result<Output> {
+) -> anyhow::Result<RunningCargo> {
     let mut rustflags = std::env::var("RUSTFLAGS").unwrap_or_default();
     write!(&mut rustflags, " {}", flags).unwrap();
 
@@ -34,38 +55,21 @@ pub fn cargo_command_with_flags(
         _ => ReleaseMode::AddRelease,
     };
 
-    let output = cargo_command(command, cargo_args, env, release_mode)?;
-    if !output.status.success() {
-        Err(anyhow::anyhow!(
-            "Cargo error ({})\n{}\n{}",
-            output.status,
-            String::from_utf8_lossy(&output.stderr).red(),
-            cargo_json_output_to_string(&output.stdout)
-                .unwrap_or_else(|error| format!("Could not parse Cargo stdout: {}", error))
-        ))
-    } else {
-        Ok(output)
-    }
+    let mut child = cargo_command(command, cargo_args, env, release_mode)?;
+    let stdout = child.stdout.take().unwrap();
+    Ok(RunningCargo {
+        child,
+        message_iter: Message::parse_stream(BufReader::new(stdout)),
+    })
 }
 
-fn cargo_json_output_to_string(output: &[u8]) -> anyhow::Result<String> {
-    let mut messages = Vec::new();
-
-    for message in Message::parse_stream(output) {
-        let message = message?;
-        write_metadata_message(&mut messages, message);
-    }
-
-    Ok(String::from_utf8(messages)?)
-}
-
-/// Run `cargo` command in release mode with the provided env variables and Cargo arguments.
+/// Spawn `cargo` command in release mode with the provided env variables and Cargo arguments.
 fn cargo_command(
     cargo_cmd: CargoCommand,
     cargo_args: Vec<String>,
     env: HashMap<String, String>,
     release_mode: ReleaseMode,
-) -> anyhow::Result<Output> {
+) -> anyhow::Result<Child> {
     let parsed_args = parse_cargo_args(cargo_args);
 
     let mut command = Command::new("cargo");
@@ -74,6 +78,9 @@ fn cargo_command(
         "--message-format",
         "json-diagnostic-rendered-ansi",
     ]);
+    command.stdin(Stdio::null());
+    command.stdout(Stdio::piped());
+    command.stderr(Stdio::inherit());
 
     match release_mode {
         ReleaseMode::AddRelease => {
@@ -101,7 +108,7 @@ fn cargo_command(
         command.env(key, value);
     }
     log::debug!("Executing cargo command: {:?}", command);
-    Ok(command.output()?)
+    Ok(command.spawn()?)
 }
 
 fn parse_cargo_args(cargo_args: Vec<String>) -> CargoArgs {
@@ -130,7 +137,10 @@ fn parse_cargo_args(cargo_args: Vec<String>) -> CargoArgs {
 }
 
 pub fn handle_metadata_message(message: Message) {
-    write_metadata_message(std::io::stdout().lock(), message);
+    let stdout = std::io::stdout();
+    let mut stdout = stdout.lock();
+    write_metadata_message(&mut stdout, message);
+    stdout.flush().unwrap();
 }
 
 fn write_metadata_message<W: Write>(mut stream: W, message: Message) {
@@ -148,7 +158,9 @@ fn write_metadata_message<W: Write>(mut stream: W, message: Message) {
             )
             .unwrap();
         }
-        _ => {}
+        _ => {
+            log::debug!("Metadata output: {:?}", message);
+        }
     }
 }
 

src/pgo/instrument.rs

@@ -1,5 +1,6 @@
-use crate::build::CargoCommand;
-use crate::build::{cargo_command_with_flags, get_artifact_kind, handle_metadata_message};
+use crate::build::{
+    cargo_command_with_flags, get_artifact_kind, handle_metadata_message, CargoCommand,
+};
 use crate::clear_directory;
 use crate::cli::cli_format_path;
 use crate::workspace::CargoContext;
@@ -46,10 +47,9 @@ pub fn pgo_instrument(ctx: CargoContext, args: PgoInstrumentArgs) -> anyhow::Res
     );
 
     let flags = format!("-Cprofile-generate={}", pgo_dir.display());
-    let output = cargo_command_with_flags(args.command, &flags, args.cargo_args)?;
-    log::debug!("Cargo stderr\n {}", String::from_utf8_lossy(&output.stderr));
+    let mut cargo = cargo_command_with_flags(args.command, &flags, args.cargo_args)?;
 
-    for message in Message::parse_stream(output.stdout.as_slice()) {
+    for message in cargo.messages() {
         let message = message?;
         match message {
             Message::CompilerArtifact(artifact) => {
@@ -88,5 +88,7 @@ it with the following environment variable: {}.",
         }
     }
 
+    cargo.check_status()?;
+
     Ok(())
 }

src/pgo/optimize.rs

@@ -50,11 +50,10 @@ pub fn pgo_optimize(ctx: CargoContext, args: PgoOptimizeArgs) -> anyhow::Result<
 
     let flags = prepare_pgo_optimization_flags(&pgo_env, &pgo_dir)?;
 
-    let output = cargo_command_with_flags(args.command, &flags, args.cargo_args)?;
-    log::debug!("Cargo stderr\n {}", String::from_utf8_lossy(&output.stderr));
+    let mut cargo = cargo_command_with_flags(args.command, &flags, args.cargo_args)?;
 
     let mut counter = MissingProfileCounter::default();
-    for message in Message::parse_stream(output.stdout.as_slice()) {
+    for message in cargo.messages() {
         let message = message?;
         match message {
             Message::CompilerArtifact(artifact) => {
@@ -91,6 +90,8 @@ pub fn pgo_optimize(ctx: CargoContext, args: PgoOptimizeArgs) -> anyhow::Result<
         }
     }
 
+    cargo.check_status()?;
+
     if counter.counter > 0 {
         log::warn!(
             "PGO profile data was not found for {} {}.",