add counters for blob bytes uploaded/downloaded to/from CAS (#12471)

Add counters to track the number of bytes uploaded to, or downloaded from, a remote CAS for blobs.

[ci skip-build-wheels]

Author: Tom Dyas

src/rust/engine/fs/store/src/remote.rs

@@ -17,7 +17,7 @@ use hashing::Digest;
 use log::Level;
 use remexec::content_addressable_storage_client::ContentAddressableStorageClient;
 use tonic::{Code, Request, Status};
-use workunit_store::{in_workunit, ObservationMetric, WorkunitMetadata};
+use workunit_store::{in_workunit, Metric, ObservationMetric, WorkunitMetadata};
 
 #[derive(Clone)]
 pub struct ByteStore {
@@ -257,7 +257,13 @@ impl ByteStore {
         workunit_store,
         workunit_name,
         workunit_metadata,
-        |_workunit| result_future,
+        |workunit| async move {
+          let result = result_future.await;
+          if result.is_ok() {
+            workunit.increment_counter(Metric::RemoteStoreBlobBytesUploaded, len as u64);
+          }
+          result
+        },
       )
       .await
     } else {
@@ -366,7 +372,16 @@ impl ByteStore {
         workunit_store_handle.store,
         workunit_name,
         workunit_metadata,
-        |_workunit| result_future,
+        |workunit| async move {
+          let result = result_future.await;
+          if result.is_ok() {
+            workunit.increment_counter(
+              Metric::RemoteStoreBlobBytesDownloaded,
+              digest.size_bytes as u64,
+            );
+          }
+          result
+        },
       )
       .await
     } else {

src/rust/engine/workunit_store/src/metrics.rs

@@ -75,6 +75,10 @@ pub enum Metric {
   RemoteExecutionSuccess,
   RemoteExecutionTimeouts,
   RemoteStoreMissingDigest,
+  /// Total number of bytes of blobs downloaded from a remote CAS.
+  RemoteStoreBlobBytesDownloaded,
+  /// Total number of bytes of blobs uploaded to a remote CAS.
+  RemoteStoreBlobBytesUploaded,
 }
 
 impl Metric {