ops: Classify HTTP/2 Go Away response as a retryable transient error

Signed-off-by: Joshua Potts <[email protected]>

Author: iamjpotts

Cargo.lock

@@ -25,6 +25,10 @@ backoff = "0.4.0"
 ed25519-dalek = { version = "2.0.0", features = ["rand_core"] }
 fraction = { version = "0.15.1", default-features = false }
 futures-core = "0.3.21"
+# Transitive dependency of tonic 0.11. Can be removed when tonic is upgraded to 0.12.
+h2_03 = { package = "h2", version = "0.3.6" }
+# Transitive dependency of tonic 0.12
+h2 = "0.4.6"
 hedera-proto = { path = "./protobufs", version = "0.13.0", features = [
   "time_0_3",
   "fraction",

Cargo.toml

@@ -41,7 +41,10 @@ use tonic::transport::Channel;
 use triomphe::Arc;
 
 use crate::client::NetworkData;
-use crate::execute::error::is_hyper_canceled;
+use crate::execute::error::{
+    is_hyper_canceled,
+    is_tonic_status_transient,
+};
 use crate::ping_query::PingQuery;
 use crate::{
     client,
@@ -357,6 +360,10 @@ fn map_tonic_error(
             }
         }
 
+        tonic::Code::Internal if is_tonic_status_transient(&status) => {
+            retry::Error::Transient(status.into())
+        }
+
         // fail immediately
         _ => retry::Error::Permanent(status.into()),
     }

src/execute.rs

@@ -67,3 +67,28 @@ mod test_is_hyper_canceled {
         assert!(!is_hyper_canceled(&input));
     }
 }
+
+pub(super) fn is_tonic_status_transient(status: &tonic::Status) -> bool {
+    let source = status
+        .source()
+        .and_then(|it| it.downcast_ref::<tonic::transport::Error>())
+        .and_then(StdError::source);
+
+    let Some(source) = source else {
+        return false;
+    };
+
+    if let Some(hyper_0) = source.downcast_ref::<hyper_0::Error>() {
+        // tonic 0.11 (current dependency)
+        let source: Option<&h2_03::Error> = hyper_0.source().and_then(|s| s.downcast_ref());
+
+        source.map(|s| s.is_go_away()).unwrap_or_default()
+    } else if let Some(hyper_1) = source.downcast_ref::<hyper::Error>() {
+        // tonic 0.12
+        let source: Option<&h2::Error> = hyper_1.source().and_then(|s| s.downcast_ref());
+
+        source.map(|s| s.is_go_away()).unwrap_or_default()
+    } else {
+        false
+    }
+}