fix: use middleware submission tweaks (#5100)

### Description Uses the short-term fixes described in this [design doc](https://www.notion.so/hyperlanexyz/State-of-EVM-provider-middleware-16c6d35200d680d8a6b5f0d32cd8c66b?pvs=4) and implemented in hyperlane-xyz/ethers-rs#23 These fixes are expected to make the relayer self-recover from nonce gaps. An expected drawback is that tx hashes are dropped more often due to the nonce manager resyncing to nonces that end up being too low by 1, which can slow down submissions. After 12h of testing, any submit queue spikes were only 1 min long and the biggest one had 7 messages - so short enough not to trigger the "rising submit queue" alert which stays pending for 6min: https://abacusworks.grafana.net/alerting/de6hwplr7eakgc ### Drive-by changes - Removes verbosity from a bunch of logs (we had `HyperlaneMessage` in a single log occur 6-7 times, but wasn't noticeable in `span`s because it was originating from args named `message` which have a clash with inner workings of the `tracing` lib) - rewords some logs because they were misleading ### Related issues Fixes: hyperlane-xyz/hyperlane-monorepo#5107 ### Backward compatibility Yes ### Testing E2E with a gas escalator that randomly dropped every other tx - the nonce manager was able to recover but was slower to submit. Then evaluated on RC for 12h as described above.
fmorency · Jan 13, 2025 · 18bb3cd · 18bb3cd
1 parent cfb4ced
commit 18bb3cd
Show file tree

Hide file tree

Showing 12 changed files with 29 additions and 26 deletions.
diff --git a/main/Cargo.lock b/main/Cargo.lock
diff --git a/main/Cargo.toml b/main/Cargo.toml
@@ -198,27 +198,27 @@ overflow-checks = true
 [workspace.dependencies.ethers]
 features = []
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2025-01-09"
+tag = "2025-01-10"
 
 [workspace.dependencies.ethers-contract]
 features = ["legacy"]
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2025-01-09"
+tag = "2025-01-10"
 
 [workspace.dependencies.ethers-core]
 features = []
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2025-01-09"
+tag = "2025-01-10"
 
 [workspace.dependencies.ethers-providers]
 features = []
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2025-01-09"
+tag = "2025-01-10"
 
 [workspace.dependencies.ethers-signers]
 features = ["aws"]
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2025-01-09"
+tag = "2025-01-10"
 
 [patch.crates-io.curve25519-dalek]
 branch = "v3.2.2-relax-zeroize"

diff --git a/main/agents/relayer/src/msg/metadata/aggregation.rs b/main/agents/relayer/src/msg/metadata/aggregation.rs
@@ -117,7 +117,7 @@ impl AggregationIsmMetadataBuilder {
 
 #[async_trait]
 impl MetadataBuilder for AggregationIsmMetadataBuilder {
-    #[instrument(err, skip(self), ret)]
+    #[instrument(err, skip(self, message), ret)]
     #[allow(clippy::blocks_in_conditions)] // TODO: `rustc` 1.80.1 clippy issue
     async fn build(
         &self,

diff --git a/main/agents/relayer/src/msg/metadata/base.rs b/main/agents/relayer/src/msg/metadata/base.rs
@@ -189,7 +189,7 @@ impl Deref for MessageMetadataBuilder {
 
 #[async_trait]
 impl MetadataBuilder for MessageMetadataBuilder {
-    #[instrument(err, skip(self), fields(destination_domain=self.destination_domain().name()))]
+    #[instrument(err, skip(self, message), fields(destination_domain=self.destination_domain().name()))]
     async fn build(
         &self,
         ism_address: H256,
@@ -228,7 +228,7 @@ impl MessageMetadataBuilder {
         }
     }
 
-    #[instrument(err, skip(self), fields(destination_domain=self.destination_domain().name()), ret)]
+    #[instrument(err, skip(self, message), fields(destination_domain=self.destination_domain().name()), ret)]
     pub async fn build_ism_and_metadata(
         &self,
         ism_address: H256,

diff --git a/main/agents/relayer/src/msg/metadata/ccip_read.rs b/main/agents/relayer/src/msg/metadata/ccip_read.rs
@@ -27,7 +27,7 @@ pub struct CcipReadIsmMetadataBuilder {
 
 #[async_trait]
 impl MetadataBuilder for CcipReadIsmMetadataBuilder {
-    #[instrument(err, skip(self))]
+    #[instrument(err, skip(self, message))]
     async fn build(
         &self,
         ism_address: H256,

diff --git a/main/agents/relayer/src/msg/metadata/routing.rs b/main/agents/relayer/src/msg/metadata/routing.rs
@@ -14,7 +14,7 @@ pub struct RoutingIsmMetadataBuilder {
 
 #[async_trait]
 impl MetadataBuilder for RoutingIsmMetadataBuilder {
-    #[instrument(err, skip(self), ret)]
+    #[instrument(err, skip(self, message), ret)]
     #[allow(clippy::blocks_in_conditions)] // TODO: `rustc` 1.80.1 clippy issue
     async fn build(
         &self,

diff --git a/main/agents/relayer/src/msg/op_submitter.rs b/main/agents/relayer/src/msg/op_submitter.rs
@@ -543,7 +543,7 @@ impl OperationBatch {
         };
 
         if !excluded_ops.is_empty() {
-            warn!(excluded_ops=?excluded_ops, "Either the batch tx would revert, or the operations would revert in the batch. Falling back to serial submission.");
+            warn!(excluded_ops=?excluded_ops, "Either operations reverted in the batch or the txid wasn't included. Falling back to serial submission.");
             OperationBatch::new(excluded_ops, self.domain)
                 .submit_serially(prepare_queue, confirm_queue, metrics)
                 .await;

diff --git a/main/agents/relayer/src/msg/pending_message.rs b/main/agents/relayer/src/msg/pending_message.rs
@@ -32,7 +32,7 @@ pub const CONFIRM_DELAY: Duration = if cfg!(any(test, feature = "test-utils")) {
     // Wait 5 seconds after submitting the message before confirming in test mode
     Duration::from_secs(5)
 } else {
-    // Wait 1 min after submitting the message before confirming in normal/production mode
+    // Wait 10 min after submitting the message before confirming in normal/production mode
     Duration::from_secs(60 * 10)
 };
 

diff --git a/main/chains/hyperlane-ethereum/src/ism/aggregation_ism.rs b/main/chains/hyperlane-ethereum/src/ism/aggregation_ism.rs
@@ -92,7 +92,7 @@ impl<M> AggregationIsm for EthereumAggregationIsm<M>
 where
     M: Middleware + 'static,
 {
-    #[instrument(err)]
+    #[instrument(err, skip(self, message))]
     #[allow(clippy::blocks_in_conditions)] // TODO: `rustc` 1.80.1 clippy issue
     async fn modules_and_threshold(
         &self,

diff --git a/main/chains/hyperlane-ethereum/src/ism/multisig_ism.rs b/main/chains/hyperlane-ethereum/src/ism/multisig_ism.rs
@@ -98,7 +98,7 @@ impl<M> MultisigIsm for EthereumMultisigIsm<M>
 where
     M: Middleware + 'static,
 {
-    #[instrument(err)]
+    #[instrument(err, skip(self, message))]
     #[allow(clippy::blocks_in_conditions)] // TODO: `rustc` 1.80.1 clippy issue
     async fn validators_and_threshold(
         &self,

diff --git a/main/chains/hyperlane-ethereum/src/ism/routing_ism.rs b/main/chains/hyperlane-ethereum/src/ism/routing_ism.rs
@@ -89,7 +89,7 @@ impl<M> RoutingIsm for EthereumRoutingIsm<M>
 where
     M: Middleware + 'static,
 {
-    #[instrument(err)]
+    #[instrument(err, skip(self, message))]
     #[allow(clippy::blocks_in_conditions)] // TODO: `rustc` 1.80.1 clippy issue
     async fn route(&self, message: &HyperlaneMessage) -> ChainResult<H256> {
         let ism = self

diff --git a/main/utils/run-locally/src/invariants/termination_invariants.rs b/main/utils/run-locally/src/invariants/termination_invariants.rs
@@ -39,7 +39,10 @@ pub fn termination_invariants_met(
     )?;
     assert!(!lengths.is_empty(), "Could not find queue length metric");
     if lengths.iter().sum::<u32>() != ZERO_MERKLE_INSERTION_KATHY_MESSAGES {
-        log!("Relayer queues not empty. Lengths: {:?}", lengths);
+        log!(
+            "Relayer queues contain more messages than the zero-merkle-insertion ones. Lengths: {:?}",
+            lengths
+        );
         return Ok(false);
     };