diff --git a/rust/main/agents/relayer/src/merkle_tree/processor.rs b/rust/main/agents/relayer/src/merkle_tree/processor.rs index e12a7fc343..0977c2cbe6 100644 --- a/rust/main/agents/relayer/src/merkle_tree/processor.rs +++ b/rust/main/agents/relayer/src/merkle_tree/processor.rs @@ -76,7 +76,7 @@ impl MerkleTreeProcessor { // Update the metrics // we assume that leaves are inserted in order so this will be monotonically increasing self.metrics - .latest_leaf_index_gauge + .latest_tree_insertion_gauge .set(insertion.index() as i64); Some(insertion) } else { @@ -89,14 +89,14 @@ impl MerkleTreeProcessor { #[derive(Debug)] pub struct MerkleTreeProcessorMetrics { - latest_leaf_index_gauge: IntGauge, + latest_tree_insertion_gauge: IntGauge, } impl MerkleTreeProcessorMetrics { pub fn new(metrics: &CoreMetrics, origin: &HyperlaneDomain) -> Self { Self { - latest_leaf_index_gauge: metrics - .latest_leaf_index() + latest_tree_insertion_gauge: metrics + .latest_tree_insertion() .with_label_values(&[origin.name()]), } } diff --git a/rust/main/hyperlane-base/src/metrics/core.rs b/rust/main/hyperlane-base/src/metrics/core.rs index 5bed85b60d..24cca269f2 100644 --- a/rust/main/hyperlane-base/src/metrics/core.rs +++ b/rust/main/hyperlane-base/src/metrics/core.rs @@ -38,7 +38,7 @@ pub struct CoreMetrics { span_counts: IntCounterVec, span_events: IntCounterVec, last_known_message_nonce: IntGaugeVec, - latest_leaf_index: IntGaugeVec, + latest_tree_insertion: IntGaugeVec, highest_seen_tree_index: IntGaugeVec, submitter_queue_length: IntGaugeVec, @@ -114,10 +114,10 @@ impl CoreMetrics { registry )?; - let latest_leaf_index = register_int_gauge_vec_with_registry!( + let latest_tree_insertion = register_int_gauge_vec_with_registry!( opts!( - namespaced!("latest_leaf_index"), - "Latest leaf index inserted into the merkle tree", + namespaced!("latest_tree_insertion"), + "Latest leaf inserted into the merkle tree", const_labels_ref ), &["origin"], @@ -198,7 +198,7 @@ impl CoreMetrics { span_counts, span_events, last_known_message_nonce, - latest_leaf_index, + latest_tree_insertion, highest_seen_tree_index, submitter_queue_length, @@ -336,8 +336,8 @@ impl CoreMetrics { /// /// Labels: /// - `origin`: Origin chain the leaf index is being tracked at. - pub fn latest_leaf_index(&self) -> IntGaugeVec { - self.latest_leaf_index.clone() + pub fn latest_tree_insertion(&self) -> IntGaugeVec { + self.latest_tree_insertion.clone() } /// Reports the current highest tree index seen by the relayer. diff --git a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs index cb851af8d2..246adad513 100644 --- a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs @@ -137,6 +137,21 @@ pub fn termination_invariants_met( return Ok(false); } + // check for each origin that the highest tree index seen by the syncer == the latest tree index inserted into the local tree + let latest_tree_insertion: Vec = fetch_metric( + RELAYER_METRICS_PORT, + "hyperlane_latest_tree_insertion", + &hashmap! {}, + )?; + let highest_seen_tree_index: Vec = fetch_metric( + RELAYER_METRICS_PORT, + "hyperlane_highest_seen_tree_index", + &hashmap! {}, + )?; + for (i, item) in latest_tree_insertion.iter().enumerate() { + assert_eq!(*item, highest_seen_tree_index[i]); + } + if let Some((solana_cli_tools_path, solana_config_path)) = solana_cli_tools_path.zip(solana_config_path) {