From 448d8c22d5a425d86d48bc25e56dbac8c2f16db2 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 30 Oct 2024 17:26:46 +0200 Subject: [PATCH 001/131] Remove height parameter, change tests --- crates/fuel-gas-price-algorithm/src/v1.rs | 57 ++++------ .../fuel-gas-price-algorithm/src/v1/tests.rs | 1 - .../v1/tests/update_da_record_data_tests.rs | 102 +++++++++++------- .../gas_price_service/src/v1/metadata.rs | 7 -- 4 files changed, 84 insertions(+), 83 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 9a9da066fee..e45e6dcae8f 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -16,14 +16,12 @@ mod tests; pub enum Error { #[error("Skipped L2 block update: expected {expected:?}, got {got:?}")] SkippedL2Block { expected: u32, got: u32 }, - #[error("Skipped DA block update: expected {expected:?}, got {got:?}")] - SkippedDABlock { expected: u32, got: u32 }, #[error("Could not calculate cost per byte: {bytes:?} bytes, {cost:?} cost")] CouldNotCalculateCostPerByte { bytes: u128, cost: u128 }, #[error("Failed to include L2 block data: {0}")] FailedTooIncludeL2BlockData(String), - #[error("L2 block expected but not found in unrecorded blocks: {0}")] - L2BlockExpectedNotFound(u32), + #[error("L2 block expected but not found in unrecorded blocks: {height:?}")] + L2BlockExpectedNotFound { height: u32 }, } // TODO: separate exec gas price and DA gas price into newtypes for clarity @@ -131,8 +129,6 @@ pub struct AlgorithmUpdaterV1 { pub max_da_gas_price_change_percent: u16, /// The cumulative reward from the DA portion of the gas price pub total_da_rewards_excess: u128, - /// The height of the last L2 block recorded on the DA chain - pub da_recorded_block_height: u32, /// The cumulative cost of recording L2 blocks on the DA chain as of the last recorded block pub latest_known_total_da_cost_excess: u128, /// The predicted cost of recording L2 blocks on the DA chain as of the last L2 block @@ -517,30 +513,19 @@ impl AlgorithmUpdaterV1 { height_range: Range, range_cost: u128, ) -> Result<(), Error> { - let expected = self.da_recorded_block_height.saturating_add(1); - let first = height_range.start; - if first != expected { - Err(Error::SkippedDABlock { - expected, - got: first, - }) - } else { - let last = height_range.end.saturating_sub(1); - let range_bytes = self.drain_l2_block_bytes_for_range(height_range)?; - let new_cost_per_byte: u128 = range_cost.checked_div(range_bytes).ok_or( - Error::CouldNotCalculateCostPerByte { - bytes: range_bytes, - cost: range_cost, - }, - )?; - self.da_recorded_block_height = last; - let new_da_block_cost = self - .latest_known_total_da_cost_excess - .saturating_add(range_cost); - self.latest_known_total_da_cost_excess = new_da_block_cost; - self.latest_da_cost_per_byte = new_cost_per_byte; - Ok(()) - } + let range_bytes = self.drain_l2_block_bytes_for_range(height_range)?; + let new_cost_per_byte: u128 = range_cost.checked_div(range_bytes).ok_or( + Error::CouldNotCalculateCostPerByte { + bytes: range_bytes, + cost: range_cost, + }, + )?; + let new_da_block_cost = self + .latest_known_total_da_cost_excess + .saturating_add(range_cost); + self.latest_known_total_da_cost_excess = new_da_block_cost; + self.latest_da_cost_per_byte = new_cost_per_byte; + Ok(()) } fn drain_l2_block_bytes_for_range( @@ -549,13 +534,11 @@ impl AlgorithmUpdaterV1 { ) -> Result { let mut total: u128 = 0; for expected_height in height_range { - let (actual_height, bytes) = self - .unrecorded_blocks - .pop_first() - .ok_or(Error::L2BlockExpectedNotFound(expected_height))?; - if actual_height != expected_height { - return Err(Error::L2BlockExpectedNotFound(expected_height)); - } + let bytes = self.unrecorded_blocks.remove(&expected_height).ok_or( + Error::L2BlockExpectedNotFound { + height: expected_height, + }, + )?; total = total.saturating_add(bytes as u128); } Ok(total) diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests.rs index e7150378428..c7d8c2c47b0 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests.rs @@ -184,7 +184,6 @@ impl UpdaterBuilder { l2_block_fullness_threshold_percent: self.l2_block_capacity_threshold.into(), total_da_rewards_excess: self.total_rewards, - da_recorded_block_height: self.da_recorded_block_height, latest_da_cost_per_byte: self.da_cost_per_byte, projected_total_da_cost: self.project_total_cost, latest_known_total_da_cost_excess: self.latest_known_total_cost, diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index 123520b4e99..e4993e8f156 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -6,43 +6,72 @@ use crate::v1::{ Error, }; -#[test] -fn update_da_record_data__increases_block() { - // given - let da_recorded_block_height = 0; - let recorded_range = 1u32..3; - let recorded_cost = 200; - let unrecorded_blocks = vec![ - BlockBytes { - height: 1, - block_bytes: 1000, - }, - BlockBytes { - height: 2, - block_bytes: 2000, - }, - ]; - let mut updater = UpdaterBuilder::new() - .with_da_recorded_block_height(da_recorded_block_height) - .with_unrecorded_blocks(unrecorded_blocks) - .build(); +// #[test] +// fn update_da_record_data__increases_block() { +// // given +// let da_recorded_block_height = 0; +// let recorded_range = 1u32..3; +// let recorded_cost = 200; +// let unrecorded_blocks = vec![ +// BlockBytes { +// height: 1, +// block_bytes: 1000, +// }, +// BlockBytes { +// height: 2, +// block_bytes: 2000, +// }, +// ]; +// let mut updater = UpdaterBuilder::new() +// .with_da_recorded_block_height(da_recorded_block_height) +// .with_unrecorded_blocks(unrecorded_blocks) +// .build(); +// +// // when +// updater +// .update_da_record_data(recorded_range, recorded_cost) +// .unwrap(); +// +// // then +// let expected = 2; +// let actual = updater.da_recorded_block_height; +// assert_eq!(expected, actual); +// } - // when - updater - .update_da_record_data(recorded_range, recorded_cost) - .unwrap(); - - // then - let expected = 2; - let actual = updater.da_recorded_block_height; - assert_eq!(expected, actual); -} +// #[test] +// fn update_da_record_data__throws_error_if_out_of_order() { +// // given +// let da_recorded_block_height = 0; +// let bad_recorded_range = 2u32..4; +// let recorded_cost = 200; +// let unrecorded_blocks = vec![BlockBytes { +// height: 1, +// block_bytes: 1000, +// }]; +// let mut updater = UpdaterBuilder::new() +// .with_da_recorded_block_height(da_recorded_block_height) +// .with_unrecorded_blocks(unrecorded_blocks) +// .build(); +// +// // when +// let actual_error = updater +// .update_da_record_data(bad_recorded_range, recorded_cost) +// .unwrap_err(); +// +// // then +// let expected_error = Error::SkippedDABlock { +// expected: 1, +// got: 2, +// }; +// assert_eq!(actual_error, expected_error); +// } #[test] -fn update_da_record_data__throws_error_if_out_of_order() { +fn update_da_record_data__throws_error_if_receives_a_block_missing_from_unrecorded_blocks( +) { // given let da_recorded_block_height = 0; - let bad_recorded_range = 2u32..4; + let recorded_range = 1u32..3; let recorded_cost = 200; let unrecorded_blocks = vec![BlockBytes { height: 1, @@ -55,14 +84,11 @@ fn update_da_record_data__throws_error_if_out_of_order() { // when let actual_error = updater - .update_da_record_data(bad_recorded_range, recorded_cost) + .update_da_record_data(recorded_range, recorded_cost) .unwrap_err(); // then - let expected_error = Error::SkippedDABlock { - expected: 1, - got: 2, - }; + let expected_error = Error::L2BlockExpectedNotFound { height: 2 }; assert_eq!(actual_error, expected_error); } @@ -186,7 +212,7 @@ fn update_da_record_data__if_da_height_matches_l2_height_projected_and_known_mat .unwrap(); // then - assert_eq!(updater.l2_block_height, updater.da_recorded_block_height); + assert_eq!(updater.unrecorded_blocks.len(), 0); assert_eq!( updater.projected_total_da_cost, updater.latest_known_total_da_cost_excess diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index 422cdf42507..0ceae95a768 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -20,8 +20,6 @@ pub struct V1Metadata { pub gas_price_factor: NonZeroU64, /// The cumulative reward from the DA portion of the gas price pub total_da_rewards_excess: u128, - /// The height of the last L2 block recorded on the DA chain - pub da_recorded_block_height: u32, /// The cumulative cost of recording L2 blocks on the DA chain as of the last recorded block pub latest_known_total_da_cost_excess: u128, /// The predicted cost of recording L2 blocks on the DA chain as of the last L2 block @@ -53,9 +51,6 @@ impl V1Metadata { .saturating_mul(config.gas_price_factor.get()), gas_price_factor: config.gas_price_factor, total_da_rewards_excess: 0, - // TODO: Set to `None` after: - // https://github.com/FuelLabs/fuel-core/issues/2397 - da_recorded_block_height: 0, latest_known_total_da_cost_excess: 0, projected_total_da_cost: 0, last_profit: 0, @@ -91,7 +86,6 @@ impl From for V1Metadata { new_scaled_da_gas_price: updater.new_scaled_da_gas_price, gas_price_factor: updater.gas_price_factor, total_da_rewards_excess: updater.total_da_rewards_excess, - da_recorded_block_height: updater.da_recorded_block_height, latest_known_total_da_cost_excess: updater.latest_known_total_da_cost_excess, projected_total_da_cost: updater.projected_total_da_cost, last_profit: updater.last_profit, @@ -119,7 +113,6 @@ pub fn v1_algorithm_from_metadata( new_scaled_da_gas_price: metadata.new_scaled_da_gas_price, gas_price_factor: metadata.gas_price_factor, total_da_rewards_excess: metadata.total_da_rewards_excess, - da_recorded_block_height: metadata.da_recorded_block_height, latest_known_total_da_cost_excess: metadata.latest_known_total_da_cost_excess, projected_total_da_cost: metadata.projected_total_da_cost, last_profit: metadata.last_profit, From c5866a37c72453694185e52e6592d86befd41334 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 30 Oct 2024 17:50:37 +0200 Subject: [PATCH 002/131] Modify to take vec instead of range --- crates/fuel-gas-price-algorithm/src/v1.rs | 17 ++--- .../v1/tests/update_da_record_data_tests.rs | 70 ++----------------- 2 files changed, 12 insertions(+), 75 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index e45e6dcae8f..bc824026c73 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -3,10 +3,7 @@ use std::{ cmp::max, collections::BTreeMap, num::NonZeroU64, - ops::{ - Div, - Range, - }, + ops::Div, }; #[cfg(test)] @@ -303,11 +300,11 @@ impl core::ops::Deref for ClampedPercentage { impl AlgorithmUpdaterV1 { pub fn update_da_record_data( &mut self, - height_range: Range, + heights: Vec, range_cost: u128, ) -> Result<(), Error> { - if !height_range.is_empty() { - self.da_block_update(height_range, range_cost)?; + if !heights.is_empty() { + self.da_block_update(heights, range_cost)?; self.recalculate_projected_cost(); } Ok(()) @@ -510,10 +507,10 @@ impl AlgorithmUpdaterV1 { fn da_block_update( &mut self, - height_range: Range, + heights: Vec, range_cost: u128, ) -> Result<(), Error> { - let range_bytes = self.drain_l2_block_bytes_for_range(height_range)?; + let range_bytes = self.drain_l2_block_bytes_for_range(heights)?; let new_cost_per_byte: u128 = range_cost.checked_div(range_bytes).ok_or( Error::CouldNotCalculateCostPerByte { bytes: range_bytes, @@ -530,7 +527,7 @@ impl AlgorithmUpdaterV1 { fn drain_l2_block_bytes_for_range( &mut self, - height_range: Range, + height_range: Vec, ) -> Result { let mut total: u128 = 0; for expected_height in height_range { diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index e4993e8f156..a18dab86fb3 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -6,72 +6,12 @@ use crate::v1::{ Error, }; -// #[test] -// fn update_da_record_data__increases_block() { -// // given -// let da_recorded_block_height = 0; -// let recorded_range = 1u32..3; -// let recorded_cost = 200; -// let unrecorded_blocks = vec![ -// BlockBytes { -// height: 1, -// block_bytes: 1000, -// }, -// BlockBytes { -// height: 2, -// block_bytes: 2000, -// }, -// ]; -// let mut updater = UpdaterBuilder::new() -// .with_da_recorded_block_height(da_recorded_block_height) -// .with_unrecorded_blocks(unrecorded_blocks) -// .build(); -// -// // when -// updater -// .update_da_record_data(recorded_range, recorded_cost) -// .unwrap(); -// -// // then -// let expected = 2; -// let actual = updater.da_recorded_block_height; -// assert_eq!(expected, actual); -// } - -// #[test] -// fn update_da_record_data__throws_error_if_out_of_order() { -// // given -// let da_recorded_block_height = 0; -// let bad_recorded_range = 2u32..4; -// let recorded_cost = 200; -// let unrecorded_blocks = vec![BlockBytes { -// height: 1, -// block_bytes: 1000, -// }]; -// let mut updater = UpdaterBuilder::new() -// .with_da_recorded_block_height(da_recorded_block_height) -// .with_unrecorded_blocks(unrecorded_blocks) -// .build(); -// -// // when -// let actual_error = updater -// .update_da_record_data(bad_recorded_range, recorded_cost) -// .unwrap_err(); -// -// // then -// let expected_error = Error::SkippedDABlock { -// expected: 1, -// got: 2, -// }; -// assert_eq!(actual_error, expected_error); -// } - #[test] fn update_da_record_data__throws_error_if_receives_a_block_missing_from_unrecorded_blocks( ) { // given let da_recorded_block_height = 0; - let recorded_range = 1u32..3; + let recorded_range = (1u32..3).collect(); let recorded_cost = 200; let unrecorded_blocks = vec![BlockBytes { height: 1, @@ -108,7 +48,7 @@ fn update_da_record_data__updates_cost_per_byte() { let new_cost_per_byte = 100; let recorded_cost = (block_bytes * new_cost_per_byte) as u128; - let recorded_range = 1u32..2; + let recorded_range = (1u32..2).collect(); // when updater .update_da_record_data(recorded_range, recorded_cost) @@ -151,7 +91,7 @@ fn update_da_record_data__updates_known_total_cost() { .with_unrecorded_blocks(unrecorded_blocks) .build(); - let recorded_range = 11u32..14; + let recorded_range = (11u32..14).collect(); let recorded_cost = 300; // when updater @@ -204,7 +144,7 @@ fn update_da_record_data__if_da_height_matches_l2_height_projected_and_known_mat let new_cost_per_byte = 100; let block_cost = block_bytes * new_cost_per_byte; - let recorded_range = 11u32..14; + let recorded_range = (11u32..14).collect(); let recorded_cost = block_cost * 3; // when updater @@ -279,7 +219,7 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g }); let min = recorded_heights.iter().min().unwrap(); let max = recorded_heights.iter().max().unwrap(); - let recorded_range = *min..(max + 1); + let recorded_range = (*min..(max + 1)).collect(); // when updater From c54f09802168189dd0bc91128c0bbd3e046a5ec2 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 30 Oct 2024 18:31:43 +0200 Subject: [PATCH 003/131] Add tests to make sure it is sane --- crates/fuel-gas-price-algorithm/src/v1.rs | 2 +- .../fuel-gas-price-algorithm/src/v1/tests.rs | 7 - .../v1/tests/update_da_record_data_tests.rs | 137 +++++++++++++++++- 3 files changed, 130 insertions(+), 16 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index bc824026c73..85b43defbbc 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -546,7 +546,7 @@ impl AlgorithmUpdaterV1 { let projection_portion: u128 = self .unrecorded_blocks .iter() - .map(|(_, &bytes)| (bytes as u128)) + .map(|(_, &bytes)| bytes as u128) .fold(0_u128, |acc, n| acc.saturating_add(n)) .saturating_mul(self.latest_da_cost_per_byte); self.projected_total_da_cost = self diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests.rs index c7d8c2c47b0..816c491cc4e 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests.rs @@ -35,7 +35,6 @@ pub struct UpdaterBuilder { l2_block_capacity_threshold: u8, total_rewards: u128, - da_recorded_block_height: u32, da_cost_per_byte: u128, project_total_cost: u128, latest_known_total_cost: u128, @@ -63,7 +62,6 @@ impl UpdaterBuilder { l2_block_capacity_threshold: 50, total_rewards: 0, - da_recorded_block_height: 0, da_cost_per_byte: 0, project_total_cost: 0, latest_known_total_cost: 0, @@ -133,11 +131,6 @@ impl UpdaterBuilder { self } - fn with_da_recorded_block_height(mut self, da_recorded_block_height: u32) -> Self { - self.da_recorded_block_height = da_recorded_block_height; - self - } - fn with_da_cost_per_byte(mut self, da_cost_per_byte: u128) -> Self { self.da_cost_per_byte = da_cost_per_byte; self diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index a18dab86fb3..c69919cfa4b 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -10,7 +10,6 @@ use crate::v1::{ fn update_da_record_data__throws_error_if_receives_a_block_missing_from_unrecorded_blocks( ) { // given - let da_recorded_block_height = 0; let recorded_range = (1u32..3).collect(); let recorded_cost = 200; let unrecorded_blocks = vec![BlockBytes { @@ -18,7 +17,6 @@ fn update_da_record_data__throws_error_if_receives_a_block_missing_from_unrecord block_bytes: 1000, }]; let mut updater = UpdaterBuilder::new() - .with_da_recorded_block_height(da_recorded_block_height) .with_unrecorded_blocks(unrecorded_blocks) .build(); @@ -64,7 +62,6 @@ fn update_da_record_data__updates_cost_per_byte() { fn update_da_record_data__updates_known_total_cost() { // given let da_cost_per_byte = 20; - let da_recorded_block_height = 10; let l2_block_height = 15; let projected_total_cost = 2000; let known_total_cost = 1500; @@ -84,7 +81,6 @@ fn update_da_record_data__updates_known_total_cost() { ]; let mut updater = UpdaterBuilder::new() .with_da_cost_per_byte(da_cost_per_byte) - .with_da_recorded_block_height(da_recorded_block_height) .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost) .with_known_total_cost(known_total_cost) @@ -108,7 +104,6 @@ fn update_da_record_data__updates_known_total_cost() { fn update_da_record_data__if_da_height_matches_l2_height_projected_and_known_match() { // given let da_cost_per_byte = 20; - let da_recorded_block_height = 10; let l2_block_height = 13; let known_total_cost = 1500; let unrecorded_blocks = vec![ @@ -133,7 +128,6 @@ fn update_da_record_data__if_da_height_matches_l2_height_projected_and_known_mat let projected_total_cost = known_total_cost + guessed_cost; let mut updater = UpdaterBuilder::new() .with_da_cost_per_byte(da_cost_per_byte as u128) - .with_da_recorded_block_height(da_recorded_block_height) .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost as u128) .with_known_total_cost(known_total_cost as u128) @@ -164,7 +158,6 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g ) { // given let da_cost_per_byte = 20; - let da_recorded_block_height = 10; let l2_block_height = 15; let original_known_total_cost = 1500; let block_bytes = 1000; @@ -202,7 +195,6 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g let projected_total_cost = original_known_total_cost + guessed_cost; let mut updater = UpdaterBuilder::new() .with_da_cost_per_byte(da_cost_per_byte as u128) - .with_da_recorded_block_height(da_recorded_block_height) .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost as u128) .with_known_total_cost(original_known_total_cost as u128) @@ -236,3 +228,132 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g let expected = new_known_total_cost + guessed_part; assert_eq!(actual, expected as u128); } + +#[test] +fn update_da_record_data__updates_known_total_cost_if_blocks_are_out_of_order() { + // given + let da_cost_per_byte = 20; + let block_bytes = 1000; + let unrecorded_blocks = vec![ + BlockBytes { + height: 1, + block_bytes, + }, + BlockBytes { + height: 2, + block_bytes, + }, + BlockBytes { + height: 3, + block_bytes, + }, + ]; + let old_known_total_cost = 500; + let old_projected_total_cost = + old_known_total_cost + (block_bytes as u128 * da_cost_per_byte * 3); + let old_da_cost_per_byte = 20; + let mut updater = UpdaterBuilder::new() + .with_da_cost_per_byte(da_cost_per_byte) + .with_unrecorded_blocks(unrecorded_blocks) + .with_da_cost_per_byte(old_da_cost_per_byte) + .with_known_total_cost(old_known_total_cost) + .with_projected_total_cost(old_projected_total_cost) + .build(); + let new_cost_per_byte = 100; + let recorded_cost = 2 * (block_bytes * new_cost_per_byte) as u128; + let recorded_range = vec![2, 3]; + + // when + updater + .update_da_record_data(recorded_range, recorded_cost) + .unwrap(); + + // and + let expected = updater.latest_known_total_da_cost_excess + + (block_bytes * new_cost_per_byte) as u128; + let actual = updater.projected_total_da_cost; + assert_eq!(actual, expected); +} +#[test] +fn update_da_record_data__updates_projected_total_cost_if_blocks_are_out_of_order() { + // given + let da_cost_per_byte = 20; + let block_bytes = 1000; + let unrecorded_blocks = vec![ + BlockBytes { + height: 1, + block_bytes, + }, + BlockBytes { + height: 2, + block_bytes, + }, + BlockBytes { + height: 3, + block_bytes, + }, + ]; + let old_known_total_cost = 500; + let old_projected_total_cost = + old_known_total_cost + (block_bytes as u128 * da_cost_per_byte * 3); + let old_da_cost_per_byte = 20; + let mut updater = UpdaterBuilder::new() + .with_da_cost_per_byte(da_cost_per_byte) + .with_unrecorded_blocks(unrecorded_blocks) + .with_da_cost_per_byte(old_da_cost_per_byte) + .with_known_total_cost(old_known_total_cost) + .with_projected_total_cost(old_projected_total_cost) + .build(); + let new_cost_per_byte = 100; + let recorded_cost = 2 * (block_bytes * new_cost_per_byte) as u128; + let recorded_range = vec![2, 3]; + + // when + updater + .update_da_record_data(recorded_range, recorded_cost) + .unwrap(); + + // then + let expected = updater.latest_known_total_da_cost_excess + + (block_bytes * new_cost_per_byte) as u128; + let actual = updater.projected_total_da_cost; + assert_eq!(actual, expected); +} + +#[test] +fn update_da_record_data__updates_unrecorded_blocks() { + // given + let da_cost_per_byte = 20; + let block_bytes = 1000; + let unrecorded_blocks = vec![ + BlockBytes { + height: 1, + block_bytes, + }, + BlockBytes { + height: 2, + block_bytes, + }, + BlockBytes { + height: 3, + block_bytes, + }, + ]; + let mut updater = UpdaterBuilder::new() + .with_da_cost_per_byte(da_cost_per_byte) + .with_unrecorded_blocks(unrecorded_blocks) + .build(); + let new_cost_per_byte = 100; + let recorded_cost = 2 * (block_bytes * new_cost_per_byte) as u128; + let recorded_range = vec![2, 3]; + + // when + updater + .update_da_record_data(recorded_range, recorded_cost) + .unwrap(); + + // then + let expected = vec![(1, block_bytes)]; + let actual: Vec<_> = updater.unrecorded_blocks.into_iter().collect(); + assert_eq!(actual, expected); +} From fcf44777e9c13573e9e7d0a345113089fac49131 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 30 Oct 2024 18:36:07 +0200 Subject: [PATCH 004/131] Rename variable --- crates/fuel-gas-price-algorithm/src/v1.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 85b43defbbc..b4b690763cb 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -510,10 +510,10 @@ impl AlgorithmUpdaterV1 { heights: Vec, range_cost: u128, ) -> Result<(), Error> { - let range_bytes = self.drain_l2_block_bytes_for_range(heights)?; - let new_cost_per_byte: u128 = range_cost.checked_div(range_bytes).ok_or( + let recorded_bytes = self.drain_l2_block_bytes_for_range(heights)?; + let new_cost_per_byte: u128 = range_cost.checked_div(recorded_bytes).ok_or( Error::CouldNotCalculateCostPerByte { - bytes: range_bytes, + bytes: recorded_bytes, cost: range_cost, }, )?; From 159d5b4c68f325d27c16643a0ffc27d212c90a95 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 30 Oct 2024 18:38:37 +0200 Subject: [PATCH 005/131] Fix analyzer --- .../gas-price-analysis/src/charts.rs | 6 +++--- .../gas-price-analysis/src/simulation.rs | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs index 6b4c82f09ff..3c8b1175b05 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs @@ -268,15 +268,15 @@ pub fn draw_profit( const PROJECTED_PROFIT_COLOR: RGBColor = RED; const PESSIMISTIC_BLOCK_COST_COLOR: RGBColor = BLUE; let actual_profit_gwei: Vec<_> = actual_profit - .into_iter() + .iter() .map(|x| x / ONE_GWEI as i128) .collect(); let projected_profit_gwei: Vec<_> = projected_profit - .into_iter() + .iter() .map(|x| x / ONE_GWEI as i128) .collect(); let pessimistic_block_costs_gwei: Vec<_> = pessimistic_block_costs - .into_iter() + .iter() .map(|x| x / ONE_GWEI as u128) .collect(); let min = *std::cmp::min( diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs index 590952207ba..efca39438c3 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs @@ -103,7 +103,6 @@ impl Simulator { // Increase to make the da price change faster max_da_gas_price_change_percent: 10, total_da_rewards_excess: 0, - da_recorded_block_height: 0, // Change to adjust the cost per byte of the DA on block 0 latest_da_cost_per_byte: 0, projected_total_da_cost: 0, @@ -162,9 +161,8 @@ impl Simulator { // Update DA blocks on the occasion there is one if let Some((range, cost)) = da_block { for height in range.to_owned() { - updater - .update_da_record_data(height..(height + 1), cost) - .unwrap(); + let block_heights = (height..(height) + 1).collect(); + updater.update_da_record_data(block_heights, cost).unwrap(); actual_costs.push(updater.latest_known_total_da_cost_excess) } } From ba080ff3989d7e5dc5b36894417048654ed2ca7c Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 30 Oct 2024 18:41:35 +0200 Subject: [PATCH 006/131] Rename some params --- .../gas-price-analysis/src/charts.rs | 6 ++---- crates/fuel-gas-price-algorithm/src/v1.rs | 16 ++++++++-------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs index 3c8b1175b05..c0ce77d261c 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs @@ -267,10 +267,8 @@ pub fn draw_profit( const ACTUAL_PROFIT_COLOR: RGBColor = BLACK; const PROJECTED_PROFIT_COLOR: RGBColor = RED; const PESSIMISTIC_BLOCK_COST_COLOR: RGBColor = BLUE; - let actual_profit_gwei: Vec<_> = actual_profit - .iter() - .map(|x| x / ONE_GWEI as i128) - .collect(); + let actual_profit_gwei: Vec<_> = + actual_profit.iter().map(|x| x / ONE_GWEI as i128).collect(); let projected_profit_gwei: Vec<_> = projected_profit .iter() .map(|x| x / ONE_GWEI as i128) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index b4b690763cb..60d2142eabd 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -301,10 +301,10 @@ impl AlgorithmUpdaterV1 { pub fn update_da_record_data( &mut self, heights: Vec, - range_cost: u128, + recording_cost: u128, ) -> Result<(), Error> { if !heights.is_empty() { - self.da_block_update(heights, range_cost)?; + self.da_block_update(heights, recording_cost)?; self.recalculate_projected_cost(); } Ok(()) @@ -508,18 +508,18 @@ impl AlgorithmUpdaterV1 { fn da_block_update( &mut self, heights: Vec, - range_cost: u128, + recording_cost: u128, ) -> Result<(), Error> { let recorded_bytes = self.drain_l2_block_bytes_for_range(heights)?; - let new_cost_per_byte: u128 = range_cost.checked_div(recorded_bytes).ok_or( + let new_cost_per_byte: u128 = recording_cost.checked_div(recorded_bytes).ok_or( Error::CouldNotCalculateCostPerByte { bytes: recorded_bytes, - cost: range_cost, + cost: recording_cost, }, )?; let new_da_block_cost = self .latest_known_total_da_cost_excess - .saturating_add(range_cost); + .saturating_add(recording_cost); self.latest_known_total_da_cost_excess = new_da_block_cost; self.latest_da_cost_per_byte = new_cost_per_byte; Ok(()) @@ -527,10 +527,10 @@ impl AlgorithmUpdaterV1 { fn drain_l2_block_bytes_for_range( &mut self, - height_range: Vec, + heights: Vec, ) -> Result { let mut total: u128 = 0; - for expected_height in height_range { + for expected_height in heights { let bytes = self.unrecorded_blocks.remove(&expected_height).ok_or( Error::L2BlockExpectedNotFound { height: expected_height, From d2b3c280c0b1abb2a0e7105172aebcbd63102d6c Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 30 Oct 2024 19:31:52 +0200 Subject: [PATCH 007/131] Use slice instead of vec --- .../gas-price-analysis/src/simulation.rs | 4 +-- crates/fuel-gas-price-algorithm/src/v1.rs | 13 +++----- .../v1/tests/update_da_record_data_tests.rs | 32 +++++++++---------- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs index efca39438c3..22d994a7a68 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs @@ -161,8 +161,8 @@ impl Simulator { // Update DA blocks on the occasion there is one if let Some((range, cost)) = da_block { for height in range.to_owned() { - let block_heights = (height..(height) + 1).collect(); - updater.update_da_record_data(block_heights, cost).unwrap(); + let block_heights: Vec = (height..(height) + 1).collect(); + updater.update_da_record_data(&block_heights, cost).unwrap(); actual_costs.push(updater.latest_known_total_da_cost_excess) } } diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 60d2142eabd..56764e0ae1c 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -300,7 +300,7 @@ impl core::ops::Deref for ClampedPercentage { impl AlgorithmUpdaterV1 { pub fn update_da_record_data( &mut self, - heights: Vec, + heights: &[u32], recording_cost: u128, ) -> Result<(), Error> { if !heights.is_empty() { @@ -507,7 +507,7 @@ impl AlgorithmUpdaterV1 { fn da_block_update( &mut self, - heights: Vec, + heights: &[u32], recording_cost: u128, ) -> Result<(), Error> { let recorded_bytes = self.drain_l2_block_bytes_for_range(heights)?; @@ -525,15 +525,12 @@ impl AlgorithmUpdaterV1 { Ok(()) } - fn drain_l2_block_bytes_for_range( - &mut self, - heights: Vec, - ) -> Result { + fn drain_l2_block_bytes_for_range(&mut self, heights: &[u32]) -> Result { let mut total: u128 = 0; for expected_height in heights { - let bytes = self.unrecorded_blocks.remove(&expected_height).ok_or( + let bytes = self.unrecorded_blocks.remove(expected_height).ok_or( Error::L2BlockExpectedNotFound { - height: expected_height, + height: *expected_height, }, )?; total = total.saturating_add(bytes as u128); diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index c69919cfa4b..8ec3356fadb 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -10,7 +10,7 @@ use crate::v1::{ fn update_da_record_data__throws_error_if_receives_a_block_missing_from_unrecorded_blocks( ) { // given - let recorded_range = (1u32..3).collect(); + let recorded_range: Vec = (1u32..3).collect(); let recorded_cost = 200; let unrecorded_blocks = vec![BlockBytes { height: 1, @@ -22,7 +22,7 @@ fn update_da_record_data__throws_error_if_receives_a_block_missing_from_unrecord // when let actual_error = updater - .update_da_record_data(recorded_range, recorded_cost) + .update_da_record_data(&recorded_range, recorded_cost) .unwrap_err(); // then @@ -46,10 +46,10 @@ fn update_da_record_data__updates_cost_per_byte() { let new_cost_per_byte = 100; let recorded_cost = (block_bytes * new_cost_per_byte) as u128; - let recorded_range = (1u32..2).collect(); + let recorded_range: Vec = (1u32..2).collect(); // when updater - .update_da_record_data(recorded_range, recorded_cost) + .update_da_record_data(&recorded_range, recorded_cost) .unwrap(); // then @@ -87,11 +87,11 @@ fn update_da_record_data__updates_known_total_cost() { .with_unrecorded_blocks(unrecorded_blocks) .build(); - let recorded_range = (11u32..14).collect(); + let recorded_range: Vec = (11u32..14).collect(); let recorded_cost = 300; // when updater - .update_da_record_data(recorded_range, recorded_cost) + .update_da_record_data(&recorded_range, recorded_cost) .unwrap(); // then @@ -138,11 +138,11 @@ fn update_da_record_data__if_da_height_matches_l2_height_projected_and_known_mat let new_cost_per_byte = 100; let block_cost = block_bytes * new_cost_per_byte; - let recorded_range = (11u32..14).collect(); + let recorded_range: Vec = (11u32..14).collect(); let recorded_cost = block_cost * 3; // when updater - .update_da_record_data(recorded_range, recorded_cost) + .update_da_record_data(&recorded_range, recorded_cost) .unwrap(); // then @@ -211,11 +211,11 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g }); let min = recorded_heights.iter().min().unwrap(); let max = recorded_heights.iter().max().unwrap(); - let recorded_range = (*min..(max + 1)).collect(); + let recorded_range: Vec = (*min..(max + 1)).collect(); // when updater - .update_da_record_data(recorded_range, recorded_cost as u128) + .update_da_record_data(&recorded_range, recorded_cost as u128) .unwrap(); // then @@ -261,11 +261,11 @@ fn update_da_record_data__updates_known_total_cost_if_blocks_are_out_of_order() .build(); let new_cost_per_byte = 100; let recorded_cost = 2 * (block_bytes * new_cost_per_byte) as u128; - let recorded_range = vec![2, 3]; + let recorded_range: Vec = vec![2, 3]; // when updater - .update_da_record_data(recorded_range, recorded_cost) + .update_da_record_data(&recorded_range, recorded_cost) .unwrap(); // and @@ -306,11 +306,11 @@ fn update_da_record_data__updates_projected_total_cost_if_blocks_are_out_of_orde .build(); let new_cost_per_byte = 100; let recorded_cost = 2 * (block_bytes * new_cost_per_byte) as u128; - let recorded_range = vec![2, 3]; + let recorded_range: Vec = vec![2, 3]; // when updater - .update_da_record_data(recorded_range, recorded_cost) + .update_da_record_data(&recorded_range, recorded_cost) .unwrap(); // then @@ -345,11 +345,11 @@ fn update_da_record_data__updates_unrecorded_blocks() { .build(); let new_cost_per_byte = 100; let recorded_cost = 2 * (block_bytes * new_cost_per_byte) as u128; - let recorded_range = vec![2, 3]; + let recorded_range: Vec = vec![2, 3]; // when updater - .update_da_record_data(recorded_range, recorded_cost) + .update_da_record_data(&recorded_range, recorded_cost) .unwrap(); // then From 5f199323422386e91373282e131c9c69e760b111 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 31 Oct 2024 11:38:35 +0200 Subject: [PATCH 008/131] Cleanup code a bit --- .../gas-price-analysis/src/charts.rs | 14 ++++++-- crates/fuel-gas-price-algorithm/src/v1.rs | 9 +++-- .../v1/tests/update_da_record_data_tests.rs | 34 +++++++++---------- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs index c0ce77d261c..7b884357dfc 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs @@ -257,6 +257,14 @@ pub fn draw_bytes_and_cost_per_block( Ok(()) } +fn one_gwei_i128() -> i128 { + i128::from(ONE_GWEI) +} + +fn one_gwei_u128() -> u128 { + u128::from(ONE_GWEI) +} + pub fn draw_profit( drawing_area: &DrawingArea, actual_profit: &[i128], @@ -268,14 +276,14 @@ pub fn draw_profit( const PROJECTED_PROFIT_COLOR: RGBColor = RED; const PESSIMISTIC_BLOCK_COST_COLOR: RGBColor = BLUE; let actual_profit_gwei: Vec<_> = - actual_profit.iter().map(|x| x / ONE_GWEI as i128).collect(); + actual_profit.iter().map(|x| x / one_gwei_i128()).collect(); let projected_profit_gwei: Vec<_> = projected_profit .iter() - .map(|x| x / ONE_GWEI as i128) + .map(|x| x / one_gwei_i128()) .collect(); let pessimistic_block_costs_gwei: Vec<_> = pessimistic_block_costs .iter() - .map(|x| x / ONE_GWEI as u128) + .map(|x| x / one_gwei_u128()) .collect(); let min = *std::cmp::min( actual_profit_gwei diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 56764e0ae1c..034898c984a 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -510,7 +510,7 @@ impl AlgorithmUpdaterV1 { heights: &[u32], recording_cost: u128, ) -> Result<(), Error> { - let recorded_bytes = self.drain_l2_block_bytes_for_range(heights)?; + let recorded_bytes = self.drain_l2_block_bytes_for_heights(heights)?; let new_cost_per_byte: u128 = recording_cost.checked_div(recorded_bytes).ok_or( Error::CouldNotCalculateCostPerByte { bytes: recorded_bytes, @@ -525,7 +525,10 @@ impl AlgorithmUpdaterV1 { Ok(()) } - fn drain_l2_block_bytes_for_range(&mut self, heights: &[u32]) -> Result { + fn drain_l2_block_bytes_for_heights( + &mut self, + heights: &[u32], + ) -> Result { let mut total: u128 = 0; for expected_height in heights { let bytes = self.unrecorded_blocks.remove(expected_height).ok_or( @@ -543,7 +546,7 @@ impl AlgorithmUpdaterV1 { let projection_portion: u128 = self .unrecorded_blocks .iter() - .map(|(_, &bytes)| bytes as u128) + .map(|(_, &bytes)| u128::from(bytes)) .fold(0_u128, |acc, n| acc.saturating_add(n)) .saturating_mul(self.latest_da_cost_per_byte); self.projected_total_da_cost = self diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index 8ec3356fadb..1da66b2903d 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -10,7 +10,7 @@ use crate::v1::{ fn update_da_record_data__throws_error_if_receives_a_block_missing_from_unrecorded_blocks( ) { // given - let recorded_range: Vec = (1u32..3).collect(); + let recorded_heights: Vec = (1u32..3).collect(); let recorded_cost = 200; let unrecorded_blocks = vec![BlockBytes { height: 1, @@ -22,7 +22,7 @@ fn update_da_record_data__throws_error_if_receives_a_block_missing_from_unrecord // when let actual_error = updater - .update_da_record_data(&recorded_range, recorded_cost) + .update_da_record_data(&recorded_heights, recorded_cost) .unwrap_err(); // then @@ -46,10 +46,10 @@ fn update_da_record_data__updates_cost_per_byte() { let new_cost_per_byte = 100; let recorded_cost = (block_bytes * new_cost_per_byte) as u128; - let recorded_range: Vec = (1u32..2).collect(); + let recorded_heights: Vec = (1u32..2).collect(); // when updater - .update_da_record_data(&recorded_range, recorded_cost) + .update_da_record_data(&recorded_heights, recorded_cost) .unwrap(); // then @@ -87,11 +87,11 @@ fn update_da_record_data__updates_known_total_cost() { .with_unrecorded_blocks(unrecorded_blocks) .build(); - let recorded_range: Vec = (11u32..14).collect(); + let recorded_heights: Vec = (11u32..14).collect(); let recorded_cost = 300; // when updater - .update_da_record_data(&recorded_range, recorded_cost) + .update_da_record_data(&recorded_heights, recorded_cost) .unwrap(); // then @@ -138,11 +138,11 @@ fn update_da_record_data__if_da_height_matches_l2_height_projected_and_known_mat let new_cost_per_byte = 100; let block_cost = block_bytes * new_cost_per_byte; - let recorded_range: Vec = (11u32..14).collect(); + let recorded_heights: Vec = (11u32..14).collect(); let recorded_cost = block_cost * 3; // when updater - .update_da_record_data(&recorded_range, recorded_cost) + .update_da_record_data(&recorded_heights, recorded_cost) .unwrap(); // then @@ -211,11 +211,11 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g }); let min = recorded_heights.iter().min().unwrap(); let max = recorded_heights.iter().max().unwrap(); - let recorded_range: Vec = (*min..(max + 1)).collect(); + let recorded_heights: Vec = (*min..(max + 1)).collect(); // when updater - .update_da_record_data(&recorded_range, recorded_cost as u128) + .update_da_record_data(&recorded_heights, recorded_cost as u128) .unwrap(); // then @@ -261,14 +261,14 @@ fn update_da_record_data__updates_known_total_cost_if_blocks_are_out_of_order() .build(); let new_cost_per_byte = 100; let recorded_cost = 2 * (block_bytes * new_cost_per_byte) as u128; - let recorded_range: Vec = vec![2, 3]; + let recorded_heights: Vec = vec![3, 2]; // when updater - .update_da_record_data(&recorded_range, recorded_cost) + .update_da_record_data(&recorded_heights, recorded_cost) .unwrap(); - // and + // then let expected = updater.latest_known_total_da_cost_excess + (block_bytes * new_cost_per_byte) as u128; let actual = updater.projected_total_da_cost; @@ -306,11 +306,11 @@ fn update_da_record_data__updates_projected_total_cost_if_blocks_are_out_of_orde .build(); let new_cost_per_byte = 100; let recorded_cost = 2 * (block_bytes * new_cost_per_byte) as u128; - let recorded_range: Vec = vec![2, 3]; + let recorded_heights: Vec = vec![3, 2]; // when updater - .update_da_record_data(&recorded_range, recorded_cost) + .update_da_record_data(&recorded_heights, recorded_cost) .unwrap(); // then @@ -345,11 +345,11 @@ fn update_da_record_data__updates_unrecorded_blocks() { .build(); let new_cost_per_byte = 100; let recorded_cost = 2 * (block_bytes * new_cost_per_byte) as u128; - let recorded_range: Vec = vec![2, 3]; + let recorded_heights: Vec = vec![3, 2]; // when updater - .update_da_record_data(&recorded_range, recorded_cost) + .update_da_record_data(&recorded_heights, recorded_cost) .unwrap(); // then From 922f3e00126b51766e344d8834a99accd5c198bf Mon Sep 17 00:00:00 2001 From: Mitchell Turner Date: Thu, 31 Oct 2024 12:31:22 +0200 Subject: [PATCH 009/131] Update crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rafał Chabowski <88321181+rafal-ch@users.noreply.github.com> --- .../src/v1/tests/update_da_record_data_tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index 1da66b2903d..f826a092a34 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -274,6 +274,7 @@ fn update_da_record_data__updates_known_total_cost_if_blocks_are_out_of_order() let actual = updater.projected_total_da_cost; assert_eq!(actual, expected); } + #[test] fn update_da_record_data__updates_projected_total_cost_if_blocks_are_out_of_order() { // given From d7e9bddde76025618676674fe43704636f77c186 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 31 Oct 2024 12:32:59 +0200 Subject: [PATCH 010/131] Pick nit --- crates/fuel-gas-price-algorithm/src/v1.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 034898c984a..8a4b98bb1af 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -17,7 +17,7 @@ pub enum Error { CouldNotCalculateCostPerByte { bytes: u128, cost: u128 }, #[error("Failed to include L2 block data: {0}")] FailedTooIncludeL2BlockData(String), - #[error("L2 block expected but not found in unrecorded blocks: {height:?}")] + #[error("L2 block expected but not found in unrecorded blocks: {height}")] L2BlockExpectedNotFound { height: u32 }, } From 2bb86fe406840c0cd2944ba0cb3356f6b790badf Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 1 Nov 2024 10:24:05 +0200 Subject: [PATCH 011/131] Optimize projected calc by caching running total --- crates/fuel-gas-price-algorithm/src/v1.rs | 14 ++++++++------ crates/fuel-gas-price-algorithm/src/v1/tests.rs | 4 ++++ .../src/v1/tests/update_l2_block_data_tests.rs | 15 +++++++++++---- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 8a4b98bb1af..bfc1433a6b5 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -145,6 +145,8 @@ pub struct AlgorithmUpdaterV1 { pub l2_activity: L2ActivityTracker, /// The unrecorded blocks that are used to calculate the projected cost of recording blocks pub unrecorded_blocks: BTreeMap, + /// Total unrecorded block bytes + pub unrecorded_blocks_bytes: u128, } /// The `L2ActivityTracker` tracks the chain activity to determine a safety mode for setting the DA price. @@ -348,6 +350,9 @@ impl AlgorithmUpdaterV1 { // metadata self.unrecorded_blocks.insert(height, block_bytes); + self.unrecorded_blocks_bytes = self + .unrecorded_blocks_bytes + .saturating_add(block_bytes as u128); Ok(()) } } @@ -538,16 +543,13 @@ impl AlgorithmUpdaterV1 { )?; total = total.saturating_add(bytes as u128); } + self.unrecorded_blocks_bytes = self.unrecorded_blocks_bytes.saturating_sub(total); Ok(total) } fn recalculate_projected_cost(&mut self) { - // add the cost of the remaining blocks - let projection_portion: u128 = self - .unrecorded_blocks - .iter() - .map(|(_, &bytes)| u128::from(bytes)) - .fold(0_u128, |acc, n| acc.saturating_add(n)) + let projection_portion = self + .unrecorded_blocks_bytes .saturating_mul(self.latest_da_cost_per_byte); self.projected_total_da_cost = self .latest_known_total_da_cost_excess diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests.rs index 816c491cc4e..9e3dc879651 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests.rs @@ -193,6 +193,10 @@ impl UpdaterBuilder { .try_into() .expect("Should never be non-zero"), l2_activity: self.l2_activity, + unrecorded_blocks_bytes: self + .unrecorded_blocks + .iter() + .fold(0u128, |acc, b| acc + u128::from(b.block_bytes)), } } } diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs index 5cb56bbde2d..8ce3b6b5d4e 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs @@ -591,9 +591,10 @@ fn update_l2_block_data__retains_existing_blocks_and_adds_l2_block_to_unrecorded { // given let starting_block = 0; + let first_block_bytes = 1200; let preexisting_block = BlockBytes { height: 0, - block_bytes: 1000, + block_bytes: first_block_bytes, }; let mut updater = UpdaterBuilder::new() @@ -604,27 +605,33 @@ fn update_l2_block_data__retains_existing_blocks_and_adds_l2_block_to_unrecorded let height = 1; let used = 50; let capacity = 100.try_into().unwrap(); - let block_bytes = 1000; + let new_block_bytes = 1000; let new_gas_price = 100; // when updater - .update_l2_block_data(height, used, capacity, block_bytes, new_gas_price) + .update_l2_block_data(height, used, capacity, new_block_bytes, new_gas_price) .unwrap(); // then let block_bytes = BlockBytes { height, - block_bytes, + block_bytes: new_block_bytes, }; let contains_block_bytes = updater.unrecorded_blocks.contains_key(&block_bytes.height); assert!(contains_block_bytes); + // and let contains_preexisting_block_bytes = updater .unrecorded_blocks .contains_key(&preexisting_block.height); assert!(contains_preexisting_block_bytes); + + // and + let expected = first_block_bytes + new_block_bytes; + let actual = updater.unrecorded_blocks_bytes; + assert_eq!(expected as u128, actual); } fn capped_l2_activity_tracker() -> L2ActivityTracker { From 03da96395a3e72ada6d4d799e5ce7507d5a07ea0 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 1 Nov 2024 10:32:59 +0200 Subject: [PATCH 012/131] Fix analyzer --- .../gas-price-analysis/src/simulation.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs index 485da6b5ba2..92003cebb19 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs @@ -113,6 +113,7 @@ impl Simulator { last_profit: 0, second_to_last_profit: 0, l2_activity: always_normal_activity, + unrecorded_blocks_bytes: 0, } } From ea14e6f5dc2608c55c64cba6570b5253bbd264ad Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 1 Nov 2024 10:43:52 +0200 Subject: [PATCH 013/131] Simplify metadata --- .../gas_price_service/src/v1/metadata.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index 0ceae95a768..3fbdf24fadf 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -22,9 +22,6 @@ pub struct V1Metadata { pub total_da_rewards_excess: u128, /// The cumulative cost of recording L2 blocks on the DA chain as of the last recorded block pub latest_known_total_da_cost_excess: u128, - /// The predicted cost of recording L2 blocks on the DA chain as of the last L2 block - /// (This value is added on top of the `latest_known_total_da_cost` if the L2 height is higher) - pub projected_total_da_cost: u128, /// The last profit pub last_profit: i128, /// The profit before last @@ -52,7 +49,6 @@ impl V1Metadata { gas_price_factor: config.gas_price_factor, total_da_rewards_excess: 0, latest_known_total_da_cost_excess: 0, - projected_total_da_cost: 0, last_profit: 0, second_to_last_profit: 0, latest_da_cost_per_byte: 0, @@ -87,7 +83,6 @@ impl From for V1Metadata { gas_price_factor: updater.gas_price_factor, total_da_rewards_excess: updater.total_da_rewards_excess, latest_known_total_da_cost_excess: updater.latest_known_total_da_cost_excess, - projected_total_da_cost: updater.projected_total_da_cost, last_profit: updater.last_profit, second_to_last_profit: updater.second_to_last_profit, latest_da_cost_per_byte: updater.latest_da_cost_per_byte, @@ -106,6 +101,16 @@ pub fn v1_algorithm_from_metadata( config.decrease_range_size, config.block_activity_threshold.into(), ); + let unrecorded_blocks_bytes: u128 = metadata + .unrecorded_blocks + .iter() + .map(|(_, size)| u128::from(*size)) + .sum(); + let projected_portion = + unrecorded_blocks_bytes.saturating_mul(metadata.latest_da_cost_per_byte); + let projected_total_da_cost = metadata + .latest_known_total_da_cost_excess + .saturating_add(projected_portion); let unrecorded_blocks = metadata.unrecorded_blocks.into_iter().collect(); AlgorithmUpdaterV1 { new_scaled_exec_price: metadata.new_scaled_exec_price, @@ -114,7 +119,7 @@ pub fn v1_algorithm_from_metadata( gas_price_factor: metadata.gas_price_factor, total_da_rewards_excess: metadata.total_da_rewards_excess, latest_known_total_da_cost_excess: metadata.latest_known_total_da_cost_excess, - projected_total_da_cost: metadata.projected_total_da_cost, + projected_total_da_cost, last_profit: metadata.last_profit, second_to_last_profit: metadata.second_to_last_profit, latest_da_cost_per_byte: metadata.latest_da_cost_per_byte, @@ -129,5 +134,6 @@ pub fn v1_algorithm_from_metadata( da_p_component: config.da_p_component, da_d_component: config.da_d_component, unrecorded_blocks, + unrecorded_blocks_bytes, } } From d983891b99b6e51109d884f9c5cf73a9adacd124 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Sun, 17 Nov 2024 15:58:23 +0700 Subject: [PATCH 014/131] Resolve logical conflicts from merge --- .../v1/tests/update_da_record_data_tests.rs | 18 +++++---------- .../src/v1/da_source_service.rs | 4 ++-- .../block_committer_costs.rs | 22 +++++++++++++------ .../gas_price_service/src/v1/metadata.rs | 14 +++++++++--- .../gas_price_service/src/v1/service.rs | 4 ++-- 5 files changed, 36 insertions(+), 26 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index 0032c278e12..c88da67601b 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -363,7 +363,6 @@ fn update_da_record_data__updates_unrecorded_blocks() { fn update_da_record_data__da_block_lowers_da_gas_price() { // given let da_cost_per_byte = 40; - let da_recorded_block_height = 10; let l2_block_height = 11; let original_known_total_cost = 150; let unrecorded_blocks = vec![BlockBytes { @@ -381,7 +380,6 @@ fn update_da_record_data__da_block_lowers_da_gas_price() { .with_da_cost_per_byte(da_cost_per_byte as u128) .with_da_p_component(da_p_component) .with_last_profit(10, 0) - .with_da_recorded_block_height(da_recorded_block_height) .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost as u128) .with_known_total_cost(original_known_total_cost as u128) @@ -398,13 +396,13 @@ fn update_da_record_data__da_block_lowers_da_gas_price() { }); let min = recorded_heights.iter().min().unwrap(); let max = recorded_heights.iter().max().unwrap(); - let recorded_range = *min..(max + 1); + let recorded_range: Vec = (*min..(max + 1)).collect(); let old_da_gas_price = updater.new_scaled_da_gas_price; // when updater - .update_da_record_data(recorded_range, recorded_cost as u128) + .update_da_record_data(&recorded_range, recorded_cost as u128) .unwrap(); // then @@ -418,7 +416,6 @@ fn update_da_record_data__da_block_lowers_da_gas_price() { fn update_da_record_data__da_block_increases_da_gas_price() { // given let da_cost_per_byte = 40; - let da_recorded_block_height = 10; let l2_block_height = 11; let original_known_total_cost = 150; let unrecorded_blocks = vec![BlockBytes { @@ -436,7 +433,6 @@ fn update_da_record_data__da_block_increases_da_gas_price() { .with_da_cost_per_byte(da_cost_per_byte as u128) .with_da_p_component(da_p_component) .with_last_profit(-10, 0) - .with_da_recorded_block_height(da_recorded_block_height) .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost as u128) .with_known_total_cost(original_known_total_cost as u128) @@ -453,13 +449,13 @@ fn update_da_record_data__da_block_increases_da_gas_price() { }); let min = recorded_heights.iter().min().unwrap(); let max = recorded_heights.iter().max().unwrap(); - let recorded_range = *min..(max + 1); + let recorded_range: Vec = (*min..(max + 1)).collect(); let old_da_gas_price = updater.new_scaled_da_gas_price; // when updater - .update_da_record_data(recorded_range, recorded_cost as u128) + .update_da_record_data(&recorded_range, recorded_cost as u128) .unwrap(); // then @@ -473,7 +469,6 @@ fn update_da_record_data__da_block_increases_da_gas_price() { fn update_da_record_data__da_block_will_not_change_da_gas_price() { // given let da_cost_per_byte = 40; - let da_recorded_block_height = 10; let l2_block_height = 11; let original_known_total_cost = 150; let unrecorded_blocks = vec![BlockBytes { @@ -491,7 +486,6 @@ fn update_da_record_data__da_block_will_not_change_da_gas_price() { .with_da_cost_per_byte(da_cost_per_byte as u128) .with_da_p_component(da_p_component) .with_last_profit(0, 0) - .with_da_recorded_block_height(da_recorded_block_height) .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost as u128) .with_known_total_cost(original_known_total_cost as u128) @@ -508,13 +502,13 @@ fn update_da_record_data__da_block_will_not_change_da_gas_price() { }); let min = recorded_heights.iter().min().unwrap(); let max = recorded_heights.iter().max().unwrap(); - let recorded_range = *min..(max + 1); + let recorded_range: Vec = (*min..(max + 1)).collect(); let old_da_gas_price = updater.new_scaled_da_gas_price; // when updater - .update_da_record_data(recorded_range, recorded_cost as u128) + .update_da_record_data(&recorded_range, recorded_cost as u128) .unwrap(); // then diff --git a/crates/services/gas_price_service/src/v1/da_source_service.rs b/crates/services/gas_price_service/src/v1/da_source_service.rs index 840345bff98..ff2c703b518 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service.rs @@ -7,7 +7,7 @@ pub mod service; #[derive(Debug, Default, Clone, Eq, Hash, PartialEq)] pub struct DaBlockCosts { - pub l2_block_range: core::ops::Range, + pub l2_blocks: Vec, pub blob_size_bytes: u32, pub blob_cost_wei: u128, } @@ -30,7 +30,7 @@ mod tests { async fn run__when_da_block_cost_source_gives_value_shared_state_is_updated() { // given let expected_da_cost = DaBlockCosts { - l2_block_range: 0..10, + l2_blocks: (0..10).collect(), blob_size_bytes: 1024 * 128, blob_cost_wei: 2, }; diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index e0629cc0edd..111e911b855 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -42,7 +42,7 @@ pub struct RawDaBlockCosts { /// Sequence number (Monotonically increasing nonce) pub sequence_number: u32, /// The range of blocks that the costs apply to - pub blocks_range: core::ops::Range, + pub blocks_heights: Vec, /// The DA block height of the last transaction for the range of blocks pub da_block_height: DaBlockHeight, /// Rolling sum cost of posting blobs (wei) @@ -54,7 +54,11 @@ pub struct RawDaBlockCosts { impl From<&RawDaBlockCosts> for DaBlockCosts { fn from(raw_da_block_costs: &RawDaBlockCosts) -> Self { DaBlockCosts { - l2_block_range: raw_da_block_costs.blocks_range.clone(), + l2_blocks: raw_da_block_costs + .blocks_heights + .clone() + .into_iter() + .collect(), blob_size_bytes: raw_da_block_costs.total_size_bytes, blob_cost_wei: raw_da_block_costs.total_cost, } @@ -198,8 +202,11 @@ mod tests { let mut value = self.value.clone(); if let Some(value) = &mut value { value.sequence_number = seq_no; - value.blocks_range = - value.blocks_range.end * seq_no..value.blocks_range.end * seq_no + 10; + value.blocks_heights = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + .to_vec() + .iter() + .map(|x| x * seq_no) + .collect(); value.da_block_height = value.da_block_height + ((seq_no + 1) as u64).into(); value.total_cost += 1; @@ -218,7 +225,7 @@ mod tests { fn test_da_block_costs() -> RawDaBlockCosts { RawDaBlockCosts { sequence_number: 1, - blocks_range: 0..10, + blocks_heights: (0..10).collect(), da_block_height: 1u64.into(), total_cost: 1, total_size_bytes: 1, @@ -254,7 +261,7 @@ mod tests { let actual = block_committer.request_da_block_cost().await.unwrap(); // then - assert_ne!(da_block_costs.blocks_range, actual.l2_block_range); + assert_ne!(da_block_costs.blocks_heights, actual.l2_blocks); } #[tokio::test] @@ -293,7 +300,8 @@ mod tests { let mut value = self.value.clone(); if let Some(value) = &mut value { value.sequence_number = seq_no; - value.blocks_range = value.blocks_range.end..value.blocks_range.end + 10; + value.blocks_heights = + value.blocks_heights.iter().map(|x| x + seq_no).collect(); value.da_block_height = value.da_block_height + 1u64.into(); value.total_cost -= 1; value.total_size_bytes -= 1; diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index 3e9665cd7b5..68253981eec 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -3,7 +3,10 @@ use fuel_gas_price_algorithm::v1::{ AlgorithmUpdaterV1, L2ActivityTracker, }; -use std::num::NonZeroU64; +use std::{ + collections::BTreeMap, + num::NonZeroU64, +}; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] pub struct V1Metadata { @@ -82,14 +85,18 @@ impl From<&V1AlgorithmConfig> for AlgorithmUpdaterV1 { value.decrease_range_size, value.block_activity_threshold.into(), ); - let unrecorded_blocks = value.unrecorded_blocks.clone().into_iter().collect(); + let unrecorded_blocks: BTreeMap<_, _> = + value.unrecorded_blocks.clone().into_iter().collect(); + let unrecorded_blocks_bytes: u128 = unrecorded_blocks + .values() + .map(|size| u128::from(*size)) + .sum(); Self { new_scaled_exec_price: value.new_exec_gas_price, l2_block_height: 0, new_scaled_da_gas_price: value.min_da_gas_price, gas_price_factor: value.gas_price_factor, total_da_rewards_excess: 0, - da_recorded_block_height: 0, latest_known_total_da_cost_excess: 0, projected_total_da_cost: 0, last_profit: 0, @@ -106,6 +113,7 @@ impl From<&V1AlgorithmConfig> for AlgorithmUpdaterV1 { da_p_component: value.da_p_component, da_d_component: value.da_d_component, unrecorded_blocks, + unrecorded_blocks_bytes, } } } diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 5ab56160334..21ff0ffe4e6 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -139,7 +139,7 @@ where da_block_costs: DaBlockCosts, ) -> anyhow::Result<()> { self.algorithm_updater.update_da_record_data( - da_block_costs.l2_block_range, + &da_block_costs.l2_blocks, da_block_costs.blob_cost_wei, )?; @@ -474,7 +474,7 @@ mod tests { let da_source = DaSourceService::new( DummyDaBlockCosts::new( Ok(DaBlockCosts { - l2_block_range: 1..2, + l2_blocks: (1..2).collect(), blob_cost_wei: 9000, blob_size_bytes: 3000, }), From 1836656b58205274d4a2b7163ef02a6e02cd55a2 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 18 Nov 2024 11:04:40 +0700 Subject: [PATCH 015/131] WIP --- crates/services/gas_price_service/src/v1.rs | 2 + .../src/v1/uninitialized_task.rs | 320 ++++++++++++++++++ 2 files changed, 322 insertions(+) create mode 100644 crates/services/gas_price_service/src/v1/uninitialized_task.rs diff --git a/crates/services/gas_price_service/src/v1.rs b/crates/services/gas_price_service/src/v1.rs index fafb7245ef8..86c20351ed9 100644 --- a/crates/services/gas_price_service/src/v1.rs +++ b/crates/services/gas_price_service/src/v1.rs @@ -2,3 +2,5 @@ pub mod algorithm; pub mod da_source_service; pub mod metadata; pub mod service; + +pub mod uninitialized_task; diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs new file mode 100644 index 00000000000..ace1d7186bb --- /dev/null +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -0,0 +1,320 @@ +use fuel_core_services::{RunnableService, ServiceRunner}; +use fuel_core_types::fuel_types::BlockHeight; +use fuel_gas_price_algorithm::v1::AlgorithmUpdaterV1; +use crate::common::l2_block_source::FuelL2BlockSource; +use crate::common::updater_metadata::UpdaterMetadata; +use crate::v1::algorithm::SharedV1Algorithm; +use crate::v1::metadata::V1AlgorithmConfig; +use crate::v1::service::GasPriceServiceV1; + +pub struct UninitializedTask { + pub config: V1AlgorithmConfig, + pub genesis_block_height: BlockHeight, + pub settings: SettingsProvider, + pub gas_price_db: GasPriceStore, + pub on_chain_db: L2DataStoreView, + pub block_stream: BoxStream, + pub(crate) shared_algo: SharedV1Algorithm, + pub(crate) algo_updater: AlgorithmUpdaterV1, + pub(crate) metadata_storage: Metadata, +} + +impl +UninitializedTask +where + L2DataStore: L2Data, + L2DataStoreView: AtomicView, + GasPriceStore: GasPriceData, + Metadata: MetadataStorage, + SettingsProvider: GasPriceSettingsProvider, +{ + pub fn new( + config: V1AlgorithmConfig, + genesis_block_height: BlockHeight, + settings: SettingsProvider, + block_stream: BoxStream, + gas_price_db: GasPriceStore, + metadata_storage: Metadata, + on_chain_db: L2DataStoreView, + ) -> anyhow::Result { + let latest_block_height: u32 = on_chain_db + .latest_view()? + .latest_height() + .unwrap_or(genesis_block_height) + .into(); + + let (algo_updater, shared_algo) = + initialize_algorithm(&config, latest_block_height, &metadata_storage)?; + + let task = Self { + config, + genesis_block_height, + settings, + gas_price_db, + on_chain_db, + block_stream, + algo_updater, + shared_algo, + metadata_storage, + }; + Ok(task) + } + + pub fn init( + mut self, + ) -> anyhow::Result, Metadata>> + { + let mut first_run = false; + let latest_block_height: u32 = self + .on_chain_db + .latest_view()? + .latest_height() + .unwrap_or(self.genesis_block_height) + .into(); + + let maybe_metadata_height = self.gas_price_db.latest_height(); + let metadata_height = if let Some(metadata_height) = maybe_metadata_height { + metadata_height.into() + } else { + first_run = true; + latest_block_height + }; + + let l2_block_source = FuelL2BlockSource::new( + self.genesis_block_height, + self.settings.clone(), + self.block_stream, + ); + + if BlockHeight::from(latest_block_height) == self.genesis_block_height + || first_run + { + let service = GasPriceServiceV1::new( + l2_block_source, + self.metadata_storage, + self.shared_algo, + self.algo_updater, + ); + Ok(service) + } else { + if latest_block_height > metadata_height { + sync_gas_price_db_with_on_chain_storage( + &self.settings, + &mut self.metadata_storage, + &self.on_chain_db, + metadata_height, + latest_block_height, + )?; + } + + let service = GasPriceServiceV1::new( + l2_block_source, + self.metadata_storage, + self.shared_algo, + self.algo_updater, + ); + Ok(service) + } + } +} + +#[async_trait::async_trait] +impl +RunnableService +for UninitializedTask +where + L2DataStore: L2Data, + L2DataStoreView: AtomicView, + GasPriceStore: GasPriceData, + Metadata: MetadataStorage, + SettingsProvider: GasPriceSettingsProvider, +{ + const NAME: &'static str = "GasPriceServiceV0"; + type SharedData = SharedV0Algorithm; + type Task = GasPriceServiceV1, Metadata>; + type TaskParams = (); + + fn shared_data(&self) -> Self::SharedData { + self.shared_algo.clone() + } + + async fn into_task( + self, + _state_watcher: &StateWatcher, + _params: Self::TaskParams, + ) -> anyhow::Result { + UninitializedTask::init(self) + } +} + +pub fn initialize_algorithm( + config: &V1AlgorithmConfig, + latest_block_height: u32, + metadata_storage: &Metadata, +) -> GasPriceResult<(AlgorithmUpdaterV1, SharedV1Algorithm)> +where + Metadata: MetadataStorage, +{ + let min_exec_gas_price = config.min_gas_price; + let exec_gas_price_change_percent = config.gas_price_change_percent; + let l2_block_fullness_threshold_percent = config.gas_price_threshold_percent; + + let algorithm_updater; + if let Some(updater_metadata) = metadata_storage + .get_metadata(&latest_block_height.into()) + .map_err(|err| GasPriceError::CouldNotInitUpdater(anyhow::anyhow!(err)))? + { + let previous_metadata: V0Metadata = updater_metadata.try_into()?; + algorithm_updater = AlgorithmUpdaterV1::new( + previous_metadata.new_exec_price, + min_exec_gas_price, + exec_gas_price_change_percent, + previous_metadata.l2_block_height, + l2_block_fullness_threshold_percent, + ); + } else { + algorithm_updater = AlgorithmUpdaterV1::new( + config.starting_gas_price, + min_exec_gas_price, + exec_gas_price_change_percent, + latest_block_height, + l2_block_fullness_threshold_percent, + ); + } + + let shared_algo = + SharedGasPriceAlgo::new_with_algorithm(algorithm_updater.algorithm()); + + Ok((algorithm_updater, shared_algo)) +} + +fn sync_gas_price_db_with_on_chain_storage< + L2DataStore, + L2DataStoreView, + Metadata, + SettingsProvider, +>( + settings: &SettingsProvider, + metadata_storage: &mut Metadata, + on_chain_db: &L2DataStoreView, + metadata_height: u32, + latest_block_height: u32, +) -> anyhow::Result<()> +where + L2DataStore: L2Data, + L2DataStoreView: AtomicView, + Metadata: MetadataStorage, + SettingsProvider: GasPriceSettingsProvider, +{ + let metadata = metadata_storage + .get_metadata(&metadata_height.into())? + .ok_or(anyhow::anyhow!( + "Expected metadata to exist for height: {metadata_height}" + ))?; + + let mut algo_updater = if let UpdaterMetadata::V1(metadata) = metadata { + Ok(AlgorithmUpdaterV1::new( + metadata.new_exec_price, + 0, + 0, + metadata.l2_block_height, + 0, + )) + } else { + Err(anyhow::anyhow!("Expected V0 metadata")) + }?; + + sync_v0_metadata( + settings, + on_chain_db, + metadata_height, + latest_block_height, + &mut algo_updater, + metadata_storage, + )?; + + Ok(()) +} + +fn sync_v0_metadata( + settings: &SettingsProvider, + on_chain_db: &L2DataStoreView, + metadata_height: u32, + latest_block_height: u32, + updater: &mut AlgorithmUpdaterV1, + metadata_storage: &mut Metadata, +) -> anyhow::Result<()> +where + L2DataStore: L2Data, + L2DataStoreView: AtomicView, + Metadata: MetadataStorage, + SettingsProvider: GasPriceSettingsProvider, +{ + let first = metadata_height.saturating_add(1); + let view = on_chain_db.latest_view()?; + for height in first..=latest_block_height { + let block = view + .get_block(&height.into())? + .ok_or(not_found!("FullBlock"))?; + let param_version = block.header().consensus_parameters_version; + + let GasPriceSettings { + gas_price_factor, + block_gas_limit, + } = settings.settings(¶m_version)?; + let block_gas_capacity = block_gas_limit.try_into()?; + + let block_gas_used = + match get_block_info(&block, gas_price_factor, block_gas_limit)? { + BlockInfo::GenesisBlock => { + Err(anyhow::anyhow!("should not be genesis block"))? + } + BlockInfo::Block { gas_used, .. } => gas_used, + }; + + updater.update_l2_block_data(height, block_gas_used, block_gas_capacity)?; + let metadata: UpdaterMetadata = updater.clone().into(); + metadata_storage.set_metadata(&metadata)?; + } + + Ok(()) +} + +pub fn new_gas_price_service_v1< + L2DataStore, + L2DataStoreView, + GasPriceStore, + Metadata, + SettingsProvider, +>( + config: GasPriceServiceConfig, + genesis_block_height: BlockHeight, + settings: SettingsProvider, + block_stream: BoxStream, + gas_price_db: GasPriceStore, + metadata: Metadata, + on_chain_db: L2DataStoreView, +) -> anyhow::Result< + ServiceRunner< + UninitializedTask, + >, +> +where + L2DataStore: L2Data, + L2DataStoreView: AtomicView, + GasPriceStore: GasPriceData, + SettingsProvider: GasPriceSettingsProvider, + Metadata: MetadataStorage, +{ + let v0_config = config.v0().ok_or(anyhow::anyhow!("Expected V0 config"))?; + let gas_price_init = UninitializedTask::new( + v0_config, + genesis_block_height, + settings, + block_stream, + gas_price_db, + metadata, + on_chain_db, + )?; + Ok(ServiceRunner::new(gas_price_init)) +} From b15a93f1d804d5f61d7227ff731aa96cbb8cfd67 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 19 Nov 2024 09:45:04 +0700 Subject: [PATCH 016/131] Get compiling --- .../src/v1/uninitialized_task.rs | 158 ++++++++++++------ 1 file changed, 108 insertions(+), 50 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index ace1d7186bb..650a21ba8a4 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -1,13 +1,63 @@ -use fuel_core_services::{RunnableService, ServiceRunner}; -use fuel_core_types::fuel_types::BlockHeight; +use crate::{ + common::{ + fuel_core_storage_adapter::{ + get_block_info, + GasPriceSettings, + GasPriceSettingsProvider, + }, + gas_price_algorithm::SharedGasPriceAlgo, + l2_block_source::FuelL2BlockSource, + updater_metadata::UpdaterMetadata, + utils::{ + BlockInfo, + Error as GasPriceError, + Result as GasPriceResult, + }, + }, + ports::{ + GasPriceData, + GasPriceServiceConfig, + L2Data, + MetadataStorage, + }, + v1::{ + algorithm::SharedV1Algorithm, + da_source_service::service::{ + DaBlockCostsSource, + DaSourceService, + }, + metadata::{ + v1_algorithm_from_metadata, + V1AlgorithmConfig, + V1Metadata, + }, + service::GasPriceServiceV1, + }, +}; +use anyhow::Error; +use fuel_core_services::{ + stream::BoxStream, + RunnableService, + ServiceRunner, + StateWatcher, +}; +use fuel_core_storage::{ + not_found, + transactional::AtomicView, +}; +use fuel_core_types::{ + fuel_types::BlockHeight, + services::block_importer::SharedImportResult, +}; use fuel_gas_price_algorithm::v1::AlgorithmUpdaterV1; -use crate::common::l2_block_source::FuelL2BlockSource; -use crate::common::updater_metadata::UpdaterMetadata; -use crate::v1::algorithm::SharedV1Algorithm; -use crate::v1::metadata::V1AlgorithmConfig; -use crate::v1::service::GasPriceServiceV1; -pub struct UninitializedTask { +pub struct UninitializedTask< + L2DataStoreView, + GasPriceStore, + Metadata, + DA, + SettingsProvider, +> { pub config: V1AlgorithmConfig, pub genesis_block_height: BlockHeight, pub settings: SettingsProvider, @@ -17,17 +67,20 @@ pub struct UninitializedTask -UninitializedTask +impl + UninitializedTask where L2DataStore: L2Data, L2DataStoreView: AtomicView, GasPriceStore: GasPriceData, Metadata: MetadataStorage, + DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, { + #[allow(clippy::too_many_arguments)] pub fn new( config: V1AlgorithmConfig, genesis_block_height: BlockHeight, @@ -35,6 +88,7 @@ where block_stream: BoxStream, gas_price_db: GasPriceStore, metadata_storage: Metadata, + da_source: DA, on_chain_db: L2DataStoreView, ) -> anyhow::Result { let latest_block_height: u32 = on_chain_db @@ -56,14 +110,16 @@ where algo_updater, shared_algo, metadata_storage, + da_source, }; Ok(task) } pub fn init( mut self, - ) -> anyhow::Result, Metadata>> - { + ) -> anyhow::Result< + GasPriceServiceV1, Metadata, DA>, + > { let mut first_run = false; let latest_block_height: u32 = self .on_chain_db @@ -86,6 +142,10 @@ where self.block_stream, ); + // TODO: Add to config + let poll_interval = None; + let da_service = DaSourceService::new(self.da_source, poll_interval); + if BlockHeight::from(latest_block_height) == self.genesis_block_height || first_run { @@ -94,6 +154,7 @@ where self.metadata_storage, self.shared_algo, self.algo_updater, + da_service, ); Ok(service) } else { @@ -112,6 +173,7 @@ where self.metadata_storage, self.shared_algo, self.algo_updater, + da_service, ); Ok(service) } @@ -119,19 +181,20 @@ where } #[async_trait::async_trait] -impl -RunnableService -for UninitializedTask +impl + RunnableService + for UninitializedTask where L2DataStore: L2Data, L2DataStoreView: AtomicView, GasPriceStore: GasPriceData, Metadata: MetadataStorage, + DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, { const NAME: &'static str = "GasPriceServiceV0"; - type SharedData = SharedV0Algorithm; - type Task = GasPriceServiceV1, Metadata>; + type SharedData = SharedV1Algorithm; + type Task = GasPriceServiceV1, Metadata, DA>; type TaskParams = (); fn shared_data(&self) -> Self::SharedData { @@ -155,31 +218,15 @@ pub fn initialize_algorithm( where Metadata: MetadataStorage, { - let min_exec_gas_price = config.min_gas_price; - let exec_gas_price_change_percent = config.gas_price_change_percent; - let l2_block_fullness_threshold_percent = config.gas_price_threshold_percent; - let algorithm_updater; if let Some(updater_metadata) = metadata_storage .get_metadata(&latest_block_height.into()) .map_err(|err| GasPriceError::CouldNotInitUpdater(anyhow::anyhow!(err)))? { - let previous_metadata: V0Metadata = updater_metadata.try_into()?; - algorithm_updater = AlgorithmUpdaterV1::new( - previous_metadata.new_exec_price, - min_exec_gas_price, - exec_gas_price_change_percent, - previous_metadata.l2_block_height, - l2_block_fullness_threshold_percent, - ); + let previous_metadata: V1Metadata = updater_metadata.try_into()?; + algorithm_updater = v1_algorithm_from_metadata(previous_metadata, config); } else { - algorithm_updater = AlgorithmUpdaterV1::new( - config.starting_gas_price, - min_exec_gas_price, - exec_gas_price_change_percent, - latest_block_height, - l2_block_fullness_threshold_percent, - ); + algorithm_updater = config.into(); } let shared_algo = @@ -213,18 +260,14 @@ where ))?; let mut algo_updater = if let UpdaterMetadata::V1(metadata) = metadata { - Ok(AlgorithmUpdaterV1::new( - metadata.new_exec_price, - 0, - 0, - metadata.l2_block_height, - 0, - )) + // let algorithm_updater = v1_algorithm_from_metadata(metadata, settings); + // Ok(algorithm_updater) + todo!() } else { - Err(anyhow::anyhow!("Expected V0 metadata")) + Err(anyhow::anyhow!("Expected V1 metadata")) }?; - sync_v0_metadata( + sync_v1_metadata( settings, on_chain_db, metadata_height, @@ -236,7 +279,7 @@ where Ok(()) } -fn sync_v0_metadata( +fn sync_v1_metadata( settings: &SettingsProvider, on_chain_db: &L2DataStoreView, metadata_height: u32, @@ -272,7 +315,16 @@ where BlockInfo::Block { gas_used, .. } => gas_used, }; - updater.update_l2_block_data(height, block_gas_used, block_gas_capacity)?; + // TODO: Set these using the block values + let block_bytes = 0; + let fee_wei = 0; + updater.update_l2_block_data( + height, + block_gas_used, + block_gas_capacity, + block_bytes, + fee_wei, + )?; let metadata: UpdaterMetadata = updater.clone().into(); metadata_storage.set_metadata(&metadata)?; } @@ -280,11 +332,14 @@ where Ok(()) } +#[allow(clippy::type_complexity)] +#[allow(clippy::too_many_arguments)] pub fn new_gas_price_service_v1< L2DataStore, L2DataStoreView, GasPriceStore, Metadata, + DA, SettingsProvider, >( config: GasPriceServiceConfig, @@ -293,10 +348,11 @@ pub fn new_gas_price_service_v1< block_stream: BoxStream, gas_price_db: GasPriceStore, metadata: Metadata, + da_source: DA, on_chain_db: L2DataStoreView, ) -> anyhow::Result< ServiceRunner< - UninitializedTask, + UninitializedTask, >, > where @@ -305,15 +361,17 @@ where GasPriceStore: GasPriceData, SettingsProvider: GasPriceSettingsProvider, Metadata: MetadataStorage, + DA: DaBlockCostsSource, { - let v0_config = config.v0().ok_or(anyhow::anyhow!("Expected V0 config"))?; + let v1_config = config.v1().ok_or(anyhow::anyhow!("Expected V1 config"))?; let gas_price_init = UninitializedTask::new( - v0_config, + v1_config, genesis_block_height, settings, block_stream, gas_price_db, metadata, + da_source, on_chain_db, )?; Ok(ServiceRunner::new(gas_price_init)) From 279191c622df480324756e3467b6e395db3f991a Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 19 Nov 2024 10:57:16 +0700 Subject: [PATCH 017/131] Start adding tests --- Cargo.lock | 1 + crates/services/gas_price_service/Cargo.toml | 1 + .../gas_price_service/src/v0/tests.rs | 2 +- crates/services/gas_price_service/src/v1.rs | 2 + .../gas_price_service/src/v1/service.rs | 2 +- .../gas_price_service/src/v1/tests.rs | 425 ++++++++++++++++++ 6 files changed, 431 insertions(+), 2 deletions(-) create mode 100644 crates/services/gas_price_service/src/v1/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 1903c106fde..e1d52a67f05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3490,6 +3490,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", + "tracing-subscriber", ] [[package]] diff --git a/crates/services/gas_price_service/Cargo.toml b/crates/services/gas_price_service/Cargo.toml index 9903bba7d03..dfd2ed149d3 100644 --- a/crates/services/gas_price_service/Cargo.toml +++ b/crates/services/gas_price_service/Cargo.toml @@ -27,6 +27,7 @@ thiserror = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } tracing = { workspace = true } +tracing-subscriber = "0.3.18" [dev-dependencies] fuel-core-services = { workspace = true, features = ["test-helpers"] } diff --git a/crates/services/gas_price_service/src/v0/tests.rs b/crates/services/gas_price_service/src/v0/tests.rs index 7526975e467..0fabc88d72f 100644 --- a/crates/services/gas_price_service/src/v0/tests.rs +++ b/crates/services/gas_price_service/src/v0/tests.rs @@ -169,7 +169,7 @@ async fn next_gas_price__affected_by_new_l2_block() { let read_algo = service.next_block_algorithm(); let initial = read_algo.next_gas_price(); - let mut watcher = StateWatcher::default(); + let mut watcher = StateWatcher::started(); // when service.run(&mut watcher).await.unwrap(); diff --git a/crates/services/gas_price_service/src/v1.rs b/crates/services/gas_price_service/src/v1.rs index 86c20351ed9..c0e83dbb6e7 100644 --- a/crates/services/gas_price_service/src/v1.rs +++ b/crates/services/gas_price_service/src/v1.rs @@ -3,4 +3,6 @@ pub mod da_source_service; pub mod metadata; pub mod service; +#[cfg(test)] +mod tests; pub mod uninitialized_task; diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 21ff0ffe4e6..497d3f83dae 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -225,7 +225,7 @@ where async fn shutdown(mut self) -> anyhow::Result<()> { // handle all the remaining l2 blocks while let Some(Ok(block)) = self.l2_block_source.get_l2_block().now_or_never() { - tracing::debug!("Updating gas price algorithm"); + tracing::debug!("Updating gas price algorithm before shutdown"); self.apply_block_info_to_gas_algorithm(block).await?; } diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs new file mode 100644 index 00000000000..e2261f155f7 --- /dev/null +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -0,0 +1,425 @@ +#![allow(non_snake_case)] +use crate::{ + common::{ + fuel_core_storage_adapter::{ + GasPriceSettings, + GasPriceSettingsProvider, + }, + l2_block_source::L2BlockSource, + updater_metadata::UpdaterMetadata, + utils::{ + BlockInfo, + Error as GasPriceError, + Result as GasPriceResult, + }, + }, + ports::{ + GasPriceData, + L2Data, + MetadataStorage, + }, + v1::{ + da_source_service::{ + service::{ + DaBlockCostsSource, + DaSourceService, + }, + DaBlockCosts, + }, + metadata::{ + V1AlgorithmConfig, + V1Metadata, + }, + service::GasPriceServiceV1, + uninitialized_task::{ + initialize_algorithm, + UninitializedTask, + }, + }, +}; +use anyhow::anyhow; +use fuel_core_services::{ + stream::{ + BoxStream, + IntoBoxStream, + }, + RunnableTask, + StateWatcher, +}; +use fuel_core_storage::{ + transactional::AtomicView, + Result as StorageResult, +}; +use fuel_core_types::{ + blockchain::{ + block::Block, + header::ConsensusParametersVersion, + }, + fuel_tx::Transaction, + fuel_types::BlockHeight, + services::block_importer::{ + ImportResult, + SharedImportResult, + }, +}; +use std::{ + num::{ + NonZero, + NonZeroU64, + }, + ops::Deref, + sync::Arc, + time::Duration, +}; +use tokio::sync::mpsc::Receiver; + +struct FakeL2BlockSource { + l2_block: Receiver, +} + +#[async_trait::async_trait] +impl L2BlockSource for FakeL2BlockSource { + async fn get_l2_block(&mut self) -> GasPriceResult { + let block = self.l2_block.recv().await.unwrap(); + Ok(block) + } +} + +struct FakeMetadata { + inner: Arc>>, +} + +impl FakeMetadata { + fn empty() -> Self { + Self { + inner: Arc::new(std::sync::Mutex::new(None)), + } + } +} + +impl MetadataStorage for FakeMetadata { + fn get_metadata(&self, _: &BlockHeight) -> GasPriceResult> { + let metadata = self.inner.lock().unwrap().clone(); + Ok(metadata) + } + + fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { + *self.inner.lock().unwrap() = Some(metadata.clone()); + Ok(()) + } +} + +struct ErroringMetadata; + +impl MetadataStorage for ErroringMetadata { + fn get_metadata(&self, _: &BlockHeight) -> GasPriceResult> { + Err(GasPriceError::CouldNotFetchMetadata { + source_error: anyhow!("boo!"), + }) + } + + fn set_metadata(&mut self, _: &UpdaterMetadata) -> GasPriceResult<()> { + Err(GasPriceError::CouldNotSetMetadata { + block_height: Default::default(), + source_error: anyhow!("boo!"), + }) + } +} + +struct FakeDABlockCost; + +#[async_trait::async_trait] +impl DaBlockCostsSource for FakeDABlockCost { + async fn request_da_block_cost(&mut self) -> anyhow::Result { + todo!() + } +} + +fn arbitrary_config() -> V1AlgorithmConfig { + V1AlgorithmConfig { + new_exec_gas_price: 100, + min_exec_gas_price: 0, + exec_gas_price_change_percent: 10, + l2_block_fullness_threshold_percent: 0, + gas_price_factor: NonZeroU64::new(100).unwrap(), + min_da_gas_price: 0, + max_da_gas_price_change_percent: 0, + da_p_component: 0, + da_d_component: 0, + normal_range_size: 0, + capped_range_size: 0, + decrease_range_size: 0, + block_activity_threshold: 0, + unrecorded_blocks: vec![], + } +} + +fn arbitrary_metadata() -> V1Metadata { + V1Metadata { + new_scaled_exec_price: 100, + l2_block_height: 0, + new_scaled_da_gas_price: 0, + gas_price_factor: NonZeroU64::new(100).unwrap(), + total_da_rewards_excess: 0, + latest_known_total_da_cost_excess: 0, + last_profit: 0, + second_to_last_profit: 0, + latest_da_cost_per_byte: 0, + unrecorded_blocks: vec![], + } +} + +fn different_arb_config() -> V1AlgorithmConfig { + V1AlgorithmConfig { + new_exec_gas_price: 200, + min_exec_gas_price: 0, + exec_gas_price_change_percent: 20, + l2_block_fullness_threshold_percent: 0, + gas_price_factor: NonZeroU64::new(100).unwrap(), + min_da_gas_price: 0, + max_da_gas_price_change_percent: 0, + da_p_component: 0, + da_d_component: 0, + normal_range_size: 0, + capped_range_size: 0, + decrease_range_size: 0, + block_activity_threshold: 0, + unrecorded_blocks: vec![], + } +} + +#[tokio::test] +async fn next_gas_price__affected_by_new_l2_block() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::DEBUG) + .try_init(); + + tracing::debug!("test test test test"); + + // given + let l2_block = BlockInfo::Block { + height: 1, + gas_used: 60, + block_gas_capacity: 100, + block_bytes: 100, + block_fees: 100, + }; + let (l2_block_sender, l2_block_receiver) = tokio::sync::mpsc::channel(1); + let l2_block_source = FakeL2BlockSource { + l2_block: l2_block_receiver, + }; + let metadata_storage = FakeMetadata::empty(); + + let config = arbitrary_config(); + let height = 0; + let (algo_updater, shared_algo) = + initialize_algorithm(&config, height, &metadata_storage).unwrap(); + let da_source = FakeDABlockCost; + let da_source_service = DaSourceService::new(da_source, None); + let mut service = GasPriceServiceV1::new( + l2_block_source, + metadata_storage, + shared_algo, + algo_updater, + da_source_service, + ); + + let read_algo = service.next_block_algorithm(); + let initial = read_algo.next_gas_price(); + let mut watcher = StateWatcher::started(); + + // when + service.run(&mut watcher).await.unwrap(); + l2_block_sender.send(l2_block).await.unwrap(); + service.shutdown().await.unwrap(); + + // then + let new = read_algo.next_gas_price(); + assert_ne!(initial, new); +} + +// #[tokio::test] +// async fn next__new_l2_block_saves_old_metadata() { +// // given +// let l2_block = BlockInfo::Block { +// height: 1, +// gas_used: 60, +// block_gas_capacity: 100, +// block_bytes: 100, +// block_fees: 100, +// }; +// let (l2_block_sender, l2_block_receiver) = tokio::sync::mpsc::channel(1); +// let l2_block_source = FakeL2BlockSource { +// l2_block: l2_block_receiver, +// }; +// let metadata_inner = Arc::new(std::sync::Mutex::new(None)); +// let metadata_storage = FakeMetadata { +// inner: metadata_inner.clone(), +// }; +// +// let config = arbitrary_config(); +// let height = 0; +// let (algo_updater, shared_algo) = +// initialize_algorithm(&config, height, &metadata_storage).unwrap(); +// +// let mut service = GasPriceServiceV1::new( +// l2_block_source, +// metadata_storage, +// shared_algo, +// algo_updater, +// ); +// +// // when +// let read_algo = service.next_block_algorithm(); +// let mut watcher = StateWatcher::default(); +// let start = read_algo.next_gas_price(); +// +// service.run(&mut watcher).await.unwrap(); +// l2_block_sender.send(l2_block).await.unwrap(); +// service.shutdown().await.unwrap(); +// +// // then +// let new = read_algo.next_gas_price(); +// assert_ne!(start, new); +// } + +#[derive(Clone)] +struct FakeSettings; + +impl GasPriceSettingsProvider for FakeSettings { + fn settings( + &self, + _param_version: &ConsensusParametersVersion, + ) -> GasPriceResult { + unimplemented!() + } +} + +#[derive(Clone)] +struct FakeGasPriceDb; + +// GasPriceData + Modifiable + KeyValueInspect +impl GasPriceData for FakeGasPriceDb { + fn latest_height(&self) -> Option { + unimplemented!() + } +} + +#[derive(Clone)] +struct FakeOnChainDb { + height: BlockHeight, +} + +impl FakeOnChainDb { + fn new(height: u32) -> Self { + Self { + height: height.into(), + } + } +} + +struct FakeL2Data { + height: BlockHeight, +} + +impl FakeL2Data { + fn new(height: BlockHeight) -> Self { + Self { height } + } +} + +impl L2Data for FakeL2Data { + fn latest_height(&self) -> StorageResult { + Ok(self.height) + } + + fn get_block( + &self, + _height: &BlockHeight, + ) -> StorageResult>> { + unimplemented!() + } +} +impl AtomicView for FakeOnChainDb { + type LatestView = FakeL2Data; + + fn latest_view(&self) -> StorageResult { + Ok(FakeL2Data::new(self.height)) + } +} + +fn empty_block_stream() -> BoxStream { + let blocks: Vec + Send + Sync>> = vec![]; + tokio_stream::iter(blocks).into_boxed() +} + +// #[tokio::test] +// async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overrides() { +// // given +// let original_metadata = arbitrary_metadata(); +// let original = UpdaterMetadata::V1(original_metadata.clone()); +// let metadata_inner = Arc::new(std::sync::Mutex::new(Some(original.clone()))); +// let metadata_storage = FakeMetadata { +// inner: metadata_inner, +// }; +// +// let different_config = different_arb_config(); +// assert_ne!( +// different_config.starting_gas_price, +// original_metadata.new_exec_price +// ); +// let different_l2_block = 1231; +// assert_ne!(different_l2_block, original_metadata.l2_block_height); +// let settings = FakeSettings; +// let block_stream = empty_block_stream(); +// let gas_price_db = FakeGasPriceDb; +// let on_chain_db = FakeOnChainDb::new(different_l2_block); +// +// // when +// let service = UninitializedTask::new( +// different_config, +// 0.into(), +// settings, +// block_stream, +// gas_price_db, +// metadata_storage, +// on_chain_db, +// ) +// .unwrap(); +// +// // then +// let V1Metadata { +// new_exec_price, +// l2_block_height, +// } = original_metadata; +// let UninitializedTask { algo_updater, .. } = service; +// assert_eq!(algo_updater.new_exec_price, new_exec_price); +// assert_eq!(algo_updater.l2_block_height, l2_block_height); +// } + +// #[tokio::test] +// async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { +// // given +// let config = arbitrary_config(); +// let different_l2_block = 1231; +// let metadata_storage = ErroringMetadata; +// let settings = FakeSettings; +// let block_stream = empty_block_stream(); +// let gas_price_db = FakeGasPriceDb; +// let on_chain_db = FakeOnChainDb::new(different_l2_block); +// +// // when +// let res = UninitializedTask::new( +// config, +// 0.into(), +// settings, +// block_stream, +// gas_price_db, +// metadata_storage, +// on_chain_db, +// ); +// +// // then +// let is_err = res.is_err(); +// assert!(is_err); +// } From 63b02664b83a829b4c82595b07b60f4b0a6cb602 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 19 Nov 2024 14:01:46 +0700 Subject: [PATCH 018/131] Get existing tests passing --- .../gas_price_service/src/v1/tests.rs | 238 +++++++++--------- 1 file changed, 122 insertions(+), 116 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index e2261f155f7..e22fe4f1a4f 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -135,7 +135,7 @@ impl DaBlockCostsSource for FakeDABlockCost { } } -fn arbitrary_config() -> V1AlgorithmConfig { +fn zero_threshold_arbitrary_config() -> V1AlgorithmConfig { V1AlgorithmConfig { new_exec_gas_price: 100, min_exec_gas_price: 0, @@ -210,7 +210,7 @@ async fn next_gas_price__affected_by_new_l2_block() { }; let metadata_storage = FakeMetadata::empty(); - let config = arbitrary_config(); + let config = zero_threshold_arbitrary_config(); let height = 0; let (algo_updater, shared_algo) = initialize_algorithm(&config, height, &metadata_storage).unwrap(); @@ -238,50 +238,49 @@ async fn next_gas_price__affected_by_new_l2_block() { assert_ne!(initial, new); } -// #[tokio::test] -// async fn next__new_l2_block_saves_old_metadata() { -// // given -// let l2_block = BlockInfo::Block { -// height: 1, -// gas_used: 60, -// block_gas_capacity: 100, -// block_bytes: 100, -// block_fees: 100, -// }; -// let (l2_block_sender, l2_block_receiver) = tokio::sync::mpsc::channel(1); -// let l2_block_source = FakeL2BlockSource { -// l2_block: l2_block_receiver, -// }; -// let metadata_inner = Arc::new(std::sync::Mutex::new(None)); -// let metadata_storage = FakeMetadata { -// inner: metadata_inner.clone(), -// }; -// -// let config = arbitrary_config(); -// let height = 0; -// let (algo_updater, shared_algo) = -// initialize_algorithm(&config, height, &metadata_storage).unwrap(); -// -// let mut service = GasPriceServiceV1::new( -// l2_block_source, -// metadata_storage, -// shared_algo, -// algo_updater, -// ); -// -// // when -// let read_algo = service.next_block_algorithm(); -// let mut watcher = StateWatcher::default(); -// let start = read_algo.next_gas_price(); -// -// service.run(&mut watcher).await.unwrap(); -// l2_block_sender.send(l2_block).await.unwrap(); -// service.shutdown().await.unwrap(); -// -// // then -// let new = read_algo.next_gas_price(); -// assert_ne!(start, new); -// } +#[tokio::test] +async fn run__new_l2_block_saves_old_metadata() { + // given + let l2_block = BlockInfo::Block { + height: 1, + gas_used: 60, + block_gas_capacity: 100, + block_bytes: 100, + block_fees: 100, + }; + let (l2_block_sender, l2_block_receiver) = tokio::sync::mpsc::channel(1); + let l2_block_source = FakeL2BlockSource { + l2_block: l2_block_receiver, + }; + let metadata_inner = Arc::new(std::sync::Mutex::new(None)); + let metadata_storage = FakeMetadata { + inner: metadata_inner.clone(), + }; + + let config = zero_threshold_arbitrary_config(); + let height = 0; + let (algo_updater, shared_algo) = + initialize_algorithm(&config, height, &metadata_storage).unwrap(); + let da_source = FakeDABlockCost; + let da_source_service = DaSourceService::new(da_source, None); + let mut service = GasPriceServiceV1::new( + l2_block_source, + metadata_storage, + shared_algo, + algo_updater, + da_source_service, + ); + let mut watcher = StateWatcher::default(); + + // when + service.run(&mut watcher).await.unwrap(); + l2_block_sender.send(l2_block).await.unwrap(); + service.shutdown().await.unwrap(); + + // then + let metadata_is_some = metadata_inner.lock().unwrap().is_some(); + assert!(metadata_is_some) +} #[derive(Clone)] struct FakeSettings; @@ -353,73 +352,80 @@ fn empty_block_stream() -> BoxStream { tokio_stream::iter(blocks).into_boxed() } -// #[tokio::test] -// async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overrides() { -// // given -// let original_metadata = arbitrary_metadata(); -// let original = UpdaterMetadata::V1(original_metadata.clone()); -// let metadata_inner = Arc::new(std::sync::Mutex::new(Some(original.clone()))); -// let metadata_storage = FakeMetadata { -// inner: metadata_inner, -// }; -// -// let different_config = different_arb_config(); -// assert_ne!( -// different_config.starting_gas_price, -// original_metadata.new_exec_price -// ); -// let different_l2_block = 1231; -// assert_ne!(different_l2_block, original_metadata.l2_block_height); -// let settings = FakeSettings; -// let block_stream = empty_block_stream(); -// let gas_price_db = FakeGasPriceDb; -// let on_chain_db = FakeOnChainDb::new(different_l2_block); -// -// // when -// let service = UninitializedTask::new( -// different_config, -// 0.into(), -// settings, -// block_stream, -// gas_price_db, -// metadata_storage, -// on_chain_db, -// ) -// .unwrap(); -// -// // then -// let V1Metadata { -// new_exec_price, -// l2_block_height, -// } = original_metadata; -// let UninitializedTask { algo_updater, .. } = service; -// assert_eq!(algo_updater.new_exec_price, new_exec_price); -// assert_eq!(algo_updater.l2_block_height, l2_block_height); -// } - -// #[tokio::test] -// async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { -// // given -// let config = arbitrary_config(); -// let different_l2_block = 1231; -// let metadata_storage = ErroringMetadata; -// let settings = FakeSettings; -// let block_stream = empty_block_stream(); -// let gas_price_db = FakeGasPriceDb; -// let on_chain_db = FakeOnChainDb::new(different_l2_block); -// -// // when -// let res = UninitializedTask::new( -// config, -// 0.into(), -// settings, -// block_stream, -// gas_price_db, -// metadata_storage, -// on_chain_db, -// ); -// -// // then -// let is_err = res.is_err(); -// assert!(is_err); -// } +#[tokio::test] +async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overrides() { + // given + let original_metadata = arbitrary_metadata(); + let original = UpdaterMetadata::V1(original_metadata.clone()); + let metadata_inner = Arc::new(std::sync::Mutex::new(Some(original.clone()))); + let metadata_storage = FakeMetadata { + inner: metadata_inner, + }; + + let different_config = different_arb_config(); + let descaleed_exec_price = + original_metadata.new_scaled_exec_price / original_metadata.gas_price_factor; + assert_ne!(different_config.new_exec_gas_price, descaleed_exec_price); + let different_l2_block = 1231; + assert_ne!(different_l2_block, original_metadata.l2_block_height); + let settings = FakeSettings; + let block_stream = empty_block_stream(); + let gas_price_db = FakeGasPriceDb; + let on_chain_db = FakeOnChainDb::new(different_l2_block); + let da_cost_source = FakeDABlockCost; + + // when + let service = UninitializedTask::new( + different_config, + 0.into(), + settings, + block_stream, + gas_price_db, + metadata_storage, + da_cost_source, + on_chain_db, + ) + .unwrap(); + + // then + let V1Metadata { + new_scaled_exec_price: original_new_scaled_exec_price, + l2_block_height, + .. + } = original_metadata; + let UninitializedTask { algo_updater, .. } = service; + assert_eq!( + algo_updater.new_scaled_exec_price, + original_new_scaled_exec_price + ); + assert_eq!(algo_updater.l2_block_height, l2_block_height); +} + +#[tokio::test] +async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { + // given + let config = zero_threshold_arbitrary_config(); + let different_l2_block = 1231; + let metadata_storage = ErroringMetadata; + let settings = FakeSettings; + let block_stream = empty_block_stream(); + let gas_price_db = FakeGasPriceDb; + let on_chain_db = FakeOnChainDb::new(different_l2_block); + let da_cost_source = FakeDABlockCost; + + // when + let res = UninitializedTask::new( + config, + 0.into(), + settings, + block_stream, + gas_price_db, + metadata_storage, + da_cost_source, + on_chain_db, + ); + + // then + let is_err = res.is_err(); + assert!(is_err); +} From 477c731d5090a254d8d2685f1867dabaf6d99bee Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 19 Nov 2024 16:12:28 +0700 Subject: [PATCH 019/131] WIP --- Cargo.lock | 1 - crates/services/gas_price_service/Cargo.toml | 1 - .../gas_price_service/src/v1/tests.rs | 92 ++++++++++++++----- 3 files changed, 71 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1d52a67f05..1903c106fde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3490,7 +3490,6 @@ dependencies = [ "tokio", "tokio-stream", "tracing", - "tracing-subscriber", ] [[package]] diff --git a/crates/services/gas_price_service/Cargo.toml b/crates/services/gas_price_service/Cargo.toml index dfd2ed149d3..9903bba7d03 100644 --- a/crates/services/gas_price_service/Cargo.toml +++ b/crates/services/gas_price_service/Cargo.toml @@ -27,7 +27,6 @@ thiserror = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } tracing = { workspace = true } -tracing-subscriber = "0.3.18" [dev-dependencies] fuel-core-services = { workspace = true, features = ["test-helpers"] } diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index e22fe4f1a4f..fe4b76892ee 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -62,14 +62,11 @@ use fuel_core_types::{ SharedImportResult, }, }; +use fuel_gas_price_algorithm::v1::AlgorithmUpdaterV1; use std::{ - num::{ - NonZero, - NonZeroU64, - }, + num::NonZeroU64, ops::Deref, sync::Arc, - time::Duration, }; use tokio::sync::mpsc::Receiver; @@ -126,12 +123,28 @@ impl MetadataStorage for ErroringMetadata { } } -struct FakeDABlockCost; +struct FakeDABlockCost { + da_block_costs: Receiver, +} + +impl FakeDABlockCost { + fn never_returns() -> Self { + let (_sender, receiver) = tokio::sync::mpsc::channel(1); + Self { + da_block_costs: receiver, + } + } + + fn new(da_block_costs: Receiver) -> Self { + Self { da_block_costs } + } +} #[async_trait::async_trait] impl DaBlockCostsSource for FakeDABlockCost { async fn request_da_block_cost(&mut self) -> anyhow::Result { - todo!() + let costs = self.da_block_costs.recv().await.unwrap(); + Ok(costs) } } @@ -190,12 +203,6 @@ fn different_arb_config() -> V1AlgorithmConfig { #[tokio::test] async fn next_gas_price__affected_by_new_l2_block() { - let _ = tracing_subscriber::fmt() - .with_max_level(tracing::Level::DEBUG) - .try_init(); - - tracing::debug!("test test test test"); - // given let l2_block = BlockInfo::Block { height: 1, @@ -214,7 +221,7 @@ async fn next_gas_price__affected_by_new_l2_block() { let height = 0; let (algo_updater, shared_algo) = initialize_algorithm(&config, height, &metadata_storage).unwrap(); - let da_source = FakeDABlockCost; + let da_source = FakeDABlockCost::never_returns(); let da_source_service = DaSourceService::new(da_source, None); let mut service = GasPriceServiceV1::new( l2_block_source, @@ -261,7 +268,7 @@ async fn run__new_l2_block_saves_old_metadata() { let height = 0; let (algo_updater, shared_algo) = initialize_algorithm(&config, height, &metadata_storage).unwrap(); - let da_source = FakeDABlockCost; + let da_source = FakeDABlockCost::never_returns(); let da_source_service = DaSourceService::new(da_source, None); let mut service = GasPriceServiceV1::new( l2_block_source, @@ -372,7 +379,7 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr let block_stream = empty_block_stream(); let gas_price_db = FakeGasPriceDb; let on_chain_db = FakeOnChainDb::new(different_l2_block); - let da_cost_source = FakeDABlockCost; + let da_cost_source = FakeDABlockCost::never_returns(); // when let service = UninitializedTask::new( @@ -388,17 +395,60 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr .unwrap(); // then + let UninitializedTask { algo_updater, .. } = service; + algo_updater_matches_values_from_old_metadata(algo_updater, original_metadata); +} + +fn algo_updater_matches_values_from_old_metadata( + algo_updater: AlgorithmUpdaterV1, + original_metadata: V1Metadata, +) { let V1Metadata { new_scaled_exec_price: original_new_scaled_exec_price, - l2_block_height, - .. + l2_block_height: original_l2_block_height, + new_scaled_da_gas_price: original_new_scaled_da_gas_price, + gas_price_factor: original_gas_price_factor, + total_da_rewards_excess: original_total_da_rewards_excess, + latest_known_total_da_cost_excess: original_latest_known_total_da_cost_excess, + last_profit: original_last_profit, + second_to_last_profit: original_second_to_last_profit, + latest_da_cost_per_byte: original_latest_da_cost_per_byte, + unrecorded_blocks: original_unrecorded_blocks, } = original_metadata; - let UninitializedTask { algo_updater, .. } = service; assert_eq!( algo_updater.new_scaled_exec_price, original_new_scaled_exec_price ); - assert_eq!(algo_updater.l2_block_height, l2_block_height); + assert_eq!(algo_updater.l2_block_height, original_l2_block_height); + assert_eq!( + algo_updater.new_scaled_da_gas_price, + original_new_scaled_da_gas_price + ); + assert_eq!(algo_updater.gas_price_factor, original_gas_price_factor); + assert_eq!( + algo_updater.total_da_rewards_excess, + original_total_da_rewards_excess + ); + assert_eq!( + algo_updater.latest_known_total_da_cost_excess, + original_latest_known_total_da_cost_excess + ); + assert_eq!(algo_updater.last_profit, original_last_profit); + assert_eq!( + algo_updater.second_to_last_profit, + original_second_to_last_profit + ); + assert_eq!( + algo_updater.latest_da_cost_per_byte, + original_latest_da_cost_per_byte + ); + assert_eq!( + algo_updater + .unrecorded_blocks + .into_iter() + .collect::>(), + original_unrecorded_blocks.into_iter().collect::>() + ); } #[tokio::test] @@ -411,7 +461,7 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { let block_stream = empty_block_stream(); let gas_price_db = FakeGasPriceDb; let on_chain_db = FakeOnChainDb::new(different_l2_block); - let da_cost_source = FakeDABlockCost; + let da_cost_source = FakeDABlockCost::never_returns(); // when let res = UninitializedTask::new( From 7ca55c7828bf92a83aa1ba46b0331ccbe6d0625d Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 19 Nov 2024 17:46:33 +0700 Subject: [PATCH 020/131] Ignore entire batches with blocks not in the unrecorded_blocks --- Cargo.lock | 1 + crates/fuel-gas-price-algorithm/Cargo.toml | 1 + crates/fuel-gas-price-algorithm/src/v1.rs | 58 +++++++++++-------- .../v1/tests/update_da_record_data_tests.rs | 20 +++---- 4 files changed, 45 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a3d5b110deb..a307d8f998b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3917,6 +3917,7 @@ dependencies = [ "rand", "serde", "thiserror 1.0.69", + "tracing", ] [[package]] diff --git a/crates/fuel-gas-price-algorithm/Cargo.toml b/crates/fuel-gas-price-algorithm/Cargo.toml index 662a8e5be08..24cb619585c 100644 --- a/crates/fuel-gas-price-algorithm/Cargo.toml +++ b/crates/fuel-gas-price-algorithm/Cargo.toml @@ -17,6 +17,7 @@ path = "src/lib.rs" [dependencies] serde = { workspace = true, features = ["derive"] } thiserror = { workspace = true } +tracing = { workspace = true } [dev-dependencies] proptest = { workspace = true } diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 28ef8a343a2..d9479521199 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -516,36 +516,46 @@ impl AlgorithmUpdaterV1 { heights: &[u32], recording_cost: u128, ) -> Result<(), Error> { - let recorded_bytes = self.drain_l2_block_bytes_for_heights(heights)?; - let new_cost_per_byte: u128 = recording_cost.checked_div(recorded_bytes).ok_or( - Error::CouldNotCalculateCostPerByte { - bytes: recorded_bytes, - cost: recording_cost, - }, - )?; - let new_da_block_cost = self - .latest_known_total_da_cost_excess - .saturating_add(recording_cost); - self.latest_known_total_da_cost_excess = new_da_block_cost; - self.latest_da_cost_per_byte = new_cost_per_byte; + let maybe_recorded_bytes = self.drain_l2_block_bytes_for_heights(heights); + if let Some(recorded_bytes) = maybe_recorded_bytes { + let new_cost_per_byte: u128 = recording_cost + .checked_div(recorded_bytes) + .ok_or(Error::CouldNotCalculateCostPerByte { + bytes: recorded_bytes, + cost: recording_cost, + })?; + let new_da_block_cost = self + .latest_known_total_da_cost_excess + .saturating_add(recording_cost); + self.latest_known_total_da_cost_excess = new_da_block_cost; + self.latest_da_cost_per_byte = new_cost_per_byte; + } Ok(()) } - fn drain_l2_block_bytes_for_heights( - &mut self, - heights: &[u32], - ) -> Result { + fn drain_l2_block_bytes_for_heights(&mut self, heights: &[u32]) -> Option { + let mut should_ignore_batch = false; let mut total: u128 = 0; for expected_height in heights { - let bytes = self.unrecorded_blocks.remove(expected_height).ok_or( - Error::L2BlockExpectedNotFound { - height: *expected_height, - }, - )?; - total = total.saturating_add(bytes as u128); + let maybe_bytes = self.unrecorded_blocks.remove(expected_height); + if let Some(bytes) = maybe_bytes { + total = total.saturating_add(bytes as u128); + } else { + tracing::error!( + "L2 block expected but not found in unrecorded blocks: {}. Ignoring batch of heights: {:?}", + expected_height, + heights, + ); + should_ignore_batch = true; + } + } + if should_ignore_batch { + None + } else { + self.unrecorded_blocks_bytes = + self.unrecorded_blocks_bytes.saturating_sub(total); + Some(total) } - self.unrecorded_blocks_bytes = self.unrecorded_blocks_bytes.saturating_sub(total); - Ok(total) } fn recalculate_projected_cost(&mut self) { diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index c88da67601b..65a6ea54daf 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -1,13 +1,10 @@ -use crate::v1::{ - tests::{ - BlockBytes, - UpdaterBuilder, - }, - Error, +use crate::v1::tests::{ + BlockBytes, + UpdaterBuilder, }; #[test] -fn update_da_record_data__throws_error_if_receives_a_block_missing_from_unrecorded_blocks( +fn update_da_record_data__ignores_batch_if_receives_a_block_missing_from_unrecorded_blocks( ) { // given let recorded_heights: Vec = (1u32..3).collect(); @@ -19,15 +16,16 @@ fn update_da_record_data__throws_error_if_receives_a_block_missing_from_unrecord let mut updater = UpdaterBuilder::new() .with_unrecorded_blocks(unrecorded_blocks) .build(); + let old = updater.algorithm(); // when - let actual_error = updater + updater .update_da_record_data(&recorded_heights, recorded_cost) - .unwrap_err(); + .unwrap(); // then - let expected_error = Error::L2BlockExpectedNotFound { height: 2 }; - assert_eq!(actual_error, expected_error); + let new = updater.algorithm(); + assert_eq!(new, old); } #[test] From 6ac41de9ef1fd499341eb99c325fb1d50b81188e Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 19 Nov 2024 17:49:59 +0700 Subject: [PATCH 021/131] Add comments to explain unexpected behavior --- crates/fuel-gas-price-algorithm/src/v1.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index d9479521199..e859b158af1 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -517,6 +517,8 @@ impl AlgorithmUpdaterV1 { recording_cost: u128, ) -> Result<(), Error> { let maybe_recorded_bytes = self.drain_l2_block_bytes_for_heights(heights); + // If we weren't able to get the bytes for all the heights, we can't update the cost per byte in a logical way. + // We will just accept the predicted cost as the real cost and move on with the algorithm if let Some(recorded_bytes) = maybe_recorded_bytes { let new_cost_per_byte: u128 = recording_cost .checked_div(recorded_bytes) @@ -533,6 +535,8 @@ impl AlgorithmUpdaterV1 { Ok(()) } + // Get the bytes for all specified heights, or get none of them. + // Always remove the blocks from the unrecorded blocks so they don't build up indefinitely fn drain_l2_block_bytes_for_heights(&mut self, heights: &[u32]) -> Option { let mut should_ignore_batch = false; let mut total: u128 = 0; From cc90cbd41f93bf79ed89e32a6d8ec6495332a1ef Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 20 Nov 2024 10:17:27 +0700 Subject: [PATCH 022/131] Fix tracing messages to be emitted at the right time --- crates/fuel-gas-price-algorithm/src/v1.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index e859b158af1..cd62e7288ed 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -531,6 +531,8 @@ impl AlgorithmUpdaterV1 { .saturating_add(recording_cost); self.latest_known_total_da_cost_excess = new_da_block_cost; self.latest_da_cost_per_byte = new_cost_per_byte; + } else { + tracing::error!("Ignoring batch of heights: {:?}", heights); } Ok(()) } @@ -546,9 +548,8 @@ impl AlgorithmUpdaterV1 { total = total.saturating_add(bytes as u128); } else { tracing::error!( - "L2 block expected but not found in unrecorded blocks: {}. Ignoring batch of heights: {:?}", + "L2 block expected but not found in unrecorded blocks: {}", expected_height, - heights, ); should_ignore_batch = true; } From 7ac01aa29d4315a5a64b85c363da214758e84e0a Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 20 Nov 2024 11:27:51 +0700 Subject: [PATCH 023/131] Fix logic to always remove recorded blocks, use predicted cost as best estimate --- crates/fuel-gas-price-algorithm/src/v1.rs | 77 +++++++++++++++-------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index cd62e7288ed..9d2536be94e 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -1,4 +1,10 @@ -use crate::utils::cumulative_percentage_change; +use crate::{ + utils::cumulative_percentage_change, + v1::BytesForHeights::{ + Complete, + Incomplete, + }, +}; use std::{ cmp::max, collections::BTreeMap, @@ -299,6 +305,13 @@ impl core::ops::Deref for ClampedPercentage { } } +enum BytesForHeights { + // Was able to find all the heights + Complete(u128), + // Was not able to find all the heights + Incomplete(u128), +} + impl AlgorithmUpdaterV1 { pub fn update_da_record_data( &mut self, @@ -516,31 +529,42 @@ impl AlgorithmUpdaterV1 { heights: &[u32], recording_cost: u128, ) -> Result<(), Error> { - let maybe_recorded_bytes = self.drain_l2_block_bytes_for_heights(heights); - // If we weren't able to get the bytes for all the heights, we can't update the cost per byte in a logical way. - // We will just accept the predicted cost as the real cost and move on with the algorithm - if let Some(recorded_bytes) = maybe_recorded_bytes { - let new_cost_per_byte: u128 = recording_cost - .checked_div(recorded_bytes) - .ok_or(Error::CouldNotCalculateCostPerByte { - bytes: recorded_bytes, - cost: recording_cost, - })?; - let new_da_block_cost = self - .latest_known_total_da_cost_excess - .saturating_add(recording_cost); - self.latest_known_total_da_cost_excess = new_da_block_cost; - self.latest_da_cost_per_byte = new_cost_per_byte; - } else { - tracing::error!("Ignoring batch of heights: {:?}", heights); - } + let bytes_for_heights = self.drain_l2_block_bytes_for_heights(heights); + match bytes_for_heights { + Complete(bytes) => { + let new_cost_per_byte: u128 = recording_cost.checked_div(bytes).ok_or( + Error::CouldNotCalculateCostPerByte { + bytes, + cost: recording_cost, + }, + )?; + let new_da_block_cost = self + .latest_known_total_da_cost_excess + .saturating_add(recording_cost); + self.latest_known_total_da_cost_excess = new_da_block_cost; + self.latest_da_cost_per_byte = new_cost_per_byte; + } + // If we weren't able to get the bytes for all the heights, we can't update the cost per byte in a logical way. + // We will just accept the predicted cost as the real cost and move on with the algorithm + // We can safely say that the cost didn't exceed the `recording_cost`, so we will take the + // minimum of the two costs + Incomplete(bytes) => { + let new_guess_recording_cost = + bytes.saturating_mul(self.latest_da_cost_per_byte); + let new_da_block_cost = self + .latest_known_total_da_cost_excess + .saturating_add(new_guess_recording_cost) + .min(recording_cost); + self.latest_known_total_da_cost_excess = new_da_block_cost; + } + }; Ok(()) } // Get the bytes for all specified heights, or get none of them. // Always remove the blocks from the unrecorded blocks so they don't build up indefinitely - fn drain_l2_block_bytes_for_heights(&mut self, heights: &[u32]) -> Option { - let mut should_ignore_batch = false; + fn drain_l2_block_bytes_for_heights(&mut self, heights: &[u32]) -> BytesForHeights { + let mut missed_some_heights = false; let mut total: u128 = 0; for expected_height in heights { let maybe_bytes = self.unrecorded_blocks.remove(expected_height); @@ -551,15 +575,14 @@ impl AlgorithmUpdaterV1 { "L2 block expected but not found in unrecorded blocks: {}", expected_height, ); - should_ignore_batch = true; + missed_some_heights = true; } } - if should_ignore_batch { - None + self.unrecorded_blocks_bytes = self.unrecorded_blocks_bytes.saturating_sub(total); + if missed_some_heights { + Incomplete(total) } else { - self.unrecorded_blocks_bytes = - self.unrecorded_blocks_bytes.saturating_sub(total); - Some(total) + Complete(total) } } From 2592ad009e75423c4cdb03630bea1762a9f34f8b Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 20 Nov 2024 12:18:53 +0700 Subject: [PATCH 024/131] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdd6424972a..9d869e620b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] ### Added +- [2442](https://github.com/FuelLabs/fuel-core/pull/2442): Add uninitialized task for V1 gas price service - [2154](https://github.com/FuelLabs/fuel-core/pull/2154): Added `Unknown` variant to `ConsensusParameters` graphql queries - [2154](https://github.com/FuelLabs/fuel-core/pull/2154): Added `Unknown` variant to `Block` graphql queries - [2154](https://github.com/FuelLabs/fuel-core/pull/2154): Added `TransactionType` type in `fuel-client` From eb233ab99441e84eeba091a1c324f7c4727a34d7 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 20 Nov 2024 12:43:25 +0700 Subject: [PATCH 025/131] Remove some TODOs --- crates/fuel-gas-price-algorithm/src/v1.rs | 13 +++++++------ .../src/common/fuel_core_storage_adapter.rs | 9 ++++++++- crates/services/gas_price_service/src/v1/tests.rs | 4 ++-- .../gas_price_service/src/v1/uninitialized_task.rs | 11 +++++++---- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 9d2536be94e..f89cfd315cf 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -1,3 +1,10 @@ +use std::{ + cmp::max, + collections::BTreeMap, + num::NonZeroU64, + ops::Div, +}; + use crate::{ utils::cumulative_percentage_change, v1::BytesForHeights::{ @@ -5,12 +12,6 @@ use crate::{ Incomplete, }, }; -use std::{ - cmp::max, - collections::BTreeMap, - num::NonZeroU64, - ops::Div, -}; #[cfg(test)] mod tests; diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs index e81ea58b2c3..4d452421ba1 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs @@ -111,7 +111,7 @@ pub fn get_block_info( Ok(info) } -fn mint_values(block: &Block) -> GasPriceResult<(u64, u64)> { +pub(crate) fn mint_values(block: &Block) -> GasPriceResult<(u64, u64)> { let mint = block .transactions() .last() @@ -121,6 +121,13 @@ fn mint_values(block: &Block) -> GasPriceResult<(u64, u64)> { })?; Ok((*mint.mint_amount(), *mint.gas_price())) } + +// TODO: Don't take a direct dependency on `Postcard` as it's not guaranteed to be the encoding format +// https://github.com/FuelLabs/fuel-core/issues/2443 +pub(crate) fn block_bytes(block: &Block) -> u64 { + Postcard::encode(block).len() as u64 +} + fn block_used_gas( fee: u64, gas_price: u64, diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index fe4b76892ee..f47f060bf2b 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -236,7 +236,7 @@ async fn next_gas_price__affected_by_new_l2_block() { let mut watcher = StateWatcher::started(); // when - service.run(&mut watcher).await.unwrap(); + service.run(&mut watcher).await; l2_block_sender.send(l2_block).await.unwrap(); service.shutdown().await.unwrap(); @@ -280,7 +280,7 @@ async fn run__new_l2_block_saves_old_metadata() { let mut watcher = StateWatcher::default(); // when - service.run(&mut watcher).await.unwrap(); + service.run(&mut watcher).await; l2_block_sender.send(l2_block).await.unwrap(); service.shutdown().await.unwrap(); diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 650a21ba8a4..068191d1543 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -1,7 +1,9 @@ use crate::{ common::{ fuel_core_storage_adapter::{ + block_bytes, get_block_info, + mint_values, GasPriceSettings, GasPriceSettingsProvider, }, @@ -46,6 +48,7 @@ use fuel_core_storage::{ transactional::AtomicView, }; use fuel_core_types::{ + fuel_tx::field::MintAmount, fuel_types::BlockHeight, services::block_importer::SharedImportResult, }; @@ -143,6 +146,7 @@ where ); // TODO: Add to config + // https://github.com/FuelLabs/fuel-core/issues/2140 let poll_interval = None; let da_service = DaSourceService::new(self.da_source, poll_interval); @@ -315,15 +319,14 @@ where BlockInfo::Block { gas_used, .. } => gas_used, }; - // TODO: Set these using the block values - let block_bytes = 0; - let fee_wei = 0; + let block_bytes = block_bytes(&block); + let (fee_wei, _) = mint_values(&block)?; updater.update_l2_block_data( height, block_gas_used, block_gas_capacity, block_bytes, - fee_wei, + fee_wei.into(), )?; let metadata: UpdaterMetadata = updater.clone().into(); metadata_storage.set_metadata(&metadata)?; From d32b34793678e224f547d56e2c12369343f3df14 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 20 Nov 2024 12:59:22 +0700 Subject: [PATCH 026/131] Remove other todo --- .../src/v1/uninitialized_task.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 068191d1543..736629cc52f 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -165,6 +165,7 @@ where if latest_block_height > metadata_height { sync_gas_price_db_with_on_chain_storage( &self.settings, + &self.config, &mut self.metadata_storage, &self.on_chain_db, metadata_height, @@ -246,6 +247,7 @@ fn sync_gas_price_db_with_on_chain_storage< SettingsProvider, >( settings: &SettingsProvider, + config: &V1AlgorithmConfig, metadata_storage: &mut Metadata, on_chain_db: &L2DataStoreView, metadata_height: u32, @@ -263,13 +265,13 @@ where "Expected metadata to exist for height: {metadata_height}" ))?; - let mut algo_updater = if let UpdaterMetadata::V1(metadata) = metadata { - // let algorithm_updater = v1_algorithm_from_metadata(metadata, settings); - // Ok(algorithm_updater) - todo!() - } else { - Err(anyhow::anyhow!("Expected V1 metadata")) - }?; + let metadata = match metadata { + UpdaterMetadata::V1(metadata) => metadata, + UpdaterMetadata::V0(metadata) => { + V1Metadata::construct_from_v0_metadata(metadata, config)? + } + }; + let mut algo_updater = v1_algorithm_from_metadata(metadata, config); sync_v1_metadata( settings, From 9f78fa6d4c94f6af9522e31f9e00d6c6d11af24c Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 20 Nov 2024 17:59:30 +0700 Subject: [PATCH 027/131] Add tests, fix bad bug --- crates/fuel-gas-price-algorithm/src/v1.rs | 8 ++-- .../v1/tests/update_da_record_data_tests.rs | 45 ++++++++++++++++++- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 9d2536be94e..c97b874eb15 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -549,12 +549,12 @@ impl AlgorithmUpdaterV1 { // We can safely say that the cost didn't exceed the `recording_cost`, so we will take the // minimum of the two costs Incomplete(bytes) => { - let new_guess_recording_cost = - bytes.saturating_mul(self.latest_da_cost_per_byte); + let new_guess_recording_cost = bytes + .saturating_mul(self.latest_da_cost_per_byte) + .min(recording_cost); let new_da_block_cost = self .latest_known_total_da_cost_excess - .saturating_add(new_guess_recording_cost) - .min(recording_cost); + .saturating_add(new_guess_recording_cost); self.latest_known_total_da_cost_excess = new_da_block_cost; } }; diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index 65a6ea54daf..6e584e7629d 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -4,17 +4,55 @@ use crate::v1::tests::{ }; #[test] -fn update_da_record_data__ignores_batch_if_receives_a_block_missing_from_unrecorded_blocks( +fn update_da_record_data__if_receives_batch_with_unknown_blocks_will_include_known_blocks_with_previous_cost( +) { + // given + let recorded_heights: Vec = (1u32..3).collect(); + let recorded_cost = 1_000_000; + let block_bytes = 1000; + let unrecorded_blocks = vec![BlockBytes { + height: 1, + block_bytes, + }]; + let cost_per_byte = 333; + let known_total_cost = 10_000; + let mut updater = UpdaterBuilder::new() + .with_unrecorded_blocks(unrecorded_blocks) + .with_da_cost_per_byte(cost_per_byte) + .with_known_total_cost(known_total_cost) + .build(); + let old = updater.algorithm(); + + // when + updater + .update_da_record_data(&recorded_heights, recorded_cost) + .unwrap(); + + // then + let new = updater.algorithm(); + assert_eq!(new, old); + let expected = known_total_cost + block_bytes as u128 * cost_per_byte; + let actual = updater.latest_known_total_da_cost_excess; + assert_eq!(expected, actual); +} + +#[test] +fn update_da_record_data__if_receives_batch_with_unknown_blocks_will_never_increase_cost_more_than_recorded_cost( ) { // given let recorded_heights: Vec = (1u32..3).collect(); let recorded_cost = 200; + let block_bytes = 1000; let unrecorded_blocks = vec![BlockBytes { height: 1, - block_bytes: 1000, + block_bytes, }]; + let cost_per_byte = 333; + let known_total_cost = 10_000; let mut updater = UpdaterBuilder::new() .with_unrecorded_blocks(unrecorded_blocks) + .with_da_cost_per_byte(cost_per_byte) + .with_known_total_cost(known_total_cost) .build(); let old = updater.algorithm(); @@ -26,6 +64,9 @@ fn update_da_record_data__ignores_batch_if_receives_a_block_missing_from_unrecor // then let new = updater.algorithm(); assert_eq!(new, old); + let expected = known_total_cost + recorded_cost; + let actual = updater.latest_known_total_da_cost_excess; + assert_eq!(expected, actual); } #[test] From fc8c42755a5369b70d3f317816a566f801aa8ffa Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 20 Nov 2024 19:35:11 +0700 Subject: [PATCH 028/131] Fix copy-paste errors --- .../gas_price_service/src/v1/tests.rs | 6 ++-- .../src/v1/uninitialized_task.rs | 32 +++---------------- 2 files changed, 8 insertions(+), 30 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index f47f060bf2b..d9d95695488 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -30,11 +30,11 @@ use crate::{ V1AlgorithmConfig, V1Metadata, }, - service::GasPriceServiceV1, - uninitialized_task::{ + service::{ initialize_algorithm, - UninitializedTask, + GasPriceServiceV1, }, + uninitialized_task::UninitializedTask, }, }; use anyhow::anyhow; diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 736629cc52f..158d7b7ccdc 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -33,7 +33,10 @@ use crate::{ V1AlgorithmConfig, V1Metadata, }, - service::GasPriceServiceV1, + service::{ + initialize_algorithm, + GasPriceServiceV1, + }, }, }; use anyhow::Error; @@ -197,7 +200,7 @@ where DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, { - const NAME: &'static str = "GasPriceServiceV0"; + const NAME: &'static str = "GasPriceServiceV1"; type SharedData = SharedV1Algorithm; type Task = GasPriceServiceV1, Metadata, DA>; type TaskParams = (); @@ -215,31 +218,6 @@ where } } -pub fn initialize_algorithm( - config: &V1AlgorithmConfig, - latest_block_height: u32, - metadata_storage: &Metadata, -) -> GasPriceResult<(AlgorithmUpdaterV1, SharedV1Algorithm)> -where - Metadata: MetadataStorage, -{ - let algorithm_updater; - if let Some(updater_metadata) = metadata_storage - .get_metadata(&latest_block_height.into()) - .map_err(|err| GasPriceError::CouldNotInitUpdater(anyhow::anyhow!(err)))? - { - let previous_metadata: V1Metadata = updater_metadata.try_into()?; - algorithm_updater = v1_algorithm_from_metadata(previous_metadata, config); - } else { - algorithm_updater = config.into(); - } - - let shared_algo = - SharedGasPriceAlgo::new_with_algorithm(algorithm_updater.algorithm()); - - Ok((algorithm_updater, shared_algo)) -} - fn sync_gas_price_db_with_on_chain_storage< L2DataStore, L2DataStoreView, From 87dc5470eb898a9d6939cca1dfa99626fab47ae0 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 20 Nov 2024 19:59:18 +0700 Subject: [PATCH 029/131] Fix broken tests --- .../services/gas_price_service/src/v0/service.rs | 2 ++ crates/services/gas_price_service/src/v0/tests.rs | 14 ++++++++------ .../services/gas_price_service/src/v1/metadata.rs | 4 +++- crates/services/gas_price_service/src/v1/tests.rs | 4 ++-- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/crates/services/gas_price_service/src/v0/service.rs b/crates/services/gas_price_service/src/v0/service.rs index 7591c8a377b..ca8fbf80d48 100644 --- a/crates/services/gas_price_service/src/v0/service.rs +++ b/crates/services/gas_price_service/src/v0/service.rs @@ -141,6 +141,7 @@ where Metadata: MetadataStorage, { async fn run(&mut self, watcher: &mut StateWatcher) -> TaskNextAction { + tracing::debug!("Starting gas price service"); tokio::select! { biased; _ = watcher.while_started() => { @@ -148,6 +149,7 @@ where TaskNextAction::Stop } l2_block_res = self.l2_block_source.get_l2_block() => { + tracing::debug!("Received L2 block"); let res = self.process_l2_block_res(l2_block_res).await; TaskNextAction::always_continue(res) } diff --git a/crates/services/gas_price_service/src/v0/tests.rs b/crates/services/gas_price_service/src/v0/tests.rs index d592ef34ec3..7ba279a08ca 100644 --- a/crates/services/gas_price_service/src/v0/tests.rs +++ b/crates/services/gas_price_service/src/v0/tests.rs @@ -167,14 +167,16 @@ async fn next_gas_price__affected_by_new_l2_block() { algo_updater, ); + tracing::debug!("service created"); + let read_algo = service.next_block_algorithm(); let initial = read_algo.next_gas_price(); let mut watcher = StateWatcher::started(); + tokio::spawn(async move { service.run(&mut watcher).await }); // when - service.run(&mut watcher).await; l2_block_sender.send(l2_block).await.unwrap(); - service.shutdown().await.unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; // then let new = read_algo.next_gas_price(); @@ -212,14 +214,14 @@ async fn next__new_l2_block_saves_old_metadata() { algo_updater, ); - // when let read_algo = service.next_block_algorithm(); - let mut watcher = StateWatcher::default(); let start = read_algo.next_gas_price(); + let mut watcher = StateWatcher::started(); + tokio::spawn(async move { service.run(&mut watcher).await }); - service.run(&mut watcher).await; + // when l2_block_sender.send(l2_block).await.unwrap(); - service.shutdown().await.unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; // then let new = read_algo.next_gas_price(); diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index 68253981eec..0976bfd3bfb 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -92,7 +92,9 @@ impl From<&V1AlgorithmConfig> for AlgorithmUpdaterV1 { .map(|size| u128::from(*size)) .sum(); Self { - new_scaled_exec_price: value.new_exec_gas_price, + new_scaled_exec_price: value + .new_exec_gas_price + .saturating_mul(value.gas_price_factor.get()), l2_block_height: 0, new_scaled_da_gas_price: value.min_da_gas_price, gas_price_factor: value.gas_price_factor, diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index d9d95695488..e5188a9c869 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -234,11 +234,11 @@ async fn next_gas_price__affected_by_new_l2_block() { let read_algo = service.next_block_algorithm(); let initial = read_algo.next_gas_price(); let mut watcher = StateWatcher::started(); + tokio::spawn(async move { service.run(&mut watcher).await }); // when - service.run(&mut watcher).await; l2_block_sender.send(l2_block).await.unwrap(); - service.shutdown().await.unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; // then let new = read_algo.next_gas_price(); From 008f5fb39c03bad38c27a314adb1970c46dcf274 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 27 Nov 2024 11:42:18 -0700 Subject: [PATCH 030/131] Correct test, use trace instead of debug --- crates/services/gas_price_service/src/v0/service.rs | 2 +- crates/services/gas_price_service/src/v0/tests.rs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/crates/services/gas_price_service/src/v0/service.rs b/crates/services/gas_price_service/src/v0/service.rs index ca8fbf80d48..2543dbc7212 100644 --- a/crates/services/gas_price_service/src/v0/service.rs +++ b/crates/services/gas_price_service/src/v0/service.rs @@ -141,7 +141,7 @@ where Metadata: MetadataStorage, { async fn run(&mut self, watcher: &mut StateWatcher) -> TaskNextAction { - tracing::debug!("Starting gas price service"); + tracing::trace!("Starting gas price service"); tokio::select! { biased; _ = watcher.while_started() => { diff --git a/crates/services/gas_price_service/src/v0/tests.rs b/crates/services/gas_price_service/src/v0/tests.rs index 7ba279a08ca..294c5029a4d 100644 --- a/crates/services/gas_price_service/src/v0/tests.rs +++ b/crates/services/gas_price_service/src/v0/tests.rs @@ -214,8 +214,6 @@ async fn next__new_l2_block_saves_old_metadata() { algo_updater, ); - let read_algo = service.next_block_algorithm(); - let start = read_algo.next_gas_price(); let mut watcher = StateWatcher::started(); tokio::spawn(async move { service.run(&mut watcher).await }); @@ -224,8 +222,8 @@ async fn next__new_l2_block_saves_old_metadata() { tokio::time::sleep(std::time::Duration::from_millis(10)).await; // then - let new = read_algo.next_gas_price(); - assert_ne!(start, new); + let metadata_has_been_updated = metadata_inner.lock().unwrap().is_some(); + assert!(metadata_has_been_updated); } #[derive(Clone)] From 24d4a5d42d1d61d8bac85c13e8d3ad3e07cfbd48 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 29 Nov 2024 13:24:39 -0700 Subject: [PATCH 031/131] Introduce unrecorded blocks abstraction without implementation --- crates/fuel-gas-price-algorithm/src/v1.rs | 25 +++- .../fuel-gas-price-algorithm/src/v1/tests.rs | 5 +- .../src/common/updater_metadata.rs | 4 +- .../gas_price_service/src/v1/metadata.rs | 109 ++++++++---------- .../gas_price_service/src/v1/service.rs | 69 +++++++---- .../gas_price_service/src/v1/tests.rs | 39 ++++--- .../src/v1/uninitialized_task.rs | 44 +++++-- .../fuel_storage_unrecorded_blocks.rs | 21 ++++ 8 files changed, 202 insertions(+), 114 deletions(-) create mode 100644 crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 09cddbb42ab..3bf8192894a 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -59,6 +59,23 @@ impl AlgorithmV1 { } } +pub type Height = u32; +pub type Bytes = u64; +pub trait UnrecordedBlocks { + fn insert(&mut self, height: Height, bytes: Bytes); + fn remove(&mut self, height: &Height) -> Option; +} + +impl UnrecordedBlocks for BTreeMap { + fn insert(&mut self, height: Height, bytes: Bytes) { + self.insert(height, bytes); + } + + fn remove(&mut self, height: &Height) -> Option { + self.remove(height) + } +} + /// The state of the algorithm used to update the gas price algorithm for each block /// /// Because there will always be a delay between blocks submitted to the L2 chain and the blocks @@ -96,10 +113,8 @@ impl AlgorithmV1 { /// The DA portion also uses a moving average of the profits over the last `avg_window` blocks /// instead of the actual profit. Setting the `avg_window` to 1 will effectively disable the /// moving average. -type Height = u32; -type Bytes = u64; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] -pub struct AlgorithmUpdaterV1 { +pub struct AlgorithmUpdaterV1 { // Execution /// The gas price (scaled by the `gas_price_factor`) to cover the execution of the next block pub new_scaled_exec_price: u64, @@ -144,7 +159,7 @@ pub struct AlgorithmUpdaterV1 { /// Activity of L2 pub l2_activity: L2ActivityTracker, /// The unrecorded blocks that are used to calculate the projected cost of recording blocks - pub unrecorded_blocks: BTreeMap, + pub unrecorded_blocks: U, /// Total unrecorded block bytes pub unrecorded_blocks_bytes: u128, } @@ -299,7 +314,7 @@ impl core::ops::Deref for ClampedPercentage { } } -impl AlgorithmUpdaterV1 { +impl AlgorithmUpdaterV1 { pub fn update_da_record_data( &mut self, heights: &[u32], diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests.rs index 9e3dc879651..3a8386b7a3a 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests.rs @@ -4,8 +4,11 @@ use crate::v1::{ AlgorithmUpdaterV1, + Bytes, + Height, L2ActivityTracker, }; +use std::collections::BTreeMap; #[cfg(test)] mod algorithm_v1_tests; @@ -162,7 +165,7 @@ impl UpdaterBuilder { self } - fn build(self) -> AlgorithmUpdaterV1 { + fn build(self) -> AlgorithmUpdaterV1> { AlgorithmUpdaterV1 { min_exec_gas_price: self.min_exec_gas_price, new_scaled_exec_price: self.starting_exec_gas_price, diff --git a/crates/services/gas_price_service/src/common/updater_metadata.rs b/crates/services/gas_price_service/src/common/updater_metadata.rs index 6ce274ea1a7..85cf4981556 100644 --- a/crates/services/gas_price_service/src/common/updater_metadata.rs +++ b/crates/services/gas_price_service/src/common/updater_metadata.rs @@ -30,8 +30,8 @@ impl From for UpdaterMetadata { } } -impl From for UpdaterMetadata { - fn from(updater: AlgorithmUpdaterV1) -> Self { +impl From> for UpdaterMetadata { + fn from(updater: AlgorithmUpdaterV1) -> Self { Self::V1(updater.into()) } } diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index 0976bfd3bfb..096781f91b5 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -1,4 +1,7 @@ -use crate::v0::metadata::V0Metadata; +use crate::{ + v0::metadata::V0Metadata, + v1::uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, +}; use fuel_gas_price_algorithm::v1::{ AlgorithmUpdaterV1, L2ActivityTracker, @@ -30,9 +33,8 @@ pub struct V1Metadata { pub second_to_last_profit: i128, /// The latest known cost per byte for recording blocks on the DA chain pub latest_da_cost_per_byte: u128, - /// List of (height, size) for l2 blocks that have not been recorded on the DA chain (that we know), - /// but have been used to estimate the cost of recording blocks on the DA chain - pub unrecorded_blocks: Vec<(u32, u64)>, + /// Track the total bytes of all unrecorded blocks + pub unrecorded_block_bytes: u128, } impl V1Metadata { @@ -54,7 +56,7 @@ impl V1Metadata { last_profit: 0, second_to_last_profit: 0, latest_da_cost_per_byte: 0, - unrecorded_blocks: vec![], + unrecorded_block_bytes: 0, }; Ok(metadata) } @@ -74,54 +76,49 @@ pub struct V1AlgorithmConfig { pub capped_range_size: u16, pub decrease_range_size: u16, pub block_activity_threshold: u8, - pub unrecorded_blocks: Vec<(u32, u64)>, } -impl From<&V1AlgorithmConfig> for AlgorithmUpdaterV1 { - fn from(value: &V1AlgorithmConfig) -> Self { - let l2_activity = L2ActivityTracker::new_full( - value.normal_range_size, - value.capped_range_size, - value.decrease_range_size, - value.block_activity_threshold.into(), - ); - let unrecorded_blocks: BTreeMap<_, _> = - value.unrecorded_blocks.clone().into_iter().collect(); - let unrecorded_blocks_bytes: u128 = unrecorded_blocks - .values() - .map(|size| u128::from(*size)) - .sum(); - Self { - new_scaled_exec_price: value - .new_exec_gas_price - .saturating_mul(value.gas_price_factor.get()), - l2_block_height: 0, - new_scaled_da_gas_price: value.min_da_gas_price, - gas_price_factor: value.gas_price_factor, - total_da_rewards_excess: 0, - latest_known_total_da_cost_excess: 0, - projected_total_da_cost: 0, - last_profit: 0, - second_to_last_profit: 0, - latest_da_cost_per_byte: 0, - l2_activity, - min_exec_gas_price: value.min_exec_gas_price, - exec_gas_price_change_percent: value.exec_gas_price_change_percent, - l2_block_fullness_threshold_percent: value - .l2_block_fullness_threshold_percent - .into(), - min_da_gas_price: value.min_da_gas_price, - max_da_gas_price_change_percent: value.max_da_gas_price_change_percent, - da_p_component: value.da_p_component, - da_d_component: value.da_d_component, - unrecorded_blocks, - unrecorded_blocks_bytes, - } +pub fn updater_from_config( + value: &V1AlgorithmConfig, + unrecorded_blocks: U, +) -> AlgorithmUpdaterV1 { + let l2_activity = L2ActivityTracker::new_full( + value.normal_range_size, + value.capped_range_size, + value.decrease_range_size, + value.block_activity_threshold.into(), + ); + let unrecorded_blocks_bytes = 0; + AlgorithmUpdaterV1 { + new_scaled_exec_price: value + .new_exec_gas_price + .saturating_mul(value.gas_price_factor.get()), + l2_block_height: 0, + new_scaled_da_gas_price: value.min_da_gas_price, + gas_price_factor: value.gas_price_factor, + total_da_rewards_excess: 0, + latest_known_total_da_cost_excess: 0, + projected_total_da_cost: 0, + last_profit: 0, + second_to_last_profit: 0, + latest_da_cost_per_byte: 0, + l2_activity, + min_exec_gas_price: value.min_exec_gas_price, + exec_gas_price_change_percent: value.exec_gas_price_change_percent, + l2_block_fullness_threshold_percent: value + .l2_block_fullness_threshold_percent + .into(), + min_da_gas_price: value.min_da_gas_price, + max_da_gas_price_change_percent: value.max_da_gas_price_change_percent, + da_p_component: value.da_p_component, + da_d_component: value.da_d_component, + unrecorded_blocks, + unrecorded_blocks_bytes, } } -impl From for V1Metadata { - fn from(updater: AlgorithmUpdaterV1) -> Self { +impl From> for V1Metadata { + fn from(updater: AlgorithmUpdaterV1) -> Self { Self { new_scaled_exec_price: updater.new_scaled_exec_price, l2_block_height: updater.l2_block_height, @@ -132,32 +129,28 @@ impl From for V1Metadata { last_profit: updater.last_profit, second_to_last_profit: updater.second_to_last_profit, latest_da_cost_per_byte: updater.latest_da_cost_per_byte, - unrecorded_blocks: updater.unrecorded_blocks.into_iter().collect(), + unrecorded_block_bytes: updater.unrecorded_blocks_bytes, } } } -pub fn v1_algorithm_from_metadata( +pub fn v1_algorithm_from_metadata( metadata: V1Metadata, config: &V1AlgorithmConfig, -) -> AlgorithmUpdaterV1 { + unrecorded_blocks: U, +) -> AlgorithmUpdaterV1 { let l2_activity = L2ActivityTracker::new_full( config.normal_range_size, config.capped_range_size, config.decrease_range_size, config.block_activity_threshold.into(), ); - let unrecorded_blocks_bytes: u128 = metadata - .unrecorded_blocks - .iter() - .map(|(_, size)| u128::from(*size)) - .sum(); + let unrecorded_blocks_bytes: u128 = metadata.unrecorded_block_bytes; let projected_portion = unrecorded_blocks_bytes.saturating_mul(metadata.latest_da_cost_per_byte); let projected_total_da_cost = metadata .latest_known_total_da_cost_excess .saturating_add(projected_portion); - let unrecorded_blocks = metadata.unrecorded_blocks.into_iter().collect(); AlgorithmUpdaterV1 { new_scaled_exec_price: metadata.new_scaled_exec_price, l2_block_height: metadata.l2_block_height, @@ -179,7 +172,7 @@ pub fn v1_algorithm_from_metadata( max_da_gas_price_change_percent: config.max_da_gas_price_change_percent, da_p_component: config.da_p_component, da_d_component: config.da_d_component, - unrecorded_blocks, unrecorded_blocks_bytes, + unrecorded_blocks, } } diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index f95649144a3..763dc0e975a 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -3,7 +3,10 @@ use crate::{ gas_price_algorithm::SharedGasPriceAlgo, l2_block_source::L2BlockSource, updater_metadata::UpdaterMetadata, - utils::BlockInfo, + utils::{ + BlockInfo, + Result as GasPriceResult, + }, }, ports::MetadataStorage, v0::metadata::V0Metadata, @@ -18,10 +21,12 @@ use crate::{ DaBlockCosts, }, metadata::{ + updater_from_config, v1_algorithm_from_metadata, V1AlgorithmConfig, V1Metadata, }, + uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, }, }; use anyhow::anyhow; @@ -37,6 +42,7 @@ use fuel_gas_price_algorithm::{ v1::{ AlgorithmUpdaterV1, AlgorithmV1, + UnrecordedBlocks, }, }; use futures::FutureExt; @@ -46,12 +52,11 @@ use tokio::sync::broadcast::{ Receiver, }; -use crate::common::utils::Result as GasPriceResult; - /// The service that updates the gas price algorithm. -pub struct GasPriceServiceV1 +pub struct GasPriceServiceV1 where DA: DaBlockCostsSource, + U: Clone, { /// The algorithm that can be used in the next block shared_algo: SharedV1Algorithm, @@ -60,18 +65,19 @@ where /// The metadata storage metadata_storage: Metadata, /// The algorithm updater - algorithm_updater: AlgorithmUpdaterV1, + algorithm_updater: AlgorithmUpdaterV1, /// the da source adapter handle da_source_adapter_handle: DaSourceService, /// The da source channel da_source_channel: Receiver, } -impl GasPriceServiceV1 +impl GasPriceServiceV1 where L2: L2BlockSource, Metadata: MetadataStorage, DA: DaBlockCostsSource, + U: UnrecordedBlocks + Clone, { async fn process_l2_block_res( &mut self, @@ -99,16 +105,17 @@ where } } -impl GasPriceServiceV1 +impl GasPriceServiceV1 where Metadata: MetadataStorage, DA: DaBlockCostsSource, + U: UnrecordedBlocks + Clone, { pub fn new( l2_block_source: L2, metadata_storage: Metadata, shared_algo: SharedV1Algorithm, - algorithm_updater: AlgorithmUpdaterV1, + algorithm_updater: AlgorithmUpdaterV1, da_source_adapter_handle: DaSourceService, ) -> Self { let da_source_channel = @@ -123,7 +130,7 @@ where } } - pub fn algorithm_updater(&self) -> &AlgorithmUpdaterV1 { + pub fn algorithm_updater(&self) -> &AlgorithmUpdaterV1 { &self.algorithm_updater } @@ -227,11 +234,12 @@ where } #[async_trait] -impl RunnableTask for GasPriceServiceV1 +impl RunnableTask for GasPriceServiceV1 where L2: L2BlockSource, Metadata: MetadataStorage, DA: DaBlockCostsSource, + U: UnrecordedBlocks + Clone + Send + Sync, { async fn run(&mut self, watcher: &mut StateWatcher) -> TaskNextAction { tokio::select! { @@ -289,26 +297,26 @@ fn convert_to_v1_metadata( } } -pub fn initialize_algorithm( +pub fn initialize_algorithm( config: &V1AlgorithmConfig, latest_block_height: u32, metadata_storage: &Metadata, -) -> crate::common::utils::Result<(AlgorithmUpdaterV1, SharedV1Algorithm)> + unrecorded_blocks: U, +) -> crate::common::utils::Result<(AlgorithmUpdaterV1, SharedV1Algorithm)> where Metadata: MetadataStorage, + U: UnrecordedBlocks + Clone, { - let algorithm_updater; - if let Some(updater_metadata) = metadata_storage + let algorithm_updater = if let Some(updater_metadata) = metadata_storage .get_metadata(&latest_block_height.into()) .map_err(|err| { crate::common::utils::Error::CouldNotInitUpdater(anyhow::anyhow!(err)) - })? - { + })? { let v1_metadata = convert_to_v1_metadata(updater_metadata, config)?; - algorithm_updater = v1_algorithm_from_metadata(v1_metadata, config); + v1_algorithm_from_metadata(v1_metadata, config, unrecorded_blocks) } else { - algorithm_updater = AlgorithmUpdaterV1::from(config); - } + updater_from_config(config, unrecorded_blocks) + }; let shared_algo = SharedGasPriceAlgo::new_with_algorithm(algorithm_updater.algorithm()); @@ -341,6 +349,7 @@ mod tests { initialize_algorithm, GasPriceServiceV1, }, + uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, }, }; use fuel_core_services::{ @@ -427,10 +436,15 @@ mod tests { capped_range_size: 100, decrease_range_size: 4, block_activity_threshold: 20, - unrecorded_blocks: vec![], }; - let (algo_updater, shared_algo) = - initialize_algorithm(&config, l2_block_height, &metadata_storage).unwrap(); + let unrecorded_blocks = FuelStorageUnrecordedBlocks; + let (algo_updater, shared_algo) = initialize_algorithm( + &config, + l2_block_height, + &metadata_storage, + unrecorded_blocks, + ) + .unwrap(); let notifier = Arc::new(tokio::sync::Notify::new()); let dummy_da_source = DaSourceService::new( @@ -494,10 +508,15 @@ mod tests { capped_range_size: 100, decrease_range_size: 4, block_activity_threshold: 20, - unrecorded_blocks: vec![(1, 100)], }; - let (algo_updater, shared_algo) = - initialize_algorithm(&config, block_height, &metadata_storage).unwrap(); + let unrecorded_blocks = FuelStorageUnrecordedBlocks; + let (algo_updater, shared_algo) = initialize_algorithm( + &config, + block_height, + &metadata_storage, + unrecorded_blocks, + ) + .unwrap(); let notifier = Arc::new(tokio::sync::Notify::new()); let da_source = DaSourceService::new( diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index e5188a9c869..37e7ab433f8 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -34,7 +34,10 @@ use crate::{ initialize_algorithm, GasPriceServiceV1, }, - uninitialized_task::UninitializedTask, + uninitialized_task::{ + fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, + UninitializedTask, + }, }, }; use anyhow::anyhow; @@ -62,7 +65,10 @@ use fuel_core_types::{ SharedImportResult, }, }; -use fuel_gas_price_algorithm::v1::AlgorithmUpdaterV1; +use fuel_gas_price_algorithm::v1::{ + AlgorithmUpdaterV1, + UnrecordedBlocks, +}; use std::{ num::NonZeroU64, ops::Deref, @@ -163,7 +169,6 @@ fn zero_threshold_arbitrary_config() -> V1AlgorithmConfig { capped_range_size: 0, decrease_range_size: 0, block_activity_threshold: 0, - unrecorded_blocks: vec![], } } @@ -178,7 +183,7 @@ fn arbitrary_metadata() -> V1Metadata { last_profit: 0, second_to_last_profit: 0, latest_da_cost_per_byte: 0, - unrecorded_blocks: vec![], + unrecorded_block_bytes: 0, } } @@ -197,7 +202,6 @@ fn different_arb_config() -> V1AlgorithmConfig { capped_range_size: 0, decrease_range_size: 0, block_activity_threshold: 0, - unrecorded_blocks: vec![], } } @@ -219,8 +223,10 @@ async fn next_gas_price__affected_by_new_l2_block() { let config = zero_threshold_arbitrary_config(); let height = 0; + let unrecorded_blocks = FuelStorageUnrecordedBlocks; let (algo_updater, shared_algo) = - initialize_algorithm(&config, height, &metadata_storage).unwrap(); + initialize_algorithm(&config, height, &metadata_storage, unrecorded_blocks) + .unwrap(); let da_source = FakeDABlockCost::never_returns(); let da_source_service = DaSourceService::new(da_source, None); let mut service = GasPriceServiceV1::new( @@ -266,8 +272,10 @@ async fn run__new_l2_block_saves_old_metadata() { let config = zero_threshold_arbitrary_config(); let height = 0; + let unrecorded_blocks = FuelStorageUnrecordedBlocks; let (algo_updater, shared_algo) = - initialize_algorithm(&config, height, &metadata_storage).unwrap(); + initialize_algorithm(&config, height, &metadata_storage, unrecorded_blocks) + .unwrap(); let da_source = FakeDABlockCost::never_returns(); let da_source_service = DaSourceService::new(da_source, None); let mut service = GasPriceServiceV1::new( @@ -380,6 +388,7 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr let gas_price_db = FakeGasPriceDb; let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); + let unrecorded_blocks = FuelStorageUnrecordedBlocks; // when let service = UninitializedTask::new( @@ -391,6 +400,7 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr metadata_storage, da_cost_source, on_chain_db, + unrecorded_blocks, ) .unwrap(); @@ -399,8 +409,8 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr algo_updater_matches_values_from_old_metadata(algo_updater, original_metadata); } -fn algo_updater_matches_values_from_old_metadata( - algo_updater: AlgorithmUpdaterV1, +fn algo_updater_matches_values_from_old_metadata( + mut algo_updater: AlgorithmUpdaterV1, original_metadata: V1Metadata, ) { let V1Metadata { @@ -413,7 +423,7 @@ fn algo_updater_matches_values_from_old_metadata( last_profit: original_last_profit, second_to_last_profit: original_second_to_last_profit, latest_da_cost_per_byte: original_latest_da_cost_per_byte, - unrecorded_blocks: original_unrecorded_blocks, + unrecorded_block_bytes: original_unrecorded_block_bytes, } = original_metadata; assert_eq!( algo_updater.new_scaled_exec_price, @@ -443,11 +453,8 @@ fn algo_updater_matches_values_from_old_metadata( original_latest_da_cost_per_byte ); assert_eq!( - algo_updater - .unrecorded_blocks - .into_iter() - .collect::>(), - original_unrecorded_blocks.into_iter().collect::>() + algo_updater.unrecorded_blocks_bytes, + original_unrecorded_block_bytes ); } @@ -462,6 +469,7 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { let gas_price_db = FakeGasPriceDb; let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); + let unrecorded_blocks = FuelStorageUnrecordedBlocks; // when let res = UninitializedTask::new( @@ -473,6 +481,7 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { metadata_storage, da_cost_source, on_chain_db, + unrecorded_blocks, ); // then diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 158d7b7ccdc..cc8f7822dbf 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -37,6 +37,7 @@ use crate::{ initialize_algorithm, GasPriceServiceV1, }, + uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, }, }; use anyhow::Error; @@ -55,7 +56,12 @@ use fuel_core_types::{ fuel_types::BlockHeight, services::block_importer::SharedImportResult, }; -use fuel_gas_price_algorithm::v1::AlgorithmUpdaterV1; +use fuel_gas_price_algorithm::v1::{ + AlgorithmUpdaterV1, + UnrecordedBlocks, +}; + +pub mod fuel_storage_unrecorded_blocks; pub struct UninitializedTask< L2DataStoreView, @@ -71,9 +77,10 @@ pub struct UninitializedTask< pub on_chain_db: L2DataStoreView, pub block_stream: BoxStream, pub(crate) shared_algo: SharedV1Algorithm, - pub(crate) algo_updater: AlgorithmUpdaterV1, + pub(crate) algo_updater: AlgorithmUpdaterV1, pub(crate) metadata_storage: Metadata, pub(crate) da_source: DA, + pub(crate) unrecorded_blocks_handle: FuelStorageUnrecordedBlocks, } impl @@ -96,6 +103,7 @@ where metadata_storage: Metadata, da_source: DA, on_chain_db: L2DataStoreView, + unrecorded_blocks: FuelStorageUnrecordedBlocks, ) -> anyhow::Result { let latest_block_height: u32 = on_chain_db .latest_view()? @@ -103,8 +111,12 @@ where .unwrap_or(genesis_block_height) .into(); - let (algo_updater, shared_algo) = - initialize_algorithm(&config, latest_block_height, &metadata_storage)?; + let (algo_updater, shared_algo) = initialize_algorithm( + &config, + latest_block_height, + &metadata_storage, + unrecorded_blocks.clone(), + )?; let task = Self { config, @@ -117,6 +129,7 @@ where shared_algo, metadata_storage, da_source, + unrecorded_blocks_handle: unrecorded_blocks, }; Ok(task) } @@ -124,7 +137,12 @@ where pub fn init( mut self, ) -> anyhow::Result< - GasPriceServiceV1, Metadata, DA>, + GasPriceServiceV1< + FuelL2BlockSource, + Metadata, + DA, + FuelStorageUnrecordedBlocks, + >, > { let mut first_run = false; let latest_block_height: u32 = self @@ -173,6 +191,7 @@ where &self.on_chain_db, metadata_height, latest_block_height, + self.unrecorded_blocks_handle, )?; } @@ -202,7 +221,12 @@ where { const NAME: &'static str = "GasPriceServiceV1"; type SharedData = SharedV1Algorithm; - type Task = GasPriceServiceV1, Metadata, DA>; + type Task = GasPriceServiceV1< + FuelL2BlockSource, + Metadata, + DA, + FuelStorageUnrecordedBlocks, + >; type TaskParams = (); fn shared_data(&self) -> Self::SharedData { @@ -230,6 +254,7 @@ fn sync_gas_price_db_with_on_chain_storage< on_chain_db: &L2DataStoreView, metadata_height: u32, latest_block_height: u32, + unrecorded_blocks: FuelStorageUnrecordedBlocks, ) -> anyhow::Result<()> where L2DataStore: L2Data, @@ -249,7 +274,8 @@ where V1Metadata::construct_from_v0_metadata(metadata, config)? } }; - let mut algo_updater = v1_algorithm_from_metadata(metadata, config); + let mut algo_updater = + v1_algorithm_from_metadata(metadata, config, unrecorded_blocks); sync_v1_metadata( settings, @@ -268,7 +294,7 @@ fn sync_v1_metadata( on_chain_db: &L2DataStoreView, metadata_height: u32, latest_block_height: u32, - updater: &mut AlgorithmUpdaterV1, + updater: &mut AlgorithmUpdaterV1, metadata_storage: &mut Metadata, ) -> anyhow::Result<()> where @@ -347,6 +373,7 @@ where DA: DaBlockCostsSource, { let v1_config = config.v1().ok_or(anyhow::anyhow!("Expected V1 config"))?; + let unrecorded_blocks = FuelStorageUnrecordedBlocks; let gas_price_init = UninitializedTask::new( v1_config, genesis_block_height, @@ -356,6 +383,7 @@ where metadata, da_source, on_chain_db, + unrecorded_blocks, )?; Ok(ServiceRunner::new(gas_price_init)) } diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs new file mode 100644 index 00000000000..5fd3ffd2a5d --- /dev/null +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs @@ -0,0 +1,21 @@ +use fuel_gas_price_algorithm::v1::UnrecordedBlocks; + +#[derive(Debug, Clone)] +pub struct FuelStorageUnrecordedBlocks; + +impl UnrecordedBlocks for FuelStorageUnrecordedBlocks { + fn insert( + &mut self, + height: fuel_gas_price_algorithm::v1::Height, + bytes: fuel_gas_price_algorithm::v1::Bytes, + ) { + todo!() + } + + fn remove( + &mut self, + height: &fuel_gas_price_algorithm::v1::Height, + ) -> Option { + todo!() + } +} From 9f6a649f9acd102f1678cb3e5d41e201aa1749cb Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 29 Nov 2024 15:44:59 -0700 Subject: [PATCH 032/131] Use fuel storage in impl --- .../gas_price_service/src/v1/service.rs | 23 ++++- .../gas_price_service/src/v1/tests.rs | 29 ++++-- .../src/v1/uninitialized_task.rs | 95 +++++++++++++++---- .../fuel_storage_unrecorded_blocks.rs | 23 ++++- .../fuel_storage_unrecorded_blocks/storage.rs | 71 ++++++++++++++ 5 files changed, 214 insertions(+), 27 deletions(-) create mode 100644 crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks/storage.rs diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 763dc0e975a..12d22cb9081 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -330,6 +330,7 @@ where mod tests { use crate::{ common::{ + fuel_core_storage_adapter::storage::GasPriceColumn, l2_block_source::L2BlockSource, updater_metadata::UpdaterMetadata, utils::{ @@ -349,13 +350,23 @@ mod tests { initialize_algorithm, GasPriceServiceV1, }, - uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, + uninitialized_task::fuel_storage_unrecorded_blocks::{ + storage::UnrecordedBlocksColumn, + FuelStorageUnrecordedBlocks, + }, }, }; use fuel_core_services::{ RunnableTask, StateWatcher, }; + use fuel_core_storage::{ + structured_storage::test::InMemoryStorage, + transactional::{ + IntoTransaction, + StorageTransaction, + }, + }; use fuel_core_types::fuel_types::BlockHeight; use std::{ num::NonZeroU64, @@ -403,6 +414,10 @@ mod tests { } } + fn database() -> StorageTransaction> { + InMemoryStorage::default().into_transaction() + } + #[tokio::test] async fn run__updates_gas_price_with_l2_block_source() { // given @@ -437,7 +452,8 @@ mod tests { decrease_range_size: 4, block_activity_threshold: 20, }; - let unrecorded_blocks = FuelStorageUnrecordedBlocks; + let inner = database(); + let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(inner); let (algo_updater, shared_algo) = initialize_algorithm( &config, l2_block_height, @@ -509,7 +525,8 @@ mod tests { decrease_range_size: 4, block_activity_threshold: 20, }; - let unrecorded_blocks = FuelStorageUnrecordedBlocks; + let inner = database(); + let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(inner); let (algo_updater, shared_algo) = initialize_algorithm( &config, block_height, diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 37e7ab433f8..02edde11f9e 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -2,6 +2,7 @@ use crate::{ common::{ fuel_core_storage_adapter::{ + storage::GasPriceColumn, GasPriceSettings, GasPriceSettingsProvider, }, @@ -35,7 +36,10 @@ use crate::{ GasPriceServiceV1, }, uninitialized_task::{ - fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, + fuel_storage_unrecorded_blocks::{ + storage::UnrecordedBlocksColumn, + FuelStorageUnrecordedBlocks, + }, UninitializedTask, }, }, @@ -50,7 +54,12 @@ use fuel_core_services::{ StateWatcher, }; use fuel_core_storage::{ - transactional::AtomicView, + structured_storage::test::InMemoryStorage, + transactional::{ + AtomicView, + IntoTransaction, + StorageTransaction, + }, Result as StorageResult, }; use fuel_core_types::{ @@ -205,6 +214,10 @@ fn different_arb_config() -> V1AlgorithmConfig { } } +fn database() -> StorageTransaction> { + InMemoryStorage::default().into_transaction() +} + #[tokio::test] async fn next_gas_price__affected_by_new_l2_block() { // given @@ -223,7 +236,8 @@ async fn next_gas_price__affected_by_new_l2_block() { let config = zero_threshold_arbitrary_config(); let height = 0; - let unrecorded_blocks = FuelStorageUnrecordedBlocks; + let inner = database(); + let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(inner); let (algo_updater, shared_algo) = initialize_algorithm(&config, height, &metadata_storage, unrecorded_blocks) .unwrap(); @@ -272,7 +286,8 @@ async fn run__new_l2_block_saves_old_metadata() { let config = zero_threshold_arbitrary_config(); let height = 0; - let unrecorded_blocks = FuelStorageUnrecordedBlocks; + let inner = database(); + let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(inner); let (algo_updater, shared_algo) = initialize_algorithm(&config, height, &metadata_storage, unrecorded_blocks) .unwrap(); @@ -388,7 +403,8 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr let gas_price_db = FakeGasPriceDb; let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); - let unrecorded_blocks = FuelStorageUnrecordedBlocks; + let inner = database(); + let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(inner); // when let service = UninitializedTask::new( @@ -469,7 +485,8 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { let gas_price_db = FakeGasPriceDb; let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); - let unrecorded_blocks = FuelStorageUnrecordedBlocks; + let inner = database(); + let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(inner); // when let res = UninitializedTask::new( diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index cc8f7822dbf..211f2fdca12 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -37,7 +37,10 @@ use crate::{ initialize_algorithm, GasPriceServiceV1, }, - uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, + uninitialized_task::fuel_storage_unrecorded_blocks::{ + storage::UnrecordedBlocksColumn, + FuelStorageUnrecordedBlocks, + }, }, }; use anyhow::Error; @@ -48,7 +51,9 @@ use fuel_core_services::{ StateWatcher, }; use fuel_core_storage::{ + kv_store::KeyValueMutate, not_found, + structured_storage::test::InMemoryStorage, transactional::AtomicView, }; use fuel_core_types::{ @@ -69,6 +74,7 @@ pub struct UninitializedTask< Metadata, DA, SettingsProvider, + UnrecordedBlockStorage, > { pub config: V1AlgorithmConfig, pub genesis_block_height: BlockHeight, @@ -77,14 +83,31 @@ pub struct UninitializedTask< pub on_chain_db: L2DataStoreView, pub block_stream: BoxStream, pub(crate) shared_algo: SharedV1Algorithm, - pub(crate) algo_updater: AlgorithmUpdaterV1, + pub(crate) algo_updater: + AlgorithmUpdaterV1>, pub(crate) metadata_storage: Metadata, pub(crate) da_source: DA, - pub(crate) unrecorded_blocks_handle: FuelStorageUnrecordedBlocks, + pub(crate) unrecorded_blocks_handle: + FuelStorageUnrecordedBlocks, } -impl - UninitializedTask +impl< + L2DataStore, + L2DataStoreView, + GasPriceStore, + Metadata, + DA, + SettingsProvider, + UnrecordedBlockStorage, + > + UninitializedTask< + L2DataStoreView, + GasPriceStore, + Metadata, + DA, + SettingsProvider, + UnrecordedBlockStorage, + > where L2DataStore: L2Data, L2DataStoreView: AtomicView, @@ -92,6 +115,8 @@ where Metadata: MetadataStorage, DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, + UnrecordedBlockStorage: + KeyValueMutate + Send + Sync + Clone, { #[allow(clippy::too_many_arguments)] pub fn new( @@ -103,7 +128,7 @@ where metadata_storage: Metadata, da_source: DA, on_chain_db: L2DataStoreView, - unrecorded_blocks: FuelStorageUnrecordedBlocks, + unrecorded_blocks: FuelStorageUnrecordedBlocks, ) -> anyhow::Result { let latest_block_height: u32 = on_chain_db .latest_view()? @@ -141,7 +166,7 @@ where FuelL2BlockSource, Metadata, DA, - FuelStorageUnrecordedBlocks, + FuelStorageUnrecordedBlocks, >, > { let mut first_run = false; @@ -208,9 +233,23 @@ where } #[async_trait::async_trait] -impl - RunnableService - for UninitializedTask +impl< + L2DataStore, + L2DataStoreView, + GasPriceStore, + Metadata, + DA, + SettingsProvider, + UnrecordedBlockStorage, + > RunnableService + for UninitializedTask< + L2DataStoreView, + GasPriceStore, + Metadata, + DA, + SettingsProvider, + UnrecordedBlockStorage, + > where L2DataStore: L2Data, L2DataStoreView: AtomicView, @@ -218,6 +257,8 @@ where Metadata: MetadataStorage, DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, + UnrecordedBlockStorage: + KeyValueMutate + Send + Sync + Clone, { const NAME: &'static str = "GasPriceServiceV1"; type SharedData = SharedV1Algorithm; @@ -225,7 +266,7 @@ where FuelL2BlockSource, Metadata, DA, - FuelStorageUnrecordedBlocks, + FuelStorageUnrecordedBlocks, >; type TaskParams = (); @@ -247,6 +288,7 @@ fn sync_gas_price_db_with_on_chain_storage< L2DataStoreView, Metadata, SettingsProvider, + UnrecordedBlockStorage, >( settings: &SettingsProvider, config: &V1AlgorithmConfig, @@ -254,13 +296,15 @@ fn sync_gas_price_db_with_on_chain_storage< on_chain_db: &L2DataStoreView, metadata_height: u32, latest_block_height: u32, - unrecorded_blocks: FuelStorageUnrecordedBlocks, + unrecorded_blocks: FuelStorageUnrecordedBlocks, ) -> anyhow::Result<()> where L2DataStore: L2Data, L2DataStoreView: AtomicView, Metadata: MetadataStorage, SettingsProvider: GasPriceSettingsProvider, + UnrecordedBlockStorage: + KeyValueMutate + Send + Sync + Clone, { let metadata = metadata_storage .get_metadata(&metadata_height.into())? @@ -289,12 +333,18 @@ where Ok(()) } -fn sync_v1_metadata( +fn sync_v1_metadata< + L2DataStore, + L2DataStoreView, + Metadata, + SettingsProvider, + UnrecordedBlockStorage, +>( settings: &SettingsProvider, on_chain_db: &L2DataStoreView, metadata_height: u32, latest_block_height: u32, - updater: &mut AlgorithmUpdaterV1, + updater: &mut AlgorithmUpdaterV1>, metadata_storage: &mut Metadata, ) -> anyhow::Result<()> where @@ -302,6 +352,8 @@ where L2DataStoreView: AtomicView, Metadata: MetadataStorage, SettingsProvider: GasPriceSettingsProvider, + UnrecordedBlockStorage: + KeyValueMutate + Send + Sync + Clone, { let first = metadata_height.saturating_add(1); let view = on_chain_db.latest_view()?; @@ -350,6 +402,7 @@ pub fn new_gas_price_service_v1< Metadata, DA, SettingsProvider, + UnrecordedBlockStorage, >( config: GasPriceServiceConfig, genesis_block_height: BlockHeight, @@ -359,9 +412,17 @@ pub fn new_gas_price_service_v1< metadata: Metadata, da_source: DA, on_chain_db: L2DataStoreView, + unrecorded_block_storage: UnrecordedBlockStorage, ) -> anyhow::Result< ServiceRunner< - UninitializedTask, + UninitializedTask< + L2DataStoreView, + GasPriceStore, + Metadata, + DA, + SettingsProvider, + UnrecordedBlockStorage, + >, >, > where @@ -371,9 +432,11 @@ where SettingsProvider: GasPriceSettingsProvider, Metadata: MetadataStorage, DA: DaBlockCostsSource, + UnrecordedBlockStorage: + KeyValueMutate + Send + Sync + Clone, { let v1_config = config.v1().ok_or(anyhow::anyhow!("Expected V1 config"))?; - let unrecorded_blocks = FuelStorageUnrecordedBlocks; + let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(unrecorded_block_storage); let gas_price_init = UninitializedTask::new( v1_config, genesis_block_height, diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs index 5fd3ffd2a5d..1cad9e6ed58 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs @@ -1,9 +1,28 @@ +use fuel_core_storage::kv_store::{ + KeyValueInspect, + KeyValueMutate, +}; use fuel_gas_price_algorithm::v1::UnrecordedBlocks; +use storage::UnrecordedBlocksColumn; + +pub mod storage; #[derive(Debug, Clone)] -pub struct FuelStorageUnrecordedBlocks; +pub struct FuelStorageUnrecordedBlocks { + inner: Storage, +} + +impl FuelStorageUnrecordedBlocks { + pub fn new(inner: Storage) -> Self { + Self { inner } + } +} -impl UnrecordedBlocks for FuelStorageUnrecordedBlocks { +impl UnrecordedBlocks for FuelStorageUnrecordedBlocks +where + Storage: KeyValueMutate, + Storage: Send + Sync, +{ fn insert( &mut self, height: fuel_gas_price_algorithm::v1::Height, diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks/storage.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks/storage.rs new file mode 100644 index 00000000000..2f36ac0ae51 --- /dev/null +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks/storage.rs @@ -0,0 +1,71 @@ +use crate::common::updater_metadata::UpdaterMetadata; +use fuel_core_storage::{ + blueprint::plain::Plain, + codec::{ + postcard::Postcard, + primitive::Primitive, + }, + kv_store::StorageColumn, + structured_storage::TableWithBlueprint, + Mappable, +}; +use fuel_core_types::fuel_types::BlockHeight; +use fuel_gas_price_algorithm::v1::Height; + +#[repr(u32)] +#[derive( + Copy, + Clone, + Debug, + strum_macros::EnumCount, + strum_macros::IntoStaticStr, + PartialEq, + Eq, + enum_iterator::Sequence, + Hash, + num_enum::TryFromPrimitive, +)] +pub enum UnrecordedBlocksColumn { + Metadata = 0, + State = 1, +} + +impl UnrecordedBlocksColumn { + /// The total count of variants in the enum. + pub const COUNT: usize = ::COUNT; + + /// Returns the `usize` representation of the `Column`. + pub fn as_u32(&self) -> u32 { + *self as u32 + } +} + +impl StorageColumn for UnrecordedBlocksColumn { + fn name(&self) -> String { + let str: &str = self.into(); + str.to_string() + } + + fn id(&self) -> u32 { + self.as_u32() + } +} + +/// The storage table for metadata of the gas price algorithm updater +pub struct UnrecordedBlocksTable; + +impl Mappable for UnrecordedBlocksTable { + type Key = Self::OwnedKey; + type OwnedKey = u32; + type Value = Self::OwnedValue; + type OwnedValue = u64; +} + +impl TableWithBlueprint for UnrecordedBlocksTable { + type Blueprint = Plain, Postcard>; + type Column = UnrecordedBlocksColumn; + + fn column() -> Self::Column { + UnrecordedBlocksColumn::State + } +} From 1261c4dbc31171760f31ae6c07a191ed35cbeea7 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 29 Nov 2024 16:18:23 -0700 Subject: [PATCH 033/131] Implement adapter --- crates/fuel-gas-price-algorithm/src/v1.rs | 40 +++++++++----- .../src/v1/uninitialized_task.rs | 45 +++++++++++---- .../fuel_storage_unrecorded_blocks.rs | 55 +++++++++++++------ 3 files changed, 98 insertions(+), 42 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 3bf8192894a..84c1a94d331 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -19,6 +19,10 @@ pub enum Error { FailedTooIncludeL2BlockData(String), #[error("L2 block expected but not found in unrecorded blocks: {height}")] L2BlockExpectedNotFound { height: u32 }, + #[error("Could not insert unrecorded block: {0}")] + CouldNotInsertUnrecordedBlock(String), + #[error("Could not remove unrecorded block: {0}")] + CouldNotRemoveUnrecordedBlock(String), } // TODO: separate exec gas price and DA gas price into newtypes for clarity @@ -62,17 +66,19 @@ impl AlgorithmV1 { pub type Height = u32; pub type Bytes = u64; pub trait UnrecordedBlocks { - fn insert(&mut self, height: Height, bytes: Bytes); - fn remove(&mut self, height: &Height) -> Option; + fn insert(&mut self, height: Height, bytes: Bytes) -> Result<(), Error>; + fn remove(&mut self, height: &Height) -> Result, Error>; } impl UnrecordedBlocks for BTreeMap { - fn insert(&mut self, height: Height, bytes: Bytes) { + fn insert(&mut self, height: Height, bytes: Bytes) -> Result<(), Error> { self.insert(height, bytes); + Ok(()) } - fn remove(&mut self, height: &Height) -> Option { - self.remove(height) + fn remove(&mut self, height: &Height) -> Result, Error> { + let value = self.remove(height); + Ok(value) } } @@ -366,7 +372,7 @@ impl AlgorithmUpdaterV1 { self.update_da_gas_price(); // metadata - self.unrecorded_blocks.insert(height, block_bytes); + self.unrecorded_blocks.insert(height, block_bytes)?; self.unrecorded_blocks_bytes = self .unrecorded_blocks_bytes .saturating_add(block_bytes as u128); @@ -558,14 +564,20 @@ impl AlgorithmUpdaterV1 { fn update_unrecorded_block_bytes(&mut self, heights: &[u32]) { let mut total: u128 = 0; for expected_height in heights { - let maybe_bytes = self.unrecorded_blocks.remove(expected_height); - if let Some(bytes) = maybe_bytes { - total = total.saturating_add(bytes as u128); - } else { - tracing::warn!( - "L2 block expected but not found in unrecorded blocks: {}", - expected_height, - ); + let res = self.unrecorded_blocks.remove(expected_height); + match res { + Ok(Some(bytes)) => { + total = total.saturating_add(bytes as u128); + } + Ok(None) => { + tracing::warn!( + "L2 block expected but not found in unrecorded blocks: {}", + expected_height, + ); + } + Err(err) => { + tracing::error!("Could not remove unrecorded block: {}", err); + } } } self.unrecorded_blocks_bytes = self.unrecorded_blocks_bytes.saturating_sub(total); diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 211f2fdca12..fb3654aed60 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -51,10 +51,16 @@ use fuel_core_services::{ StateWatcher, }; use fuel_core_storage::{ - kv_store::KeyValueMutate, + kv_store::{ + KeyValueInspect, + KeyValueMutate, + }, not_found, structured_storage::test::InMemoryStorage, - transactional::AtomicView, + transactional::{ + AtomicView, + Modifiable, + }, }; use fuel_core_types::{ fuel_tx::field::MintAmount, @@ -115,8 +121,11 @@ where Metadata: MetadataStorage, DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, - UnrecordedBlockStorage: - KeyValueMutate + Send + Sync + Clone, + UnrecordedBlockStorage: KeyValueInspect + + Modifiable + + Send + + Sync + + Clone, { #[allow(clippy::too_many_arguments)] pub fn new( @@ -257,8 +266,11 @@ where Metadata: MetadataStorage, DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, - UnrecordedBlockStorage: - KeyValueMutate + Send + Sync + Clone, + UnrecordedBlockStorage: KeyValueInspect + + Modifiable + + Send + + Sync + + Clone, { const NAME: &'static str = "GasPriceServiceV1"; type SharedData = SharedV1Algorithm; @@ -303,8 +315,11 @@ where L2DataStoreView: AtomicView, Metadata: MetadataStorage, SettingsProvider: GasPriceSettingsProvider, - UnrecordedBlockStorage: - KeyValueMutate + Send + Sync + Clone, + UnrecordedBlockStorage: KeyValueInspect + + Modifiable + + Send + + Sync + + Clone, { let metadata = metadata_storage .get_metadata(&metadata_height.into())? @@ -352,8 +367,11 @@ where L2DataStoreView: AtomicView, Metadata: MetadataStorage, SettingsProvider: GasPriceSettingsProvider, - UnrecordedBlockStorage: - KeyValueMutate + Send + Sync + Clone, + UnrecordedBlockStorage: KeyValueInspect + + Modifiable + + Send + + Sync + + Clone, { let first = metadata_height.saturating_add(1); let view = on_chain_db.latest_view()?; @@ -432,8 +450,11 @@ where SettingsProvider: GasPriceSettingsProvider, Metadata: MetadataStorage, DA: DaBlockCostsSource, - UnrecordedBlockStorage: - KeyValueMutate + Send + Sync + Clone, + UnrecordedBlockStorage: KeyValueInspect + + Modifiable + + Send + + Sync + + Clone, { let v1_config = config.v1().ok_or(anyhow::anyhow!("Expected V1 config"))?; let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(unrecorded_block_storage); diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs index 1cad9e6ed58..8b6a3111215 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs @@ -1,8 +1,21 @@ -use fuel_core_storage::kv_store::{ - KeyValueInspect, - KeyValueMutate, +use crate::v1::uninitialized_task::fuel_storage_unrecorded_blocks::storage::UnrecordedBlocksTable; +use fuel_core_storage::{ + kv_store::{ + KeyValueInspect, + KeyValueMutate, + }, + transactional::{ + Modifiable, + WriteTransaction, + }, + StorageAsMut, + StorageAsRef, +}; +use fuel_core_types::fuel_merkle::storage::StorageMutateInfallible; +use fuel_gas_price_algorithm::{ + v1, + v1::UnrecordedBlocks, }; -use fuel_gas_price_algorithm::v1::UnrecordedBlocks; use storage::UnrecordedBlocksColumn; pub mod storage; @@ -20,21 +33,31 @@ impl FuelStorageUnrecordedBlocks { impl UnrecordedBlocks for FuelStorageUnrecordedBlocks where - Storage: KeyValueMutate, + Storage: KeyValueInspect + Modifiable, Storage: Send + Sync, { - fn insert( - &mut self, - height: fuel_gas_price_algorithm::v1::Height, - bytes: fuel_gas_price_algorithm::v1::Bytes, - ) { - todo!() + fn insert(&mut self, height: v1::Height, bytes: v1::Bytes) -> Result<(), v1::Error> { + let mut tx = self.inner.write_transaction(); + tx.storage_as_mut::() + .insert(&height, &bytes) + .and_then(|_| tx.commit()) + .map_err(|err| { + v1::Error::CouldNotInsertUnrecordedBlock(format!("Error: {:?}", err)) + })?; + Ok(()) } - fn remove( - &mut self, - height: &fuel_gas_price_algorithm::v1::Height, - ) -> Option { - todo!() + fn remove(&mut self, height: &v1::Height) -> Result, v1::Error> { + let mut tx = self.inner.write_transaction(); + let bytes = tx + .storage_as_mut::() + .take(height) + .map_err(|err| { + v1::Error::CouldNotRemoveUnrecordedBlock(format!("Error: {:?}", err)) + })?; + tx.commit().map_err(|err| { + v1::Error::CouldNotRemoveUnrecordedBlock(format!("Error: {:?}", err)) + })?; + Ok(bytes) } } From 58f749bbdebe074ee4377e81a9f14a800c88b0ff Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 29 Nov 2024 16:26:50 -0700 Subject: [PATCH 034/131] Add simple tests --- .../fuel_storage_unrecorded_blocks.rs | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs index 8b6a3111215..824891ddae7 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs @@ -61,3 +61,49 @@ where Ok(bytes) } } + +#[allow(non_snake_case)] +#[cfg(test)] +mod tests { + use super::*; + use fuel_core_storage::{ + structured_storage::test::InMemoryStorage, + transactional::{ + IntoTransaction, + StorageTransaction, + }, + }; + + fn database() -> StorageTransaction> { + InMemoryStorage::default().into_transaction() + } + + #[test] + fn insert__remove__round_trip() { + // given + let mut storage = FuelStorageUnrecordedBlocks::new(database()); + let height = 8; + let bytes = 100; + + // when + storage.insert(height, bytes).unwrap(); + + // then + let expected = Some(bytes.into()); + let actual = storage.remove(&height.into()).unwrap(); + assert_eq!(expected, actual); + } + + #[test] + fn remove__if_not_inserted_returns_none() { + // given + let mut storage = FuelStorageUnrecordedBlocks::new(database()); + let height = 8; + + // when + let maybe_value = storage.remove(&height).unwrap(); + + // then + assert!(maybe_value.is_none()); + } +} From f969e88d90abc7970eae7e9ad755e6a0a4996b57 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 29 Nov 2024 16:29:10 -0700 Subject: [PATCH 035/131] Fix test, update CHANGELOG --- CHANGELOG.md | 1 + .../src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddf434b5091..470c60f677f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [2389](https://github.com/FuelLabs/fuel-core/pull/2389): Fix construction of reverse iterator in RocksDB. ### Changed +- [2468](https://github.com/FuelLabs/fuel-core/pull/2468): Abstract unrecorded blocks concept for V1 algorithm, create new storage impl - [2295](https://github.com/FuelLabs/fuel-core/pull/2295): `CombinedDb::from_config` now respects `state_rewind_policy` with tmp RocksDB. - [2378](https://github.com/FuelLabs/fuel-core/pull/2378): Use cached hash of the topic instead of calculating it on each publishing gossip message. - [2429](https://github.com/FuelLabs/fuel-core/pull/2429): Introduce custom enum for representing result of running service tasks diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs index 824891ddae7..1398e5530c8 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs @@ -87,10 +87,10 @@ mod tests { // when storage.insert(height, bytes).unwrap(); + let actual = storage.remove(&height.into()).unwrap(); // then let expected = Some(bytes.into()); - let actual = storage.remove(&height.into()).unwrap(); assert_eq!(expected, actual); } From b2051ce6621fc60ae08e9c96f118b0b12c707510 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 29 Nov 2024 16:34:19 -0700 Subject: [PATCH 036/131] update comment --- crates/services/gas_price_service/src/v1/metadata.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index 096781f91b5..b8a884fb2cb 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -33,7 +33,8 @@ pub struct V1Metadata { pub second_to_last_profit: i128, /// The latest known cost per byte for recording blocks on the DA chain pub latest_da_cost_per_byte: u128, - /// Track the total bytes of all unrecorded blocks + /// Track the total bytes of all l2 blocks that have not been recorded on the DA chain (that we know), + /// but have been used to estimate the cost of recording blocks on the DA chain pub unrecorded_block_bytes: u128, } From 68a21ded42c5ef12abfb38ac725cfea57c476039 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 29 Nov 2024 16:36:08 -0700 Subject: [PATCH 037/131] Make generic arg consistent --- .../services/gas_price_service/src/common/updater_metadata.rs | 4 ++-- crates/services/gas_price_service/src/v1/metadata.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/services/gas_price_service/src/common/updater_metadata.rs b/crates/services/gas_price_service/src/common/updater_metadata.rs index 85cf4981556..4b1bb66de97 100644 --- a/crates/services/gas_price_service/src/common/updater_metadata.rs +++ b/crates/services/gas_price_service/src/common/updater_metadata.rs @@ -30,8 +30,8 @@ impl From for UpdaterMetadata { } } -impl From> for UpdaterMetadata { - fn from(updater: AlgorithmUpdaterV1) -> Self { +impl From> for UpdaterMetadata { + fn from(updater: AlgorithmUpdaterV1) -> Self { Self::V1(updater.into()) } } diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index b8a884fb2cb..914eddf7c7d 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -118,8 +118,8 @@ pub fn updater_from_config( } } -impl From> for V1Metadata { - fn from(updater: AlgorithmUpdaterV1) -> Self { +impl From> for V1Metadata { + fn from(updater: AlgorithmUpdaterV1) -> Self { Self { new_scaled_exec_price: updater.new_scaled_exec_price, l2_block_height: updater.l2_block_height, From 9276cc91cca2d1d47566f87dfb80b58c2f1a4dc0 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 29 Nov 2024 16:38:17 -0700 Subject: [PATCH 038/131] Appease Clippy-sama --- .../v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs index 1398e5530c8..f87da12e4dd 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs @@ -87,10 +87,10 @@ mod tests { // when storage.insert(height, bytes).unwrap(); - let actual = storage.remove(&height.into()).unwrap(); + let actual = storage.remove(&height).unwrap(); // then - let expected = Some(bytes.into()); + let expected = Some(bytes); assert_eq!(expected, actual); } From 3679422cd01e092737d495bc8f51e1a1325e9a90 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 29 Nov 2024 17:00:36 -0700 Subject: [PATCH 039/131] Remove estra import --- crates/services/gas_price_service/src/v1/uninitialized_task.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index fb3654aed60..90e4cfc1bc1 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -56,7 +56,6 @@ use fuel_core_storage::{ KeyValueMutate, }, not_found, - structured_storage::test::InMemoryStorage, transactional::{ AtomicView, Modifiable, From b47a2baa58895c392e21a341f93d4d54bf945a2a Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Sat, 30 Nov 2024 14:35:25 -0700 Subject: [PATCH 040/131] Integrate with V1 algo --- .../database_description/gas_price.rs | 2 +- crates/fuel-core/src/service/sub_services.rs | 27 ++++--- .../fuel_core_storage_adapter/storage.rs | 19 +++++ .../gas_price_service/src/v1/algorithm.rs | 2 +- .../gas_price_service/src/v1/service.rs | 7 +- .../gas_price_service/src/v1/tests.rs | 7 +- .../src/v1/uninitialized_task.rs | 41 ++++------- .../fuel_storage_unrecorded_blocks.rs | 12 ++-- .../fuel_storage_unrecorded_blocks/storage.rs | 71 ------------------- tests/tests/gas_price.rs | 3 + 10 files changed, 64 insertions(+), 127 deletions(-) delete mode 100644 crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks/storage.rs diff --git a/crates/fuel-core/src/database/database_description/gas_price.rs b/crates/fuel-core/src/database/database_description/gas_price.rs index 66d015b46a8..5b8284013f3 100644 --- a/crates/fuel-core/src/database/database_description/gas_price.rs +++ b/crates/fuel-core/src/database/database_description/gas_price.rs @@ -10,7 +10,7 @@ impl DatabaseDescription for GasPriceDatabase { type Height = BlockHeight; fn version() -> u32 { - 0 + 1 } fn name() -> String { diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index 4fd45534b24..82fbc4835f8 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -38,9 +38,13 @@ use crate::{ SubServices, }, }; -use fuel_core_gas_price_service::v0::uninitialized_task::{ - new_gas_price_service_v0, - AlgorithmV0, +use fuel_core_gas_price_service::v1::{ + algorithm::AlgorithmV1, + da_source_service::block_committer_costs::{ + BlockCommitterDaBlockCosts, + BlockCommitterHttpApi, + }, + uninitialized_task::new_gas_price_service_v1, }; use fuel_core_poa::{ signer::SignMode, @@ -71,7 +75,7 @@ pub type BlockProducerService = fuel_core_producer::block_producer::Producer< Database, TxPoolAdapter, ExecutorAdapter, - FuelGasPriceProvider, + FuelGasPriceProvider, ConsensusParametersProvider, >; @@ -184,18 +188,23 @@ pub fn init_sub_services( let block_stream = importer_adapter.events_shared_result(); let metadata = StructuredStorage::new(database.gas_price().clone()); - let gas_price_service_v0 = new_gas_price_service_v0( + let committer = BlockCommitterHttpApi::new("lolz".to_string()); + let da_source = BlockCommitterDaBlockCosts::new(committer, None); + let on_chain_db = database.on_chain().clone(); + + let gas_price_service = new_gas_price_service_v1( config.clone().into(), genesis_block_height, settings, block_stream, database.gas_price().clone(), metadata, - database.on_chain().clone(), + da_source, + on_chain_db, + database.gas_price().clone(), )?; - let gas_price_provider = - FuelGasPriceProvider::new(gas_price_service_v0.shared.clone()); + let gas_price_provider = FuelGasPriceProvider::new(gas_price_service.shared.clone()); let txpool = fuel_core_txpool::new_service( chain_id, config.txpool.clone(), @@ -332,7 +341,7 @@ pub fn init_sub_services( #[allow(unused_mut)] // `FuelService` starts and shutdowns all sub-services in the `services` order let mut services: SubServices = vec![ - Box::new(gas_price_service_v0), + Box::new(gas_price_service), Box::new(txpool), Box::new(consensus_parameters_provider_service), ]; diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs index 43d6835dcc1..d3c2fb72eea 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs @@ -27,6 +27,7 @@ use fuel_core_types::fuel_types::BlockHeight; pub enum GasPriceColumn { Metadata = 0, State = 1, + UnrecordedBlocks = 2, } impl GasPriceColumn { @@ -68,3 +69,21 @@ impl TableWithBlueprint for GasPriceMetadata { GasPriceColumn::State } } + +pub struct UnrecordedBlocksTable; + +impl Mappable for UnrecordedBlocksTable { + type Key = Self::OwnedKey; + type OwnedKey = u32; + type Value = Self::OwnedValue; + type OwnedValue = u64; +} + +impl TableWithBlueprint for UnrecordedBlocksTable { + type Blueprint = Plain, Primitive<8>>; + type Column = GasPriceColumn; + + fn column() -> Self::Column { + GasPriceColumn::UnrecordedBlocks + } +} diff --git a/crates/services/gas_price_service/src/v1/algorithm.rs b/crates/services/gas_price_service/src/v1/algorithm.rs index c7f87aed2ad..86a2b4ba98d 100644 --- a/crates/services/gas_price_service/src/v1/algorithm.rs +++ b/crates/services/gas_price_service/src/v1/algorithm.rs @@ -3,7 +3,7 @@ use crate::common::gas_price_algorithm::{ SharedGasPriceAlgo, }; use fuel_core_types::fuel_types::BlockHeight; -use fuel_gas_price_algorithm::v1::AlgorithmV1; +pub use fuel_gas_price_algorithm::v1::AlgorithmV1; impl GasPriceAlgorithm for AlgorithmV1 { fn next_gas_price(&self) -> u64 { diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 12d22cb9081..83cbcc7cb07 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -350,10 +350,7 @@ mod tests { initialize_algorithm, GasPriceServiceV1, }, - uninitialized_task::fuel_storage_unrecorded_blocks::{ - storage::UnrecordedBlocksColumn, - FuelStorageUnrecordedBlocks, - }, + uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, }, }; use fuel_core_services::{ @@ -414,7 +411,7 @@ mod tests { } } - fn database() -> StorageTransaction> { + fn database() -> StorageTransaction> { InMemoryStorage::default().into_transaction() } diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 02edde11f9e..26447865981 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -36,10 +36,7 @@ use crate::{ GasPriceServiceV1, }, uninitialized_task::{ - fuel_storage_unrecorded_blocks::{ - storage::UnrecordedBlocksColumn, - FuelStorageUnrecordedBlocks, - }, + fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, UninitializedTask, }, }, @@ -214,7 +211,7 @@ fn different_arb_config() -> V1AlgorithmConfig { } } -fn database() -> StorageTransaction> { +fn database() -> StorageTransaction> { InMemoryStorage::default().into_transaction() } diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 90e4cfc1bc1..4dadd9c2626 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -4,6 +4,7 @@ use crate::{ block_bytes, get_block_info, mint_values, + storage::GasPriceColumn, GasPriceSettings, GasPriceSettingsProvider, }, @@ -37,10 +38,7 @@ use crate::{ initialize_algorithm, GasPriceServiceV1, }, - uninitialized_task::fuel_storage_unrecorded_blocks::{ - storage::UnrecordedBlocksColumn, - FuelStorageUnrecordedBlocks, - }, + uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, }, }; use anyhow::Error; @@ -120,11 +118,8 @@ where Metadata: MetadataStorage, DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, - UnrecordedBlockStorage: KeyValueInspect - + Modifiable - + Send - + Sync - + Clone, + UnrecordedBlockStorage: + KeyValueInspect + Modifiable + Send + Sync + Clone, { #[allow(clippy::too_many_arguments)] pub fn new( @@ -265,11 +260,8 @@ where Metadata: MetadataStorage, DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, - UnrecordedBlockStorage: KeyValueInspect - + Modifiable - + Send - + Sync - + Clone, + UnrecordedBlockStorage: + KeyValueInspect + Modifiable + Send + Sync + Clone, { const NAME: &'static str = "GasPriceServiceV1"; type SharedData = SharedV1Algorithm; @@ -314,11 +306,8 @@ where L2DataStoreView: AtomicView, Metadata: MetadataStorage, SettingsProvider: GasPriceSettingsProvider, - UnrecordedBlockStorage: KeyValueInspect - + Modifiable - + Send - + Sync - + Clone, + UnrecordedBlockStorage: + KeyValueInspect + Modifiable + Send + Sync + Clone, { let metadata = metadata_storage .get_metadata(&metadata_height.into())? @@ -366,11 +355,8 @@ where L2DataStoreView: AtomicView, Metadata: MetadataStorage, SettingsProvider: GasPriceSettingsProvider, - UnrecordedBlockStorage: KeyValueInspect - + Modifiable - + Send - + Sync - + Clone, + UnrecordedBlockStorage: + KeyValueInspect + Modifiable + Send + Sync + Clone, { let first = metadata_height.saturating_add(1); let view = on_chain_db.latest_view()?; @@ -449,11 +435,8 @@ where SettingsProvider: GasPriceSettingsProvider, Metadata: MetadataStorage, DA: DaBlockCostsSource, - UnrecordedBlockStorage: KeyValueInspect - + Modifiable - + Send - + Sync - + Clone, + UnrecordedBlockStorage: + KeyValueInspect + Modifiable + Send + Sync + Clone, { let v1_config = config.v1().ok_or(anyhow::anyhow!("Expected V1 config"))?; let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(unrecorded_block_storage); diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs index f87da12e4dd..6e3985476ed 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs @@ -1,4 +1,7 @@ -use crate::v1::uninitialized_task::fuel_storage_unrecorded_blocks::storage::UnrecordedBlocksTable; +use crate::common::fuel_core_storage_adapter::storage::{ + GasPriceColumn, + UnrecordedBlocksTable, +}; use fuel_core_storage::{ kv_store::{ KeyValueInspect, @@ -16,9 +19,6 @@ use fuel_gas_price_algorithm::{ v1, v1::UnrecordedBlocks, }; -use storage::UnrecordedBlocksColumn; - -pub mod storage; #[derive(Debug, Clone)] pub struct FuelStorageUnrecordedBlocks { @@ -33,7 +33,7 @@ impl FuelStorageUnrecordedBlocks { impl UnrecordedBlocks for FuelStorageUnrecordedBlocks where - Storage: KeyValueInspect + Modifiable, + Storage: KeyValueInspect + Modifiable, Storage: Send + Sync, { fn insert(&mut self, height: v1::Height, bytes: v1::Bytes) -> Result<(), v1::Error> { @@ -74,7 +74,7 @@ mod tests { }, }; - fn database() -> StorageTransaction> { + fn database() -> StorageTransaction> { InMemoryStorage::default().into_transaction() } diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks/storage.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks/storage.rs deleted file mode 100644 index 2f36ac0ae51..00000000000 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks/storage.rs +++ /dev/null @@ -1,71 +0,0 @@ -use crate::common::updater_metadata::UpdaterMetadata; -use fuel_core_storage::{ - blueprint::plain::Plain, - codec::{ - postcard::Postcard, - primitive::Primitive, - }, - kv_store::StorageColumn, - structured_storage::TableWithBlueprint, - Mappable, -}; -use fuel_core_types::fuel_types::BlockHeight; -use fuel_gas_price_algorithm::v1::Height; - -#[repr(u32)] -#[derive( - Copy, - Clone, - Debug, - strum_macros::EnumCount, - strum_macros::IntoStaticStr, - PartialEq, - Eq, - enum_iterator::Sequence, - Hash, - num_enum::TryFromPrimitive, -)] -pub enum UnrecordedBlocksColumn { - Metadata = 0, - State = 1, -} - -impl UnrecordedBlocksColumn { - /// The total count of variants in the enum. - pub const COUNT: usize = ::COUNT; - - /// Returns the `usize` representation of the `Column`. - pub fn as_u32(&self) -> u32 { - *self as u32 - } -} - -impl StorageColumn for UnrecordedBlocksColumn { - fn name(&self) -> String { - let str: &str = self.into(); - str.to_string() - } - - fn id(&self) -> u32 { - self.as_u32() - } -} - -/// The storage table for metadata of the gas price algorithm updater -pub struct UnrecordedBlocksTable; - -impl Mappable for UnrecordedBlocksTable { - type Key = Self::OwnedKey; - type OwnedKey = u32; - type Value = Self::OwnedValue; - type OwnedValue = u64; -} - -impl TableWithBlueprint for UnrecordedBlocksTable { - type Blueprint = Plain, Postcard>; - type Column = UnrecordedBlocksColumn; - - fn column() -> Self::Column { - UnrecordedBlocksColumn::State - } -} diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 5b24470ca34..4f1990d7c3f 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -416,3 +416,6 @@ async fn startup__can_override_gas_price_values_by_changing_config() { assert_eq!(l2_block_height, new_height); recovered_driver.kill().await; } + +#[test] +fn produce_block__l1_committed_block_effects_gas_price() {} From 9fbdf1a772ef15945e0baa3994f19de2461871c0 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Sat, 30 Nov 2024 15:51:19 -0700 Subject: [PATCH 041/131] WIP add test --- Cargo.lock | 272 +++++++++++++++++- .../block_committer_costs.rs | 2 +- tests/Cargo.toml | 1 + tests/tests/gas_price.rs | 43 ++- 4 files changed, 314 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87073221f28..940c83fd259 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -619,6 +619,21 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "atomic" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" + +[[package]] +name = "atomic" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +dependencies = [ + "bytemuck", +] + [[package]] name = "atomic-polyfill" version = "1.0.3" @@ -1116,6 +1131,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" +[[package]] +name = "binascii" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72" + [[package]] name = "bindgen" version = "0.65.1" @@ -1252,6 +1273,12 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" +[[package]] +name = "bytemuck" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" + [[package]] name = "byteorder" version = "1.5.0" @@ -2294,6 +2321,39 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "devise" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d90b0c4c777a2cad215e3c7be59ac7c15adf45cf76317009b7d096d46f651d" +dependencies = [ + "devise_codegen", + "devise_core", +] + +[[package]] +name = "devise_codegen" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71b28680d8be17a570a2334922518be6adc3f58ecc880cbb404eaeb8624fd867" +dependencies = [ + "devise_core", + "quote", +] + +[[package]] +name = "devise_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b035a542cf7abf01f2e3c4d5a7acbaebfefe120ae4efc7bde3df98186e4b8af7" +dependencies = [ + "bitflags 2.6.0", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.89", +] + [[package]] name = "diff" version = "0.1.13" @@ -3030,6 +3090,20 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic 0.6.0", + "pear", + "serde", + "toml 0.8.19", + "uncased", + "version_check", +] + [[package]] name = "findshlibs" version = "0.10.2" @@ -3730,6 +3804,7 @@ dependencies = [ "proptest", "rand", "reqwest 0.12.9", + "rocket", "rstest", "serde_json", "spki", @@ -4272,6 +4347,19 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generator" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e" +dependencies = [ + "cc", + "libc", + "log", + "rustversion", + "windows 0.48.0", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -5043,7 +5131,7 @@ dependencies = [ "smol", "system-configuration 0.6.1", "tokio", - "windows", + "windows 0.53.0", ] [[package]] @@ -5168,6 +5256,12 @@ dependencies = [ "web-time", ] +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + [[package]] name = "inout" version = "0.1.3" @@ -6143,6 +6237,21 @@ dependencies = [ "logos-codegen", ] +[[package]] +name = "loom" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "serde", + "serde_json", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru" version = "0.12.5" @@ -6321,6 +6430,8 @@ dependencies = [ "memchr", "mime", "spin 0.9.8", + "tokio", + "tokio-util", "version_check", ] @@ -6897,6 +7008,29 @@ dependencies = [ "hmac 0.12.1", ] +[[package]] +name = "pear" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi 1.0.1", +] + +[[package]] +name = "pear_codegen" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.89", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -8041,6 +8175,87 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rocket" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a516907296a31df7dc04310e7043b61d71954d703b603cc6867a026d7e72d73f" +dependencies = [ + "async-stream", + "async-trait", + "atomic 0.5.3", + "binascii", + "bytes", + "either", + "figment", + "futures", + "indexmap 2.6.0", + "log", + "memchr", + "multer", + "num_cpus", + "parking_lot", + "pin-project-lite", + "rand", + "ref-cast", + "rocket_codegen", + "rocket_http", + "serde", + "state", + "tempfile", + "time", + "tokio", + "tokio-stream", + "tokio-util", + "ubyte", + "version_check", + "yansi 1.0.1", +] + +[[package]] +name = "rocket_codegen" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "575d32d7ec1a9770108c879fc7c47815a80073f96ca07ff9525a94fcede1dd46" +dependencies = [ + "devise", + "glob", + "indexmap 2.6.0", + "proc-macro2", + "quote", + "rocket_http", + "syn 2.0.89", + "unicode-xid", + "version_check", +] + +[[package]] +name = "rocket_http" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e274915a20ee3065f611c044bd63c40757396b6dbc057d6046aec27f14f882b9" +dependencies = [ + "cookie", + "either", + "futures", + "http 0.2.12", + "hyper 0.14.31", + "indexmap 2.6.0", + "log", + "memchr", + "pear", + "percent-encoding", + "pin-project-lite", + "ref-cast", + "serde", + "smallvec", + "stable-pattern", + "state", + "time", + "tokio", + "uncased", +] + [[package]] name = "rocksdb" version = "0.21.0" @@ -8362,6 +8577,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -8840,12 +9061,30 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" +[[package]] +name = "stable-pattern" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4564168c00635f88eaed410d5efa8131afa8d8699a612c80c455a0ba05c21045" +dependencies = [ + "memchr", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "state" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b8c4a4445d81357df8b1a650d0d0d6fbbbfe99d064aa5e02f3e4022061476d8" +dependencies = [ + "loom", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -9785,6 +10024,15 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ubyte" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea" +dependencies = [ + "serde", +] + [[package]] name = "ucd-trie" version = "0.1.7" @@ -9809,6 +10057,16 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "serde", + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.17" @@ -10395,6 +10653,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows" version = "0.53.0" @@ -10788,6 +11055,9 @@ name = "yansi" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +dependencies = [ + "is-terminal", +] [[package]] name = "yasna" diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index 111e911b855..b88e29bcc2c 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -15,7 +15,7 @@ use serde::{ }; #[async_trait::async_trait] -trait BlockCommitterApi: Send + Sync { +pub trait BlockCommitterApi: Send + Sync { /// Used on first run to get the latest costs and seqno async fn get_latest_costs(&self) -> DaBlockCostsResult>; /// Used to get the costs for a specific seqno diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 291f41ab7d7..019eb833478 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -63,6 +63,7 @@ postcard = { workspace = true } primitive-types = { workspace = true, default-features = false } rand = { workspace = true } reqwest = { workspace = true } +rocket = "0.5.1" rstest = "0.15" serde_json = { workspace = true } spki = "0.7.3" diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 4f1990d7c3f..3b9ced7ba7f 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -417,5 +417,44 @@ async fn startup__can_override_gas_price_values_by_changing_config() { recovered_driver.kill().await; } -#[test] -fn produce_block__l1_committed_block_effects_gas_price() {} +mod dumb_server { + use rocket::{ + get, + routes, + }; + #[get("/")] + fn hello() -> String { + format!("Hello") + } + + fn rocket() -> rocket::Rocket { + rocket::build().mount("/hello", routes![hello]) + } + pub struct DumbHttpCommitterServer { + _handle: tokio::task::JoinHandle<()>, + } + + impl DumbHttpCommitterServer { + pub async fn new() -> Self { + let _handle = tokio::spawn(async move { + rocket().launch().await.unwrap(); + }); + Self { _handle } + } + } +} + +use dumb_server::DumbHttpCommitterServer; +use fuel_core_gas_price_service::v1::da_source_service::block_committer_costs::{ + BlockCommitterApi, + BlockCommitterHttpApi, +}; +#[tokio::test] +async fn produce_block__l1_committed_block_effects_gas_price() { + let _handle = DumbHttpCommitterServer::new(); + let api = BlockCommitterHttpApi::new("/hello".to_string()); + let res = api.get_latest_costs().await; + dbg!(&res); + let costs = res.unwrap(); + dbg!(costs); +} From f7f60cd01e8bf9e2d0c45701f0977088b4840f68 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Sun, 1 Dec 2024 14:02:57 -0700 Subject: [PATCH 042/131] Add working fake http server --- Cargo.lock | 345 ++++-------------- .../block_committer_costs.rs | 9 +- tests/Cargo.toml | 3 +- tests/tests/gas_price.rs | 71 ++-- 4 files changed, 113 insertions(+), 315 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 940c83fd259..d137f890c3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -251,6 +251,16 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "155a5a185e42c6b77ac7b88a15143d930a9e9727a5b7b77eed417404ab15c247" +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "assert_cmd" version = "2.0.16" @@ -619,21 +629,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "atomic" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" - -[[package]] -name = "atomic" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" -dependencies = [ - "bytemuck", -] - [[package]] name = "atomic-polyfill" version = "1.0.3" @@ -929,7 +924,7 @@ dependencies = [ "aws-smithy-types", "bytes", "fastrand 2.2.0", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "http-body 1.0.1", @@ -1131,12 +1126,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" -[[package]] -name = "binascii" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72" - [[package]] name = "bindgen" version = "0.65.1" @@ -1273,12 +1262,6 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" -[[package]] -name = "bytemuck" -version = "1.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" - [[package]] name = "byteorder" version = "1.5.0" @@ -1624,6 +1607,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -2321,39 +2314,6 @@ dependencies = [ "syn 2.0.89", ] -[[package]] -name = "devise" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d90b0c4c777a2cad215e3c7be59ac7c15adf45cf76317009b7d096d46f651d" -dependencies = [ - "devise_codegen", - "devise_core", -] - -[[package]] -name = "devise_codegen" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b28680d8be17a570a2334922518be6adc3f58ecc880cbb404eaeb8624fd867" -dependencies = [ - "devise_core", - "quote", -] - -[[package]] -name = "devise_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b035a542cf7abf01f2e3c4d5a7acbaebfefe120ae4efc7bde3df98186e4b8af7" -dependencies = [ - "bitflags 2.6.0", - "proc-macro2", - "proc-macro2-diagnostics", - "quote", - "syn 2.0.89", -] - [[package]] name = "diff" version = "0.1.13" @@ -3090,20 +3050,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" -[[package]] -name = "figment" -version = "0.10.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" -dependencies = [ - "atomic 0.6.0", - "pear", - "serde", - "toml 0.8.19", - "uncased", - "version_check", -] - [[package]] name = "findshlibs" version = "0.10.2" @@ -3798,13 +3744,13 @@ dependencies = [ "insta", "itertools 0.12.1", "k256", + "mockito", "postcard", "pretty_assertions", "primitive-types", "proptest", "rand", "reqwest 0.12.9", - "rocket", "rstest", "serde_json", "spki", @@ -3813,6 +3759,7 @@ dependencies = [ "test-helpers", "tokio", "tracing", + "tracing-subscriber", ] [[package]] @@ -4347,19 +4294,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "generator" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e" -dependencies = [ - "cc", - "libc", - "log", - "rustversion", - "windows 0.48.0", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -4471,6 +4405,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", + "indexmap 2.6.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.4.1" @@ -4817,7 +4770,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -4840,9 +4793,11 @@ dependencies = [ "bytes", "futures-channel", "futures-util", + "h2 0.4.7", "http 1.1.0", "http-body 1.0.1", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -5131,7 +5086,7 @@ dependencies = [ "smol", "system-configuration 0.6.1", "tokio", - "windows 0.53.0", + "windows", ] [[package]] @@ -5256,12 +5211,6 @@ dependencies = [ "web-time", ] -[[package]] -name = "inlinable_string" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" - [[package]] name = "inout" version = "0.1.3" @@ -6237,21 +6186,6 @@ dependencies = [ "logos-codegen", ] -[[package]] -name = "loom" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5" -dependencies = [ - "cfg-if", - "generator", - "scoped-tls", - "serde", - "serde_json", - "tracing", - "tracing-subscriber", -] - [[package]] name = "lru" version = "0.12.5" @@ -6416,6 +6350,30 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "mockito" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "652cd6d169a36eaf9d1e6bce1a221130439a966d7f27858af66a33a66e9c4ee2" +dependencies = [ + "assert-json-diff", + "bytes", + "colored", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.5.1", + "hyper-util", + "log", + "rand", + "regex", + "serde_json", + "serde_urlencoded", + "similar", + "tokio", +] + [[package]] name = "multer" version = "3.1.0" @@ -6430,8 +6388,6 @@ dependencies = [ "memchr", "mime", "spin 0.9.8", - "tokio", - "tokio-util", "version_check", ] @@ -7008,29 +6964,6 @@ dependencies = [ "hmac 0.12.1", ] -[[package]] -name = "pear" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" -dependencies = [ - "inlinable_string", - "pear_codegen", - "yansi 1.0.1", -] - -[[package]] -name = "pear_codegen" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" -dependencies = [ - "proc-macro2", - "proc-macro2-diagnostics", - "quote", - "syn 2.0.89", -] - [[package]] name = "peeking_take_while" version = "0.1.2" @@ -8005,7 +7938,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.31", @@ -8175,87 +8108,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "rocket" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a516907296a31df7dc04310e7043b61d71954d703b603cc6867a026d7e72d73f" -dependencies = [ - "async-stream", - "async-trait", - "atomic 0.5.3", - "binascii", - "bytes", - "either", - "figment", - "futures", - "indexmap 2.6.0", - "log", - "memchr", - "multer", - "num_cpus", - "parking_lot", - "pin-project-lite", - "rand", - "ref-cast", - "rocket_codegen", - "rocket_http", - "serde", - "state", - "tempfile", - "time", - "tokio", - "tokio-stream", - "tokio-util", - "ubyte", - "version_check", - "yansi 1.0.1", -] - -[[package]] -name = "rocket_codegen" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "575d32d7ec1a9770108c879fc7c47815a80073f96ca07ff9525a94fcede1dd46" -dependencies = [ - "devise", - "glob", - "indexmap 2.6.0", - "proc-macro2", - "quote", - "rocket_http", - "syn 2.0.89", - "unicode-xid", - "version_check", -] - -[[package]] -name = "rocket_http" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e274915a20ee3065f611c044bd63c40757396b6dbc057d6046aec27f14f882b9" -dependencies = [ - "cookie", - "either", - "futures", - "http 0.2.12", - "hyper 0.14.31", - "indexmap 2.6.0", - "log", - "memchr", - "pear", - "percent-encoding", - "pin-project-lite", - "ref-cast", - "serde", - "smallvec", - "stable-pattern", - "state", - "time", - "tokio", - "uncased", -] - [[package]] name = "rocksdb" version = "0.21.0" @@ -8577,12 +8429,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - [[package]] name = "scopeguard" version = "1.2.0" @@ -9061,30 +8907,12 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" -[[package]] -name = "stable-pattern" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4564168c00635f88eaed410d5efa8131afa8d8699a612c80c455a0ba05c21045" -dependencies = [ - "memchr", -] - [[package]] name = "stable_deref_trait" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" -[[package]] -name = "state" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b8c4a4445d81357df8b1a650d0d0d6fbbbfe99d064aa5e02f3e4022061476d8" -dependencies = [ - "loom", -] - [[package]] name = "static_assertions" version = "1.1.0" @@ -10024,15 +9852,6 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" -[[package]] -name = "ubyte" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea" -dependencies = [ - "serde", -] - [[package]] name = "ucd-trie" version = "0.1.7" @@ -10057,16 +9876,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" -[[package]] -name = "uncased" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" -dependencies = [ - "serde", - "version_check", -] - [[package]] name = "unicode-bidi" version = "0.3.17" @@ -10653,15 +10462,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows" version = "0.53.0" @@ -11055,9 +10855,6 @@ name = "yansi" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" -dependencies = [ - "is-terminal", -] [[package]] name = "yasna" diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index b88e29bcc2c..3387041f2f9 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -135,13 +135,8 @@ impl BlockCommitterHttpApi { #[async_trait::async_trait] impl BlockCommitterApi for BlockCommitterHttpApi { async fn get_latest_costs(&self) -> DaBlockCostsResult> { - let response = self - .client - .get(&self.url) - .send() - .await? - .json::() - .await?; + let val = self.client.get(&self.url).send().await?; + let response = val.json::().await?; Ok(Some(response)) } diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 019eb833478..2e5555242ae 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -59,11 +59,11 @@ hyper = { workspace = true, features = ["server"] } insta = { workspace = true } itertools = { workspace = true } k256 = { version = "0.13.3", features = ["ecdsa-core"] } +mockito = "1.6.1" postcard = { workspace = true } primitive-types = { workspace = true, default-features = false } rand = { workspace = true } reqwest = { workspace = true } -rocket = "0.5.1" rstest = "0.15" serde_json = { workspace = true } spki = "0.7.3" @@ -75,6 +75,7 @@ tokio = { workspace = true, features = [ "rt-multi-thread", "test-util", ] } +tracing-subscriber.workspace = true [dev-dependencies] pretty_assertions = "1.4" diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 3b9ced7ba7f..4165cbff410 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -29,6 +29,7 @@ use fuel_core_storage::{ StorageAsRef, }; use fuel_core_types::{ + blockchain::primitives::DaBlockHeight, fuel_asm::*, fuel_crypto::{ coins_bip32::ecdsa::signature::rand_core::SeedableRng, @@ -417,44 +418,48 @@ async fn startup__can_override_gas_price_values_by_changing_config() { recovered_driver.kill().await; } -mod dumb_server { - use rocket::{ - get, - routes, - }; - #[get("/")] - fn hello() -> String { - format!("Hello") - } - - fn rocket() -> rocket::Rocket { - rocket::build().mount("/hello", routes![hello]) - } - pub struct DumbHttpCommitterServer { - _handle: tokio::task::JoinHandle<()>, - } - - impl DumbHttpCommitterServer { - pub async fn new() -> Self { - let _handle = tokio::spawn(async move { - rocket().launch().await.unwrap(); - }); - Self { _handle } - } - } -} - -use dumb_server::DumbHttpCommitterServer; use fuel_core_gas_price_service::v1::da_source_service::block_committer_costs::{ BlockCommitterApi, BlockCommitterHttpApi, + RawDaBlockCosts, }; -#[tokio::test] -async fn produce_block__l1_committed_block_effects_gas_price() { - let _handle = DumbHttpCommitterServer::new(); - let api = BlockCommitterHttpApi::new("/hello".to_string()); - let res = api.get_latest_costs().await; + +#[test] +fn produce_block__l1_committed_block_effects_gas_price() { + let mock = FakeServer::new(); + let url = mock.url(); + let api = BlockCommitterHttpApi::new(url); + let rt = tokio::runtime::Runtime::new().unwrap(); + let res = rt.block_on(api.get_latest_costs()); dbg!(&res); let costs = res.unwrap(); dbg!(costs); } + +struct FakeServer { + server: mockito::ServerGuard, +} + +impl FakeServer { + fn new() -> Self { + let mut server = mockito::Server::new(); + let costs = RawDaBlockCosts { + sequence_number: 1, + blocks_heights: vec![1, 2, 3], + da_block_height: DaBlockHeight(100), + total_cost: 100, + total_size_bytes: 100, + }; + let body = serde_json::to_string(&costs).unwrap(); + let _mock = server + .mock("GET", "/") + .with_status(201) + .with_body(body) + .create(); + Self { server } + } + + fn url(&self) -> String { + self.server.url() + } +} From 1598c2d33deff8fc77695037e157b6dfd2bc5e0c Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 2 Dec 2024 11:58:04 -0700 Subject: [PATCH 043/131] WIP add new config values --- .../tests/integration_tests.rs | 2 +- bin/fuel-core/src/cli/run.rs | 38 ++++++--- .../service/adapters/gas_price_adapters.rs | 19 +++-- crates/fuel-core/src/service/config.rs | 55 +++++++++---- crates/fuel-core/src/service/sub_services.rs | 2 +- .../services/gas_price_service/src/ports.rs | 33 +++++++- .../block_committer_costs.rs | 54 ++++++++----- .../gas_price_service/src/v1/metadata.rs | 4 +- tests/test-helpers/src/builder.rs | 2 +- tests/test-helpers/src/fuel_core_driver.rs | 4 +- tests/tests/chain.rs | 2 +- tests/tests/gas_price.rs | 80 ++++++++++++------- 12 files changed, 208 insertions(+), 87 deletions(-) diff --git a/bin/e2e-test-client/tests/integration_tests.rs b/bin/e2e-test-client/tests/integration_tests.rs index bb62d4e8dc3..720a4be4939 100644 --- a/bin/e2e-test-client/tests/integration_tests.rs +++ b/bin/e2e-test-client/tests/integration_tests.rs @@ -136,7 +136,7 @@ fn dev_config() -> Config { let reader = reader.with_chain_config(chain_config); let mut config = Config::local_node_with_reader(reader); - config.starting_gas_price = 1; + config.starting_exec_gas_price = 1; config.block_producer.coinbase_recipient = Some( ContractId::from_str( "0x7777777777777777777777777777777777777777777777777777777777777777", diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index 06d00219b50..cc5ef9765bb 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -190,12 +190,12 @@ pub struct Command { pub native_executor_version: Option, /// The starting gas price for the network - #[arg(long = "starting-gas-price", default_value = "0", env)] - pub starting_gas_price: u64, + #[arg(long = "starting-exec-gas-price", default_value = "0", env)] + pub starting_exec_gas_price: u64, /// The percentage change in gas price per block #[arg(long = "gas-price-change-percent", default_value = "0", env)] - pub gas_price_change_percent: u64, + pub gas_price_change_percent: u16, /// The minimum allowed gas price #[arg(long = "min-gas-price", default_value = "0", env)] @@ -203,7 +203,15 @@ pub struct Command { /// The percentage threshold for gas price increase #[arg(long = "gas-price-threshold-percent", default_value = "50", env)] - pub gas_price_threshold_percent: u64, + pub gas_price_threshold_percent: u8, + + // Minimum DA gas price + #[arg(long = "min-da-gas-price", default_value = "0", env)] + pub min_da_gas_price: u64, + + /// The URL for the DA Block Committer info + #[arg(long = "da-committer-url", env)] + pub da_committer_url: Option, /// The signing key used when producing blocks. /// Setting via the `CONSENSUS_KEY_SECRET` ENV var is preferred. @@ -299,10 +307,12 @@ impl Command { debug, utxo_validation, native_executor_version, - starting_gas_price, + starting_exec_gas_price: starting_gas_price, gas_price_change_percent, min_gas_price, gas_price_threshold_percent, + min_da_gas_price, + da_committer_url, consensus_key, #[cfg(feature = "aws-kms")] consensus_aws_kms, @@ -588,10 +598,10 @@ impl Command { coinbase_recipient, metrics: disabled_metrics.is_enabled(Module::Producer), }, - starting_gas_price, - gas_price_change_percent, - min_gas_price, - gas_price_threshold_percent, + starting_exec_gas_price: starting_gas_price, + exec_gas_price_change_percent: gas_price_change_percent, + min_exec_gas_price: min_gas_price, + exec_gas_price_threshold_percent: gas_price_threshold_percent, block_importer, da_compression, #[cfg(feature = "relayer")] @@ -606,6 +616,16 @@ impl Command { min_connected_reserved_peers, time_until_synced: time_until_synced.into(), memory_pool_size, + da_gas_price_factor: NonZeroU64::new(100).expect("100 is not zero"), + min_da_gas_price, + max_da_gas_price_change_percent: 0, + da_p_component: 0, + da_d_component: 0, + activity_normal_range_size: 100, + activity_capped_range_size: 0, + activity_decrease_range_size: 0, + da_committer_url, + block_activity_threshold: 0, }; Ok(config) } diff --git a/crates/fuel-core/src/service/adapters/gas_price_adapters.rs b/crates/fuel-core/src/service/adapters/gas_price_adapters.rs index 857c6ded988..2af4c597528 100644 --- a/crates/fuel-core/src/service/adapters/gas_price_adapters.rs +++ b/crates/fuel-core/src/service/adapters/gas_price_adapters.rs @@ -64,11 +64,20 @@ impl GasPriceData for Database { impl From for GasPriceServiceConfig { fn from(value: Config) -> Self { - GasPriceServiceConfig::new_v0( - value.starting_gas_price, - value.min_gas_price, - value.gas_price_change_percent, - value.gas_price_threshold_percent, + GasPriceServiceConfig::new_v1( + value.starting_exec_gas_price, + value.min_exec_gas_price, + value.exec_gas_price_change_percent, + value.exec_gas_price_threshold_percent, + value.da_gas_price_factor, + value.min_da_gas_price, + value.max_da_gas_price_change_percent, + value.da_p_component, + value.da_d_component, + value.activity_normal_range_size, + value.activity_capped_range_size, + value.activity_decrease_range_size, + value.block_activity_threshold, ) } } diff --git a/crates/fuel-core/src/service/config.rs b/crates/fuel-core/src/service/config.rs index 0e093c2b8b6..069b9dd40c7 100644 --- a/crates/fuel-core/src/service/config.rs +++ b/crates/fuel-core/src/service/config.rs @@ -1,10 +1,10 @@ +use clap::ValueEnum; +use fuel_core_poa::signer::SignMode; use std::{ + num::NonZeroU64, path::PathBuf, time::Duration, }; - -use clap::ValueEnum; -use fuel_core_poa::signer::SignMode; use strum_macros::{ Display, EnumString, @@ -57,10 +57,11 @@ pub struct Config { pub vm: VMConfig, pub txpool: TxPoolConfig, pub block_producer: fuel_core_producer::Config, - pub starting_gas_price: u64, - pub gas_price_change_percent: u64, - pub min_gas_price: u64, - pub gas_price_threshold_percent: u64, + pub starting_exec_gas_price: u64, + pub exec_gas_price_change_percent: u16, + pub min_exec_gas_price: u64, + pub exec_gas_price_threshold_percent: u8, + pub da_committer_url: Option, pub da_compression: DaCompressionConfig, pub block_importer: fuel_core_importer::Config, #[cfg(feature = "relayer")] @@ -78,6 +79,15 @@ pub struct Config { pub time_until_synced: Duration, /// The size of the memory pool in number of `MemoryInstance`s. pub memory_pool_size: usize, + pub da_gas_price_factor: NonZeroU64, + pub min_da_gas_price: u64, + pub max_da_gas_price_change_percent: u16, + pub da_p_component: i64, + pub da_d_component: i64, + pub activity_normal_range_size: u16, + pub activity_capped_range_size: u16, + pub activity_decrease_range_size: u16, + pub block_activity_threshold: u8, } impl Config { @@ -96,14 +106,17 @@ impl Config { chain_config: ChainConfig, state_config: StateConfig, ) -> Self { - Self::local_node_with_reader(SnapshotReader::new_in_memory( - chain_config, - state_config, - )) + Self::local_node_with_reader( + SnapshotReader::new_in_memory(chain_config, state_config), + None, + ) } #[cfg(feature = "test-helpers")] - pub fn local_node_with_reader(snapshot_reader: SnapshotReader) -> Self { + pub fn local_node_with_reader( + snapshot_reader: SnapshotReader, + da_committer_url: Option, + ) -> Self { let block_importer = fuel_core_importer::Config::new(false); let latest_block = snapshot_reader.last_block_config(); // In tests, we always want to use the native executor as a default configuration. @@ -171,10 +184,10 @@ impl Config { ..Default::default() }, da_compression: DaCompressionConfig::Disabled, - starting_gas_price, - gas_price_change_percent, - min_gas_price, - gas_price_threshold_percent, + starting_exec_gas_price: starting_gas_price, + exec_gas_price_change_percent: gas_price_change_percent, + min_exec_gas_price: min_gas_price, + exec_gas_price_threshold_percent: gas_price_threshold_percent, block_importer, #[cfg(feature = "relayer")] relayer: None, @@ -190,6 +203,16 @@ impl Config { min_connected_reserved_peers: 0, time_until_synced: Duration::ZERO, memory_pool_size: 4, + da_gas_price_factor: NonZeroU64::new(100).expect("100 is not zero"), + min_da_gas_price: 0, + max_da_gas_price_change_percent: 0, + da_p_component: 0, + da_d_component: 0, + activity_normal_range_size: 0, + activity_capped_range_size: 0, + activity_decrease_range_size: 0, + da_committer_url, + block_activity_threshold: 0, } } diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index 82fbc4835f8..8f8314ff6f5 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -188,7 +188,7 @@ pub fn init_sub_services( let block_stream = importer_adapter.events_shared_result(); let metadata = StructuredStorage::new(database.gas_price().clone()); - let committer = BlockCommitterHttpApi::new("lolz".to_string()); + let committer = BlockCommitterHttpApi::new(config.da_committer_url.clone()); let da_source = BlockCommitterDaBlockCosts::new(committer, None); let on_chain_db = database.on_chain().clone(); diff --git a/crates/services/gas_price_service/src/ports.rs b/crates/services/gas_price_service/src/ports.rs index cb829ab21fb..c9e2524e87b 100644 --- a/crates/services/gas_price_service/src/ports.rs +++ b/crates/services/gas_price_service/src/ports.rs @@ -12,6 +12,7 @@ use fuel_core_types::{ fuel_tx::Transaction, fuel_types::BlockHeight, }; +use std::num::NonZeroU64; pub trait L2Data: Send + Sync { fn latest_height(&self) -> StorageResult; @@ -54,8 +55,36 @@ impl GasPriceServiceConfig { }) } - pub fn new_v1(metadata: V1AlgorithmConfig) -> Self { - Self::V1(metadata) + pub fn new_v1( + new_exec_gas_price: u64, + min_exec_gas_price: u64, + exec_gas_price_change_percent: u16, + l2_block_fullness_threshold_percent: u8, + gas_price_factor: NonZeroU64, + min_da_gas_price: u64, + max_da_gas_price_change_percent: u16, + da_p_component: i64, + da_d_component: i64, + normal_range_size: u16, + capped_range_size: u16, + decrease_range_size: u16, + block_activity_threshold: u8, + ) -> Self { + Self::V1(V1AlgorithmConfig { + new_exec_gas_price, + min_exec_gas_price, + exec_gas_price_change_percent, + l2_block_fullness_threshold_percent, + gas_price_factor, + min_da_gas_price, + max_da_gas_price_change_percent, + da_p_component, + da_d_component, + normal_range_size, + capped_range_size, + decrease_range_size, + block_activity_threshold, + }) } /// Extract V0AlgorithmConfig if it is of V0 version diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index 3387041f2f9..9d3cbf6f503 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -120,11 +120,11 @@ where pub struct BlockCommitterHttpApi { client: reqwest::Client, - url: String, + url: Option, } impl BlockCommitterHttpApi { - pub fn new(url: String) -> Self { + pub fn new(url: Option) -> Self { Self { client: reqwest::Client::new(), url, @@ -135,37 +135,49 @@ impl BlockCommitterHttpApi { #[async_trait::async_trait] impl BlockCommitterApi for BlockCommitterHttpApi { async fn get_latest_costs(&self) -> DaBlockCostsResult> { - let val = self.client.get(&self.url).send().await?; - let response = val.json::().await?; - Ok(Some(response)) + if let Some(url) = &self.url { + let val = self.client.get(url).send().await?; + let response = val.json::().await?; + Ok(Some(response)) + } else { + Ok(None) + } } async fn get_costs_by_seqno( &self, number: u32, ) -> DaBlockCostsResult> { - let response = self - .client - .get(&format!("{}/{}", self.url, number)) - .send() - .await? - .json::() - .await?; - Ok(Some(response)) + if let Some(url) = &self.url { + let response = self + .client + .get(&format!("{}/{}", url, number)) + .send() + .await? + .json::() + .await?; + Ok(Some(response)) + } else { + Ok(None) + } } async fn get_cost_bundles_by_range( &self, range: core::ops::Range, ) -> DaBlockCostsResult>> { - let response = self - .client - .get(&format!("{}/{}-{}", self.url, range.start, range.end)) - .send() - .await? - .json::>() - .await?; - Ok(response.into_iter().map(Some).collect()) + if let Some(url) = &self.url { + let response = self + .client + .get(&format!("{}/{}-{}", url, range.start, range.end)) + .send() + .await? + .json::>() + .await?; + Ok(response.into_iter().map(Some).collect()) + } else { + Ok(vec![]) + } } } diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index 914eddf7c7d..08e4fbf5cc5 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -95,7 +95,9 @@ pub fn updater_from_config( .new_exec_gas_price .saturating_mul(value.gas_price_factor.get()), l2_block_height: 0, - new_scaled_da_gas_price: value.min_da_gas_price, + new_scaled_da_gas_price: value + .min_da_gas_price + .saturating_mul(value.gas_price_factor.get()), gas_price_factor: value.gas_price_factor, total_da_rewards_excess: 0, latest_known_total_da_cost_excess: 0, diff --git a/tests/test-helpers/src/builder.rs b/tests/test-helpers/src/builder.rs index 0d16bc48dc6..ec0e0eb6c53 100644 --- a/tests/test-helpers/src/builder.rs +++ b/tests/test-helpers/src/builder.rs @@ -235,7 +235,7 @@ impl TestSetupBuilder { utxo_validation: self.utxo_validation, txpool: fuel_core_txpool::config::Config::default(), block_production: self.trigger, - starting_gas_price: self.starting_gas_price, + starting_exec_gas_price: self.starting_gas_price, ..Config::local_node_with_configs(chain_conf, state) }; assert_eq!(config.combined_db_config.database_type, DbType::RocksDb); diff --git a/tests/test-helpers/src/fuel_core_driver.rs b/tests/test-helpers/src/fuel_core_driver.rs index a290712e5d2..4f2f3f8363b 100644 --- a/tests/test-helpers/src/fuel_core_driver.rs +++ b/tests/test-helpers/src/fuel_core_driver.rs @@ -22,7 +22,7 @@ impl FuelCoreDriver { pub async fn spawn_feeless(extra_args: &[&str]) -> anyhow::Result { let mut args = vec![ - "--starting-gas-price", + "--starting-exec-gas-price", "0", "--gas-price-change-percent", "0", @@ -36,7 +36,7 @@ impl FuelCoreDriver { extra_args: &[&str], ) -> anyhow::Result { let mut args = vec![ - "--starting-gas-price", + "--starting-exec-gas-price", "0", "--gas-price-change-percent", "0", diff --git a/tests/tests/chain.rs b/tests/tests/chain.rs index 43470f7c0d3..7e9b627a3f8 100644 --- a/tests/tests/chain.rs +++ b/tests/tests/chain.rs @@ -141,7 +141,7 @@ async fn network_operates_with_non_zero_base_asset_id() { let node_config = Config { debug: true, utxo_validation: true, - starting_gas_price, + starting_exec_gas_price: starting_gas_price, ..Config::local_node_with_configs(chain_config, state_config) }; diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 4165cbff410..af46a03db6a 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -104,7 +104,7 @@ async fn latest_gas_price__for_single_block_should_be_starting_gas_price() { // given let mut config = Config::local_node(); let starting_gas_price = 982; - config.starting_gas_price = starting_gas_price; + config.starting_exec_gas_price = starting_gas_price; let srv = FuelService::from_database(Database::default(), config.clone()) .await .unwrap(); @@ -138,9 +138,9 @@ async fn produce_block__raises_gas_price() { let percent = 10; let threshold = 50; node_config.block_producer.coinbase_recipient = Some([5; 32].into()); - node_config.starting_gas_price = starting_gas_price; - node_config.gas_price_change_percent = percent; - node_config.gas_price_threshold_percent = threshold; + node_config.starting_exec_gas_price = starting_gas_price; + node_config.exec_gas_price_change_percent = percent; + node_config.exec_gas_price_threshold_percent = threshold; node_config.block_production = Trigger::Never; let srv = FuelService::new_node(node_config.clone()).await.unwrap(); @@ -159,7 +159,7 @@ async fn produce_block__raises_gas_price() { let _ = client.produce_blocks(1, None).await.unwrap(); // then - let change = starting_gas_price * percent / 100; + let change = starting_gas_price * percent as u64 / 100; let expected = starting_gas_price + change; let latest = client.latest_gas_price().await.unwrap(); let actual = latest.gas_price; @@ -183,9 +183,9 @@ async fn produce_block__lowers_gas_price() { let percent = 10; let threshold = 50; node_config.block_producer.coinbase_recipient = Some([5; 32].into()); - node_config.starting_gas_price = starting_gas_price; - node_config.gas_price_change_percent = percent; - node_config.gas_price_threshold_percent = threshold; + node_config.starting_exec_gas_price = starting_gas_price; + node_config.exec_gas_price_change_percent = percent; + node_config.exec_gas_price_threshold_percent = threshold; node_config.block_production = Trigger::Never; let srv = FuelService::new_node(node_config.clone()).await.unwrap(); @@ -204,7 +204,7 @@ async fn produce_block__lowers_gas_price() { let _ = client.produce_blocks(1, None).await.unwrap(); // then - let change = starting_gas_price * percent / 100; + let change = starting_gas_price * percent as u64 / 100; let expected = starting_gas_price - change; let latest = client.latest_gas_price().await.unwrap(); let actual = latest.gas_price; @@ -217,10 +217,10 @@ async fn estimate_gas_price__is_greater_than_actual_price_at_desired_height() { let mut node_config = Config::local_node(); let starting_gas_price = 1000; let percent = 10; - node_config.starting_gas_price = starting_gas_price; - node_config.gas_price_change_percent = percent; + node_config.starting_exec_gas_price = starting_gas_price; + node_config.exec_gas_price_change_percent = percent; // Always increase - node_config.gas_price_threshold_percent = 0; + node_config.exec_gas_price_threshold_percent = 0; let srv = FuelService::new_node(node_config.clone()).await.unwrap(); let client = FuelClient::from(srv.bound_address); @@ -248,8 +248,8 @@ async fn estimate_gas_price__returns_min_gas_price_if_starting_gas_price_is_zero // Given let mut node_config = Config::local_node(); - node_config.min_gas_price = MIN_GAS_PRICE; - node_config.starting_gas_price = 0; + node_config.min_exec_gas_price = MIN_GAS_PRICE; + node_config.starting_exec_gas_price = 0; let srv = FuelService::new_node(node_config.clone()).await.unwrap(); let client = FuelClient::from(srv.bound_address); @@ -268,7 +268,7 @@ async fn latest_gas_price__if_node_restarts_gets_latest_value() { "--debug", "--poa-instant", "true", - "--starting-gas-price", + "--starting-exec-gas-price", "1000", "--gas-price-change-percent", "10", @@ -276,7 +276,7 @@ async fn latest_gas_price__if_node_restarts_gets_latest_value() { "0", ]; let driver = FuelCoreDriver::spawn(&args).await.unwrap(); - let starting = driver.node.shared.config.starting_gas_price; + let starting = driver.node.shared.config.starting_exec_gas_price; let arb_blocks_to_produce = 10; for _ in 0..arb_blocks_to_produce { driver.client.produce_blocks(1, None).await.unwrap(); @@ -418,22 +418,48 @@ async fn startup__can_override_gas_price_values_by_changing_config() { recovered_driver.kill().await; } -use fuel_core_gas_price_service::v1::da_source_service::block_committer_costs::{ - BlockCommitterApi, - BlockCommitterHttpApi, - RawDaBlockCosts, -}; +use fuel_core_gas_price_service::v1::da_source_service::block_committer_costs::RawDaBlockCosts; #[test] fn produce_block__l1_committed_block_effects_gas_price() { + let rt = tokio::runtime::Runtime::new().unwrap(); + // set up chain with single unrecorded block + let args = vec![ + "--debug", + "--poa-instant", + "true", + "--min-da-gas-price", + "100", + ]; + let (first_gas_price, temp_dir) = rt.block_on(async { + let driver = FuelCoreDriver::spawn(&args).await.unwrap(); + driver.client.produce_blocks(1, None).await.unwrap(); + let first_gas_price = driver.client.latest_gas_price().await.unwrap().gas_price; + let temp_dir = driver.kill().await; + (first_gas_price, temp_dir) + }); + + assert_eq!(100, first_gas_price); + + // set up chain with single recorded block let mock = FakeServer::new(); let url = mock.url(); - let api = BlockCommitterHttpApi::new(url); - let rt = tokio::runtime::Runtime::new().unwrap(); - let res = rt.block_on(api.get_latest_costs()); - dbg!(&res); - let costs = res.unwrap(); - dbg!(costs); + + let args = vec![ + "--debug", + "--poa-instant", + "true", + "--da-committer-url", + &url, + "--min-da-gas-price", + "100", + ]; + + rt.block_on(async { + let _driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) + .await + .unwrap(); + }); } struct FakeServer { From 434f7644868fb085c40f58890569bc709374c400 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 2 Dec 2024 13:25:58 -0700 Subject: [PATCH 044/131] WIP --- bin/fuel-core/src/cli/run.rs | 23 +++++++++++++++++++---- crates/services/src/service.rs | 2 +- tests/tests/gas_price.rs | 24 +++++++++++++----------- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index cc5ef9765bb..8fabb729f0f 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -205,10 +205,22 @@ pub struct Command { #[arg(long = "gas-price-threshold-percent", default_value = "50", env)] pub gas_price_threshold_percent: u8, - // Minimum DA gas price + /// Minimum DA gas price #[arg(long = "min-da-gas-price", default_value = "0", env)] pub min_da_gas_price: u64, + /// P component of DA gas price calculation + #[arg(long = "da-p-component", default_value = "0", env)] + pub da_p_component: i64, + + /// D component of DA gas price calculation + #[arg(long = "da-d-component", default_value = "0", env)] + pub da_d_component: i64, + + /// Maximum DA gas price change percent + #[arg(long = "max-da-gas-price-change-percent", default_value = "0", env)] + pub max_da_gas_price_change_percent: u16, + /// The URL for the DA Block Committer info #[arg(long = "da-committer-url", env)] pub da_committer_url: Option, @@ -312,6 +324,9 @@ impl Command { min_gas_price, gas_price_threshold_percent, min_da_gas_price, + da_p_component, + da_d_component, + max_da_gas_price_change_percent, da_committer_url, consensus_key, #[cfg(feature = "aws-kms")] @@ -618,9 +633,9 @@ impl Command { memory_pool_size, da_gas_price_factor: NonZeroU64::new(100).expect("100 is not zero"), min_da_gas_price, - max_da_gas_price_change_percent: 0, - da_p_component: 0, - da_d_component: 0, + max_da_gas_price_change_percent, + da_p_component, + da_d_component, activity_normal_range_size: 100, activity_capped_range_size: 0, activity_decrease_range_size: 0, diff --git a/crates/services/src/service.rs b/crates/services/src/service.rs index 5b298ffe5e0..93b1b179ff6 100644 --- a/crates/services/src/service.rs +++ b/crates/services/src/service.rs @@ -356,7 +356,7 @@ async fn run( let mut task = service .into_task(&state, params) .await - .expect("The initialization of the service failed."); + .expect("The initialization of the service failed"); sender.send_if_modified(|s| { if s.starting() { diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index af46a03db6a..cd341948ff4 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -422,9 +422,13 @@ use fuel_core_gas_price_service::v1::da_source_service::block_committer_costs::R #[test] fn produce_block__l1_committed_block_effects_gas_price() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::DEBUG) + .try_init(); + let rt = tokio::runtime::Runtime::new().unwrap(); // set up chain with single unrecorded block - let args = vec![ + let mut args = vec![ "--debug", "--poa-instant", "true", @@ -435,6 +439,7 @@ fn produce_block__l1_committed_block_effects_gas_price() { let driver = FuelCoreDriver::spawn(&args).await.unwrap(); driver.client.produce_blocks(1, None).await.unwrap(); let first_gas_price = driver.client.latest_gas_price().await.unwrap().gas_price; + tokio::time::sleep(Duration::from_millis(100)).await; let temp_dir = driver.kill().await; (first_gas_price, temp_dir) }); @@ -445,20 +450,17 @@ fn produce_block__l1_committed_block_effects_gas_price() { let mock = FakeServer::new(); let url = mock.url(); - let args = vec![ - "--debug", - "--poa-instant", - "true", - "--da-committer-url", - &url, - "--min-da-gas-price", - "100", - ]; + // add the da committer url to the args + args.extend(&["--da-committer-url", &url]); rt.block_on(async { - let _driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) + let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) .await .unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + driver.client.produce_blocks(1, None).await.unwrap(); + let new_gas_price = driver.client.latest_gas_price().await.unwrap().gas_price; + assert_ne!(first_gas_price, new_gas_price); }); } From 6af95894174bccc1c6e50a6b79a9925f80b2f9e5 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 2 Dec 2024 18:57:02 -0700 Subject: [PATCH 045/131] Restructure the algo methods to accomodate storage txs --- crates/fuel-gas-price-algorithm/src/v1.rs | 27 +- .../fuel-gas-price-algorithm/src/v1/tests.rs | 25 +- .../v1/tests/update_da_record_data_tests.rs | 437 ++++++++++-------- .../v1/tests/update_l2_block_data_tests.rs | 131 ++++-- .../services/gas_price_service/src/ports.rs | 7 + .../gas_price_service/src/v1/metadata.rs | 4 +- .../src/v1/uninitialized_task.rs | 8 + 7 files changed, 401 insertions(+), 238 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 84c1a94d331..fb83324b16c 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -2,6 +2,7 @@ use crate::utils::cumulative_percentage_change; use std::{ cmp::max, collections::BTreeMap, + marker::PhantomData, num::NonZeroU64, ops::Div, }; @@ -164,10 +165,10 @@ pub struct AlgorithmUpdaterV1 { pub latest_da_cost_per_byte: u128, /// Activity of L2 pub l2_activity: L2ActivityTracker, - /// The unrecorded blocks that are used to calculate the projected cost of recording blocks - pub unrecorded_blocks: U, /// Total unrecorded block bytes pub unrecorded_blocks_bytes: u128, + /// The unrecorded blocks that are used to calculate the projected cost of recording blocks + pub _unrecorded_blocks: PhantomData, } /// The `L2ActivityTracker` tracks the chain activity to determine a safety mode for setting the DA price. @@ -326,9 +327,15 @@ impl AlgorithmUpdaterV1 { heights: &[u32], recorded_bytes: u32, recording_cost: u128, + unrecorded_blocks: &mut U, ) -> Result<(), Error> { if !heights.is_empty() { - self.da_block_update(heights, recorded_bytes as u128, recording_cost)?; + self.da_block_update( + heights, + recorded_bytes as u128, + recording_cost, + unrecorded_blocks, + )?; self.recalculate_projected_cost(); self.update_da_gas_price(); } @@ -342,6 +349,7 @@ impl AlgorithmUpdaterV1 { capacity: NonZeroU64, block_bytes: u64, fee_wei: u128, + unrecorded_blocks: &mut U, ) -> Result<(), Error> { let expected = self.l2_block_height.saturating_add(1); if height != expected { @@ -372,7 +380,7 @@ impl AlgorithmUpdaterV1 { self.update_da_gas_price(); // metadata - self.unrecorded_blocks.insert(height, block_bytes)?; + unrecorded_blocks.insert(height, block_bytes)?; self.unrecorded_blocks_bytes = self .unrecorded_blocks_bytes .saturating_add(block_bytes as u128); @@ -538,8 +546,9 @@ impl AlgorithmUpdaterV1 { heights: &[u32], recorded_bytes: u128, recording_cost: u128, + unrecorded_blocks: &mut U, ) -> Result<(), Error> { - self.update_unrecorded_block_bytes(heights); + self.update_unrecorded_block_bytes(heights, unrecorded_blocks); let new_da_block_cost = self .latest_known_total_da_cost_excess @@ -561,10 +570,14 @@ impl AlgorithmUpdaterV1 { // Get the bytes for all specified heights, or get none of them. // Always remove the blocks from the unrecorded blocks so they don't build up indefinitely - fn update_unrecorded_block_bytes(&mut self, heights: &[u32]) { + fn update_unrecorded_block_bytes( + &mut self, + heights: &[u32], + unrecorded_blocks: &mut U, + ) { let mut total: u128 = 0; for expected_height in heights { - let res = self.unrecorded_blocks.remove(expected_height); + let res = unrecorded_blocks.remove(expected_height); match res { Ok(Some(bytes)) => { total = total.saturating_add(bytes as u128); diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests.rs index 3a8386b7a3a..48d72f677f4 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests.rs @@ -41,7 +41,7 @@ pub struct UpdaterBuilder { da_cost_per_byte: u128, project_total_cost: u128, latest_known_total_cost: u128, - unrecorded_blocks: Vec, + unrecorded_blocks_bytes: u64, last_profit: i128, second_to_last_profit: i128, da_gas_price_factor: u64, @@ -68,7 +68,7 @@ impl UpdaterBuilder { da_cost_per_byte: 0, project_total_cost: 0, latest_known_total_cost: 0, - unrecorded_blocks: vec![], + unrecorded_blocks_bytes: 0, last_profit: 0, second_to_last_profit: 0, da_gas_price_factor: 1, @@ -149,8 +149,14 @@ impl UpdaterBuilder { self } - fn with_unrecorded_blocks(mut self, unrecorded_blocks: Vec) -> Self { - self.unrecorded_blocks = unrecorded_blocks; + fn with_unrecorded_blocks( + mut self, + unrecorded_blocks: &BTreeMap, + ) -> Self { + let unrecorded_block_bytes = unrecorded_blocks + .iter() + .fold(0u64, |acc, (_, bytes)| acc + bytes); + self.unrecorded_blocks_bytes = unrecorded_block_bytes; self } @@ -183,11 +189,6 @@ impl UpdaterBuilder { latest_da_cost_per_byte: self.da_cost_per_byte, projected_total_da_cost: self.project_total_cost, latest_known_total_da_cost_excess: self.latest_known_total_cost, - unrecorded_blocks: self - .unrecorded_blocks - .iter() - .map(|b| (b.height, b.block_bytes)) - .collect(), last_profit: self.last_profit, second_to_last_profit: self.second_to_last_profit, min_da_gas_price: self.min_da_gas_price, @@ -196,10 +197,8 @@ impl UpdaterBuilder { .try_into() .expect("Should never be non-zero"), l2_activity: self.l2_activity, - unrecorded_blocks_bytes: self - .unrecorded_blocks - .iter() - .fold(0u128, |acc, b| acc + u128::from(b.block_bytes)), + unrecorded_blocks_bytes: self.unrecorded_blocks_bytes as u128, + _unrecorded_blocks: Default::default(), } } } diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index 4b87b0d4ad0..f8804916e61 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -1,7 +1,5 @@ -use crate::v1::tests::{ - BlockBytes, - UpdaterBuilder, -}; +use crate::v1::tests::UpdaterBuilder; +use std::collections::BTreeMap; #[test] fn update_da_record_data__if_receives_batch_with_unknown_blocks_will_include_known_blocks_with_previous_cost( @@ -11,14 +9,11 @@ fn update_da_record_data__if_receives_batch_with_unknown_blocks_will_include_kno let recorded_cost = 1_000_000; let recorded_bytes = 500; let block_bytes = 1000; - let unrecorded_blocks = vec![BlockBytes { - height: 1, - block_bytes, - }]; + let mut unrecorded_blocks: BTreeMap<_, _> = [(1, block_bytes)].into_iter().collect(); let cost_per_byte = 333; let known_total_cost = 10_000; let mut updater = UpdaterBuilder::new() - .with_unrecorded_blocks(unrecorded_blocks) + .with_unrecorded_blocks(&unrecorded_blocks) .with_da_cost_per_byte(cost_per_byte) .with_known_total_cost(known_total_cost) .build(); @@ -26,7 +21,12 @@ fn update_da_record_data__if_receives_batch_with_unknown_blocks_will_include_kno // when updater - .update_da_record_data(&recorded_heights, recorded_bytes, recorded_cost) + .update_da_record_data( + &recorded_heights, + recorded_bytes, + recorded_cost, + &mut unrecorded_blocks, + ) .unwrap(); // then @@ -45,14 +45,11 @@ fn update_da_record_data__if_receives_batch_with_unknown_blocks_will_never_incre let recorded_cost = 200; let block_bytes = 1000; let recorded_bytes = 500; - let unrecorded_blocks = vec![BlockBytes { - height: 1, - block_bytes, - }]; + let mut unrecorded_blocks: BTreeMap<_, _> = [(1, block_bytes)].into_iter().collect(); let cost_per_byte = 333; let known_total_cost = 10_000; let mut updater = UpdaterBuilder::new() - .with_unrecorded_blocks(unrecorded_blocks) + .with_unrecorded_blocks(&unrecorded_blocks) .with_da_cost_per_byte(cost_per_byte) .with_known_total_cost(known_total_cost) .build(); @@ -60,7 +57,12 @@ fn update_da_record_data__if_receives_batch_with_unknown_blocks_will_never_incre // when updater - .update_da_record_data(&recorded_heights, recorded_bytes, recorded_cost) + .update_da_record_data( + &recorded_heights, + recorded_bytes, + recorded_cost, + &mut unrecorded_blocks, + ) .unwrap(); // then @@ -76,13 +78,10 @@ fn update_da_record_data__updates_cost_per_byte() { // given let da_cost_per_byte = 20; let block_bytes = 1000; - let unrecorded_blocks = vec![BlockBytes { - height: 1, - block_bytes, - }]; + let mut unrecorded_blocks: BTreeMap<_, _> = [(1, block_bytes)].into_iter().collect(); let mut updater = UpdaterBuilder::new() .with_da_cost_per_byte(da_cost_per_byte) - .with_unrecorded_blocks(unrecorded_blocks) + .with_unrecorded_blocks(&unrecorded_blocks) .build(); let new_cost_per_byte = 100; @@ -91,7 +90,12 @@ fn update_da_record_data__updates_cost_per_byte() { let recorded_heights: Vec = (1u32..2).collect(); // when updater - .update_da_record_data(&recorded_heights, recorded_bytes, recorded_cost) + .update_da_record_data( + &recorded_heights, + recorded_bytes, + recorded_cost, + &mut unrecorded_blocks, + ) .unwrap(); // then @@ -107,26 +111,28 @@ fn update_da_record_data__updates_known_total_cost() { let l2_block_height = 15; let projected_total_cost = 2000; let known_total_cost = 1500; - let unrecorded_blocks = vec![ - BlockBytes { - height: 11, - block_bytes: 1000, - }, - BlockBytes { - height: 12, - block_bytes: 2000, - }, - BlockBytes { - height: 13, - block_bytes: 1500, - }, - ]; + // let unrecorded_blocks = vec![ + // BlockBytes { + // height: 11, + // block_bytes: 1000, + // }, + // BlockBytes { + // height: 12, + // block_bytes: 2000, + // }, + // BlockBytes { + // height: 13, + // block_bytes: 1500, + // }, + // ]; + let mut unrecorded_blocks: BTreeMap<_, _> = + [(11, 1000), (12, 2000), (13, 1500)].into_iter().collect(); let mut updater = UpdaterBuilder::new() .with_da_cost_per_byte(da_cost_per_byte) .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost) .with_known_total_cost(known_total_cost) - .with_unrecorded_blocks(unrecorded_blocks) + .with_unrecorded_blocks(&unrecorded_blocks) .build(); let recorded_heights: Vec = (11u32..14).collect(); @@ -134,7 +140,12 @@ fn update_da_record_data__updates_known_total_cost() { let recorded_cost = 300; // when updater - .update_da_record_data(&recorded_heights, recorded_bytes, recorded_cost) + .update_da_record_data( + &recorded_heights, + recorded_bytes, + recorded_cost, + &mut unrecorded_blocks, + ) .unwrap(); // then @@ -149,24 +160,26 @@ fn update_da_record_data__if_da_height_matches_l2_height_projected_and_known_mat let da_cost_per_byte = 20; let l2_block_height = 13; let known_total_cost = 1500; - let unrecorded_blocks = vec![ - BlockBytes { - height: 11, - block_bytes: 1000, - }, - BlockBytes { - height: 12, - block_bytes: 2000, - }, - BlockBytes { - height: 13, - block_bytes: 1500, - }, - ]; + // let unrecorded_blocks = vec![ + // BlockBytes { + // height: 11, + // block_bytes: 1000, + // }, + // BlockBytes { + // height: 12, + // block_bytes: 2000, + // }, + // BlockBytes { + // height: 13, + // block_bytes: 1500, + // }, + // ]; + let mut unrecorded_blocks: BTreeMap<_, _> = + [(11, 1000), (12, 2000), (13, 1500)].into_iter().collect(); let guessed_cost: u64 = unrecorded_blocks - .iter() - .map(|block| block.block_bytes * da_cost_per_byte) + .values() + .map(|bytes| bytes * da_cost_per_byte) .sum(); let projected_total_cost = known_total_cost + guessed_cost; let mut updater = UpdaterBuilder::new() @@ -174,7 +187,7 @@ fn update_da_record_data__if_da_height_matches_l2_height_projected_and_known_mat .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost as u128) .with_known_total_cost(known_total_cost as u128) - .with_unrecorded_blocks(unrecorded_blocks) + .with_unrecorded_blocks(&unrecorded_blocks) .build(); let block_bytes = 1000; @@ -186,11 +199,16 @@ fn update_da_record_data__if_da_height_matches_l2_height_projected_and_known_mat let recorded_cost = block_cost * 3; // when updater - .update_da_record_data(&recorded_heights, recorded_bytes, recorded_cost) + .update_da_record_data( + &recorded_heights, + recorded_bytes, + recorded_cost, + &mut unrecorded_blocks, + ) .unwrap(); // then - assert_eq!(updater.unrecorded_blocks.len(), 0); + assert_eq!(unrecorded_blocks.len(), 0); assert_eq!( updater.projected_total_da_cost, updater.latest_known_total_da_cost_excess @@ -205,35 +223,30 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g let l2_block_height = 15; let original_known_total_cost: u128 = 1500; let block_bytes = 1000; - let mut unrecorded_blocks = vec![ - BlockBytes { - height: 11, - block_bytes: 1000, - }, - BlockBytes { - height: 12, - block_bytes: 2000, - }, - BlockBytes { - height: 13, - block_bytes: 1500, - }, - ]; + let remaining = vec![(14, block_bytes), (15, block_bytes)]; + // let mut unrecorded_blocks = vec![ + // BlockBytes { + // height: 11, + // block_bytes: 1000, + // }, + // BlockBytes { + // height: 12, + // block_bytes: 2000, + // }, + // BlockBytes { + // height: 13, + // block_bytes: 1500, + // }, + // ]; + let mut pairs = vec![(11, 1000), (12, 2000), (13, 1500)]; + + pairs.extend(remaining.clone()); + + let mut unrecorded_blocks: BTreeMap<_, _> = pairs.into_iter().collect(); - let remaining = vec![ - BlockBytes { - height: 14, - block_bytes, - }, - BlockBytes { - height: 15, - block_bytes, - }, - ]; - unrecorded_blocks.extend(remaining.clone()); let guessed_cost: u128 = unrecorded_blocks - .iter() - .map(|block| block.block_bytes as u128 * da_cost_per_byte) + .values() + .map(|bytes| *bytes as u128 * da_cost_per_byte) .sum(); let projected_total_cost: u128 = original_known_total_cost + guessed_cost; let mut updater = UpdaterBuilder::new() @@ -241,7 +254,7 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost) .with_known_total_cost(original_known_total_cost) - .with_unrecorded_blocks(unrecorded_blocks) + .with_unrecorded_blocks(&unrecorded_blocks) .build(); let new_cost_per_byte = 100; @@ -252,7 +265,12 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g // when updater - .update_da_record_data(&recorded_heights, recorded_bytes, recorded_cost) + .update_da_record_data( + &recorded_heights, + recorded_bytes, + recorded_cost, + &mut unrecorded_blocks, + ) .unwrap(); // then @@ -260,7 +278,7 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g let new_known_total_cost = original_known_total_cost + recorded_cost; let guessed_part: u128 = remaining .iter() - .map(|block| block.block_bytes as u128 * new_cost_per_byte) + .map(|(_, bytes)| *bytes as u128 * new_cost_per_byte) .sum(); let expected = new_known_total_cost + guessed_part; assert_eq!(actual, expected); @@ -271,27 +289,31 @@ fn update_da_record_data__updates_known_total_cost_if_blocks_are_out_of_order() // given let da_cost_per_byte = 20; let block_bytes = 1000; - let unrecorded_blocks = vec![ - BlockBytes { - height: 1, - block_bytes, - }, - BlockBytes { - height: 2, - block_bytes, - }, - BlockBytes { - height: 3, - block_bytes, - }, - ]; + // let unrecorded_blocks = vec![ + // BlockBytes { + // height: 1, + // block_bytes, + // }, + // BlockBytes { + // height: 2, + // block_bytes, + // }, + // BlockBytes { + // height: 3, + // block_bytes, + // }, + // ]; + let mut unrecorded_blocks: BTreeMap<_, _> = + [(1, block_bytes), (2, block_bytes), (3, block_bytes)] + .into_iter() + .collect(); let old_known_total_cost = 500; let old_projected_total_cost = old_known_total_cost + (block_bytes as u128 * da_cost_per_byte * 3); let old_da_cost_per_byte = 20; let mut updater = UpdaterBuilder::new() .with_da_cost_per_byte(da_cost_per_byte) - .with_unrecorded_blocks(unrecorded_blocks) + .with_unrecorded_blocks(&unrecorded_blocks) .with_da_cost_per_byte(old_da_cost_per_byte) .with_known_total_cost(old_known_total_cost) .with_projected_total_cost(old_projected_total_cost) @@ -303,7 +325,12 @@ fn update_da_record_data__updates_known_total_cost_if_blocks_are_out_of_order() // when updater - .update_da_record_data(&recorded_heights, recorded_bytes, recorded_cost as u128) + .update_da_record_data( + &recorded_heights, + recorded_bytes, + recorded_cost as u128, + &mut unrecorded_blocks, + ) .unwrap(); // then @@ -318,27 +345,31 @@ fn update_da_record_data__updates_projected_total_cost_if_blocks_are_out_of_orde // given let da_cost_per_byte = 20; let block_bytes = 1000; - let unrecorded_blocks = vec![ - BlockBytes { - height: 1, - block_bytes, - }, - BlockBytes { - height: 2, - block_bytes, - }, - BlockBytes { - height: 3, - block_bytes, - }, - ]; + // let unrecorded_blocks = vec![ + // BlockBytes { + // height: 1, + // block_bytes, + // }, + // BlockBytes { + // height: 2, + // block_bytes, + // }, + // BlockBytes { + // height: 3, + // block_bytes, + // }, + // ]; + let mut unrecorded_blocks: BTreeMap<_, _> = + [(1, block_bytes), (2, block_bytes), (3, block_bytes)] + .into_iter() + .collect(); let old_known_total_cost = 500; let old_projected_total_cost = old_known_total_cost + (block_bytes as u128 * da_cost_per_byte * 3); let old_da_cost_per_byte = 20; let mut updater = UpdaterBuilder::new() .with_da_cost_per_byte(da_cost_per_byte) - .with_unrecorded_blocks(unrecorded_blocks) + .with_unrecorded_blocks(&unrecorded_blocks) .with_da_cost_per_byte(old_da_cost_per_byte) .with_known_total_cost(old_known_total_cost) .with_projected_total_cost(old_projected_total_cost) @@ -350,7 +381,12 @@ fn update_da_record_data__updates_projected_total_cost_if_blocks_are_out_of_orde // when updater - .update_da_record_data(&recorded_heights, recorded_bytes, recorded_cost as u128) + .update_da_record_data( + &recorded_heights, + recorded_bytes, + recorded_cost as u128, + &mut unrecorded_blocks, + ) .unwrap(); // then @@ -365,23 +401,27 @@ fn update_da_record_data__updates_unrecorded_blocks() { // given let da_cost_per_byte = 20; let block_bytes = 1000; - let unrecorded_blocks = vec![ - BlockBytes { - height: 1, - block_bytes, - }, - BlockBytes { - height: 2, - block_bytes, - }, - BlockBytes { - height: 3, - block_bytes, - }, - ]; + // let unrecorded_blocks = vec![ + // BlockBytes { + // height: 1, + // block_bytes, + // }, + // BlockBytes { + // height: 2, + // block_bytes, + // }, + // BlockBytes { + // height: 3, + // block_bytes, + // }, + // ]; + let mut unrecorded_blocks: BTreeMap<_, _> = + [(1, block_bytes), (2, block_bytes), (3, block_bytes)] + .into_iter() + .collect(); let mut updater = UpdaterBuilder::new() .with_da_cost_per_byte(da_cost_per_byte) - .with_unrecorded_blocks(unrecorded_blocks) + .with_unrecorded_blocks(&unrecorded_blocks) .build(); let new_cost_per_byte = 100; let recorded_bytes = 500; @@ -390,12 +430,17 @@ fn update_da_record_data__updates_unrecorded_blocks() { // when updater - .update_da_record_data(&recorded_heights, recorded_bytes, recorded_cost) + .update_da_record_data( + &recorded_heights, + recorded_bytes, + recorded_cost, + &mut unrecorded_blocks, + ) .unwrap(); // then let expected = vec![(1, block_bytes)]; - let actual: Vec<_> = updater.unrecorded_blocks.into_iter().collect(); + let actual: Vec<_> = unrecorded_blocks.into_iter().collect(); assert_eq!(actual, expected); } @@ -405,14 +450,15 @@ fn update_da_record_data__da_block_lowers_da_gas_price() { let da_cost_per_byte = 40; let l2_block_height = 11; let original_known_total_cost = 150; - let unrecorded_blocks = vec![BlockBytes { - height: 11, - block_bytes: 3000, - }]; + // let unrecorded_blocks = vec![BlockBytes { + // height: 11, + // block_bytes: 3000, + // }]; + let mut unrecorded_blocks: BTreeMap<_, _> = [(11, 3000)].into_iter().collect(); let da_p_component = 2; let guessed_cost: u64 = unrecorded_blocks - .iter() - .map(|block| block.block_bytes * da_cost_per_byte) + .values() + .map(|bytes| bytes * da_cost_per_byte) .sum(); let projected_total_cost = original_known_total_cost + guessed_cost; @@ -423,19 +469,19 @@ fn update_da_record_data__da_block_lowers_da_gas_price() { .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost as u128) .with_known_total_cost(original_known_total_cost as u128) - .with_unrecorded_blocks(unrecorded_blocks.clone()) + .with_unrecorded_blocks(&unrecorded_blocks) .build(); let new_cost_per_byte = 100; - let (recorded_heights, recorded_cost) = - unrecorded_blocks - .iter() - .fold((vec![], 0), |(mut range, cost), block| { - range.push(block.height); - (range, cost + block.block_bytes * new_cost_per_byte) - }); - let min = recorded_heights.iter().min().unwrap(); - let max = recorded_heights.iter().max().unwrap(); + let (recorded_heights, recorded_cost) = unrecorded_blocks.iter().fold( + (vec![], 0), + |(mut range, cost), (height, bytes)| { + range.push(height); + (range, cost + bytes * new_cost_per_byte) + }, + ); + let min = *recorded_heights.iter().min().unwrap(); + let max = *recorded_heights.iter().max().unwrap(); let recorded_range: Vec = (*min..(max + 1)).collect(); let recorded_bytes = 500; @@ -443,7 +489,12 @@ fn update_da_record_data__da_block_lowers_da_gas_price() { // when updater - .update_da_record_data(&recorded_range, recorded_bytes, recorded_cost as u128) + .update_da_record_data( + &recorded_range, + recorded_bytes, + recorded_cost as u128, + &mut unrecorded_blocks, + ) .unwrap(); // then @@ -459,14 +510,15 @@ fn update_da_record_data__da_block_increases_da_gas_price() { let da_cost_per_byte = 40; let l2_block_height = 11; let original_known_total_cost = 150; - let unrecorded_blocks = vec![BlockBytes { - height: 11, - block_bytes: 3000, - }]; + // let unrecorded_blocks = vec![BlockBytes { + // height: 11, + // block_bytes: 3000, + // }]; + let mut unrecorded_blocks: BTreeMap<_, _> = [(11, 3000)].into_iter().collect(); let da_p_component = 2; let guessed_cost: u64 = unrecorded_blocks - .iter() - .map(|block| block.block_bytes * da_cost_per_byte) + .values() + .map(|bytes| bytes * da_cost_per_byte) .sum(); let projected_total_cost = original_known_total_cost + guessed_cost; @@ -477,19 +529,24 @@ fn update_da_record_data__da_block_increases_da_gas_price() { .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost as u128) .with_known_total_cost(original_known_total_cost as u128) - .with_unrecorded_blocks(unrecorded_blocks.clone()) + .with_unrecorded_blocks(&unrecorded_blocks) .build(); let new_cost_per_byte = 100; - let (recorded_heights, recorded_cost) = - unrecorded_blocks + let (recorded_heights, recorded_cost) = unrecorded_blocks + // .iter() + // .fold((vec![], 0), |(mut range, cost), block| { + // range.push(block.height); + // (range, cost + block.block_bytes * new_cost_per_byte) + // }); .iter() - .fold((vec![], 0), |(mut range, cost), block| { - range.push(block.height); - (range, cost + block.block_bytes * new_cost_per_byte) + .fold((vec![], 0), |(mut range, cost), (height, bytes)| { + range.push(height); + (range, cost + bytes * new_cost_per_byte) }); - let min = recorded_heights.iter().min().unwrap(); - let max = recorded_heights.iter().max().unwrap(); + + let min = *recorded_heights.iter().min().unwrap(); + let max = *recorded_heights.iter().max().unwrap(); let recorded_range: Vec = (*min..(max + 1)).collect(); let recorded_bytes = 500; @@ -497,7 +554,12 @@ fn update_da_record_data__da_block_increases_da_gas_price() { // when updater - .update_da_record_data(&recorded_range, recorded_bytes, recorded_cost as u128) + .update_da_record_data( + &recorded_range, + recorded_bytes, + recorded_cost as u128, + &mut unrecorded_blocks, + ) .unwrap(); // then @@ -513,14 +575,14 @@ fn update_da_record_data__da_block_will_not_change_da_gas_price() { let da_cost_per_byte = 40; let l2_block_height = 11; let original_known_total_cost = 150; - let unrecorded_blocks = vec![BlockBytes { - height: 11, - block_bytes: 3000, - }]; + // let unrecorded_blocks = vec![BlockBytes { height: 11, + // block_bytes: 3000, + // }]; + let mut unrecorded_blocks: BTreeMap<_, _> = [(11, 3000)].into_iter().collect(); let da_p_component = 2; let guessed_cost: u64 = unrecorded_blocks - .iter() - .map(|block| block.block_bytes * da_cost_per_byte) + .values() + .map(|bytes| bytes * da_cost_per_byte) .sum(); let projected_total_cost = original_known_total_cost + guessed_cost; @@ -531,19 +593,19 @@ fn update_da_record_data__da_block_will_not_change_da_gas_price() { .with_l2_block_height(l2_block_height) .with_projected_total_cost(projected_total_cost as u128) .with_known_total_cost(original_known_total_cost as u128) - .with_unrecorded_blocks(unrecorded_blocks.clone()) + .with_unrecorded_blocks(&unrecorded_blocks) .build(); let new_cost_per_byte = 100; - let (recorded_heights, recorded_cost) = - unrecorded_blocks - .iter() - .fold((vec![], 0), |(mut range, cost), block| { - range.push(block.height); - (range, cost + block.block_bytes * new_cost_per_byte) - }); - let min = recorded_heights.iter().min().unwrap(); - let max = recorded_heights.iter().max().unwrap(); + let (recorded_heights, recorded_cost) = unrecorded_blocks.iter().fold( + (vec![], 0), + |(mut range, cost), (height, bytes)| { + range.push(height); + (range, cost + bytes * new_cost_per_byte) + }, + ); + let min = *recorded_heights.iter().min().unwrap(); + let max = *recorded_heights.iter().max().unwrap(); let recorded_range: Vec = (*min..(max + 1)).collect(); let recorded_bytes = 500; @@ -551,7 +613,12 @@ fn update_da_record_data__da_block_will_not_change_da_gas_price() { // when updater - .update_da_record_data(&recorded_range, recorded_bytes, recorded_cost as u128) + .update_da_record_data( + &recorded_range, + recorded_bytes, + recorded_cost as u128, + &mut unrecorded_blocks, + ) .unwrap(); // then diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs index 8ce3b6b5d4e..25bfab5c860 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs @@ -3,9 +3,12 @@ use crate::v1::{ BlockBytes, UpdaterBuilder, }, + Bytes, Error, + Height, L2ActivityTracker, }; +use std::collections::BTreeMap; fn decrease_l2_activity() -> L2ActivityTracker { let normal = 1; @@ -62,6 +65,10 @@ fn positive_profit_updater_builder() -> UpdaterBuilder { .with_exec_gas_price_change_percent(0) } +fn empty_unrecorded_blocks() -> BTreeMap { + BTreeMap::new() +} + #[test] fn update_l2_block_data__updates_l2_block() { // given @@ -79,7 +86,14 @@ fn update_l2_block_data__updates_l2_block() { // when updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data( + height, + used, + capacity, + block_bytes, + fee, + &mut empty_unrecorded_blocks(), + ) .unwrap(); // then @@ -104,7 +118,14 @@ fn update_l2_block_data__skipped_block_height_throws_error() { // when let actual_error = updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data( + height, + used, + capacity, + block_bytes, + fee, + &mut empty_unrecorded_blocks(), + ) .unwrap_err(); // then @@ -128,10 +149,11 @@ fn update_l2_block_data__updates_projected_cost() { let capacity = 100.try_into().unwrap(); let block_bytes = 1000; let fee = 100; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -155,10 +177,18 @@ fn update_l2_block_data__updates_the_total_reward_value() { let capacity = 100.try_into().unwrap(); let block_bytes = 1000; let fee = 10_000; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater - .update_l2_block_data(height, gas_used, capacity, block_bytes, fee) + .update_l2_block_data( + height, + gas_used, + capacity, + block_bytes, + fee, + unrecorded_blocks, + ) .unwrap(); // then @@ -183,10 +213,11 @@ fn update_l2_block_data__even_threshold_will_not_change_exec_gas_price() { let capacity = 100.try_into().unwrap(); let block_bytes = 1000; let fee = 200; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -212,10 +243,11 @@ fn update_l2_block_data__below_threshold_will_decrease_exec_gas_price() { let capacity = 100.try_into().unwrap(); let block_bytes = 1000; let fee = 200; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -243,10 +275,11 @@ fn update_l2_block_data__above_threshold_will_increase_exec_gas_price() { let capacity = 100.try_into().unwrap(); let block_bytes = 1000; let fee = 200; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -275,10 +308,11 @@ fn update_l2_block_data__exec_price_will_not_go_below_min() { let capacity = 100.try_into().unwrap(); let block_bytes = 1000; let fee = 200; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -305,10 +339,11 @@ fn update_l2_block_data__updates_last_and_last_last_profit() { let capacity = 100.try_into().unwrap(); let block_bytes = 1000; let fee = 0; // No fee so it's easier to calculate profit + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -326,6 +361,7 @@ fn update_l2_block_data__positive_profit_decrease_gas_price() { // given let mut updater = positive_profit_updater_builder().build(); let old_gas_price = updater.algorithm().calculate(); + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when let block_bytes = 500u64; @@ -336,6 +372,7 @@ fn update_l2_block_data__positive_profit_decrease_gas_price() { 100.try_into().unwrap(), block_bytes, 200, + unrecorded_blocks, ) .unwrap(); @@ -374,6 +411,7 @@ fn update_l2_block_data__price_does_not_decrease_more_than_max_percent() { .with_last_profit(last_profit, last_last_profit) .with_da_max_change_percent(max_da_change_percent) .build(); + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when let height = updater.l2_block_height + 1; @@ -382,7 +420,7 @@ fn update_l2_block_data__price_does_not_decrease_more_than_max_percent() { let block_bytes = 1000; let fee = 200; updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -408,6 +446,7 @@ fn update_l2_block_data__da_price_does_not_increase_more_than_max_percent() { let last_last_profit = 0; let max_da_change_percent = 5; let large_starting_reward = 0; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); let mut updater = UpdaterBuilder::new() .with_starting_exec_gas_price(starting_exec_gas_price) .with_da_p_component(da_p_component) @@ -428,7 +467,7 @@ fn update_l2_block_data__da_price_does_not_increase_more_than_max_percent() { let block_bytes = 1000; let fee = 200; updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -454,6 +493,7 @@ fn update_l2_block_data__never_drops_below_minimum_da_gas_price() { let last_profit = i128::MAX; let avg_window = 10; let large_reward = u128::MAX; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); let mut updater = UpdaterBuilder::new() .with_starting_exec_gas_price(starting_exec_gas_price) .with_min_exec_gas_price(starting_exec_gas_price) @@ -477,6 +517,7 @@ fn update_l2_block_data__never_drops_below_minimum_da_gas_price() { 100.try_into().unwrap(), 1000, fee, + unrecorded_blocks, ) .unwrap(); @@ -497,6 +538,7 @@ fn update_l2_block_data__even_profit_maintains_price() { let da_gas_price_denominator = 1; let block_bytes = 500u64; let starting_reward = starting_cost; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); let mut updater = UpdaterBuilder::new() .with_starting_exec_gas_price(starting_exec_gas_price) .with_starting_da_gas_price(starting_da_gas_price) @@ -518,6 +560,7 @@ fn update_l2_block_data__even_profit_maintains_price() { 100.try_into().unwrap(), block_bytes, total_fee.into(), + unrecorded_blocks, ) .unwrap(); let algo = updater.algorithm(); @@ -534,6 +577,7 @@ fn update_l2_block_data__negative_profit_increase_gas_price() { let mut updater = negative_profit_updater_builder().build(); let algo = updater.algorithm(); let old_gas_price = algo.calculate(); + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when let height = updater.l2_block_height + 1; @@ -542,7 +586,7 @@ fn update_l2_block_data__negative_profit_increase_gas_price() { let block_bytes = 500u64; let fee = 0; updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -570,10 +614,18 @@ fn update_l2_block_data__adds_l2_block_to_unrecorded_blocks() { let capacity = 100.try_into().unwrap(); let block_bytes = 1000; let new_gas_price = 100; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater - .update_l2_block_data(height, used, capacity, block_bytes, new_gas_price) + .update_l2_block_data( + height, + used, + capacity, + block_bytes, + new_gas_price, + unrecorded_blocks, + ) .unwrap(); // then @@ -582,7 +634,7 @@ fn update_l2_block_data__adds_l2_block_to_unrecorded_blocks() { block_bytes, }; let expected = block_bytes.block_bytes; - let actual = updater.unrecorded_blocks.get(&block_bytes.height).unwrap(); + let actual = unrecorded_blocks.get(&block_bytes.height).unwrap(); assert_eq!(expected, *actual); } @@ -592,14 +644,16 @@ fn update_l2_block_data__retains_existing_blocks_and_adds_l2_block_to_unrecorded // given let starting_block = 0; let first_block_bytes = 1200; - let preexisting_block = BlockBytes { - height: 0, - block_bytes: first_block_bytes, - }; + // let preexisting_block = BlockBytes { + // height: 0, + // block_bytes: first_block_bytes, + // }; + let unrecorded_blocks: BTreeMap<_, _> = + vec![(0, first_block_bytes)].into_iter().collect(); let mut updater = UpdaterBuilder::new() .with_l2_block_height(starting_block) - .with_unrecorded_blocks(vec![preexisting_block.clone()]) + .with_unrecorded_blocks(&unrecorded_blocks) .build(); let height = 1; @@ -607,10 +661,18 @@ fn update_l2_block_data__retains_existing_blocks_and_adds_l2_block_to_unrecorded let capacity = 100.try_into().unwrap(); let new_block_bytes = 1000; let new_gas_price = 100; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater - .update_l2_block_data(height, used, capacity, new_block_bytes, new_gas_price) + .update_l2_block_data( + height, + used, + capacity, + new_block_bytes, + new_gas_price, + unrecorded_blocks, + ) .unwrap(); // then @@ -618,14 +680,11 @@ fn update_l2_block_data__retains_existing_blocks_and_adds_l2_block_to_unrecorded height, block_bytes: new_block_bytes, }; - let contains_block_bytes = - updater.unrecorded_blocks.contains_key(&block_bytes.height); + let contains_block_bytes = unrecorded_blocks.contains_key(&block_bytes.height); assert!(contains_block_bytes); // and - let contains_preexisting_block_bytes = updater - .unrecorded_blocks - .contains_key(&preexisting_block.height); + let contains_preexisting_block_bytes = unrecorded_blocks.contains_key(&0); assert!(contains_preexisting_block_bytes); // and @@ -653,6 +712,7 @@ fn update_l2_block_data__da_gas_price_wants_to_increase_will_hold_if_activity_in .build(); let algo = updater.algorithm(); let old_gas_price = algo.calculate(); + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when let height = updater.l2_block_height + 1; @@ -661,7 +721,7 @@ fn update_l2_block_data__da_gas_price_wants_to_increase_will_hold_if_activity_in let block_bytes = 500u64; let fee = 0; updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -679,6 +739,7 @@ fn update_l2_block_data__da_gas_price_wants_to_decrease_will_decrease_if_activit .with_activity(capped_activity) .build(); let old_gas_price = updater.algorithm().calculate(); + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when let block_bytes = 500u64; @@ -689,6 +750,7 @@ fn update_l2_block_data__da_gas_price_wants_to_decrease_will_decrease_if_activit 100.try_into().unwrap(), block_bytes, 200, + unrecorded_blocks, ) .unwrap(); @@ -712,6 +774,7 @@ fn update_l2_block_data__da_gas_price_wants_to_increase_will_decrease_if_activit .build(); let algo = updater.algorithm(); let old_gas_price = algo.calculate(); + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when let height = updater.l2_block_height + 1; @@ -720,7 +783,7 @@ fn update_l2_block_data__da_gas_price_wants_to_increase_will_decrease_if_activit let block_bytes = 500u64; let fee = 0; updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -743,6 +806,7 @@ fn update_l2_block_data__da_gas_price_wants_to_decrease_will_decrease_if_activit .with_activity(decrease_activity) .build(); let old_gas_price = updater.algorithm().calculate(); + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when let block_bytes = 500u64; @@ -753,6 +817,7 @@ fn update_l2_block_data__da_gas_price_wants_to_decrease_will_decrease_if_activit 100.try_into().unwrap(), block_bytes, 200, + unrecorded_blocks, ) .unwrap(); @@ -786,10 +851,11 @@ fn update_l2_block_data__above_threshold_increase_activity() { let capacity = 100.try_into().unwrap(); let block_bytes = 1000; let fee = 200; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -818,10 +884,11 @@ fn update_l2_block_data__below_threshold_decrease_activity() { let capacity = 100.try_into().unwrap(); let block_bytes = 1000; let fee = 200; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -859,10 +926,11 @@ fn update_l2_block_data__if_activity_at_max_will_stop_increasing() { let capacity = 100.try_into().unwrap(); let block_bytes = 1000; let fee = 200; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then @@ -891,10 +959,11 @@ fn update_l2_block_data__if_activity_is_zero_will_stop_decreasing() { let capacity = 100.try_into().unwrap(); let block_bytes = 1000; let fee = 200; + let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater - .update_l2_block_data(height, used, capacity, block_bytes, fee) + .update_l2_block_data(height, used, capacity, block_bytes, fee, unrecorded_blocks) .unwrap(); // then diff --git a/crates/services/gas_price_service/src/ports.rs b/crates/services/gas_price_service/src/ports.rs index cb829ab21fb..a0ad8f4b185 100644 --- a/crates/services/gas_price_service/src/ports.rs +++ b/crates/services/gas_price_service/src/ports.rs @@ -27,6 +27,13 @@ pub trait MetadataStorage: Send + Sync { fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> Result<()>; } +pub trait TransactionableStorage: Send + Sync { + type Transaction; + + fn begin_transaction(&self) -> Result; + fn commit_transaction(&self, transaction: Self::Transaction) -> Result<()>; +} + /// Provides the latest block height. /// This is used to determine the latest block height that has been processed by the gas price service. /// We need this to fetch the gas price data for the latest block. diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index 914eddf7c7d..ac780a80aa5 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -113,8 +113,8 @@ pub fn updater_from_config( max_da_gas_price_change_percent: value.max_da_gas_price_change_percent, da_p_component: value.da_p_component, da_d_component: value.da_d_component, - unrecorded_blocks, unrecorded_blocks_bytes, + _unrecorded_blocks: Default::default(), } } @@ -174,6 +174,6 @@ pub fn v1_algorithm_from_metadata( da_p_component: config.da_p_component, da_d_component: config.da_d_component, unrecorded_blocks_bytes, - unrecorded_blocks, + _unrecorded_blocks: Default::default(), } } diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 90e4cfc1bc1..f94949fb87a 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -21,6 +21,7 @@ use crate::{ GasPriceServiceConfig, L2Data, MetadataStorage, + TransactionableStorage, }, v1::{ algorithm::SharedV1Algorithm, @@ -353,6 +354,7 @@ fn sync_v1_metadata< Metadata, SettingsProvider, UnrecordedBlockStorage, + StorageTxGenerator, >( settings: &SettingsProvider, on_chain_db: &L2DataStoreView, @@ -360,6 +362,7 @@ fn sync_v1_metadata< latest_block_height: u32, updater: &mut AlgorithmUpdaterV1>, metadata_storage: &mut Metadata, + storage_tx_generator: &mut StorageTxGenerator, ) -> anyhow::Result<()> where L2DataStore: L2Data, @@ -371,6 +374,8 @@ where + Send + Sync + Clone, + StorageTxGenerator: TransactionableStorage, + ::Transaction: UnrecordedBlocks, { let first = metadata_height.saturating_add(1); let view = on_chain_db.latest_view()?; @@ -396,13 +401,16 @@ where let block_bytes = block_bytes(&block); let (fee_wei, _) = mint_values(&block)?; + let mut tx = storage_tx_generator.begin_transaction()?; updater.update_l2_block_data( height, block_gas_used, block_gas_capacity, block_bytes, fee_wei.into(), + &mut tx, )?; + storage_tx_generator.commit_transaction(tx)?; let metadata: UpdaterMetadata = updater.clone().into(); metadata_storage.set_metadata(&metadata)?; } From b9d6571d5d9a5b63596bbfa226a49be8f01e7e3e Mon Sep 17 00:00:00 2001 From: Mitchell Turner Date: Tue, 3 Dec 2024 10:49:17 -0700 Subject: [PATCH 046/131] Update crates/services/gas_price_service/src/v0/service.rs Co-authored-by: Green Baneling --- crates/services/gas_price_service/src/v0/service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/services/gas_price_service/src/v0/service.rs b/crates/services/gas_price_service/src/v0/service.rs index 2543dbc7212..557d8cd6842 100644 --- a/crates/services/gas_price_service/src/v0/service.rs +++ b/crates/services/gas_price_service/src/v0/service.rs @@ -141,7 +141,7 @@ where Metadata: MetadataStorage, { async fn run(&mut self, watcher: &mut StateWatcher) -> TaskNextAction { - tracing::trace!("Starting gas price service"); + tracing::trace!("Call of `run` function of the gas price service v0"); tokio::select! { biased; _ = watcher.while_started() => { From 770998e39f172a16eec20077dd1ef949dfb23db2 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 3 Dec 2024 10:49:44 -0700 Subject: [PATCH 047/131] WIP --- crates/services/gas_price_service/src/v1/service.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 12d22cb9081..dc47ec02e11 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -70,6 +70,8 @@ where da_source_adapter_handle: DaSourceService, /// The da source channel da_source_channel: Receiver, + /// Buffer of block costs from the DA chain + da_block_costs_buffer: Vec, } impl GasPriceServiceV1 @@ -253,8 +255,9 @@ where TaskNextAction::always_continue(res) } da_block_costs = self.da_source_channel.recv() => { - let res = self.process_da_block_costs_res(da_block_costs).await; - TaskNextAction::always_continue(res) + self.da_block_costs_buffer.push(da_block_costs?); + // let res = self.process_da_block_costs_res(da_block_costs).await; + TaskNextAction::Continue } } } From 0c4640e41105af269b794d7564483b50bd4f988e Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 3 Dec 2024 11:18:01 -0700 Subject: [PATCH 048/131] Simplify generic constraints --- .../gas_price_service/src/v1/uninitialized_task.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 158d7b7ccdc..60ed5184ef5 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -319,7 +319,6 @@ where #[allow(clippy::too_many_arguments)] pub fn new_gas_price_service_v1< L2DataStore, - L2DataStoreView, GasPriceStore, Metadata, DA, @@ -332,15 +331,15 @@ pub fn new_gas_price_service_v1< gas_price_db: GasPriceStore, metadata: Metadata, da_source: DA, - on_chain_db: L2DataStoreView, + on_chain_db: L2DataStore, ) -> anyhow::Result< ServiceRunner< - UninitializedTask, + UninitializedTask, >, > where - L2DataStore: L2Data, - L2DataStoreView: AtomicView, + L2DataStore: AtomicView, + L2DataStore::LatestView: L2Data, GasPriceStore: GasPriceData, SettingsProvider: GasPriceSettingsProvider, Metadata: MetadataStorage, From 5d5d48b850e7ad25467b3b2402ae4831d9984894 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 3 Dec 2024 14:30:27 -0700 Subject: [PATCH 049/131] WIP adding transactionable storage --- crates/fuel-gas-price-algorithm/src/v1.rs | 15 +- .../fuel-gas-price-algorithm/src/v1/tests.rs | 3 +- .../src/common/fuel_core_storage_adapter.rs | 30 +- .../src/common/updater_metadata.rs | 4 +- .../gas_price_service/src/common/utils.rs | 2 + .../services/gas_price_service/src/ports.rs | 12 +- .../gas_price_service/src/v1/metadata.rs | 16 +- .../gas_price_service/src/v1/service.rs | 261 ++++++++++-------- .../gas_price_service/src/v1/tests.rs | 4 +- .../src/v1/uninitialized_task.rs | 152 +++------- 10 files changed, 243 insertions(+), 256 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index fb83324b16c..a0b51129512 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -2,7 +2,6 @@ use crate::utils::cumulative_percentage_change; use std::{ cmp::max, collections::BTreeMap, - marker::PhantomData, num::NonZeroU64, ops::Div, }; @@ -121,7 +120,7 @@ impl UnrecordedBlocks for BTreeMap { /// instead of the actual profit. Setting the `avg_window` to 1 will effectively disable the /// moving average. #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] -pub struct AlgorithmUpdaterV1 { +pub struct AlgorithmUpdaterV1 { // Execution /// The gas price (scaled by the `gas_price_factor`) to cover the execution of the next block pub new_scaled_exec_price: u64, @@ -167,8 +166,6 @@ pub struct AlgorithmUpdaterV1 { pub l2_activity: L2ActivityTracker, /// Total unrecorded block bytes pub unrecorded_blocks_bytes: u128, - /// The unrecorded blocks that are used to calculate the projected cost of recording blocks - pub _unrecorded_blocks: PhantomData, } /// The `L2ActivityTracker` tracks the chain activity to determine a safety mode for setting the DA price. @@ -321,8 +318,8 @@ impl core::ops::Deref for ClampedPercentage { } } -impl AlgorithmUpdaterV1 { - pub fn update_da_record_data( +impl AlgorithmUpdaterV1 { + pub fn update_da_record_data( &mut self, heights: &[u32], recorded_bytes: u32, @@ -342,7 +339,7 @@ impl AlgorithmUpdaterV1 { Ok(()) } - pub fn update_l2_block_data( + pub fn update_l2_block_data( &mut self, height: u32, used: u64, @@ -541,7 +538,7 @@ impl AlgorithmUpdaterV1 { .saturating_div(100) } - fn da_block_update( + fn da_block_update( &mut self, heights: &[u32], recorded_bytes: u128, @@ -570,7 +567,7 @@ impl AlgorithmUpdaterV1 { // Get the bytes for all specified heights, or get none of them. // Always remove the blocks from the unrecorded blocks so they don't build up indefinitely - fn update_unrecorded_block_bytes( + fn update_unrecorded_block_bytes( &mut self, heights: &[u32], unrecorded_blocks: &mut U, diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests.rs index 48d72f677f4..c2dcaba36fa 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests.rs @@ -171,7 +171,7 @@ impl UpdaterBuilder { self } - fn build(self) -> AlgorithmUpdaterV1> { + fn build(self) -> AlgorithmUpdaterV1 { AlgorithmUpdaterV1 { min_exec_gas_price: self.min_exec_gas_price, new_scaled_exec_price: self.starting_exec_gas_price, @@ -198,7 +198,6 @@ impl UpdaterBuilder { .expect("Should never be non-zero"), l2_activity: self.l2_activity, unrecorded_blocks_bytes: self.unrecorded_blocks_bytes as u128, - _unrecorded_blocks: Default::default(), } } } diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs index 4d452421ba1..370d375deff 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs @@ -14,7 +14,10 @@ use crate::{ }, updater_metadata::UpdaterMetadata, }, - ports::MetadataStorage, + ports::{ + MetadataStorage, + TransactionableStorage, + }, }; use fuel_core_storage::{ codec::{ @@ -25,6 +28,7 @@ use fuel_core_storage::{ structured_storage::StructuredStorage, transactional::{ Modifiable, + StorageTransaction, WriteTransaction, }, StorageAsMut, @@ -82,6 +86,30 @@ where } } +impl TransactionableStorage for StructuredStorage +where + Storage: Modifiable + Send + Sync, +{ + type Transaction<'a> = StorageTransaction<&'a mut Self> where Self: 'a; + + fn begin_transaction<'a>(&'a mut self) -> GasPriceResult> + where + Self: 'a, + { + Ok(self.write_transaction()) + } + + fn commit_transaction<'a>(transaction: Self::Transaction<'a>) -> GasPriceResult<()> + where + Self: 'a, + { + transaction + .commit() + .map_err(|err| GasPriceError::CouldNotCommit(err.into()))?; + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq)] pub struct GasPriceSettings { pub gas_price_factor: u64, diff --git a/crates/services/gas_price_service/src/common/updater_metadata.rs b/crates/services/gas_price_service/src/common/updater_metadata.rs index 4b1bb66de97..6ce274ea1a7 100644 --- a/crates/services/gas_price_service/src/common/updater_metadata.rs +++ b/crates/services/gas_price_service/src/common/updater_metadata.rs @@ -30,8 +30,8 @@ impl From for UpdaterMetadata { } } -impl From> for UpdaterMetadata { - fn from(updater: AlgorithmUpdaterV1) -> Self { +impl From for UpdaterMetadata { + fn from(updater: AlgorithmUpdaterV1) -> Self { Self::V1(updater.into()) } } diff --git a/crates/services/gas_price_service/src/common/utils.rs b/crates/services/gas_price_service/src/common/utils.rs index a3813e53e4f..b57fc2608a2 100644 --- a/crates/services/gas_price_service/src/common/utils.rs +++ b/crates/services/gas_price_service/src/common/utils.rs @@ -19,6 +19,8 @@ pub enum Error { CouldNotInitUpdater(anyhow::Error), #[error("Failed to convert metadata to concrete type. THere is no migration path for this metadata version")] CouldNotConvertMetadata, // todo(https://github.com/FuelLabs/fuel-core/issues/2286) + #[error("Failed to commit to storage: {0:?}")] + CouldNotCommit(anyhow::Error), } pub type Result = core::result::Result; diff --git a/crates/services/gas_price_service/src/ports.rs b/crates/services/gas_price_service/src/ports.rs index a0ad8f4b185..a071969cab4 100644 --- a/crates/services/gas_price_service/src/ports.rs +++ b/crates/services/gas_price_service/src/ports.rs @@ -28,10 +28,16 @@ pub trait MetadataStorage: Send + Sync { } pub trait TransactionableStorage: Send + Sync { - type Transaction; + type Transaction<'a> + where + Self: 'a; - fn begin_transaction(&self) -> Result; - fn commit_transaction(&self, transaction: Self::Transaction) -> Result<()>; + fn begin_transaction<'a>(&'a mut self) -> Result> + where + Self: 'a; + fn commit_transaction<'a>(transaction: Self::Transaction<'a>) -> Result<()> + where + Self: 'a; } /// Provides the latest block height. diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index ac780a80aa5..d1e7de6375f 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -79,10 +79,7 @@ pub struct V1AlgorithmConfig { pub block_activity_threshold: u8, } -pub fn updater_from_config( - value: &V1AlgorithmConfig, - unrecorded_blocks: U, -) -> AlgorithmUpdaterV1 { +pub fn updater_from_config(value: &V1AlgorithmConfig) -> AlgorithmUpdaterV1 { let l2_activity = L2ActivityTracker::new_full( value.normal_range_size, value.capped_range_size, @@ -114,12 +111,11 @@ pub fn updater_from_config( da_p_component: value.da_p_component, da_d_component: value.da_d_component, unrecorded_blocks_bytes, - _unrecorded_blocks: Default::default(), } } -impl From> for V1Metadata { - fn from(updater: AlgorithmUpdaterV1) -> Self { +impl From for V1Metadata { + fn from(updater: AlgorithmUpdaterV1) -> Self { Self { new_scaled_exec_price: updater.new_scaled_exec_price, l2_block_height: updater.l2_block_height, @@ -135,11 +131,10 @@ impl From> for V1Metadata { } } -pub fn v1_algorithm_from_metadata( +pub fn v1_algorithm_from_metadata( metadata: V1Metadata, config: &V1AlgorithmConfig, - unrecorded_blocks: U, -) -> AlgorithmUpdaterV1 { +) -> AlgorithmUpdaterV1 { let l2_activity = L2ActivityTracker::new_full( config.normal_range_size, config.capped_range_size, @@ -174,6 +169,5 @@ pub fn v1_algorithm_from_metadata( da_p_component: config.da_p_component, da_d_component: config.da_d_component, unrecorded_blocks_bytes, - _unrecorded_blocks: Default::default(), } } diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index dc47ec02e11..549b2e48a8c 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -1,3 +1,24 @@ +use std::num::NonZeroU64; + +use anyhow::anyhow; +use async_trait::async_trait; +use fuel_core_services::{ + RunnableService, + RunnableTask, + StateWatcher, + TaskNextAction, +}; +use fuel_gas_price_algorithm::{ + v0::AlgorithmUpdaterV0, + v1::{ + AlgorithmUpdaterV1, + AlgorithmV1, + UnrecordedBlocks, + }, +}; +use futures::FutureExt; +use tokio::sync::broadcast::Receiver; + use crate::{ common::{ gas_price_algorithm::SharedGasPriceAlgo, @@ -8,7 +29,10 @@ use crate::{ Result as GasPriceResult, }, }, - ports::MetadataStorage, + ports::{ + MetadataStorage, + TransactionableStorage, + }, v0::metadata::V0Metadata, v1::{ algorithm::SharedV1Algorithm, @@ -29,62 +53,43 @@ use crate::{ uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, }, }; -use anyhow::anyhow; -use async_trait::async_trait; -use fuel_core_services::{ - RunnableService, - RunnableTask, - StateWatcher, - TaskNextAction, -}; -use fuel_gas_price_algorithm::{ - v0::AlgorithmUpdaterV0, - v1::{ - AlgorithmUpdaterV1, - AlgorithmV1, - UnrecordedBlocks, - }, -}; -use futures::FutureExt; -use std::num::NonZeroU64; -use tokio::sync::broadcast::{ - error::RecvError, - Receiver, -}; /// The service that updates the gas price algorithm. -pub struct GasPriceServiceV1 +pub struct GasPriceServiceV1 where DA: DaBlockCostsSource, - U: Clone, + StorageTxProvider: TransactionableStorage, { /// The algorithm that can be used in the next block shared_algo: SharedV1Algorithm, /// The L2 block source l2_block_source: L2, - /// The metadata storage - metadata_storage: Metadata, /// The algorithm updater - algorithm_updater: AlgorithmUpdaterV1, + algorithm_updater: AlgorithmUpdaterV1, /// the da source adapter handle da_source_adapter_handle: DaSourceService, /// The da source channel da_source_channel: Receiver, /// Buffer of block costs from the DA chain da_block_costs_buffer: Vec, + /// Storage transaction provider for metadata and unrecorded blocks + storage_tx_provider: StorageTxProvider, } -impl GasPriceServiceV1 +impl GasPriceServiceV1 where L2: L2BlockSource, - Metadata: MetadataStorage, DA: DaBlockCostsSource, - U: UnrecordedBlocks + Clone, + StorageTxProvider: TransactionableStorage, + // StorageTxProvider::Transaction<'a>: UnrecordedBlocks + MetadataStorage, { - async fn process_l2_block_res( - &mut self, + async fn commit_block_data_to_algorithm<'a>( + &'a mut self, l2_block_res: GasPriceResult, - ) -> anyhow::Result<()> { + ) -> anyhow::Result<()> + where + StorageTxProvider::Transaction<'a>: MetadataStorage + UnrecordedBlocks, + { tracing::info!("Received L2 block result: {:?}", l2_block_res); let block = l2_block_res?; @@ -93,46 +98,46 @@ where Ok(()) } - async fn process_da_block_costs_res( - &mut self, - da_block_costs: Result, - ) -> anyhow::Result<()> { - tracing::info!("Received DA block costs: {:?}", da_block_costs); - let da_block_costs = da_block_costs?; - - tracing::debug!("Updating DA block costs"); - self.apply_da_block_costs_to_gas_algorithm(da_block_costs) - .await?; - Ok(()) - } + // async fn process_da_block_costs_res( + // &mut self, + // da_block_costs: Result, + // ) -> anyhow::Result<()> { + // tracing::info!("Received DA block costs: {:?}", da_block_costs); + // let da_block_costs = da_block_costs?; + // + // tracing::debug!("Updating DA block costs"); + // self.apply_da_block_costs_to_gas_algorithm(da_block_costs) + // .await?; + // Ok(()) + // } } -impl GasPriceServiceV1 +impl GasPriceServiceV1 where - Metadata: MetadataStorage, DA: DaBlockCostsSource, - U: UnrecordedBlocks + Clone, + StorageTxProvider: TransactionableStorage, { pub fn new( l2_block_source: L2, - metadata_storage: Metadata, shared_algo: SharedV1Algorithm, - algorithm_updater: AlgorithmUpdaterV1, + algorithm_updater: AlgorithmUpdaterV1, da_source_adapter_handle: DaSourceService, + storage_tx_provider: StorageTxProvider, ) -> Self { let da_source_channel = da_source_adapter_handle.shared_data().clone().subscribe(); Self { shared_algo, l2_block_source, - metadata_storage, algorithm_updater, da_source_adapter_handle, da_source_channel, + da_block_costs_buffer: Vec::new(), + storage_tx_provider, } } - pub fn algorithm_updater(&self) -> &AlgorithmUpdaterV1 { + pub fn algorithm_updater(&self) -> &AlgorithmUpdaterV1 { &self.algorithm_updater } @@ -152,21 +157,26 @@ where .ok_or_else(|| anyhow!("Block gas capacity must be non-zero")) } - async fn set_metadata(&mut self) -> anyhow::Result<()> { - let metadata: UpdaterMetadata = self.algorithm_updater.clone().into(); - self.metadata_storage - .set_metadata(&metadata) - .map_err(|err| anyhow!(err)) - } + // async fn set_metadata( + // &mut self, + // tx: &mut StorageTxProvider::Transaction, + // ) -> anyhow::Result<()> { + // let metadata: UpdaterMetadata = self.algorithm_updater.clone().into(); + // tx.set_metadata(&metadata).map_err(|err| anyhow!(err)) + // } - async fn handle_normal_block( - &mut self, + async fn handle_normal_block<'a>( + &'a mut self, height: u32, gas_used: u64, block_gas_capacity: u64, block_bytes: u64, block_fees: u64, - ) -> anyhow::Result<()> { + ) -> anyhow::Result<()> + where + StorageTxProvider::Transaction<'a>: UnrecordedBlocks, + { + let mut storage_tx = self.storage_tx_provider.begin_transaction()?; let capacity = self.validate_block_gas_capacity(block_gas_capacity)?; self.algorithm_updater.update_l2_block_data( @@ -175,33 +185,44 @@ where capacity, block_bytes, block_fees as u128, + &mut storage_tx, )?; - self.set_metadata().await?; + StorageTxProvider::commit_transaction(storage_tx)?; Ok(()) } - async fn handle_da_block_costs( + async fn handle_da_block_costs<'a>( &mut self, da_block_costs: DaBlockCosts, - ) -> anyhow::Result<()> { + unrecorded_blocks: &mut StorageTxProvider::Transaction<'a>, + ) -> anyhow::Result<()> + where + StorageTxProvider::Transaction<'a>: UnrecordedBlocks, + { self.algorithm_updater.update_da_record_data( &da_block_costs.l2_blocks, da_block_costs.blob_size_bytes, da_block_costs.blob_cost_wei, + unrecorded_blocks, )?; - self.set_metadata().await?; Ok(()) } - async fn apply_block_info_to_gas_algorithm( - &mut self, + async fn apply_block_info_to_gas_algorithm<'a>( + &'a mut self, l2_block: BlockInfo, - ) -> anyhow::Result<()> { + ) -> anyhow::Result<()> + where + StorageTxProvider::Transaction<'a>: MetadataStorage + UnrecordedBlocks, + { match l2_block { BlockInfo::GenesisBlock => { - self.set_metadata().await?; + let metadata: UpdaterMetadata = self.algorithm_updater.clone().into(); + let mut tx = self.storage_tx_provider.begin_transaction()?; + tx.set_metadata(&metadata).map_err(|err| anyhow!(err))?; + StorageTxProvider::commit_transaction(tx)?; } BlockInfo::Block { height, @@ -221,27 +242,30 @@ where } } - self.update(self.algorithm_updater.algorithm()).await; + let new_algo = self.algorithm_updater.algorithm(); + // self.update(new_algo).await; + self.shared_algo.update(new_algo).await; Ok(()) } - async fn apply_da_block_costs_to_gas_algorithm( - &mut self, - da_block_costs: DaBlockCosts, - ) -> anyhow::Result<()> { - self.handle_da_block_costs(da_block_costs).await?; - self.update(self.algorithm_updater.algorithm()).await; - Ok(()) - } + // async fn apply_da_block_costs_to_gas_algorithm( + // &mut self, + // da_block_costs: DaBlockCosts, + // ) -> anyhow::Result<()> { + // self.handle_da_block_costs(da_block_costs).await?; + // self.update(self.algorithm_updater.algorithm()).await; + // Ok(()) + // } } #[async_trait] -impl RunnableTask for GasPriceServiceV1 +impl RunnableTask + for GasPriceServiceV1 where L2: L2BlockSource, - Metadata: MetadataStorage, DA: DaBlockCostsSource, - U: UnrecordedBlocks + Clone + Send + Sync, + StorageTxProvider: TransactionableStorage<'static>, + StorageTxProvider::Transaction: UnrecordedBlocks + MetadataStorage, { async fn run(&mut self, watcher: &mut StateWatcher) -> TaskNextAction { tokio::select! { @@ -251,13 +275,20 @@ where TaskNextAction::Stop } l2_block_res = self.l2_block_source.get_l2_block() => { - let res = self.process_l2_block_res(l2_block_res).await; + let res = self.commit_block_data_to_algorithm(l2_block_res).await; TaskNextAction::always_continue(res) } - da_block_costs = self.da_source_channel.recv() => { - self.da_block_costs_buffer.push(da_block_costs?); - // let res = self.process_da_block_costs_res(da_block_costs).await; - TaskNextAction::Continue + da_block_costs_res = self.da_source_channel.recv() => { + match da_block_costs_res { + Ok(da_block_costs) => { + self.da_block_costs_buffer.push(da_block_costs); + TaskNextAction::Continue + }, + Err(err) => { + let err = anyhow!("Error receiving DA block costs: {:?}", err); + TaskNextAction::ErrorContinue(err) + } + } } } } @@ -269,12 +300,6 @@ where self.apply_block_info_to_gas_algorithm(block).await?; } - while let Ok(da_block_costs) = self.da_source_channel.try_recv() { - tracing::debug!("Updating DA block costs"); - self.apply_da_block_costs_to_gas_algorithm(da_block_costs) - .await?; - } - // run shutdown hooks for internal services self.da_source_adapter_handle.shutdown().await?; @@ -300,15 +325,13 @@ fn convert_to_v1_metadata( } } -pub fn initialize_algorithm( +pub fn initialize_algorithm( config: &V1AlgorithmConfig, latest_block_height: u32, metadata_storage: &Metadata, - unrecorded_blocks: U, -) -> crate::common::utils::Result<(AlgorithmUpdaterV1, SharedV1Algorithm)> +) -> crate::common::utils::Result<(AlgorithmUpdaterV1, SharedV1Algorithm)> where Metadata: MetadataStorage, - U: UnrecordedBlocks + Clone, { let algorithm_updater = if let Some(updater_metadata) = metadata_storage .get_metadata(&latest_block_height.into()) @@ -316,9 +339,9 @@ where crate::common::utils::Error::CouldNotInitUpdater(anyhow::anyhow!(err)) })? { let v1_metadata = convert_to_v1_metadata(updater_metadata, config)?; - v1_algorithm_from_metadata(v1_metadata, config, unrecorded_blocks) + v1_algorithm_from_metadata(v1_metadata, config) } else { - updater_from_config(config, unrecorded_blocks) + updater_from_config(config) }; let shared_algo = @@ -331,9 +354,29 @@ where #[allow(non_snake_case)] #[cfg(test)] mod tests { + use std::{ + num::NonZeroU64, + sync::Arc, + time::Duration, + }; + + use tokio::sync::mpsc; + + use fuel_core_services::{ + RunnableTask, + StateWatcher, + }; + use fuel_core_storage::{ + structured_storage::test::InMemoryStorage, + transactional::{ + IntoTransaction, + StorageTransaction, + }, + }; + use fuel_core_types::fuel_types::BlockHeight; + use crate::{ common::{ - fuel_core_storage_adapter::storage::GasPriceColumn, l2_block_source::L2BlockSource, updater_metadata::UpdaterMetadata, utils::{ @@ -359,24 +402,6 @@ mod tests { }, }, }; - use fuel_core_services::{ - RunnableTask, - StateWatcher, - }; - use fuel_core_storage::{ - structured_storage::test::InMemoryStorage, - transactional::{ - IntoTransaction, - StorageTransaction, - }, - }; - use fuel_core_types::fuel_types::BlockHeight; - use std::{ - num::NonZeroU64, - sync::Arc, - time::Duration, - }; - use tokio::sync::mpsc; struct FakeL2BlockSource { l2_block: mpsc::Receiver, diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 02edde11f9e..d5e94682478 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -425,8 +425,8 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr algo_updater_matches_values_from_old_metadata(algo_updater, original_metadata); } -fn algo_updater_matches_values_from_old_metadata( - mut algo_updater: AlgorithmUpdaterV1, +fn algo_updater_matches_values_from_old_metadata( + mut algo_updater: AlgorithmUpdaterV1, original_metadata: V1Metadata, ) { let V1Metadata { diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index f662582384f..2d270f92c9b 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -60,6 +60,7 @@ use fuel_core_storage::{ transactional::{ AtomicView, Modifiable, + StorageTransaction, }, }; use fuel_core_types::{ @@ -77,10 +78,9 @@ pub mod fuel_storage_unrecorded_blocks; pub struct UninitializedTask< L2DataStoreView, GasPriceStore, - Metadata, DA, SettingsProvider, - UnrecordedBlockStorage, + PersistedData, > { pub config: V1AlgorithmConfig, pub genesis_block_height: BlockHeight, @@ -89,43 +89,29 @@ pub struct UninitializedTask< pub on_chain_db: L2DataStoreView, pub block_stream: BoxStream, pub(crate) shared_algo: SharedV1Algorithm, - pub(crate) algo_updater: - AlgorithmUpdaterV1>, - pub(crate) metadata_storage: Metadata, + pub(crate) algo_updater: AlgorithmUpdaterV1, pub(crate) da_source: DA, - pub(crate) unrecorded_blocks_handle: - FuelStorageUnrecordedBlocks, + pub persisted_data: PersistedData, } impl< L2DataStore, L2DataStoreView, GasPriceStore, - Metadata, DA, SettingsProvider, - UnrecordedBlockStorage, - > - UninitializedTask< - L2DataStoreView, - GasPriceStore, - Metadata, - DA, - SettingsProvider, - UnrecordedBlockStorage, + PersistedData, > + UninitializedTask where L2DataStore: L2Data, L2DataStoreView: AtomicView, GasPriceStore: GasPriceData, - Metadata: MetadataStorage, + PersistedData: MetadataStorage, + PersistedData: TransactionableStorage<'static>, + PersistedData::Transaction: MetadataStorage + UnrecordedBlocks, DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, - UnrecordedBlockStorage: KeyValueInspect - + Modifiable - + Send - + Sync - + Clone, { #[allow(clippy::too_many_arguments)] pub fn new( @@ -134,10 +120,9 @@ where settings: SettingsProvider, block_stream: BoxStream, gas_price_db: GasPriceStore, - metadata_storage: Metadata, da_source: DA, on_chain_db: L2DataStoreView, - unrecorded_blocks: FuelStorageUnrecordedBlocks, + persisted_data: PersistedData, ) -> anyhow::Result { let latest_block_height: u32 = on_chain_db .latest_view()? @@ -145,12 +130,8 @@ where .unwrap_or(genesis_block_height) .into(); - let (algo_updater, shared_algo) = initialize_algorithm( - &config, - latest_block_height, - &metadata_storage, - unrecorded_blocks.clone(), - )?; + let (algo_updater, shared_algo) = + initialize_algorithm(&config, latest_block_height, &persisted_data)?; let task = Self { config, @@ -161,9 +142,8 @@ where block_stream, algo_updater, shared_algo, - metadata_storage, da_source, - unrecorded_blocks_handle: unrecorded_blocks, + persisted_data, }; Ok(task) } @@ -171,12 +151,7 @@ where pub fn init( mut self, ) -> anyhow::Result< - GasPriceServiceV1< - FuelL2BlockSource, - Metadata, - DA, - FuelStorageUnrecordedBlocks, - >, + GasPriceServiceV1, DA, PersistedData>, > { let mut first_run = false; let latest_block_height: u32 = self @@ -210,10 +185,10 @@ where { let service = GasPriceServiceV1::new( l2_block_source, - self.metadata_storage, self.shared_algo, self.algo_updater, da_service, + self.persisted_data, ); Ok(service) } else { @@ -221,20 +196,19 @@ where sync_gas_price_db_with_on_chain_storage( &self.settings, &self.config, - &mut self.metadata_storage, &self.on_chain_db, metadata_height, latest_block_height, - self.unrecorded_blocks_handle, + &mut self.persisted_data, )?; } let service = GasPriceServiceV1::new( l2_block_source, - self.metadata_storage, self.shared_algo, self.algo_updater, da_service, + self.persisted_data, ); Ok(service) } @@ -246,40 +220,30 @@ impl< L2DataStore, L2DataStoreView, GasPriceStore, - Metadata, DA, SettingsProvider, - UnrecordedBlockStorage, + PersistedData, > RunnableService for UninitializedTask< L2DataStoreView, GasPriceStore, - Metadata, DA, SettingsProvider, - UnrecordedBlockStorage, + PersistedData, > where L2DataStore: L2Data, L2DataStoreView: AtomicView, GasPriceStore: GasPriceData, - Metadata: MetadataStorage, DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, - UnrecordedBlockStorage: KeyValueInspect - + Modifiable - + Send - + Sync - + Clone, + PersistedData: MetadataStorage + TransactionableStorage<'static>, + >::Transaction: + MetadataStorage + UnrecordedBlocks, { const NAME: &'static str = "GasPriceServiceV1"; type SharedData = SharedV1Algorithm; - type Task = GasPriceServiceV1< - FuelL2BlockSource, - Metadata, - DA, - FuelStorageUnrecordedBlocks, - >; + type Task = GasPriceServiceV1, DA, PersistedData>; type TaskParams = (); fn shared_data(&self) -> Self::SharedData { @@ -298,30 +262,25 @@ where fn sync_gas_price_db_with_on_chain_storage< L2DataStore, L2DataStoreView, - Metadata, SettingsProvider, - UnrecordedBlockStorage, + PersistedData, >( settings: &SettingsProvider, config: &V1AlgorithmConfig, - metadata_storage: &mut Metadata, on_chain_db: &L2DataStoreView, metadata_height: u32, latest_block_height: u32, - unrecorded_blocks: FuelStorageUnrecordedBlocks, + persisted_data: &mut PersistedData, ) -> anyhow::Result<()> where L2DataStore: L2Data, L2DataStoreView: AtomicView, - Metadata: MetadataStorage, SettingsProvider: GasPriceSettingsProvider, - UnrecordedBlockStorage: KeyValueInspect - + Modifiable - + Send - + Sync - + Clone, + PersistedData: MetadataStorage + TransactionableStorage<'static>, + >::Transaction: + MetadataStorage + UnrecordedBlocks, { - let metadata = metadata_storage + let metadata = persisted_data .get_metadata(&metadata_height.into())? .ok_or(anyhow::anyhow!( "Expected metadata to exist for height: {metadata_height}" @@ -333,8 +292,7 @@ where V1Metadata::construct_from_v0_metadata(metadata, config)? } }; - let mut algo_updater = - v1_algorithm_from_metadata(metadata, config, unrecorded_blocks); + let mut algo_updater = v1_algorithm_from_metadata(metadata, config); sync_v1_metadata( settings, @@ -342,40 +300,27 @@ where metadata_height, latest_block_height, &mut algo_updater, - metadata_storage, + persisted_data, )?; Ok(()) } -fn sync_v1_metadata< - L2DataStore, - L2DataStoreView, - Metadata, - SettingsProvider, - UnrecordedBlockStorage, - StorageTxGenerator, ->( +fn sync_v1_metadata( settings: &SettingsProvider, on_chain_db: &L2DataStoreView, metadata_height: u32, latest_block_height: u32, - updater: &mut AlgorithmUpdaterV1>, - metadata_storage: &mut Metadata, + updater: &mut AlgorithmUpdaterV1, storage_tx_generator: &mut StorageTxGenerator, ) -> anyhow::Result<()> where L2DataStore: L2Data, L2DataStoreView: AtomicView, - Metadata: MetadataStorage, SettingsProvider: GasPriceSettingsProvider, - UnrecordedBlockStorage: KeyValueInspect - + Modifiable - + Send - + Sync - + Clone, - StorageTxGenerator: TransactionableStorage, - ::Transaction: UnrecordedBlocks, + StorageTxGenerator: TransactionableStorage<'static>, + >::Transaction: + MetadataStorage + UnrecordedBlocks, { let first = metadata_height.saturating_add(1); let view = on_chain_db.latest_view()?; @@ -410,9 +355,9 @@ where fee_wei.into(), &mut tx, )?; - storage_tx_generator.commit_transaction(tx)?; let metadata: UpdaterMetadata = updater.clone().into(); - metadata_storage.set_metadata(&metadata)?; + tx.set_metadata(&metadata)?; + storage_tx_generator.commit_transaction(tx)?; } Ok(()) @@ -423,29 +368,26 @@ where pub fn new_gas_price_service_v1< L2DataStore, GasPriceStore, - Metadata, DA, SettingsProvider, - UnrecordedBlockStorage, + PersistedData, >( config: GasPriceServiceConfig, genesis_block_height: BlockHeight, settings: SettingsProvider, block_stream: BoxStream, gas_price_db: GasPriceStore, - metadata: Metadata, da_source: DA, on_chain_db: L2DataStore, - unrecorded_block_storage: UnrecordedBlockStorage, + persisted_data: PersistedData, ) -> anyhow::Result< ServiceRunner< UninitializedTask< L2DataStore, GasPriceStore, - Metadata, DA, SettingsProvider, - UnrecordedBlockStorage, + PersistedData, >, >, > @@ -454,26 +396,20 @@ where L2DataStore::LatestView: L2Data, GasPriceStore: GasPriceData, SettingsProvider: GasPriceSettingsProvider, - Metadata: MetadataStorage, DA: DaBlockCostsSource, - UnrecordedBlockStorage: KeyValueInspect - + Modifiable - + Send - + Sync - + Clone, + PersistedData: MetadataStorage + TransactionableStorage<'static>, + PersistedData::Transaction: MetadataStorage + UnrecordedBlocks, { let v1_config = config.v1().ok_or(anyhow::anyhow!("Expected V1 config"))?; - let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(unrecorded_block_storage); let gas_price_init = UninitializedTask::new( v1_config, genesis_block_height, settings, block_stream, gas_price_db, - metadata, da_source, on_chain_db, - unrecorded_blocks, + persisted_data, )?; Ok(ServiceRunner::new(gas_price_init)) } From 3c0a0ffe23ed09bdbe0918c06d6fd3e2adc25b2b Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 3 Dec 2024 17:07:47 -0700 Subject: [PATCH 050/131] WIP fixing lifetime issues --- .../src/common/fuel_core_storage_adapter.rs | 81 ++++++++++++++++++- .../fuel_core_storage_adapter/storage.rs | 21 +++++ .../gas_price_service/src/v1/metadata.rs | 10 +-- .../gas_price_service/src/v1/service.rs | 51 +++++------- .../gas_price_service/src/v1/tests.rs | 27 ++----- .../src/v1/uninitialized_task.rs | 41 +++++----- .../fuel_storage_unrecorded_blocks.rs | 12 +-- .../fuel_storage_unrecorded_blocks/storage.rs | 71 ---------------- 8 files changed, 158 insertions(+), 156 deletions(-) delete mode 100644 crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks/storage.rs diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs index 370d375deff..5581212db3b 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs @@ -11,6 +11,7 @@ use crate::{ fuel_core_storage_adapter::storage::{ GasPriceColumn, GasPriceMetadata, + UnrecordedBlocksTable, }, updater_metadata::UpdaterMetadata, }, @@ -47,6 +48,12 @@ use fuel_core_types::{ Transaction, }, }; +use fuel_gas_price_algorithm::v1::{ + Bytes, + Error, + Height, + UnrecordedBlocks, +}; use std::cmp::min; #[cfg(test)] @@ -86,17 +93,55 @@ where } } +pub struct WrappedStorageTransaction<'a, Storage> { + inner: StorageTransaction<&'a mut StructuredStorage>, +} + +impl<'a, Storage> WrappedStorageTransaction<'a, Storage> { + fn wrap(inner: StorageTransaction<&'a mut StructuredStorage>) -> Self { + Self { inner } + } +} + +impl<'a, Storage> UnrecordedBlocks for WrappedStorageTransaction<'a, Storage> +where + Storage: KeyValueInspect + Modifiable + Send + Sync, +{ + fn insert(&mut self, height: Height, bytes: Bytes) -> Result<(), Error> { + self.inner + .storage_as_mut::() + .insert(&height, &bytes) + .map_err(|err| { + Error::CouldNotInsertUnrecordedBlock(format!("Error: {:?}", err)) + })?; + Ok(()) + } + + fn remove(&mut self, height: &Height) -> Result, Error> { + let bytes = self + .inner + .storage_as_mut::() + .take(height) + .map_err(|err| { + Error::CouldNotRemoveUnrecordedBlock(format!("Error: {:?}", err)) + })?; + Ok(bytes) + } +} + impl TransactionableStorage for StructuredStorage where Storage: Modifiable + Send + Sync, { - type Transaction<'a> = StorageTransaction<&'a mut Self> where Self: 'a; + type Transaction<'a> = WrappedStorageTransaction<'a, Storage> where Self: 'a; fn begin_transaction<'a>(&'a mut self) -> GasPriceResult> where Self: 'a, { - Ok(self.write_transaction()) + let tx = self.write_transaction(); + let wrapped = WrappedStorageTransaction::wrap(tx); + Ok(wrapped) } fn commit_transaction<'a>(transaction: Self::Transaction<'a>) -> GasPriceResult<()> @@ -104,12 +149,44 @@ where Self: 'a, { transaction + .inner .commit() .map_err(|err| GasPriceError::CouldNotCommit(err.into()))?; Ok(()) } } +impl<'a, Storage> MetadataStorage for WrappedStorageTransaction<'a, Storage> +where + Storage: KeyValueInspect + Modifiable + Send + Sync, +{ + fn get_metadata( + &self, + block_height: &BlockHeight, + ) -> GasPriceResult> { + let metadata = self + .inner + .storage::() + .get(block_height) + .map_err(|err| GasPriceError::CouldNotFetchMetadata { + source_error: err.into(), + })?; + Ok(metadata.map(|inner| inner.into_owned())) + } + + fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { + let block_height = metadata.l2_block_height(); + self.inner + .storage_as_mut::() + .insert(&block_height, metadata) + .map_err(|err| GasPriceError::CouldNotSetMetadata { + block_height, + source_error: err.into(), + })?; + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq)] pub struct GasPriceSettings { pub gas_price_factor: u64, diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs index 43d6835dcc1..4fb8fc8f28f 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs @@ -27,6 +27,7 @@ use fuel_core_types::fuel_types::BlockHeight; pub enum GasPriceColumn { Metadata = 0, State = 1, + UnrecordedBlocks = 2, } impl GasPriceColumn { @@ -68,3 +69,23 @@ impl TableWithBlueprint for GasPriceMetadata { GasPriceColumn::State } } + +/// The storage for all the unrecorded blocks from gas price algorithm, used for guessing the cost +/// for future blocks to be recorded on the DA chain +pub struct UnrecordedBlocksTable; + +impl Mappable for UnrecordedBlocksTable { + type Key = Self::OwnedKey; + type OwnedKey = u32; + type Value = Self::OwnedValue; + type OwnedValue = u64; +} + +impl TableWithBlueprint for UnrecordedBlocksTable { + type Blueprint = Plain, Postcard>; + type Column = GasPriceColumn; + + fn column() -> Self::Column { + GasPriceColumn::UnrecordedBlocks + } +} diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index d1e7de6375f..2ae3fbd9910 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -1,15 +1,9 @@ -use crate::{ - v0::metadata::V0Metadata, - v1::uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, -}; +use crate::v0::metadata::V0Metadata; use fuel_gas_price_algorithm::v1::{ AlgorithmUpdaterV1, L2ActivityTracker, }; -use std::{ - collections::BTreeMap, - num::NonZeroU64, -}; +use std::num::NonZeroU64; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] pub struct V1Metadata { diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 549b2e48a8c..14deaeddd7e 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -150,7 +150,6 @@ where } fn validate_block_gas_capacity( - &self, block_gas_capacity: u64, ) -> anyhow::Result { NonZeroU64::new(block_gas_capacity) @@ -177,7 +176,7 @@ where StorageTxProvider::Transaction<'a>: UnrecordedBlocks, { let mut storage_tx = self.storage_tx_provider.begin_transaction()?; - let capacity = self.validate_block_gas_capacity(block_gas_capacity)?; + let capacity = Self::validate_block_gas_capacity(block_gas_capacity)?; self.algorithm_updater.update_l2_block_data( height, @@ -189,6 +188,8 @@ where )?; StorageTxProvider::commit_transaction(storage_tx)?; + let new_algo = self.algorithm_updater.algorithm(); + self.shared_algo.update(new_algo).await; Ok(()) } @@ -223,6 +224,9 @@ where let mut tx = self.storage_tx_provider.begin_transaction()?; tx.set_metadata(&metadata).map_err(|err| anyhow!(err))?; StorageTxProvider::commit_transaction(tx)?; + let new_algo = self.algorithm_updater.algorithm(); + // self.update(new_algo).await; + self.shared_algo.update(new_algo).await; } BlockInfo::Block { height, @@ -242,9 +246,6 @@ where } } - let new_algo = self.algorithm_updater.algorithm(); - // self.update(new_algo).await; - self.shared_algo.update(new_algo).await; Ok(()) } @@ -262,10 +263,10 @@ where impl RunnableTask for GasPriceServiceV1 where - L2: L2BlockSource, - DA: DaBlockCostsSource, - StorageTxProvider: TransactionableStorage<'static>, - StorageTxProvider::Transaction: UnrecordedBlocks + MetadataStorage, + L2: L2BlockSource + 'static, + DA: DaBlockCostsSource + 'static, + StorageTxProvider: TransactionableStorage + 'static, + StorageTxProvider::Transaction<'static>: UnrecordedBlocks + MetadataStorage, { async fn run(&mut self, watcher: &mut StateWatcher) -> TaskNextAction { tokio::select! { @@ -377,6 +378,7 @@ mod tests { use crate::{ common::{ + fuel_core_storage_adapter::storage::GasPriceColumn, l2_block_source::L2BlockSource, updater_metadata::UpdaterMetadata, utils::{ @@ -396,10 +398,7 @@ mod tests { initialize_algorithm, GasPriceServiceV1, }, - uninitialized_task::fuel_storage_unrecorded_blocks::{ - storage::UnrecordedBlocksColumn, - FuelStorageUnrecordedBlocks, - }, + uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, }, }; @@ -442,7 +441,7 @@ mod tests { } } - fn database() -> StorageTransaction> { + fn database() -> StorageTransaction> { InMemoryStorage::default().into_transaction() } @@ -481,14 +480,8 @@ mod tests { block_activity_threshold: 20, }; let inner = database(); - let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(inner); - let (algo_updater, shared_algo) = initialize_algorithm( - &config, - l2_block_height, - &metadata_storage, - unrecorded_blocks, - ) - .unwrap(); + let (algo_updater, shared_algo) = + initialize_algorithm(&config, l2_block_height, &metadata_storage).unwrap(); let notifier = Arc::new(tokio::sync::Notify::new()); let dummy_da_source = DaSourceService::new( @@ -501,10 +494,10 @@ mod tests { let mut service = GasPriceServiceV1::new( l2_block_source, - metadata_storage, shared_algo, algo_updater, dummy_da_source, + inner, ); let read_algo = service.next_block_algorithm(); let mut watcher = StateWatcher::default(); @@ -554,14 +547,8 @@ mod tests { block_activity_threshold: 20, }; let inner = database(); - let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(inner); - let (algo_updater, shared_algo) = initialize_algorithm( - &config, - block_height, - &metadata_storage, - unrecorded_blocks, - ) - .unwrap(); + let (algo_updater, shared_algo) = + initialize_algorithm(&config, block_height, &metadata_storage).unwrap(); let notifier = Arc::new(tokio::sync::Notify::new()); let da_source = DaSourceService::new( @@ -579,10 +566,10 @@ mod tests { let mut service = GasPriceServiceV1::new( l2_block_source, - metadata_storage, shared_algo, algo_updater, da_source, + inner, ); let read_algo = service.next_block_algorithm(); let initial_price = read_algo.next_gas_price(); diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index d5e94682478..f15d37acd88 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -36,10 +36,7 @@ use crate::{ GasPriceServiceV1, }, uninitialized_task::{ - fuel_storage_unrecorded_blocks::{ - storage::UnrecordedBlocksColumn, - FuelStorageUnrecordedBlocks, - }, + fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, UninitializedTask, }, }, @@ -214,7 +211,7 @@ fn different_arb_config() -> V1AlgorithmConfig { } } -fn database() -> StorageTransaction> { +fn database() -> StorageTransaction> { InMemoryStorage::default().into_transaction() } @@ -237,18 +234,16 @@ async fn next_gas_price__affected_by_new_l2_block() { let config = zero_threshold_arbitrary_config(); let height = 0; let inner = database(); - let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(inner); let (algo_updater, shared_algo) = - initialize_algorithm(&config, height, &metadata_storage, unrecorded_blocks) - .unwrap(); + initialize_algorithm(&config, height, &metadata_storage).unwrap(); let da_source = FakeDABlockCost::never_returns(); let da_source_service = DaSourceService::new(da_source, None); let mut service = GasPriceServiceV1::new( l2_block_source, - metadata_storage, shared_algo, algo_updater, da_source_service, + inner, ); let read_algo = service.next_block_algorithm(); @@ -287,18 +282,16 @@ async fn run__new_l2_block_saves_old_metadata() { let config = zero_threshold_arbitrary_config(); let height = 0; let inner = database(); - let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(inner); let (algo_updater, shared_algo) = - initialize_algorithm(&config, height, &metadata_storage, unrecorded_blocks) - .unwrap(); + initialize_algorithm(&config, height, &metadata_storage).unwrap(); let da_source = FakeDABlockCost::never_returns(); let da_source_service = DaSourceService::new(da_source, None); let mut service = GasPriceServiceV1::new( l2_block_source, - metadata_storage, shared_algo, algo_updater, da_source_service, + inner, ); let mut watcher = StateWatcher::default(); @@ -404,7 +397,6 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); let inner = database(); - let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(inner); // when let service = UninitializedTask::new( @@ -413,10 +405,9 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr settings, block_stream, gas_price_db, - metadata_storage, da_cost_source, on_chain_db, - unrecorded_blocks, + inner, ) .unwrap(); @@ -486,7 +477,6 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); let inner = database(); - let unrecorded_blocks = FuelStorageUnrecordedBlocks::new(inner); // when let res = UninitializedTask::new( @@ -495,10 +485,9 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { settings, block_stream, gas_price_db, - metadata_storage, da_cost_source, on_chain_db, - unrecorded_blocks, + inner, ); // then diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 2d270f92c9b..f18052005d6 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -38,10 +38,7 @@ use crate::{ initialize_algorithm, GasPriceServiceV1, }, - uninitialized_task::fuel_storage_unrecorded_blocks::{ - storage::UnrecordedBlocksColumn, - FuelStorageUnrecordedBlocks, - }, + uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, }, }; use anyhow::Error; @@ -95,6 +92,7 @@ pub struct UninitializedTask< } impl< + 'a, L2DataStore, L2DataStoreView, GasPriceStore, @@ -108,8 +106,8 @@ where L2DataStoreView: AtomicView, GasPriceStore: GasPriceData, PersistedData: MetadataStorage, - PersistedData: TransactionableStorage<'static>, - PersistedData::Transaction: MetadataStorage + UnrecordedBlocks, + PersistedData: TransactionableStorage + 'a, + PersistedData::Transaction<'a>: MetadataStorage + UnrecordedBlocks, DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, { @@ -235,10 +233,10 @@ where L2DataStore: L2Data, L2DataStoreView: AtomicView, GasPriceStore: GasPriceData, - DA: DaBlockCostsSource, - SettingsProvider: GasPriceSettingsProvider, - PersistedData: MetadataStorage + TransactionableStorage<'static>, - >::Transaction: + DA: DaBlockCostsSource + 'static, + SettingsProvider: GasPriceSettingsProvider + 'static, + PersistedData: MetadataStorage + TransactionableStorage + 'static, + ::Transaction<'static>: MetadataStorage + UnrecordedBlocks, { const NAME: &'static str = "GasPriceServiceV1"; @@ -260,6 +258,7 @@ where } fn sync_gas_price_db_with_on_chain_storage< + 'a, L2DataStore, L2DataStoreView, SettingsProvider, @@ -276,8 +275,8 @@ where L2DataStore: L2Data, L2DataStoreView: AtomicView, SettingsProvider: GasPriceSettingsProvider, - PersistedData: MetadataStorage + TransactionableStorage<'static>, - >::Transaction: + PersistedData: MetadataStorage + TransactionableStorage + 'a, + ::Transaction<'a>: MetadataStorage + UnrecordedBlocks, { let metadata = persisted_data @@ -306,7 +305,13 @@ where Ok(()) } -fn sync_v1_metadata( +fn sync_v1_metadata< + 'a, + L2DataStore, + L2DataStoreView, + SettingsProvider, + StorageTxGenerator, +>( settings: &SettingsProvider, on_chain_db: &L2DataStoreView, metadata_height: u32, @@ -318,8 +323,8 @@ where L2DataStore: L2Data, L2DataStoreView: AtomicView, SettingsProvider: GasPriceSettingsProvider, - StorageTxGenerator: TransactionableStorage<'static>, - >::Transaction: + StorageTxGenerator: TransactionableStorage + 'a, + ::Transaction<'a>: MetadataStorage + UnrecordedBlocks, { let first = metadata_height.saturating_add(1); @@ -357,7 +362,7 @@ where )?; let metadata: UpdaterMetadata = updater.clone().into(); tx.set_metadata(&metadata)?; - storage_tx_generator.commit_transaction(tx)?; + StorageTxGenerator::commit_transaction(tx)?; } Ok(()) @@ -397,8 +402,8 @@ where GasPriceStore: GasPriceData, SettingsProvider: GasPriceSettingsProvider, DA: DaBlockCostsSource, - PersistedData: MetadataStorage + TransactionableStorage<'static>, - PersistedData::Transaction: MetadataStorage + UnrecordedBlocks, + PersistedData: MetadataStorage + TransactionableStorage + 'static, + PersistedData::Transaction<'static>: MetadataStorage + UnrecordedBlocks, { let v1_config = config.v1().ok_or(anyhow::anyhow!("Expected V1 config"))?; let gas_price_init = UninitializedTask::new( diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs index f87da12e4dd..6e3985476ed 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs @@ -1,4 +1,7 @@ -use crate::v1::uninitialized_task::fuel_storage_unrecorded_blocks::storage::UnrecordedBlocksTable; +use crate::common::fuel_core_storage_adapter::storage::{ + GasPriceColumn, + UnrecordedBlocksTable, +}; use fuel_core_storage::{ kv_store::{ KeyValueInspect, @@ -16,9 +19,6 @@ use fuel_gas_price_algorithm::{ v1, v1::UnrecordedBlocks, }; -use storage::UnrecordedBlocksColumn; - -pub mod storage; #[derive(Debug, Clone)] pub struct FuelStorageUnrecordedBlocks { @@ -33,7 +33,7 @@ impl FuelStorageUnrecordedBlocks { impl UnrecordedBlocks for FuelStorageUnrecordedBlocks where - Storage: KeyValueInspect + Modifiable, + Storage: KeyValueInspect + Modifiable, Storage: Send + Sync, { fn insert(&mut self, height: v1::Height, bytes: v1::Bytes) -> Result<(), v1::Error> { @@ -74,7 +74,7 @@ mod tests { }, }; - fn database() -> StorageTransaction> { + fn database() -> StorageTransaction> { InMemoryStorage::default().into_transaction() } diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks/storage.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks/storage.rs deleted file mode 100644 index 2f36ac0ae51..00000000000 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks/storage.rs +++ /dev/null @@ -1,71 +0,0 @@ -use crate::common::updater_metadata::UpdaterMetadata; -use fuel_core_storage::{ - blueprint::plain::Plain, - codec::{ - postcard::Postcard, - primitive::Primitive, - }, - kv_store::StorageColumn, - structured_storage::TableWithBlueprint, - Mappable, -}; -use fuel_core_types::fuel_types::BlockHeight; -use fuel_gas_price_algorithm::v1::Height; - -#[repr(u32)] -#[derive( - Copy, - Clone, - Debug, - strum_macros::EnumCount, - strum_macros::IntoStaticStr, - PartialEq, - Eq, - enum_iterator::Sequence, - Hash, - num_enum::TryFromPrimitive, -)] -pub enum UnrecordedBlocksColumn { - Metadata = 0, - State = 1, -} - -impl UnrecordedBlocksColumn { - /// The total count of variants in the enum. - pub const COUNT: usize = ::COUNT; - - /// Returns the `usize` representation of the `Column`. - pub fn as_u32(&self) -> u32 { - *self as u32 - } -} - -impl StorageColumn for UnrecordedBlocksColumn { - fn name(&self) -> String { - let str: &str = self.into(); - str.to_string() - } - - fn id(&self) -> u32 { - self.as_u32() - } -} - -/// The storage table for metadata of the gas price algorithm updater -pub struct UnrecordedBlocksTable; - -impl Mappable for UnrecordedBlocksTable { - type Key = Self::OwnedKey; - type OwnedKey = u32; - type Value = Self::OwnedValue; - type OwnedValue = u64; -} - -impl TableWithBlueprint for UnrecordedBlocksTable { - type Blueprint = Plain, Postcard>; - type Column = UnrecordedBlocksColumn; - - fn column() -> Self::Column { - UnrecordedBlocksColumn::State - } -} From 36bbb4dc12adf7ec5f3fd1f23411d3c63f289e80 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 3 Dec 2024 18:45:51 -0700 Subject: [PATCH 051/131] Incremental improvements to compiler errors --- .../gas_price_service/src/v1/uninitialized_task.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index f18052005d6..e7db1b968ef 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -269,7 +269,7 @@ fn sync_gas_price_db_with_on_chain_storage< on_chain_db: &L2DataStoreView, metadata_height: u32, latest_block_height: u32, - persisted_data: &mut PersistedData, + persisted_data: &'a mut PersistedData, ) -> anyhow::Result<()> where L2DataStore: L2Data, @@ -317,7 +317,7 @@ fn sync_v1_metadata< metadata_height: u32, latest_block_height: u32, updater: &mut AlgorithmUpdaterV1, - storage_tx_generator: &mut StorageTxGenerator, + storage_tx_generator: &'a mut StorageTxGenerator, ) -> anyhow::Result<()> where L2DataStore: L2Data, @@ -329,6 +329,7 @@ where { let first = metadata_height.saturating_add(1); let view = on_chain_db.latest_view()?; + let mut tx = storage_tx_generator.begin_transaction()?; for height in first..=latest_block_height { let block = view .get_block(&height.into())? @@ -351,7 +352,6 @@ where let block_bytes = block_bytes(&block); let (fee_wei, _) = mint_values(&block)?; - let mut tx = storage_tx_generator.begin_transaction()?; updater.update_l2_block_data( height, block_gas_used, @@ -362,8 +362,8 @@ where )?; let metadata: UpdaterMetadata = updater.clone().into(); tx.set_metadata(&metadata)?; - StorageTxGenerator::commit_transaction(tx)?; } + StorageTxGenerator::commit_transaction(tx)?; Ok(()) } From 1447d90f6fd6338e890cdfa589161d1c18fb5e3f Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 4 Dec 2024 08:48:55 -0700 Subject: [PATCH 052/131] Fix compilation errors with lifetimes --- .../src/common/fuel_core_storage_adapter.rs | 10 ++-------- crates/services/gas_price_service/src/ports.rs | 8 ++------ crates/services/gas_price_service/src/v1/service.rs | 6 +++--- .../gas_price_service/src/v1/uninitialized_task.rs | 9 ++++----- 4 files changed, 11 insertions(+), 22 deletions(-) diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs index 5581212db3b..13cfcf9ad57 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs @@ -135,19 +135,13 @@ where { type Transaction<'a> = WrappedStorageTransaction<'a, Storage> where Self: 'a; - fn begin_transaction<'a>(&'a mut self) -> GasPriceResult> - where - Self: 'a, - { + fn begin_transaction(&mut self) -> GasPriceResult> { let tx = self.write_transaction(); let wrapped = WrappedStorageTransaction::wrap(tx); Ok(wrapped) } - fn commit_transaction<'a>(transaction: Self::Transaction<'a>) -> GasPriceResult<()> - where - Self: 'a, - { + fn commit_transaction(transaction: Self::Transaction<'_>) -> GasPriceResult<()> { transaction .inner .commit() diff --git a/crates/services/gas_price_service/src/ports.rs b/crates/services/gas_price_service/src/ports.rs index a071969cab4..62001673f5a 100644 --- a/crates/services/gas_price_service/src/ports.rs +++ b/crates/services/gas_price_service/src/ports.rs @@ -32,12 +32,8 @@ pub trait TransactionableStorage: Send + Sync { where Self: 'a; - fn begin_transaction<'a>(&'a mut self) -> Result> - where - Self: 'a; - fn commit_transaction<'a>(transaction: Self::Transaction<'a>) -> Result<()> - where - Self: 'a; + fn begin_transaction(&mut self) -> Result>; + fn commit_transaction(transaction: Self::Transaction<'_>) -> Result<()>; } /// Provides the latest block height. diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 14deaeddd7e..082f1d1f0bf 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -263,10 +263,10 @@ where impl RunnableTask for GasPriceServiceV1 where - L2: L2BlockSource + 'static, - DA: DaBlockCostsSource + 'static, + L2: L2BlockSource, + DA: DaBlockCostsSource, StorageTxProvider: TransactionableStorage + 'static, - StorageTxProvider::Transaction<'static>: UnrecordedBlocks + MetadataStorage, + for<'a> StorageTxProvider::Transaction<'a>: UnrecordedBlocks + MetadataStorage, { async fn run(&mut self, watcher: &mut StateWatcher) -> TaskNextAction { tokio::select! { diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index e7db1b968ef..19e99854a29 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -92,7 +92,6 @@ pub struct UninitializedTask< } impl< - 'a, L2DataStore, L2DataStoreView, GasPriceStore, @@ -106,8 +105,8 @@ where L2DataStoreView: AtomicView, GasPriceStore: GasPriceData, PersistedData: MetadataStorage, - PersistedData: TransactionableStorage + 'a, - PersistedData::Transaction<'a>: MetadataStorage + UnrecordedBlocks, + PersistedData: TransactionableStorage, + for<'a> PersistedData::Transaction<'a>: MetadataStorage + UnrecordedBlocks, DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, { @@ -236,7 +235,7 @@ where DA: DaBlockCostsSource + 'static, SettingsProvider: GasPriceSettingsProvider + 'static, PersistedData: MetadataStorage + TransactionableStorage + 'static, - ::Transaction<'static>: + for<'a> ::Transaction<'a>: MetadataStorage + UnrecordedBlocks, { const NAME: &'static str = "GasPriceServiceV1"; @@ -403,7 +402,7 @@ where SettingsProvider: GasPriceSettingsProvider, DA: DaBlockCostsSource, PersistedData: MetadataStorage + TransactionableStorage + 'static, - PersistedData::Transaction<'static>: MetadataStorage + UnrecordedBlocks, + for<'a> PersistedData::Transaction<'a>: MetadataStorage + UnrecordedBlocks, { let v1_config = config.v1().ok_or(anyhow::anyhow!("Expected V1 config"))?; let gas_price_init = UninitializedTask::new( From 755f7b0f34d7fc608a6dc9c58455cced3bff1463 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 4 Dec 2024 11:20:07 -0700 Subject: [PATCH 053/131] Add passing test --- Cargo.lock | 1 + crates/fuel-gas-price-algorithm/src/v1.rs | 5 ++ crates/services/gas_price_service/Cargo.toml | 1 + .../services/gas_price_service/src/ports.rs | 13 ++-- .../gas_price_service/src/v1/service.rs | 63 +++++++++++++++---- 5 files changed, 66 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 56a3736a59f..a541eae8dd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,6 +3455,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", + "tracing-subscriber", ] [[package]] diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index a0b51129512..c812382fee6 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -350,6 +350,11 @@ impl AlgorithmUpdaterV1 { ) -> Result<(), Error> { let expected = self.l2_block_height.saturating_add(1); if height != expected { + tracing::debug!( + "Skipped L2 block update: expected {}, got {}", + expected, + height + ); Err(Error::SkippedL2Block { expected, got: height, diff --git a/crates/services/gas_price_service/Cargo.toml b/crates/services/gas_price_service/Cargo.toml index 9903bba7d03..4eab33d503b 100644 --- a/crates/services/gas_price_service/Cargo.toml +++ b/crates/services/gas_price_service/Cargo.toml @@ -32,3 +32,4 @@ tracing = { workspace = true } fuel-core-services = { workspace = true, features = ["test-helpers"] } fuel-core-storage = { workspace = true, features = ["test-helpers"] } fuel-core-types = { path = "./../../types", features = ["test-helpers"] } +tracing-subscriber = { workspace = true } diff --git a/crates/services/gas_price_service/src/ports.rs b/crates/services/gas_price_service/src/ports.rs index 62001673f5a..d693ebc6b13 100644 --- a/crates/services/gas_price_service/src/ports.rs +++ b/crates/services/gas_price_service/src/ports.rs @@ -1,3 +1,10 @@ +use fuel_core_storage::Result as StorageResult; +use fuel_core_types::{ + blockchain::block::Block, + fuel_tx::Transaction, + fuel_types::BlockHeight, +}; + use crate::{ common::{ updater_metadata::UpdaterMetadata, @@ -6,12 +13,6 @@ use crate::{ v0::metadata::V0AlgorithmConfig, v1::metadata::V1AlgorithmConfig, }; -use fuel_core_storage::Result as StorageResult; -use fuel_core_types::{ - blockchain::block::Block, - fuel_tx::Transaction, - fuel_types::BlockHeight, -}; pub trait L2Data: Send + Sync { fn latest_height(&self) -> StorageResult; diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 082f1d1f0bf..831a3a531d2 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -178,6 +178,16 @@ where let mut storage_tx = self.storage_tx_provider.begin_transaction()?; let capacity = Self::validate_block_gas_capacity(block_gas_capacity)?; + for da_block_costs in self.da_block_costs_buffer.drain(..) { + tracing::debug!("Updating DA block costs: {:?}", da_block_costs); + self.algorithm_updater.update_da_record_data( + &da_block_costs.l2_blocks, + da_block_costs.blob_size_bytes, + da_block_costs.blob_cost_wei, + &mut storage_tx, + )?; + } + self.algorithm_updater.update_l2_block_data( height, gas_used, @@ -187,6 +197,7 @@ where &mut storage_tx, )?; + tracing::debug!("Committing transaction"); StorageTxProvider::commit_transaction(storage_tx)?; let new_algo = self.algorithm_updater.algorithm(); self.shared_algo.update(new_algo).await; @@ -276,10 +287,12 @@ where TaskNextAction::Stop } l2_block_res = self.l2_block_source.get_l2_block() => { + tracing::debug!("Received L2 block result: {:?}", l2_block_res); let res = self.commit_block_data_to_algorithm(l2_block_res).await; TaskNextAction::always_continue(res) } da_block_costs_res = self.da_source_channel.recv() => { + tracing::debug!("Received DA block costs: {:?}", da_block_costs_res); match da_block_costs_res { Ok(da_block_costs) => { self.da_block_costs_buffer.push(da_block_costs); @@ -372,13 +385,20 @@ mod tests { transactional::{ IntoTransaction, StorageTransaction, + WriteTransaction, }, + StorageAsMut, }; use fuel_core_types::fuel_types::BlockHeight; use crate::{ common::{ - fuel_core_storage_adapter::storage::GasPriceColumn, + fuel_core_storage_adapter::storage::{ + GasPriceColumn, + GasPriceColumn::UnrecordedBlocks, + UnrecordedBlocksTable, + }, + gas_price_algorithm::SharedGasPriceAlgo, l2_block_source::L2BlockSource, updater_metadata::UpdaterMetadata, utils::{ @@ -393,7 +413,10 @@ mod tests { service::DaSourceService, DaBlockCosts, }, - metadata::V1AlgorithmConfig, + metadata::{ + updater_from_config, + V1AlgorithmConfig, + }, service::{ initialize_algorithm, GasPriceServiceV1, @@ -515,9 +538,13 @@ mod tests { #[tokio::test] async fn run__updates_gas_price_with_da_block_cost_source() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::DEBUG) + .try_init(); + // given - let block_height = 1; - let l2_block = BlockInfo::Block { + let block_height = 2; + let l2_block_2 = BlockInfo::Block { height: block_height, gas_used: 60, block_gas_capacity: 100, @@ -531,14 +558,15 @@ mod tests { }; let metadata_storage = FakeMetadata::empty(); + // Configured so exec gas price doesn't change, only da gas price let config = V1AlgorithmConfig { new_exec_gas_price: 100, min_exec_gas_price: 50, - exec_gas_price_change_percent: 20, + exec_gas_price_change_percent: 0, l2_block_fullness_threshold_percent: 20, gas_price_factor: NonZeroU64::new(10).unwrap(), - min_da_gas_price: 100, - max_da_gas_price_change_percent: 50, + min_da_gas_price: 0, + max_da_gas_price_change_percent: 100, da_p_component: 4, da_d_component: 2, normal_range_size: 10, @@ -546,9 +574,18 @@ mod tests { decrease_range_size: 4, block_activity_threshold: 20, }; - let inner = database(); - let (algo_updater, shared_algo) = - initialize_algorithm(&config, block_height, &metadata_storage).unwrap(); + let mut inner = database(); + let mut tx = inner.write_transaction(); + tx.storage_as_mut::() + .insert(&BlockHeight::from(1), &100) + .unwrap(); + tx.commit().unwrap(); + let mut algo_updater = updater_from_config(&config); + let shared_algo = + SharedGasPriceAlgo::new_with_algorithm(algo_updater.algorithm()); + algo_updater.l2_block_height = block_height - 1; + algo_updater.last_profit = 10_000; + algo_updater.new_scaled_da_gas_price = 10_000_000; let notifier = Arc::new(tokio::sync::Notify::new()); let da_source = DaSourceService::new( @@ -562,7 +599,7 @@ mod tests { ), Some(Duration::from_millis(1)), ); - let mut watcher = StateWatcher::default(); + let mut watcher = StateWatcher::started(); let mut service = GasPriceServiceV1::new( l2_block_source, @@ -583,6 +620,10 @@ mod tests { .run(&mut da_source_watcher) .await; + service.run(&mut watcher).await; + tokio::time::sleep(Duration::from_millis(100)).await; + l2_block_sender.send(l2_block_2).await.unwrap(); + // when service.run(&mut watcher).await; tokio::time::sleep(Duration::from_millis(100)).await; From 05ddee68508a04de57eb5f439f5027fe219300c6 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 4 Dec 2024 13:27:10 -0700 Subject: [PATCH 054/131] Fix another test --- crates/fuel-gas-price-algorithm/src/v1.rs | 16 +++++ .../gas_price_service/src/v1/metadata.rs | 1 + .../gas_price_service/src/v1/tests.rs | 64 +++++++++++++++---- 3 files changed, 69 insertions(+), 12 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index c812382fee6..8a160a468b5 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -288,6 +288,22 @@ impl L2ActivityTracker { pub fn current_activity(&self) -> u16 { self.chain_activity } + + pub fn max_activity(&self) -> u16 { + self.max_activity + } + + pub fn capped_activity_threshold(&self) -> u16 { + self.capped_activity_threshold + } + + pub fn decrease_activity_threshold(&self) -> u16 { + self.decrease_activity_threshold + } + + pub fn block_activity_threshold(&self) -> ClampedPercentage { + self.block_activity_threshold.clone() + } } /// A value that represents a value between 0 and 100. Higher values are clamped to 100 diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index 2ae3fbd9910..bf33cca6ffd 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -57,6 +57,7 @@ impl V1Metadata { } } +#[derive(Debug, Clone, PartialEq)] pub struct V1AlgorithmConfig { pub new_exec_gas_price: u64, pub min_exec_gas_price: u64, diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index f15d37acd88..864c021a3ae 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -2,7 +2,10 @@ use crate::{ common::{ fuel_core_storage_adapter::{ - storage::GasPriceColumn, + storage::{ + GasPriceColumn, + GasPriceMetadata, + }, GasPriceSettings, GasPriceSettingsProvider, }, @@ -51,13 +54,16 @@ use fuel_core_services::{ StateWatcher, }; use fuel_core_storage::{ + iter::IteratorOverTable, structured_storage::test::InMemoryStorage, transactional::{ AtomicView, IntoTransaction, StorageTransaction, + WriteTransaction, }, Result as StorageResult, + StorageAsMut, }; use fuel_core_types::{ blockchain::{ @@ -215,6 +221,20 @@ fn database() -> StorageTransaction> { InMemoryStorage::default().into_transaction() } +fn database_with_metadata( + metadata: &V1Metadata, +) -> StorageTransaction> { + let mut db = database(); + let mut tx = db.write_transaction(); + let height = metadata.l2_block_height.into(); + let metadata = UpdaterMetadata::V1(metadata.clone()); + tx.storage_as_mut::() + .insert(&height, &metadata) + .unwrap(); + tx.commit().unwrap(); + db +} + #[tokio::test] async fn next_gas_price__affected_by_new_l2_block() { // given @@ -379,28 +399,21 @@ fn empty_block_stream() -> BoxStream { async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overrides() { // given let original_metadata = arbitrary_metadata(); - let original = UpdaterMetadata::V1(original_metadata.clone()); - let metadata_inner = Arc::new(std::sync::Mutex::new(Some(original.clone()))); - let metadata_storage = FakeMetadata { - inner: metadata_inner, - }; let different_config = different_arb_config(); let descaleed_exec_price = original_metadata.new_scaled_exec_price / original_metadata.gas_price_factor; assert_ne!(different_config.new_exec_gas_price, descaleed_exec_price); - let different_l2_block = 1231; - assert_ne!(different_l2_block, original_metadata.l2_block_height); + let different_l2_block = 0; let settings = FakeSettings; let block_stream = empty_block_stream(); let gas_price_db = FakeGasPriceDb; let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); - let inner = database(); - + let inner = database_with_metadata(&original_metadata); // when let service = UninitializedTask::new( - different_config, + different_config.clone(), 0.into(), settings, block_stream, @@ -413,7 +426,11 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr // then let UninitializedTask { algo_updater, .. } = service; - algo_updater_matches_values_from_old_metadata(algo_updater, original_metadata); + algo_updater_matches_values_from_old_metadata( + algo_updater.clone(), + original_metadata.clone(), + ); + algo_updater_override_values_match(algo_updater, different_config); } fn algo_updater_matches_values_from_old_metadata( @@ -465,6 +482,29 @@ fn algo_updater_matches_values_from_old_metadata( ); } +fn algo_updater_override_values_match( + algo_updater: AlgorithmUpdaterV1, + config: V1AlgorithmConfig, +) { + assert_eq!(algo_updater.min_exec_gas_price, config.min_exec_gas_price); + assert_eq!( + algo_updater.exec_gas_price_change_percent, + config.exec_gas_price_change_percent + ); + assert_eq!( + algo_updater.l2_block_fullness_threshold_percent, + config.l2_block_fullness_threshold_percent.into() + ); + assert_eq!(algo_updater.gas_price_factor, config.gas_price_factor); + assert_eq!(algo_updater.min_da_gas_price, config.min_da_gas_price); + assert_eq!( + algo_updater.max_da_gas_price_change_percent, + config.max_da_gas_price_change_percent + ); + assert_eq!(algo_updater.da_p_component, config.da_p_component); + assert_eq!(algo_updater.da_d_component, config.da_d_component); +} + #[tokio::test] async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { // given From d9bbe368e26fb812d2549a0063671fa68869eabb Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 4 Dec 2024 13:33:55 -0700 Subject: [PATCH 055/131] Fix another test --- .../gas_price_service/src/v1/tests.rs | 50 +++++++++++++++++-- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 864c021a3ae..51734f6ed7e 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -21,6 +21,7 @@ use crate::{ GasPriceData, L2Data, MetadataStorage, + TransactionableStorage, }, v1::{ da_source_service::{ @@ -79,6 +80,9 @@ use fuel_core_types::{ }; use fuel_gas_price_algorithm::v1::{ AlgorithmUpdaterV1, + Bytes, + Error, + Height, UnrecordedBlocks, }; use std::{ @@ -124,9 +128,9 @@ impl MetadataStorage for FakeMetadata { } } -struct ErroringMetadata; +struct ErroringPersistedData; -impl MetadataStorage for ErroringMetadata { +impl MetadataStorage for ErroringPersistedData { fn get_metadata(&self, _: &BlockHeight) -> GasPriceResult> { Err(GasPriceError::CouldNotFetchMetadata { source_error: anyhow!("boo!"), @@ -141,6 +145,43 @@ impl MetadataStorage for ErroringMetadata { } } +struct UnimplementedStorageTx; + +impl TransactionableStorage for ErroringPersistedData { + type Transaction<'a> = UnimplementedStorageTx; + + fn begin_transaction(&mut self) -> GasPriceResult> { + todo!() + } + + fn commit_transaction(transaction: Self::Transaction<'_>) -> GasPriceResult<()> { + todo!() + } +} + +impl MetadataStorage for UnimplementedStorageTx { + fn get_metadata( + &self, + block_height: &BlockHeight, + ) -> GasPriceResult> { + todo!() + } + + fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { + todo!() + } +} + +impl UnrecordedBlocks for UnimplementedStorageTx { + fn insert(&mut self, height: Height, bytes: Bytes) -> Result<(), Error> { + todo!() + } + + fn remove(&mut self, height: &Height) -> Result, Error> { + todo!() + } +} + struct FakeDABlockCost { da_block_costs: Receiver, } @@ -510,13 +551,12 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { // given let config = zero_threshold_arbitrary_config(); let different_l2_block = 1231; - let metadata_storage = ErroringMetadata; + let erroring_persisted_data = ErroringPersistedData; let settings = FakeSettings; let block_stream = empty_block_stream(); let gas_price_db = FakeGasPriceDb; let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); - let inner = database(); // when let res = UninitializedTask::new( @@ -527,7 +567,7 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { gas_price_db, da_cost_source, on_chain_db, - inner, + erroring_persisted_data, ); // then From b70b2eb7ec98a9618733c5c4363a44778e6cd2d3 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 4 Dec 2024 13:59:40 -0700 Subject: [PATCH 056/131] Fix remaining test --- .../gas_price_service/src/v1/service.rs | 8 +++-- .../gas_price_service/src/v1/tests.rs | 30 +++++++++++-------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 831a3a531d2..a0ddfa00ea6 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -73,7 +73,7 @@ where /// Buffer of block costs from the DA chain da_block_costs_buffer: Vec, /// Storage transaction provider for metadata and unrecorded blocks - storage_tx_provider: StorageTxProvider, + pub(crate) storage_tx_provider: StorageTxProvider, } impl GasPriceServiceV1 @@ -173,7 +173,7 @@ where block_fees: u64, ) -> anyhow::Result<()> where - StorageTxProvider::Transaction<'a>: UnrecordedBlocks, + StorageTxProvider::Transaction<'a>: UnrecordedBlocks + MetadataStorage, { let mut storage_tx = self.storage_tx_provider.begin_transaction()?; let capacity = Self::validate_block_gas_capacity(block_gas_capacity)?; @@ -198,6 +198,10 @@ where )?; tracing::debug!("Committing transaction"); + let metadata = self.algorithm_updater.clone().into(); + storage_tx + .set_metadata(&metadata) + .map_err(|err| anyhow!(err))?; StorageTxProvider::commit_transaction(storage_tx)?; let new_algo = self.algorithm_updater.algorithm(); self.shared_algo.update(new_algo).await; diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 51734f6ed7e..65b9b9f4496 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -24,6 +24,7 @@ use crate::{ TransactionableStorage, }, v1::{ + algorithm::SharedV1Algorithm, da_source_service::{ service::{ DaBlockCostsSource, @@ -32,6 +33,7 @@ use crate::{ DaBlockCosts, }, metadata::{ + updater_from_config, V1AlgorithmConfig, V1Metadata, }, @@ -324,8 +326,9 @@ async fn next_gas_price__affected_by_new_l2_block() { #[tokio::test] async fn run__new_l2_block_saves_old_metadata() { // given + let height = 1; let l2_block = BlockInfo::Block { - height: 1, + height, gas_used: 60, block_gas_capacity: 100, block_bytes: 100, @@ -335,16 +338,11 @@ async fn run__new_l2_block_saves_old_metadata() { let l2_block_source = FakeL2BlockSource { l2_block: l2_block_receiver, }; - let metadata_inner = Arc::new(std::sync::Mutex::new(None)); - let metadata_storage = FakeMetadata { - inner: metadata_inner.clone(), - }; let config = zero_threshold_arbitrary_config(); - let height = 0; let inner = database(); - let (algo_updater, shared_algo) = - initialize_algorithm(&config, height, &metadata_storage).unwrap(); + let algo_updater = updater_from_config(&config); + let shared_algo = SharedV1Algorithm::new_with_algorithm(algo_updater.algorithm()); let da_source = FakeDABlockCost::never_returns(); let da_source_service = DaSourceService::new(da_source, None); let mut service = GasPriceServiceV1::new( @@ -354,16 +352,22 @@ async fn run__new_l2_block_saves_old_metadata() { da_source_service, inner, ); - let mut watcher = StateWatcher::default(); + let mut watcher = StateWatcher::started(); // when - service.run(&mut watcher).await; l2_block_sender.send(l2_block).await.unwrap(); - service.shutdown().await.unwrap(); + service.run(&mut watcher).await; // then - let metadata_is_some = metadata_inner.lock().unwrap().is_some(); - assert!(metadata_is_some) + let metadata_is_some = service + .storage_tx_provider + .get_metadata(&height.into()) + .unwrap() + .is_some(); + assert!(metadata_is_some); + + // cleanup + service.shutdown().await.unwrap(); } #[derive(Clone)] From 8009b07a7d150bfbed4d5a4d2c14494298f0d46f Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 4 Dec 2024 14:07:37 -0700 Subject: [PATCH 057/131] Remove commented code --- .../v1/tests/update_da_record_data_tests.rs | 109 ++---------------- .../v1/tests/update_l2_block_data_tests.rs | 4 - .../gas_price_service/src/v1/service.rs | 30 ----- 3 files changed, 7 insertions(+), 136 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index f8804916e61..c262352a311 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -111,20 +111,6 @@ fn update_da_record_data__updates_known_total_cost() { let l2_block_height = 15; let projected_total_cost = 2000; let known_total_cost = 1500; - // let unrecorded_blocks = vec![ - // BlockBytes { - // height: 11, - // block_bytes: 1000, - // }, - // BlockBytes { - // height: 12, - // block_bytes: 2000, - // }, - // BlockBytes { - // height: 13, - // block_bytes: 1500, - // }, - // ]; let mut unrecorded_blocks: BTreeMap<_, _> = [(11, 1000), (12, 2000), (13, 1500)].into_iter().collect(); let mut updater = UpdaterBuilder::new() @@ -160,20 +146,6 @@ fn update_da_record_data__if_da_height_matches_l2_height_projected_and_known_mat let da_cost_per_byte = 20; let l2_block_height = 13; let known_total_cost = 1500; - // let unrecorded_blocks = vec![ - // BlockBytes { - // height: 11, - // block_bytes: 1000, - // }, - // BlockBytes { - // height: 12, - // block_bytes: 2000, - // }, - // BlockBytes { - // height: 13, - // block_bytes: 1500, - // }, - // ]; let mut unrecorded_blocks: BTreeMap<_, _> = [(11, 1000), (12, 2000), (13, 1500)].into_iter().collect(); @@ -224,20 +196,6 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g let original_known_total_cost: u128 = 1500; let block_bytes = 1000; let remaining = vec![(14, block_bytes), (15, block_bytes)]; - // let mut unrecorded_blocks = vec![ - // BlockBytes { - // height: 11, - // block_bytes: 1000, - // }, - // BlockBytes { - // height: 12, - // block_bytes: 2000, - // }, - // BlockBytes { - // height: 13, - // block_bytes: 1500, - // }, - // ]; let mut pairs = vec![(11, 1000), (12, 2000), (13, 1500)]; pairs.extend(remaining.clone()); @@ -289,20 +247,6 @@ fn update_da_record_data__updates_known_total_cost_if_blocks_are_out_of_order() // given let da_cost_per_byte = 20; let block_bytes = 1000; - // let unrecorded_blocks = vec![ - // BlockBytes { - // height: 1, - // block_bytes, - // }, - // BlockBytes { - // height: 2, - // block_bytes, - // }, - // BlockBytes { - // height: 3, - // block_bytes, - // }, - // ]; let mut unrecorded_blocks: BTreeMap<_, _> = [(1, block_bytes), (2, block_bytes), (3, block_bytes)] .into_iter() @@ -345,20 +289,6 @@ fn update_da_record_data__updates_projected_total_cost_if_blocks_are_out_of_orde // given let da_cost_per_byte = 20; let block_bytes = 1000; - // let unrecorded_blocks = vec![ - // BlockBytes { - // height: 1, - // block_bytes, - // }, - // BlockBytes { - // height: 2, - // block_bytes, - // }, - // BlockBytes { - // height: 3, - // block_bytes, - // }, - // ]; let mut unrecorded_blocks: BTreeMap<_, _> = [(1, block_bytes), (2, block_bytes), (3, block_bytes)] .into_iter() @@ -401,20 +331,6 @@ fn update_da_record_data__updates_unrecorded_blocks() { // given let da_cost_per_byte = 20; let block_bytes = 1000; - // let unrecorded_blocks = vec![ - // BlockBytes { - // height: 1, - // block_bytes, - // }, - // BlockBytes { - // height: 2, - // block_bytes, - // }, - // BlockBytes { - // height: 3, - // block_bytes, - // }, - // ]; let mut unrecorded_blocks: BTreeMap<_, _> = [(1, block_bytes), (2, block_bytes), (3, block_bytes)] .into_iter() @@ -510,10 +426,6 @@ fn update_da_record_data__da_block_increases_da_gas_price() { let da_cost_per_byte = 40; let l2_block_height = 11; let original_known_total_cost = 150; - // let unrecorded_blocks = vec![BlockBytes { - // height: 11, - // block_bytes: 3000, - // }]; let mut unrecorded_blocks: BTreeMap<_, _> = [(11, 3000)].into_iter().collect(); let da_p_component = 2; let guessed_cost: u64 = unrecorded_blocks @@ -533,17 +445,13 @@ fn update_da_record_data__da_block_increases_da_gas_price() { .build(); let new_cost_per_byte = 100; - let (recorded_heights, recorded_cost) = unrecorded_blocks - // .iter() - // .fold((vec![], 0), |(mut range, cost), block| { - // range.push(block.height); - // (range, cost + block.block_bytes * new_cost_per_byte) - // }); - .iter() - .fold((vec![], 0), |(mut range, cost), (height, bytes)| { - range.push(height); - (range, cost + bytes * new_cost_per_byte) - }); + let (recorded_heights, recorded_cost) = unrecorded_blocks.iter().fold( + (vec![], 0), + |(mut range, cost), (height, bytes)| { + range.push(height); + (range, cost + bytes * new_cost_per_byte) + }, + ); let min = *recorded_heights.iter().min().unwrap(); let max = *recorded_heights.iter().max().unwrap(); @@ -575,9 +483,6 @@ fn update_da_record_data__da_block_will_not_change_da_gas_price() { let da_cost_per_byte = 40; let l2_block_height = 11; let original_known_total_cost = 150; - // let unrecorded_blocks = vec![BlockBytes { height: 11, - // block_bytes: 3000, - // }]; let mut unrecorded_blocks: BTreeMap<_, _> = [(11, 3000)].into_iter().collect(); let da_p_component = 2; let guessed_cost: u64 = unrecorded_blocks diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs index 25bfab5c860..25a01b0eaba 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs @@ -644,10 +644,6 @@ fn update_l2_block_data__retains_existing_blocks_and_adds_l2_block_to_unrecorded // given let starting_block = 0; let first_block_bytes = 1200; - // let preexisting_block = BlockBytes { - // height: 0, - // block_bytes: first_block_bytes, - // }; let unrecorded_blocks: BTreeMap<_, _> = vec![(0, first_block_bytes)].into_iter().collect(); diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index a0ddfa00ea6..db7cfbd3fe5 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -97,19 +97,6 @@ where self.apply_block_info_to_gas_algorithm(block).await?; Ok(()) } - - // async fn process_da_block_costs_res( - // &mut self, - // da_block_costs: Result, - // ) -> anyhow::Result<()> { - // tracing::info!("Received DA block costs: {:?}", da_block_costs); - // let da_block_costs = da_block_costs?; - // - // tracing::debug!("Updating DA block costs"); - // self.apply_da_block_costs_to_gas_algorithm(da_block_costs) - // .await?; - // Ok(()) - // } } impl GasPriceServiceV1 @@ -156,14 +143,6 @@ where .ok_or_else(|| anyhow!("Block gas capacity must be non-zero")) } - // async fn set_metadata( - // &mut self, - // tx: &mut StorageTxProvider::Transaction, - // ) -> anyhow::Result<()> { - // let metadata: UpdaterMetadata = self.algorithm_updater.clone().into(); - // tx.set_metadata(&metadata).map_err(|err| anyhow!(err)) - // } - async fn handle_normal_block<'a>( &'a mut self, height: u32, @@ -263,15 +242,6 @@ where Ok(()) } - - // async fn apply_da_block_costs_to_gas_algorithm( - // &mut self, - // da_block_costs: DaBlockCosts, - // ) -> anyhow::Result<()> { - // self.handle_da_block_costs(da_block_costs).await?; - // self.update(self.algorithm_updater.algorithm()).await; - // Ok(()) - // } } #[async_trait] From 0bb7e7a6b3fb2378d04628e7a9f122db39a0babe Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 4 Dec 2024 14:11:04 -0700 Subject: [PATCH 058/131] More cleanup --- crates/fuel-gas-price-algorithm/src/v1.rs | 5 ----- crates/services/gas_price_service/src/v1/service.rs | 6 ------ 2 files changed, 11 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 8a160a468b5..6b36786d4a2 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -366,11 +366,6 @@ impl AlgorithmUpdaterV1 { ) -> Result<(), Error> { let expected = self.l2_block_height.saturating_add(1); if height != expected { - tracing::debug!( - "Skipped L2 block update: expected {}, got {}", - expected, - height - ); Err(Error::SkippedL2Block { expected, got: height, diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index db7cfbd3fe5..13c94fc3c2a 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -81,7 +81,6 @@ where L2: L2BlockSource, DA: DaBlockCostsSource, StorageTxProvider: TransactionableStorage, - // StorageTxProvider::Transaction<'a>: UnrecordedBlocks + MetadataStorage, { async fn commit_block_data_to_algorithm<'a>( &'a mut self, @@ -176,7 +175,6 @@ where &mut storage_tx, )?; - tracing::debug!("Committing transaction"); let metadata = self.algorithm_updater.clone().into(); storage_tx .set_metadata(&metadata) @@ -512,10 +510,6 @@ mod tests { #[tokio::test] async fn run__updates_gas_price_with_da_block_cost_source() { - let _ = tracing_subscriber::fmt() - .with_max_level(tracing::Level::DEBUG) - .try_init(); - // given let block_height = 2; let l2_block_2 = BlockInfo::Block { From ed9dab485a25ee11acd1582716d00e2b8833e5d4 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 4 Dec 2024 14:19:27 -0700 Subject: [PATCH 059/131] Remove unused dep --- Cargo.lock | 1 - crates/services/gas_price_service/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a541eae8dd7..56a3736a59f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,7 +3455,6 @@ dependencies = [ "tokio", "tokio-stream", "tracing", - "tracing-subscriber", ] [[package]] diff --git a/crates/services/gas_price_service/Cargo.toml b/crates/services/gas_price_service/Cargo.toml index 4eab33d503b..9903bba7d03 100644 --- a/crates/services/gas_price_service/Cargo.toml +++ b/crates/services/gas_price_service/Cargo.toml @@ -32,4 +32,3 @@ tracing = { workspace = true } fuel-core-services = { workspace = true, features = ["test-helpers"] } fuel-core-storage = { workspace = true, features = ["test-helpers"] } fuel-core-types = { path = "./../../types", features = ["test-helpers"] } -tracing-subscriber = { workspace = true } From 6c5df4cf994daf6fd6d27474f79392465c9592a7 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 4 Dec 2024 14:21:30 -0700 Subject: [PATCH 060/131] Update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 750d298b91a..c5f9e0e8c03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [2389](https://github.com/FuelLabs/fuel-core/pull/2389): Fix construction of reverse iterator in RocksDB. ### Changed -- [2468](https://github.com/FuelLabs/fuel-core/pull/2468): Abstract unrecorded blocks concept for V1 algorithm, create new storage impl +- [2468](https://github.com/FuelLabs/fuel-core/pull/2468): Abstract unrecorded blocks concept for V1 algorithm, create new storage impl. Introduce `TransactionableStorage` trait to allow atomic changes to the storage. - [2295](https://github.com/FuelLabs/fuel-core/pull/2295): `CombinedDb::from_config` now respects `state_rewind_policy` with tmp RocksDB. - [2378](https://github.com/FuelLabs/fuel-core/pull/2378): Use cached hash of the topic instead of calculating it on each publishing gossip message. - [2429](https://github.com/FuelLabs/fuel-core/pull/2429): Introduce custom enum for representing result of running service tasks From a6caf6cbf3ec7081c007a8f3c0f8e6ba554aa56a Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 4 Dec 2024 15:00:24 -0700 Subject: [PATCH 061/131] Add test for updating sequence number --- .../src/common/fuel_core_storage_adapter.rs | 44 ++++- .../fuel_core_storage_adapter/storage.rs | 19 ++ .../services/gas_price_service/src/ports.rs | 9 + .../src/v1/da_source_service.rs | 2 + .../block_committer_costs.rs | 1 + .../gas_price_service/src/v1/service.rs | 183 ++++++++++++++---- .../gas_price_service/src/v1/tests.rs | 5 + .../src/v1/uninitialized_task.rs | 6 +- 8 files changed, 220 insertions(+), 49 deletions(-) diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs index 13cfcf9ad57..54e3eabd65c 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs @@ -11,11 +11,13 @@ use crate::{ fuel_core_storage_adapter::storage::{ GasPriceColumn, GasPriceMetadata, + SequenceNumberTable, UnrecordedBlocksTable, }, updater_metadata::UpdaterMetadata, }, ports::{ + DaSequenceNumberTracker, MetadataStorage, TransactionableStorage, }, @@ -93,17 +95,17 @@ where } } -pub struct WrappedStorageTransaction<'a, Storage> { +pub struct NewStorageTransaction<'a, Storage> { inner: StorageTransaction<&'a mut StructuredStorage>, } -impl<'a, Storage> WrappedStorageTransaction<'a, Storage> { +impl<'a, Storage> NewStorageTransaction<'a, Storage> { fn wrap(inner: StorageTransaction<&'a mut StructuredStorage>) -> Self { Self { inner } } } -impl<'a, Storage> UnrecordedBlocks for WrappedStorageTransaction<'a, Storage> +impl<'a, Storage> UnrecordedBlocks for NewStorageTransaction<'a, Storage> where Storage: KeyValueInspect + Modifiable + Send + Sync, { @@ -133,11 +135,11 @@ impl TransactionableStorage for StructuredStorage where Storage: Modifiable + Send + Sync, { - type Transaction<'a> = WrappedStorageTransaction<'a, Storage> where Self: 'a; + type Transaction<'a> = NewStorageTransaction<'a, Storage> where Self: 'a; fn begin_transaction(&mut self) -> GasPriceResult> { let tx = self.write_transaction(); - let wrapped = WrappedStorageTransaction::wrap(tx); + let wrapped = NewStorageTransaction::wrap(tx); Ok(wrapped) } @@ -150,7 +152,7 @@ where } } -impl<'a, Storage> MetadataStorage for WrappedStorageTransaction<'a, Storage> +impl<'a, Storage> MetadataStorage for NewStorageTransaction<'a, Storage> where Storage: KeyValueInspect + Modifiable + Send + Sync, { @@ -181,6 +183,36 @@ where } } +impl<'a, Storage> DaSequenceNumberTracker for NewStorageTransaction<'a, Storage> +where + Storage: KeyValueInspect + Modifiable + Send + Sync, +{ + fn get_sequence_number( + &self, + block_height: &BlockHeight, + ) -> GasPriceResult> { + let sequence_number = self + .inner + .storage::() + .get(block_height) + .map_err(|err| GasPriceError::CouldNotFetchDARecord(err.into()))? + .map(|no| *no); + Ok(sequence_number) + } + + fn set_sequence_number( + &mut self, + block_height: &BlockHeight, + sequence_number: u32, + ) -> GasPriceResult<()> { + self.inner + .storage_as_mut::() + .insert(block_height, &sequence_number) + .map_err(|err| GasPriceError::CouldNotFetchDARecord(err.into()))?; + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq)] pub struct GasPriceSettings { pub gas_price_factor: u64, diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs index 4fb8fc8f28f..e6cb745b36c 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs @@ -28,6 +28,7 @@ pub enum GasPriceColumn { Metadata = 0, State = 1, UnrecordedBlocks = 2, + SequenceNumber = 3, } impl GasPriceColumn { @@ -89,3 +90,21 @@ impl TableWithBlueprint for UnrecordedBlocksTable { GasPriceColumn::UnrecordedBlocks } } + +pub struct SequenceNumberTable; + +impl Mappable for SequenceNumberTable { + type Key = Self::OwnedKey; + type OwnedKey = u32; + type Value = Self::OwnedValue; + type OwnedValue = u32; +} + +impl TableWithBlueprint for SequenceNumberTable { + type Blueprint = Plain, Postcard>; + type Column = GasPriceColumn; + + fn column() -> Self::Column { + GasPriceColumn::SequenceNumber + } +} diff --git a/crates/services/gas_price_service/src/ports.rs b/crates/services/gas_price_service/src/ports.rs index d693ebc6b13..2a451d068e1 100644 --- a/crates/services/gas_price_service/src/ports.rs +++ b/crates/services/gas_price_service/src/ports.rs @@ -28,6 +28,15 @@ pub trait MetadataStorage: Send + Sync { fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> Result<()>; } +pub trait DaSequenceNumberTracker: Send + Sync { + fn get_sequence_number(&self, block_height: &BlockHeight) -> Result>; + fn set_sequence_number( + &mut self, + block_height: &BlockHeight, + sequence_number: u32, + ) -> Result<()>; +} + pub trait TransactionableStorage: Send + Sync { type Transaction<'a> where diff --git a/crates/services/gas_price_service/src/v1/da_source_service.rs b/crates/services/gas_price_service/src/v1/da_source_service.rs index ff2c703b518..e649d3f4222 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service.rs @@ -7,6 +7,7 @@ pub mod service; #[derive(Debug, Default, Clone, Eq, Hash, PartialEq)] pub struct DaBlockCosts { + pub sequence_number: u32, pub l2_blocks: Vec, pub blob_size_bytes: u32, pub blob_cost_wei: u128, @@ -30,6 +31,7 @@ mod tests { async fn run__when_da_block_cost_source_gives_value_shared_state_is_updated() { // given let expected_da_cost = DaBlockCosts { + sequence_number: 1, l2_blocks: (0..10).collect(), blob_size_bytes: 1024 * 128, blob_cost_wei: 2, diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index 59498849996..a937e5a5a45 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -54,6 +54,7 @@ pub struct RawDaBlockCosts { impl From<&RawDaBlockCosts> for DaBlockCosts { fn from(raw_da_block_costs: &RawDaBlockCosts) -> Self { DaBlockCosts { + sequence_number: raw_da_block_costs.sequence_number, l2_blocks: raw_da_block_costs .blocks_heights .clone() diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 13c94fc3c2a..c66134b164e 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -1,24 +1,5 @@ use std::num::NonZeroU64; -use anyhow::anyhow; -use async_trait::async_trait; -use fuel_core_services::{ - RunnableService, - RunnableTask, - StateWatcher, - TaskNextAction, -}; -use fuel_gas_price_algorithm::{ - v0::AlgorithmUpdaterV0, - v1::{ - AlgorithmUpdaterV1, - AlgorithmV1, - UnrecordedBlocks, - }, -}; -use futures::FutureExt; -use tokio::sync::broadcast::Receiver; - use crate::{ common::{ gas_price_algorithm::SharedGasPriceAlgo, @@ -30,6 +11,7 @@ use crate::{ }, }, ports::{ + DaSequenceNumberTracker, MetadataStorage, TransactionableStorage, }, @@ -53,6 +35,25 @@ use crate::{ uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, }, }; +use anyhow::anyhow; +use async_trait::async_trait; +use fuel_core_services::{ + RunnableService, + RunnableTask, + StateWatcher, + TaskNextAction, +}; +use fuel_core_types::fuel_types::BlockHeight; +use fuel_gas_price_algorithm::{ + v0::AlgorithmUpdaterV0, + v1::{ + AlgorithmUpdaterV1, + AlgorithmV1, + UnrecordedBlocks, + }, +}; +use futures::FutureExt; +use tokio::sync::broadcast::Receiver; /// The service that updates the gas price algorithm. pub struct GasPriceServiceV1 @@ -87,7 +88,8 @@ where l2_block_res: GasPriceResult, ) -> anyhow::Result<()> where - StorageTxProvider::Transaction<'a>: MetadataStorage + UnrecordedBlocks, + StorageTxProvider::Transaction<'a>: + MetadataStorage + UnrecordedBlocks + DaSequenceNumberTracker, { tracing::info!("Received L2 block result: {:?}", l2_block_res); let block = l2_block_res?; @@ -151,7 +153,8 @@ where block_fees: u64, ) -> anyhow::Result<()> where - StorageTxProvider::Transaction<'a>: UnrecordedBlocks + MetadataStorage, + StorageTxProvider::Transaction<'a>: + UnrecordedBlocks + MetadataStorage + DaSequenceNumberTracker, { let mut storage_tx = self.storage_tx_provider.begin_transaction()?; let capacity = Self::validate_block_gas_capacity(block_gas_capacity)?; @@ -164,6 +167,12 @@ where da_block_costs.blob_cost_wei, &mut storage_tx, )?; + storage_tx + .set_sequence_number( + &BlockHeight::from(height), + da_block_costs.sequence_number, + ) + .map_err(|err| anyhow!(err))?; } self.algorithm_updater.update_l2_block_data( @@ -185,30 +194,13 @@ where Ok(()) } - async fn handle_da_block_costs<'a>( - &mut self, - da_block_costs: DaBlockCosts, - unrecorded_blocks: &mut StorageTxProvider::Transaction<'a>, - ) -> anyhow::Result<()> - where - StorageTxProvider::Transaction<'a>: UnrecordedBlocks, - { - self.algorithm_updater.update_da_record_data( - &da_block_costs.l2_blocks, - da_block_costs.blob_size_bytes, - da_block_costs.blob_cost_wei, - unrecorded_blocks, - )?; - - Ok(()) - } - async fn apply_block_info_to_gas_algorithm<'a>( &'a mut self, l2_block: BlockInfo, ) -> anyhow::Result<()> where - StorageTxProvider::Transaction<'a>: MetadataStorage + UnrecordedBlocks, + StorageTxProvider::Transaction<'a>: + MetadataStorage + UnrecordedBlocks + DaSequenceNumberTracker, { match l2_block { BlockInfo::GenesisBlock => { @@ -249,7 +241,8 @@ where L2: L2BlockSource, DA: DaBlockCostsSource, StorageTxProvider: TransactionableStorage + 'static, - for<'a> StorageTxProvider::Transaction<'a>: UnrecordedBlocks + MetadataStorage, + for<'a> StorageTxProvider::Transaction<'a>: + UnrecordedBlocks + MetadataStorage + DaSequenceNumberTracker, { async fn run(&mut self, watcher: &mut StateWatcher) -> TaskNextAction { tokio::select! { @@ -368,6 +361,7 @@ mod tests { fuel_core_storage_adapter::storage::{ GasPriceColumn, GasPriceColumn::UnrecordedBlocks, + SequenceNumberTable, UnrecordedBlocksTable, }, gas_price_algorithm::SharedGasPriceAlgo, @@ -559,6 +553,7 @@ mod tests { let da_source = DaSourceService::new( DummyDaBlockCosts::new( Ok(DaBlockCosts { + sequence_number: 1, l2_blocks: (1..2).collect(), blob_cost_wei: 9000, blob_size_bytes: 3000, @@ -601,4 +596,110 @@ mod tests { let actual_price = read_algo.next_gas_price(); assert_ne!(initial_price, actual_price); } + + fn arbitrary_v1_algorithm_config() -> V1AlgorithmConfig { + V1AlgorithmConfig { + new_exec_gas_price: 100, + min_exec_gas_price: 50, + exec_gas_price_change_percent: 0, + l2_block_fullness_threshold_percent: 20, + gas_price_factor: NonZeroU64::new(10).unwrap(), + min_da_gas_price: 0, + max_da_gas_price_change_percent: 100, + da_p_component: 4, + da_d_component: 2, + normal_range_size: 10, + capped_range_size: 100, + decrease_range_size: 4, + block_activity_threshold: 20, + } + } + + #[tokio::test] + async fn run__responses_from_da_service_update_sequence_number_in_storage() { + // given + let sequence_number = 1234; + let block_height = 2; + let l2_block_2 = BlockInfo::Block { + height: block_height, + gas_used: 60, + block_gas_capacity: 100, + block_bytes: 100, + block_fees: 100, + }; + + let (l2_block_sender, l2_block_receiver) = mpsc::channel(1); + let l2_block_source = FakeL2BlockSource { + l2_block: l2_block_receiver, + }; + + let metadata_storage = FakeMetadata::empty(); + // Configured so exec gas price doesn't change, only da gas price + let config = arbitrary_v1_algorithm_config(); + let mut inner = database(); + let mut tx = inner.write_transaction(); + tx.storage_as_mut::() + .insert(&BlockHeight::from(1), &100) + .unwrap(); + tx.commit().unwrap(); + let mut algo_updater = updater_from_config(&config); + let shared_algo = + SharedGasPriceAlgo::new_with_algorithm(algo_updater.algorithm()); + algo_updater.l2_block_height = block_height - 1; + algo_updater.last_profit = 10_000; + algo_updater.new_scaled_da_gas_price = 10_000_000; + + let notifier = Arc::new(tokio::sync::Notify::new()); + let da_source = DaSourceService::new( + DummyDaBlockCosts::new( + Ok(DaBlockCosts { + sequence_number, + l2_blocks: (1..2).collect(), + blob_cost_wei: 9000, + blob_size_bytes: 3000, + }), + notifier.clone(), + ), + Some(Duration::from_millis(1)), + ); + let mut watcher = StateWatcher::started(); + + let mut service = GasPriceServiceV1::new( + l2_block_source, + shared_algo, + algo_updater, + da_source, + inner, + ); + let read_algo = service.next_block_algorithm(); + let initial_price = read_algo.next_gas_price(); + + // the RunnableTask depends on the handle passed to it for the da block cost source to already be running, + // which is the responsibility of the UninitializedTask in the `into_task` method of the RunnableService + // here we mimic that behaviour by running the da block cost service. + let mut da_source_watcher = StateWatcher::started(); + service + .da_source_adapter_handle + .run(&mut da_source_watcher) + .await; + + service.run(&mut watcher).await; + tokio::time::sleep(Duration::from_millis(100)).await; + l2_block_sender.send(l2_block_2).await.unwrap(); + + // when + service.run(&mut watcher).await; + tokio::time::sleep(Duration::from_millis(100)).await; + + // then + let latest_sequence_number = service + .storage_tx_provider + .storage::() + .get(&BlockHeight::from(block_height)) + .unwrap() + .unwrap(); + assert_eq!(*latest_sequence_number, sequence_number); + + service.shutdown().await.unwrap(); + } } diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 65b9b9f4496..36e23853e80 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -578,3 +578,8 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { let is_err = res.is_err(); assert!(is_err); } + +#[tokio::test] +async fn uninitialized_task__starts_da_service_with_sequence_number_in_storage() { + todo!() +} diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 19e99854a29..c30f6ce9767 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -17,6 +17,7 @@ use crate::{ }, }, ports::{ + DaSequenceNumberTracker, GasPriceData, GasPriceServiceConfig, L2Data, @@ -236,7 +237,7 @@ where SettingsProvider: GasPriceSettingsProvider + 'static, PersistedData: MetadataStorage + TransactionableStorage + 'static, for<'a> ::Transaction<'a>: - MetadataStorage + UnrecordedBlocks, + MetadataStorage + UnrecordedBlocks + DaSequenceNumberTracker, { const NAME: &'static str = "GasPriceServiceV1"; type SharedData = SharedV1Algorithm; @@ -402,7 +403,8 @@ where SettingsProvider: GasPriceSettingsProvider, DA: DaBlockCostsSource, PersistedData: MetadataStorage + TransactionableStorage + 'static, - for<'a> PersistedData::Transaction<'a>: MetadataStorage + UnrecordedBlocks, + for<'a> PersistedData::Transaction<'a>: + MetadataStorage + UnrecordedBlocks + DaSequenceNumberTracker, { let v1_config = config.v1().ok_or(anyhow::anyhow!("Expected V1 config"))?; let gas_price_init = UninitializedTask::new( From 5dcfcb7885a795268c216bd774c6f85d013bf308 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 4 Dec 2024 16:10:07 -0700 Subject: [PATCH 062/131] Add test for reading sequence number --- .../block_committer_costs.rs | 7 +- .../src/v1/da_source_service/dummy_costs.rs | 4 + .../src/v1/da_source_service/service.rs | 1 + .../gas_price_service/src/v1/tests.rs | 149 +++++++++++++++--- .../src/v1/uninitialized_task.rs | 24 ++- 5 files changed, 158 insertions(+), 27 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index a937e5a5a45..4ff70796b0a 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -15,7 +15,7 @@ use serde::{ }; #[async_trait::async_trait] -trait BlockCommitterApi: Send + Sync { +pub trait BlockCommitterApi: Send + Sync { /// Used on first run to get the latest costs and seqno async fn get_latest_costs(&self) -> DaBlockCostsResult>; /// Used to get the costs for a specific seqno @@ -117,6 +117,11 @@ where self.last_raw_da_block_costs = Some(raw_da_block_costs.clone()); Ok(da_block_costs) } + async fn set_last_value(&mut self, sequence_number: u32) -> DaBlockCostsResult<()> { + self.last_raw_da_block_costs = + self.client.get_costs_by_seqno(sequence_number).await?; + Ok(()) + } } pub struct BlockCommitterHttpApi { diff --git a/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs index 8fe22a89e0b..ddeb88e4d0e 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs @@ -34,4 +34,8 @@ impl DaBlockCostsSource for DummyDaBlockCosts { } } } + + async fn set_last_value(&mut self, sequence_number: u32) -> DaBlockCostsResult<()> { + todo!() + } } diff --git a/crates/services/gas_price_service/src/v1/da_source_service/service.rs b/crates/services/gas_price_service/src/v1/da_source_service/service.rs index 328f73e6a20..7800d80489c 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/service.rs @@ -72,6 +72,7 @@ where #[async_trait::async_trait] pub trait DaBlockCostsSource: Send + Sync { async fn request_da_block_cost(&mut self) -> Result; + async fn set_last_value(&mut self, sequence_number: u32) -> Result<()>; } #[async_trait::async_trait] diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 36e23853e80..0abeb5e8ded 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -8,6 +8,7 @@ use crate::{ }, GasPriceSettings, GasPriceSettingsProvider, + NewStorageTransaction, }, l2_block_source::L2BlockSource, updater_metadata::UpdaterMetadata, @@ -18,6 +19,7 @@ use crate::{ }, }, ports::{ + DaSequenceNumberTracker, GasPriceData, L2Data, MetadataStorage, @@ -26,6 +28,10 @@ use crate::{ v1::{ algorithm::SharedV1Algorithm, da_source_service::{ + block_committer_costs::{ + BlockCommitterApi, + RawDaBlockCosts, + }, service::{ DaBlockCostsSource, DaSourceService, @@ -47,7 +53,10 @@ use crate::{ }, }, }; -use anyhow::anyhow; +use anyhow::{ + anyhow, + Result, +}; use fuel_core_services::{ stream::{ BoxStream, @@ -73,6 +82,7 @@ use fuel_core_types::{ block::Block, header::ConsensusParametersVersion, }, + fuel_asm::op::exp, fuel_tx::Transaction, fuel_types::BlockHeight, services::block_importer::{ @@ -89,8 +99,14 @@ use fuel_gas_price_algorithm::v1::{ }; use std::{ num::NonZeroU64, - ops::Deref, - sync::Arc, + ops::{ + Deref, + Range, + }, + sync::{ + Arc, + Mutex, + }, }; use tokio::sync::mpsc::Receiver; @@ -156,7 +172,7 @@ impl TransactionableStorage for ErroringPersistedData { todo!() } - fn commit_transaction(transaction: Self::Transaction<'_>) -> GasPriceResult<()> { + fn commit_transaction(_transaction: Self::Transaction<'_>) -> GasPriceResult<()> { todo!() } } @@ -164,28 +180,46 @@ impl TransactionableStorage for ErroringPersistedData { impl MetadataStorage for UnimplementedStorageTx { fn get_metadata( &self, - block_height: &BlockHeight, + _block_height: &BlockHeight, ) -> GasPriceResult> { - todo!() + unimplemented!() } - fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { - todo!() + fn set_metadata(&mut self, _metadata: &UpdaterMetadata) -> GasPriceResult<()> { + unimplemented!() } } impl UnrecordedBlocks for UnimplementedStorageTx { - fn insert(&mut self, height: Height, bytes: Bytes) -> Result<(), Error> { - todo!() + fn insert(&mut self, _height: Height, _bytes: Bytes) -> Result<(), Error> { + unimplemented!() } - fn remove(&mut self, height: &Height) -> Result, Error> { - todo!() + fn remove(&mut self, _height: &Height) -> Result, Error> { + unimplemented!() + } +} + +impl DaSequenceNumberTracker for UnimplementedStorageTx { + fn get_sequence_number( + &self, + _block_height: &BlockHeight, + ) -> GasPriceResult> { + unimplemented!() + } + + fn set_sequence_number( + &mut self, + _block_height: &BlockHeight, + _sequence_number: u32, + ) -> GasPriceResult<()> { + unimplemented!() } } struct FakeDABlockCost { da_block_costs: Receiver, + sequence_number: Arc>>, } impl FakeDABlockCost { @@ -193,20 +227,42 @@ impl FakeDABlockCost { let (_sender, receiver) = tokio::sync::mpsc::channel(1); Self { da_block_costs: receiver, + sequence_number: Arc::new(Mutex::new(None)), } } fn new(da_block_costs: Receiver) -> Self { - Self { da_block_costs } + Self { + da_block_costs, + sequence_number: Arc::new(Mutex::new(None)), + } + } + + fn never_returns_with_handle_to_sequence_number() -> (Self, Arc>>) { + let (_sender, receiver) = tokio::sync::mpsc::channel(1); + let sequence_number = Arc::new(Mutex::new(None)); + let service = Self { + da_block_costs: receiver, + sequence_number: sequence_number.clone(), + }; + (service, sequence_number) } } #[async_trait::async_trait] impl DaBlockCostsSource for FakeDABlockCost { - async fn request_da_block_cost(&mut self) -> anyhow::Result { + async fn request_da_block_cost(&mut self) -> Result { let costs = self.da_block_costs.recv().await.unwrap(); Ok(costs) } + + async fn set_last_value(&mut self, sequence_number: u32) -> Result<()> { + self.sequence_number + .lock() + .unwrap() + .replace(sequence_number); + Ok(()) + } } fn zero_threshold_arbitrary_config() -> V1AlgorithmConfig { @@ -383,12 +439,26 @@ impl GasPriceSettingsProvider for FakeSettings { } #[derive(Clone)] -struct FakeGasPriceDb; +struct FakeGasPriceDb { + height: Option, +} + +impl FakeGasPriceDb { + fn new(height: u32) -> Self { + Self { + height: Some(height.into()), + } + } + + fn empty() -> Self { + Self { height: None } + } +} // GasPriceData + Modifiable + KeyValueInspect impl GasPriceData for FakeGasPriceDb { fn latest_height(&self) -> Option { - unimplemented!() + self.height } } @@ -452,7 +522,7 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr let different_l2_block = 0; let settings = FakeSettings; let block_stream = empty_block_stream(); - let gas_price_db = FakeGasPriceDb; + let gas_price_db = FakeGasPriceDb::empty(); let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); let inner = database_with_metadata(&original_metadata); @@ -558,7 +628,7 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { let erroring_persisted_data = ErroringPersistedData; let settings = FakeSettings; let block_stream = empty_block_stream(); - let gas_price_db = FakeGasPriceDb; + let gas_price_db = FakeGasPriceDb::empty(); let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); @@ -580,6 +650,45 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { } #[tokio::test] -async fn uninitialized_task__starts_da_service_with_sequence_number_in_storage() { - todo!() +async fn uninitialized_task__init__starts_da_service_with_sequence_number_in_storage() { + // given + let block_height = 1; + let sequence_number: u32 = 123; + let original_metadata = arbitrary_metadata(); + + let different_config = different_arb_config(); + let descaleed_exec_price = + original_metadata.new_scaled_exec_price / original_metadata.gas_price_factor; + assert_ne!(different_config.new_exec_gas_price, descaleed_exec_price); + let different_l2_block = 0; + let settings = FakeSettings; + let block_stream = empty_block_stream(); + let gas_price_db = FakeGasPriceDb::new(block_height); + let on_chain_db = FakeOnChainDb::new(different_l2_block); + let (da_cost_source, sequence_number_handle) = + FakeDABlockCost::never_returns_with_handle_to_sequence_number(); + let mut inner = database_with_metadata(&original_metadata); + let mut tx = inner.begin_transaction().unwrap(); + tx.set_sequence_number(&block_height.into(), sequence_number) + .unwrap(); + StorageTransaction::commit_transaction(tx).unwrap(); + let service = UninitializedTask::new( + different_config.clone(), + 0.into(), + settings, + block_stream, + gas_price_db, + da_cost_source, + on_chain_db, + inner, + ) + .unwrap(); + + // when + service.init().await.unwrap(); + + // then + let actual = sequence_number_handle.lock().unwrap(); + let expected = Some(sequence_number); + assert_eq!(*actual, expected); } diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index c30f6ce9767..56d6cae7eeb 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -26,9 +26,15 @@ use crate::{ }, v1::{ algorithm::SharedV1Algorithm, - da_source_service::service::{ - DaBlockCostsSource, - DaSourceService, + da_source_service::{ + block_committer_costs::{ + BlockCommitterApi, + BlockCommitterDaBlockCosts, + }, + service::{ + DaBlockCostsSource, + DaSourceService, + }, }, metadata::{ v1_algorithm_from_metadata, @@ -107,7 +113,8 @@ where GasPriceStore: GasPriceData, PersistedData: MetadataStorage, PersistedData: TransactionableStorage, - for<'a> PersistedData::Transaction<'a>: MetadataStorage + UnrecordedBlocks, + for<'a> PersistedData::Transaction<'a>: + MetadataStorage + UnrecordedBlocks + DaSequenceNumberTracker, DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, { @@ -146,7 +153,7 @@ where Ok(task) } - pub fn init( + pub async fn init( mut self, ) -> anyhow::Result< GasPriceServiceV1, DA, PersistedData>, @@ -176,6 +183,11 @@ where // TODO: Add to config // https://github.com/FuelLabs/fuel-core/issues/2140 let poll_interval = None; + let tx = self.persisted_data.begin_transaction()?; + if let Some(sequence_number) = tx.get_sequence_number(&metadata_height.into())? { + self.da_source.set_last_value(sequence_number).await?; + } + drop(tx); let da_service = DaSourceService::new(self.da_source, poll_interval); if BlockHeight::from(latest_block_height) == self.genesis_block_height @@ -253,7 +265,7 @@ where _state_watcher: &StateWatcher, _params: Self::TaskParams, ) -> anyhow::Result { - UninitializedTask::init(self) + UninitializedTask::init(self).await } } From a587b8ad8b1984e3a83029866a7de276849005a9 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 4 Dec 2024 16:36:38 -0700 Subject: [PATCH 063/131] Separate traits out into get and set traits --- .../src/common/fuel_core_storage_adapter.rs | 106 ++++++++++++------ .../services/gas_price_service/src/ports.rs | 14 ++- .../gas_price_service/src/v0/service.rs | 30 +++-- .../gas_price_service/src/v0/tests.rs | 31 ++--- .../src/v0/uninitialized_task.rs | 15 +-- .../gas_price_service/src/v1/service.rs | 35 +++--- .../gas_price_service/src/v1/tests.rs | 69 +++++++----- .../src/v1/uninitialized_task.rs | 33 +++--- 8 files changed, 205 insertions(+), 128 deletions(-) diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs index 54e3eabd65c..f1c38d504d2 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs @@ -17,8 +17,10 @@ use crate::{ updater_metadata::UpdaterMetadata, }, ports::{ - DaSequenceNumberTracker, - MetadataStorage, + GetDaSequenceNumber, + GetMetadataStorage, + SetDaSequenceNumber, + SetMetadataStorage, TransactionableStorage, }, }; @@ -63,10 +65,28 @@ mod metadata_tests; pub mod storage; -impl MetadataStorage for StructuredStorage +impl SetMetadataStorage for StructuredStorage where Storage: KeyValueInspect + Modifiable, Storage: Send + Sync, +{ + fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { + let block_height = metadata.l2_block_height(); + let mut tx = self.write_transaction(); + tx.storage_as_mut::() + .insert(&block_height, metadata) + .and_then(|_| tx.commit()) + .map_err(|err| GasPriceError::CouldNotSetMetadata { + block_height, + source_error: err.into(), + })?; + Ok(()) + } +} + +impl GetMetadataStorage for StructuredStorage +where + Storage: KeyValueInspect + Send + Sync, { fn get_metadata( &self, @@ -80,18 +100,22 @@ where })?; Ok(metadata.map(|inner| inner.into_owned())) } +} - fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { - let block_height = metadata.l2_block_height(); - let mut tx = self.write_transaction(); - tx.storage_as_mut::() - .insert(&block_height, metadata) - .and_then(|_| tx.commit()) - .map_err(|err| GasPriceError::CouldNotSetMetadata { - block_height, - source_error: err.into(), - })?; - Ok(()) +impl GetDaSequenceNumber for StructuredStorage +where + Storage: KeyValueInspect + Send + Sync, +{ + fn get_sequence_number( + &self, + block_height: &BlockHeight, + ) -> GasPriceResult> { + let sequence_number = self + .storage::() + .get(block_height) + .map_err(|err| GasPriceError::CouldNotFetchDARecord(err.into()))? + .map(|no| *no); + Ok(sequence_number) } } @@ -152,9 +176,25 @@ where } } -impl<'a, Storage> MetadataStorage for NewStorageTransaction<'a, Storage> +impl<'a, Storage> SetMetadataStorage for NewStorageTransaction<'a, Storage> where Storage: KeyValueInspect + Modifiable + Send + Sync, +{ + fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { + let block_height = metadata.l2_block_height(); + self.inner + .storage_as_mut::() + .insert(&block_height, metadata) + .map_err(|err| GasPriceError::CouldNotSetMetadata { + block_height, + source_error: err.into(), + })?; + Ok(()) + } +} +impl<'a, Storage> GetMetadataStorage for NewStorageTransaction<'a, Storage> +where + Storage: KeyValueInspect + Send + Sync, { fn get_metadata( &self, @@ -169,21 +209,25 @@ where })?; Ok(metadata.map(|inner| inner.into_owned())) } +} - fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { - let block_height = metadata.l2_block_height(); +impl<'a, Storage> SetDaSequenceNumber for NewStorageTransaction<'a, Storage> +where + Storage: KeyValueInspect + Modifiable + Send + Sync, +{ + fn set_sequence_number( + &mut self, + block_height: &BlockHeight, + sequence_number: u32, + ) -> GasPriceResult<()> { self.inner - .storage_as_mut::() - .insert(&block_height, metadata) - .map_err(|err| GasPriceError::CouldNotSetMetadata { - block_height, - source_error: err.into(), - })?; + .storage_as_mut::() + .insert(block_height, &sequence_number) + .map_err(|err| GasPriceError::CouldNotFetchDARecord(err.into()))?; Ok(()) } } - -impl<'a, Storage> DaSequenceNumberTracker for NewStorageTransaction<'a, Storage> +impl<'a, Storage> GetDaSequenceNumber for NewStorageTransaction<'a, Storage> where Storage: KeyValueInspect + Modifiable + Send + Sync, { @@ -199,18 +243,6 @@ where .map(|no| *no); Ok(sequence_number) } - - fn set_sequence_number( - &mut self, - block_height: &BlockHeight, - sequence_number: u32, - ) -> GasPriceResult<()> { - self.inner - .storage_as_mut::() - .insert(block_height, &sequence_number) - .map_err(|err| GasPriceError::CouldNotFetchDARecord(err.into()))?; - Ok(()) - } } #[derive(Debug, Clone, PartialEq)] diff --git a/crates/services/gas_price_service/src/ports.rs b/crates/services/gas_price_service/src/ports.rs index 2a451d068e1..9b92156bb69 100644 --- a/crates/services/gas_price_service/src/ports.rs +++ b/crates/services/gas_price_service/src/ports.rs @@ -22,14 +22,16 @@ pub trait L2Data: Send + Sync { ) -> StorageResult>>; } -pub trait MetadataStorage: Send + Sync { +pub trait SetMetadataStorage: Send + Sync { + fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> Result<()>; +} + +pub trait GetMetadataStorage: Send + Sync { fn get_metadata(&self, block_height: &BlockHeight) -> Result>; - fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> Result<()>; } -pub trait DaSequenceNumberTracker: Send + Sync { - fn get_sequence_number(&self, block_height: &BlockHeight) -> Result>; +pub trait SetDaSequenceNumber: Send + Sync { fn set_sequence_number( &mut self, block_height: &BlockHeight, @@ -37,6 +39,10 @@ pub trait DaSequenceNumberTracker: Send + Sync { ) -> Result<()>; } +pub trait GetDaSequenceNumber: Send + Sync { + fn get_sequence_number(&self, block_height: &BlockHeight) -> Result>; +} + pub trait TransactionableStorage: Send + Sync { type Transaction<'a> where diff --git a/crates/services/gas_price_service/src/v0/service.rs b/crates/services/gas_price_service/src/v0/service.rs index 557d8cd6842..c4237b1141e 100644 --- a/crates/services/gas_price_service/src/v0/service.rs +++ b/crates/services/gas_price_service/src/v0/service.rs @@ -4,7 +4,10 @@ use crate::{ updater_metadata::UpdaterMetadata, utils::BlockInfo, }, - ports::MetadataStorage, + ports::{ + GetMetadataStorage, + SetMetadataStorage, + }, v0::algorithm::SharedV0Algorithm, }; use anyhow::anyhow; @@ -35,7 +38,7 @@ pub struct GasPriceServiceV0 { impl GasPriceServiceV0 where - Metadata: MetadataStorage, + Metadata: GetMetadataStorage + SetMetadataStorage, { pub fn new( l2_block_source: L2, @@ -120,7 +123,7 @@ where impl GasPriceServiceV0 where L2: L2BlockSource, - Metadata: MetadataStorage, + Metadata: GetMetadataStorage + SetMetadataStorage, { async fn process_l2_block_res( &mut self, @@ -138,7 +141,7 @@ where impl RunnableTask for GasPriceServiceV0 where L2: L2BlockSource, - Metadata: MetadataStorage, + Metadata: GetMetadataStorage + SetMetadataStorage, { async fn run(&mut self, watcher: &mut StateWatcher) -> TaskNextAction { tracing::trace!("Call of `run` function of the gas price service v0"); @@ -178,7 +181,10 @@ mod tests { Result as GasPriceResult, }, }, - ports::MetadataStorage, + ports::{ + GetMetadataStorage, + SetMetadataStorage, + }, v0::{ metadata::V0AlgorithmConfig, service::GasPriceServiceV0, @@ -217,7 +223,14 @@ mod tests { } } - impl MetadataStorage for FakeMetadata { + impl SetMetadataStorage for FakeMetadata { + fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { + *self.inner.lock().unwrap() = Some(metadata.clone()); + Ok(()) + } + } + + impl GetMetadataStorage for FakeMetadata { fn get_metadata( &self, _: &BlockHeight, @@ -225,11 +238,6 @@ mod tests { let metadata = self.inner.lock().unwrap().clone(); Ok(metadata) } - - fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { - *self.inner.lock().unwrap() = Some(metadata.clone()); - Ok(()) - } } #[tokio::test] diff --git a/crates/services/gas_price_service/src/v0/tests.rs b/crates/services/gas_price_service/src/v0/tests.rs index 294c5029a4d..d29116e59d8 100644 --- a/crates/services/gas_price_service/src/v0/tests.rs +++ b/crates/services/gas_price_service/src/v0/tests.rs @@ -16,8 +16,9 @@ use crate::{ }, ports::{ GasPriceData, + GetMetadataStorage, L2Data, - MetadataStorage, + SetMetadataStorage, }, v0::{ metadata::{ @@ -86,27 +87,23 @@ impl FakeMetadata { } } -impl MetadataStorage for FakeMetadata { - fn get_metadata(&self, _: &BlockHeight) -> GasPriceResult> { - let metadata = self.inner.lock().unwrap().clone(); - Ok(metadata) - } - +impl SetMetadataStorage for FakeMetadata { fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { *self.inner.lock().unwrap() = Some(metadata.clone()); Ok(()) } } -struct ErroringMetadata; - -impl MetadataStorage for ErroringMetadata { +impl GetMetadataStorage for FakeMetadata { fn get_metadata(&self, _: &BlockHeight) -> GasPriceResult> { - Err(GasPriceError::CouldNotFetchMetadata { - source_error: anyhow!("boo!"), - }) + let metadata = self.inner.lock().unwrap().clone(); + Ok(metadata) } +} + +struct ErroringMetadata; +impl SetMetadataStorage for ErroringMetadata { fn set_metadata(&mut self, _: &UpdaterMetadata) -> GasPriceResult<()> { Err(GasPriceError::CouldNotSetMetadata { block_height: Default::default(), @@ -115,6 +112,14 @@ impl MetadataStorage for ErroringMetadata { } } +impl GetMetadataStorage for ErroringMetadata { + fn get_metadata(&self, _: &BlockHeight) -> GasPriceResult> { + Err(GasPriceError::CouldNotFetchMetadata { + source_error: anyhow!("boo!"), + }) + } +} + fn arbitrary_config() -> V0AlgorithmConfig { V0AlgorithmConfig { starting_gas_price: 100, diff --git a/crates/services/gas_price_service/src/v0/uninitialized_task.rs b/crates/services/gas_price_service/src/v0/uninitialized_task.rs index 569739605c7..0dd0e395d68 100644 --- a/crates/services/gas_price_service/src/v0/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v0/uninitialized_task.rs @@ -18,7 +18,7 @@ use crate::{ GasPriceData, GasPriceServiceConfig, L2Data, - MetadataStorage, + SetMetadataStorage, }, v0::{ algorithm::SharedV0Algorithm, @@ -45,6 +45,7 @@ use fuel_core_types::{ }; use fuel_gas_price_algorithm::v0::AlgorithmUpdaterV0; +use crate::ports::GetMetadataStorage; pub use fuel_gas_price_algorithm::v0::AlgorithmV0; pub struct UninitializedTask { @@ -65,7 +66,7 @@ where L2DataStore: L2Data, L2DataStoreView: AtomicView, GasPriceStore: GasPriceData, - Metadata: MetadataStorage, + Metadata: GetMetadataStorage + SetMetadataStorage, SettingsProvider: GasPriceSettingsProvider, { pub fn new( @@ -166,7 +167,7 @@ where L2DataStore: L2Data, L2DataStoreView: AtomicView, GasPriceStore: GasPriceData, - Metadata: MetadataStorage, + Metadata: GetMetadataStorage + SetMetadataStorage, SettingsProvider: GasPriceSettingsProvider, { const NAME: &'static str = "GasPriceServiceV0"; @@ -193,7 +194,7 @@ pub fn initialize_algorithm( metadata_storage: &Metadata, ) -> GasPriceResult<(AlgorithmUpdaterV0, SharedV0Algorithm)> where - Metadata: MetadataStorage, + Metadata: GetMetadataStorage + SetMetadataStorage, { let min_exec_gas_price = config.min_gas_price; let exec_gas_price_change_percent = config.gas_price_change_percent; @@ -243,7 +244,7 @@ fn sync_gas_price_db_with_on_chain_storage< where L2DataStore: L2Data, L2DataStoreView: AtomicView, - Metadata: MetadataStorage, + Metadata: GetMetadataStorage + SetMetadataStorage, SettingsProvider: GasPriceSettingsProvider, { let metadata = metadata_storage @@ -287,7 +288,7 @@ fn sync_v0_metadata( where L2DataStore: L2Data, L2DataStoreView: AtomicView, - Metadata: MetadataStorage, + Metadata: SetMetadataStorage, SettingsProvider: GasPriceSettingsProvider, { let first = metadata_height.saturating_add(1); @@ -344,7 +345,7 @@ where L2DataStoreView: AtomicView, GasPriceStore: GasPriceData, SettingsProvider: GasPriceSettingsProvider, - Metadata: MetadataStorage, + Metadata: GetMetadataStorage + SetMetadataStorage, { let v0_config = config.v0().ok_or(anyhow::anyhow!("Expected V0 config"))?; let gas_price_init = UninitializedTask::new( diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index c66134b164e..11fa4ff0cc7 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -11,8 +11,9 @@ use crate::{ }, }, ports::{ - DaSequenceNumberTracker, - MetadataStorage, + GetMetadataStorage, + SetDaSequenceNumber, + SetMetadataStorage, TransactionableStorage, }, v0::metadata::V0Metadata, @@ -89,7 +90,7 @@ where ) -> anyhow::Result<()> where StorageTxProvider::Transaction<'a>: - MetadataStorage + UnrecordedBlocks + DaSequenceNumberTracker, + SetMetadataStorage + UnrecordedBlocks + SetDaSequenceNumber, { tracing::info!("Received L2 block result: {:?}", l2_block_res); let block = l2_block_res?; @@ -154,7 +155,7 @@ where ) -> anyhow::Result<()> where StorageTxProvider::Transaction<'a>: - UnrecordedBlocks + MetadataStorage + DaSequenceNumberTracker, + UnrecordedBlocks + SetMetadataStorage + SetDaSequenceNumber, { let mut storage_tx = self.storage_tx_provider.begin_transaction()?; let capacity = Self::validate_block_gas_capacity(block_gas_capacity)?; @@ -200,7 +201,7 @@ where ) -> anyhow::Result<()> where StorageTxProvider::Transaction<'a>: - MetadataStorage + UnrecordedBlocks + DaSequenceNumberTracker, + SetMetadataStorage + UnrecordedBlocks + SetDaSequenceNumber, { match l2_block { BlockInfo::GenesisBlock => { @@ -242,7 +243,7 @@ where DA: DaBlockCostsSource, StorageTxProvider: TransactionableStorage + 'static, for<'a> StorageTxProvider::Transaction<'a>: - UnrecordedBlocks + MetadataStorage + DaSequenceNumberTracker, + UnrecordedBlocks + SetMetadataStorage + SetDaSequenceNumber, { async fn run(&mut self, watcher: &mut StateWatcher) -> TaskNextAction { tokio::select! { @@ -310,7 +311,7 @@ pub fn initialize_algorithm( metadata_storage: &Metadata, ) -> crate::common::utils::Result<(AlgorithmUpdaterV1, SharedV1Algorithm)> where - Metadata: MetadataStorage, + Metadata: GetMetadataStorage, { let algorithm_updater = if let Some(updater_metadata) = metadata_storage .get_metadata(&latest_block_height.into()) @@ -372,7 +373,10 @@ mod tests { Result as GasPriceResult, }, }, - ports::MetadataStorage, + ports::{ + GetMetadataStorage, + SetMetadataStorage, + }, v1::{ da_source_service::{ dummy_costs::DummyDaBlockCosts, @@ -415,7 +419,14 @@ mod tests { } } - impl MetadataStorage for FakeMetadata { + impl SetMetadataStorage for FakeMetadata { + fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { + *self.inner.lock().unwrap() = Some(metadata.clone()); + Ok(()) + } + } + + impl GetMetadataStorage for FakeMetadata { fn get_metadata( &self, _: &BlockHeight, @@ -423,13 +434,7 @@ mod tests { let metadata = self.inner.lock().unwrap().clone(); Ok(metadata) } - - fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { - *self.inner.lock().unwrap() = Some(metadata.clone()); - Ok(()) - } } - fn database() -> StorageTransaction> { InMemoryStorage::default().into_transaction() } diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 0abeb5e8ded..ad522b16405 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -19,10 +19,12 @@ use crate::{ }, }, ports::{ - DaSequenceNumberTracker, GasPriceData, + GetDaSequenceNumber, + GetMetadataStorage, L2Data, - MetadataStorage, + SetDaSequenceNumber, + SetMetadataStorage, TransactionableStorage, }, v1::{ @@ -134,32 +136,43 @@ impl FakeMetadata { } } -impl MetadataStorage for FakeMetadata { - fn get_metadata(&self, _: &BlockHeight) -> GasPriceResult> { - let metadata = self.inner.lock().unwrap().clone(); - Ok(metadata) - } - +impl SetMetadataStorage for FakeMetadata { fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { *self.inner.lock().unwrap() = Some(metadata.clone()); Ok(()) } } +impl GetMetadataStorage for FakeMetadata { + fn get_metadata(&self, _: &BlockHeight) -> GasPriceResult> { + let metadata = self.inner.lock().unwrap().clone(); + Ok(metadata) + } +} struct ErroringPersistedData; -impl MetadataStorage for ErroringPersistedData { +impl SetMetadataStorage for ErroringPersistedData { + fn set_metadata(&mut self, _: &UpdaterMetadata) -> GasPriceResult<()> { + Err(GasPriceError::CouldNotSetMetadata { + block_height: Default::default(), + source_error: anyhow!("boo!"), + }) + } +} +impl GetMetadataStorage for ErroringPersistedData { fn get_metadata(&self, _: &BlockHeight) -> GasPriceResult> { Err(GasPriceError::CouldNotFetchMetadata { source_error: anyhow!("boo!"), }) } +} - fn set_metadata(&mut self, _: &UpdaterMetadata) -> GasPriceResult<()> { - Err(GasPriceError::CouldNotSetMetadata { - block_height: Default::default(), - source_error: anyhow!("boo!"), - }) +impl GetDaSequenceNumber for ErroringPersistedData { + fn get_sequence_number( + &self, + _block_height: &BlockHeight, + ) -> GasPriceResult> { + Err(GasPriceError::CouldNotFetchDARecord(anyhow!("boo!"))) } } @@ -177,17 +190,18 @@ impl TransactionableStorage for ErroringPersistedData { } } -impl MetadataStorage for UnimplementedStorageTx { +impl SetMetadataStorage for UnimplementedStorageTx { + fn set_metadata(&mut self, _metadata: &UpdaterMetadata) -> GasPriceResult<()> { + unimplemented!() + } +} +impl GetMetadataStorage for UnimplementedStorageTx { fn get_metadata( &self, _block_height: &BlockHeight, ) -> GasPriceResult> { unimplemented!() } - - fn set_metadata(&mut self, _metadata: &UpdaterMetadata) -> GasPriceResult<()> { - unimplemented!() - } } impl UnrecordedBlocks for UnimplementedStorageTx { @@ -200,14 +214,7 @@ impl UnrecordedBlocks for UnimplementedStorageTx { } } -impl DaSequenceNumberTracker for UnimplementedStorageTx { - fn get_sequence_number( - &self, - _block_height: &BlockHeight, - ) -> GasPriceResult> { - unimplemented!() - } - +impl SetDaSequenceNumber for UnimplementedStorageTx { fn set_sequence_number( &mut self, _block_height: &BlockHeight, @@ -216,6 +223,14 @@ impl DaSequenceNumberTracker for UnimplementedStorageTx { unimplemented!() } } +impl GetDaSequenceNumber for UnimplementedStorageTx { + fn get_sequence_number( + &self, + _block_height: &BlockHeight, + ) -> GasPriceResult> { + unimplemented!() + } +} struct FakeDABlockCost { da_block_costs: Receiver, diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 56d6cae7eeb..2721545952c 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -17,11 +17,13 @@ use crate::{ }, }, ports::{ - DaSequenceNumberTracker, GasPriceData, GasPriceServiceConfig, + GetDaSequenceNumber, + GetMetadataStorage, L2Data, - MetadataStorage, + SetDaSequenceNumber, + SetMetadataStorage, TransactionableStorage, }, v1::{ @@ -111,10 +113,10 @@ where L2DataStore: L2Data, L2DataStoreView: AtomicView, GasPriceStore: GasPriceData, - PersistedData: MetadataStorage, + PersistedData: GetMetadataStorage + GetDaSequenceNumber, PersistedData: TransactionableStorage, for<'a> PersistedData::Transaction<'a>: - MetadataStorage + UnrecordedBlocks + DaSequenceNumberTracker, + SetMetadataStorage + UnrecordedBlocks + SetDaSequenceNumber, DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, { @@ -183,11 +185,12 @@ where // TODO: Add to config // https://github.com/FuelLabs/fuel-core/issues/2140 let poll_interval = None; - let tx = self.persisted_data.begin_transaction()?; - if let Some(sequence_number) = tx.get_sequence_number(&metadata_height.into())? { + if let Some(sequence_number) = self + .persisted_data + .get_sequence_number(&metadata_height.into())? + { self.da_source.set_last_value(sequence_number).await?; } - drop(tx); let da_service = DaSourceService::new(self.da_source, poll_interval); if BlockHeight::from(latest_block_height) == self.genesis_block_height @@ -247,9 +250,10 @@ where GasPriceStore: GasPriceData, DA: DaBlockCostsSource + 'static, SettingsProvider: GasPriceSettingsProvider + 'static, - PersistedData: MetadataStorage + TransactionableStorage + 'static, + PersistedData: + GetMetadataStorage + GetDaSequenceNumber + TransactionableStorage + 'static, for<'a> ::Transaction<'a>: - MetadataStorage + UnrecordedBlocks + DaSequenceNumberTracker, + SetMetadataStorage + UnrecordedBlocks + SetDaSequenceNumber, { const NAME: &'static str = "GasPriceServiceV1"; type SharedData = SharedV1Algorithm; @@ -287,9 +291,9 @@ where L2DataStore: L2Data, L2DataStoreView: AtomicView, SettingsProvider: GasPriceSettingsProvider, - PersistedData: MetadataStorage + TransactionableStorage + 'a, + PersistedData: GetMetadataStorage + TransactionableStorage + 'a, ::Transaction<'a>: - MetadataStorage + UnrecordedBlocks, + SetMetadataStorage + UnrecordedBlocks, { let metadata = persisted_data .get_metadata(&metadata_height.into())? @@ -337,7 +341,7 @@ where SettingsProvider: GasPriceSettingsProvider, StorageTxGenerator: TransactionableStorage + 'a, ::Transaction<'a>: - MetadataStorage + UnrecordedBlocks, + SetMetadataStorage + UnrecordedBlocks, { let first = metadata_height.saturating_add(1); let view = on_chain_db.latest_view()?; @@ -414,9 +418,10 @@ where GasPriceStore: GasPriceData, SettingsProvider: GasPriceSettingsProvider, DA: DaBlockCostsSource, - PersistedData: MetadataStorage + TransactionableStorage + 'static, + PersistedData: + GetMetadataStorage + GetDaSequenceNumber + TransactionableStorage + 'static, for<'a> PersistedData::Transaction<'a>: - MetadataStorage + UnrecordedBlocks + DaSequenceNumberTracker, + SetMetadataStorage + UnrecordedBlocks + SetDaSequenceNumber, { let v1_config = config.v1().ok_or(anyhow::anyhow!("Expected V1 config"))?; let gas_price_init = UninitializedTask::new( From 37235aed399064cd4d582605ca475cc3a340f716 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 4 Dec 2024 17:15:14 -0700 Subject: [PATCH 064/131] Fix broken test --- .../src/v1/tests/update_l2_block_data_tests.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs index 25a01b0eaba..6f0dbc4debe 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs @@ -644,8 +644,9 @@ fn update_l2_block_data__retains_existing_blocks_and_adds_l2_block_to_unrecorded // given let starting_block = 0; let first_block_bytes = 1200; - let unrecorded_blocks: BTreeMap<_, _> = - vec![(0, first_block_bytes)].into_iter().collect(); + let mut unrecorded_blocks: BTreeMap<_, _> = vec![(starting_block, first_block_bytes)] + .into_iter() + .collect(); let mut updater = UpdaterBuilder::new() .with_l2_block_height(starting_block) @@ -657,7 +658,6 @@ fn update_l2_block_data__retains_existing_blocks_and_adds_l2_block_to_unrecorded let capacity = 100.try_into().unwrap(); let new_block_bytes = 1000; let new_gas_price = 100; - let unrecorded_blocks = &mut empty_unrecorded_blocks(); // when updater @@ -667,7 +667,7 @@ fn update_l2_block_data__retains_existing_blocks_and_adds_l2_block_to_unrecorded capacity, new_block_bytes, new_gas_price, - unrecorded_blocks, + &mut unrecorded_blocks, ) .unwrap(); @@ -680,7 +680,8 @@ fn update_l2_block_data__retains_existing_blocks_and_adds_l2_block_to_unrecorded assert!(contains_block_bytes); // and - let contains_preexisting_block_bytes = unrecorded_blocks.contains_key(&0); + let contains_preexisting_block_bytes = + unrecorded_blocks.contains_key(&starting_block); assert!(contains_preexisting_block_bytes); // and From 96f6b9df4f4b90e86862cd56a8489d194bee8425 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 5 Dec 2024 12:29:11 -0700 Subject: [PATCH 065/131] Cleanup extra code, remove unused imports, hide imlementations. --- crates/fuel-gas-price-algorithm/src/v1.rs | 6 ++++-- .../gas_price_service/src/v1/da_source_service.rs | 1 + .../src/v1/da_source_service/block_committer_costs.rs | 2 +- .../src/v1/da_source_service/dummy_costs.rs | 2 +- crates/services/gas_price_service/src/v1/service.rs | 11 ++++++++--- crates/services/gas_price_service/src/v1/tests.rs | 6 +----- .../gas_price_service/src/v1/uninitialized_task.rs | 5 +---- 7 files changed, 17 insertions(+), 16 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 6b36786d4a2..bffec4934c4 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -302,12 +302,14 @@ impl L2ActivityTracker { } pub fn block_activity_threshold(&self) -> ClampedPercentage { - self.block_activity_threshold.clone() + self.block_activity_threshold } } /// A value that represents a value between 0 and 100. Higher values are clamped to 100 -#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, PartialOrd)] +#[derive( + serde::Serialize, serde::Deserialize, Debug, Copy, Clone, PartialEq, PartialOrd, +)] pub struct ClampedPercentage { value: u8, } diff --git a/crates/services/gas_price_service/src/v1/da_source_service.rs b/crates/services/gas_price_service/src/v1/da_source_service.rs index e649d3f4222..13f5e0c8045 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service.rs @@ -2,6 +2,7 @@ use crate::v1::da_source_service::service::DaBlockCostsSource; use std::time::Duration; pub mod block_committer_costs; +#[cfg(test)] pub mod dummy_costs; pub mod service; diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index 4ff70796b0a..5c9b37b7fb6 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -15,7 +15,7 @@ use serde::{ }; #[async_trait::async_trait] -pub trait BlockCommitterApi: Send + Sync { +trait BlockCommitterApi: Send + Sync { /// Used on first run to get the latest costs and seqno async fn get_latest_costs(&self) -> DaBlockCostsResult>; /// Used to get the costs for a specific seqno diff --git a/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs index ddeb88e4d0e..234c253383f 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs @@ -36,6 +36,6 @@ impl DaBlockCostsSource for DummyDaBlockCosts { } async fn set_last_value(&mut self, sequence_number: u32) -> DaBlockCostsResult<()> { - todo!() + unimplemented!("This is a dummy implementation"); } } diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 11fa4ff0cc7..fbd4959e01b 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -75,7 +75,7 @@ where /// Buffer of block costs from the DA chain da_block_costs_buffer: Vec, /// Storage transaction provider for metadata and unrecorded blocks - pub(crate) storage_tx_provider: StorageTxProvider, + storage_tx_provider: StorageTxProvider, } impl GasPriceServiceV1 @@ -134,6 +134,11 @@ where self.shared_algo.clone() } + #[cfg(test)] + pub fn storage_tx_provider(&self) -> &StorageTxProvider { + &self.storage_tx_provider + } + async fn update(&mut self, new_algorithm: AlgorithmV1) { self.shared_algo.update(new_algorithm).await; } @@ -157,8 +162,8 @@ where StorageTxProvider::Transaction<'a>: UnrecordedBlocks + SetMetadataStorage + SetDaSequenceNumber, { - let mut storage_tx = self.storage_tx_provider.begin_transaction()?; let capacity = Self::validate_block_gas_capacity(block_gas_capacity)?; + let mut storage_tx = self.storage_tx_provider.begin_transaction()?; for da_block_costs in self.da_block_costs_buffer.drain(..) { tracing::debug!("Updating DA block costs: {:?}", da_block_costs); @@ -241,7 +246,7 @@ impl RunnableTask where L2: L2BlockSource, DA: DaBlockCostsSource, - StorageTxProvider: TransactionableStorage + 'static, + StorageTxProvider: TransactionableStorage, for<'a> StorageTxProvider::Transaction<'a>: UnrecordedBlocks + SetMetadataStorage + SetDaSequenceNumber, { diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index ad522b16405..9be9e33be10 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -30,10 +30,6 @@ use crate::{ v1::{ algorithm::SharedV1Algorithm, da_source_service::{ - block_committer_costs::{ - BlockCommitterApi, - RawDaBlockCosts, - }, service::{ DaBlockCostsSource, DaSourceService, @@ -431,7 +427,7 @@ async fn run__new_l2_block_saves_old_metadata() { // then let metadata_is_some = service - .storage_tx_provider + .storage_tx_provider() .get_metadata(&height.into()) .unwrap() .is_some(); diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 2721545952c..63691e49b91 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -29,10 +29,7 @@ use crate::{ v1::{ algorithm::SharedV1Algorithm, da_source_service::{ - block_committer_costs::{ - BlockCommitterApi, - BlockCommitterDaBlockCosts, - }, + block_committer_costs::BlockCommitterDaBlockCosts, service::{ DaBlockCostsSource, DaSourceService, From 95f85f5cce3438ab28ca23370db86535f9b7536f Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 5 Dec 2024 16:00:10 -0700 Subject: [PATCH 066/131] Fix compilation --- bin/e2e-test-client/tests/integration_tests.rs | 2 +- crates/fuel-core/src/service/sub_services.rs | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bin/e2e-test-client/tests/integration_tests.rs b/bin/e2e-test-client/tests/integration_tests.rs index 720a4be4939..328381238b1 100644 --- a/bin/e2e-test-client/tests/integration_tests.rs +++ b/bin/e2e-test-client/tests/integration_tests.rs @@ -135,7 +135,7 @@ fn dev_config() -> Config { chain_config.genesis_state_transition_version = Some(LATEST_STATE_TRANSITION_VERSION); let reader = reader.with_chain_config(chain_config); - let mut config = Config::local_node_with_reader(reader); + let mut config = Config::local_node_with_reader(reader, None); config.starting_exec_gas_price = 1; config.block_producer.coinbase_recipient = Some( ContractId::from_str( diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index 3d30249f19d..ed84c5ea318 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -40,7 +40,10 @@ use crate::{ }; use fuel_core_gas_price_service::v1::{ algorithm::AlgorithmV1, - da_source_service::block_committer_costs::BlockCommitterHttpApi, + da_source_service::block_committer_costs::{ + BlockCommitterDaBlockCosts, + BlockCommitterHttpApi, + }, uninitialized_task::new_gas_price_service_v1, }; use fuel_core_poa::{ @@ -183,9 +186,10 @@ pub fn init_sub_services( let genesis_block_height = *genesis_block.header().height(); let settings = consensus_parameters_provider.clone(); let block_stream = importer_adapter.events_shared_result(); - let metadata = StructuredStorage::new(database.gas_price().clone()); + let persisted_data = StructuredStorage::new(database.gas_price().clone()); - let committer = BlockCommitterHttpApi::new(config.da_committer_url.clone()); + let committer_api = BlockCommitterHttpApi::new(config.da_committer_url.clone()); + let da_source = BlockCommitterDaBlockCosts::new(committer_api, None); let gas_price_service = new_gas_price_service_v1( config.clone().into(), @@ -194,7 +198,7 @@ pub fn init_sub_services( block_stream, database.gas_price().clone(), da_source, - on_chain_db, + database.on_chain().clone(), persisted_data, )?; From eb91cebd6347d839e3e46c63dbe506f7d066cbe4 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 5 Dec 2024 18:50:00 -0700 Subject: [PATCH 067/131] Use `ServiceRunner` for da service --- bin/fuel-core/src/cli/run.rs | 6 ++ .../service/adapters/gas_price_adapters.rs | 1 + crates/fuel-core/src/service/config.rs | 2 + crates/fuel-core/src/service/sub_services.rs | 1 + .../services/gas_price_service/src/ports.rs | 2 + .../src/v1/da_source_service/service.rs | 2 + .../gas_price_service/src/v1/metadata.rs | 2 + .../gas_price_service/src/v1/service.rs | 64 +++++++++++-------- .../gas_price_service/src/v1/tests.rs | 13 +++- .../src/v1/uninitialized_task.rs | 19 ++++-- tests/tests/gas_price.rs | 9 +-- 11 files changed, 83 insertions(+), 38 deletions(-) diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index 8fabb729f0f..a2bf44b3b4d 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -225,6 +225,10 @@ pub struct Command { #[arg(long = "da-committer-url", env)] pub da_committer_url: Option, + /// The interval at which the `DaSourceService` polls for new data + #[arg(long = "da-poll-interval", env)] + pub da_poll_interval: Option, + /// The signing key used when producing blocks. /// Setting via the `CONSENSUS_KEY_SECRET` ENV var is preferred. #[arg(long = "consensus-key", env = "CONSENSUS_KEY_SECRET")] @@ -328,6 +332,7 @@ impl Command { da_d_component, max_da_gas_price_change_percent, da_committer_url, + da_poll_interval, consensus_key, #[cfg(feature = "aws-kms")] consensus_aws_kms, @@ -641,6 +646,7 @@ impl Command { activity_decrease_range_size: 0, da_committer_url, block_activity_threshold: 0, + da_poll_interval, }; Ok(config) } diff --git a/crates/fuel-core/src/service/adapters/gas_price_adapters.rs b/crates/fuel-core/src/service/adapters/gas_price_adapters.rs index 2af4c597528..fe131ddadbf 100644 --- a/crates/fuel-core/src/service/adapters/gas_price_adapters.rs +++ b/crates/fuel-core/src/service/adapters/gas_price_adapters.rs @@ -78,6 +78,7 @@ impl From for GasPriceServiceConfig { value.activity_capped_range_size, value.activity_decrease_range_size, value.block_activity_threshold, + value.da_poll_interval, ) } } diff --git a/crates/fuel-core/src/service/config.rs b/crates/fuel-core/src/service/config.rs index 069b9dd40c7..78a83081b9d 100644 --- a/crates/fuel-core/src/service/config.rs +++ b/crates/fuel-core/src/service/config.rs @@ -62,6 +62,7 @@ pub struct Config { pub min_exec_gas_price: u64, pub exec_gas_price_threshold_percent: u8, pub da_committer_url: Option, + pub da_poll_interval: Option, pub da_compression: DaCompressionConfig, pub block_importer: fuel_core_importer::Config, #[cfg(feature = "relayer")] @@ -213,6 +214,7 @@ impl Config { activity_decrease_range_size: 0, da_committer_url, block_activity_threshold: 0, + da_poll_interval: Some(10), } } diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index ed84c5ea318..50ef1cd3e17 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -188,6 +188,7 @@ pub fn init_sub_services( let block_stream = importer_adapter.events_shared_result(); let persisted_data = StructuredStorage::new(database.gas_price().clone()); + tracing::debug!("da_committer_url: {:?}", config.da_committer_url); let committer_api = BlockCommitterHttpApi::new(config.da_committer_url.clone()); let da_source = BlockCommitterDaBlockCosts::new(committer_api, None); diff --git a/crates/services/gas_price_service/src/ports.rs b/crates/services/gas_price_service/src/ports.rs index 9fc3daf6f30..798d6595c30 100644 --- a/crates/services/gas_price_service/src/ports.rs +++ b/crates/services/gas_price_service/src/ports.rs @@ -95,6 +95,7 @@ impl GasPriceServiceConfig { capped_range_size: u16, decrease_range_size: u16, block_activity_threshold: u8, + da_poll_interval: Option, ) -> Self { Self::V1(V1AlgorithmConfig { new_exec_gas_price, @@ -110,6 +111,7 @@ impl GasPriceServiceConfig { capped_range_size, decrease_range_size, block_activity_threshold, + da_poll_interval, }) } diff --git a/crates/services/gas_price_service/src/v1/da_source_service/service.rs b/crates/services/gas_price_service/src/v1/da_source_service/service.rs index 7800d80489c..207879adc83 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/service.rs @@ -110,12 +110,14 @@ where /// This function polls the source according to a polling interval /// described by the DaBlockCostsService async fn run(&mut self, state_watcher: &mut StateWatcher) -> TaskNextAction { + tracing::debug!("111111111111111111111111111111111"); tokio::select! { biased; _ = state_watcher.while_started() => { TaskNextAction::Stop } _ = self.poll_interval.tick() => { + tracing::debug!("Polling DaSourceService for block costs"); let da_block_costs_res = self.process_block_costs().await; TaskNextAction::always_continue(da_block_costs_res) } diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index a158d2f2126..b157ec2464a 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -72,6 +72,8 @@ pub struct V1AlgorithmConfig { pub capped_range_size: u16, pub decrease_range_size: u16, pub block_activity_threshold: u8, + /// The interval at which the `DaSourceService` polls for new data + pub da_poll_interval: Option, } pub fn updater_from_config(value: &V1AlgorithmConfig) -> AlgorithmUpdaterV1 { diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index dc2ece43e79..773a8a8a113 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -41,6 +41,8 @@ use async_trait::async_trait; use fuel_core_services::{ RunnableService, RunnableTask, + Service, + ServiceRunner, StateWatcher, TaskNextAction, }; @@ -59,7 +61,7 @@ use tokio::sync::broadcast::Receiver; /// The service that updates the gas price algorithm. pub struct GasPriceServiceV1 where - DA: DaBlockCostsSource, + DA: DaBlockCostsSource + 'static, StorageTxProvider: TransactionableStorage, { /// The algorithm that can be used in the next block @@ -69,7 +71,7 @@ where /// The algorithm updater algorithm_updater: AlgorithmUpdaterV1, /// the da source adapter handle - da_source_adapter_handle: DaSourceService, + da_source_adapter_handle: ServiceRunner>, /// The da source channel da_source_channel: Receiver, /// Buffer of block costs from the DA chain @@ -110,11 +112,10 @@ where l2_block_source: L2, shared_algo: SharedV1Algorithm, algorithm_updater: AlgorithmUpdaterV1, - da_source_adapter_handle: DaSourceService, + da_source_adapter_handle: ServiceRunner>, storage_tx_provider: StorageTxProvider, ) -> Self { - let da_source_channel = - da_source_adapter_handle.shared_data().clone().subscribe(); + let da_source_channel = da_source_adapter_handle.shared.clone().subscribe(); Self { shared_algo, l2_block_source, @@ -259,11 +260,13 @@ where } l2_block_res = self.l2_block_source.get_l2_block() => { tracing::debug!("Received L2 block result: {:?}", l2_block_res); + tracing::debug!("2222222222222222222222222222222222222"); let res = self.commit_block_data_to_algorithm(l2_block_res).await; TaskNextAction::always_continue(res) } da_block_costs_res = self.da_source_channel.recv() => { tracing::debug!("Received DA block costs: {:?}", da_block_costs_res); + tracing::debug!("999999999999999999999999999999999999"); match da_block_costs_res { Ok(da_block_costs) => { self.da_block_costs_buffer.push(da_block_costs); @@ -286,7 +289,7 @@ where } // run shutdown hooks for internal services - self.da_source_adapter_handle.shutdown().await?; + self.da_source_adapter_handle.stop_and_await().await?; Ok(()) } @@ -349,6 +352,8 @@ mod tests { use fuel_core_services::{ RunnableTask, + Service, + ServiceRunner, StateWatcher, }; use fuel_core_storage::{ @@ -478,6 +483,7 @@ mod tests { capped_range_size: 100, decrease_range_size: 4, block_activity_threshold: 20, + da_poll_interval: None, }; let inner = database(); let (algo_updater, shared_algo) = @@ -491,12 +497,14 @@ mod tests { ), None, ); + let da_service_runner = ServiceRunner::new(dummy_da_source); + da_service_runner.start_and_await().await.unwrap(); let mut service = GasPriceServiceV1::new( l2_block_source, shared_algo, algo_updater, - dummy_da_source, + da_service_runner, inner, ); let read_algo = service.next_block_algorithm(); @@ -546,6 +554,7 @@ mod tests { capped_range_size: 100, decrease_range_size: 4, block_activity_threshold: 20, + da_poll_interval: None, }; let mut inner = database(); let mut tx = inner.write_transaction(); @@ -574,25 +583,27 @@ mod tests { Some(Duration::from_millis(1)), ); let mut watcher = StateWatcher::started(); + let da_service_runner = ServiceRunner::new(da_source); + da_service_runner.start_and_await().await.unwrap(); let mut service = GasPriceServiceV1::new( l2_block_source, shared_algo, algo_updater, - da_source, + da_service_runner, inner, ); let read_algo = service.next_block_algorithm(); let initial_price = read_algo.next_gas_price(); - // the RunnableTask depends on the handle passed to it for the da block cost source to already be running, - // which is the responsibility of the UninitializedTask in the `into_task` method of the RunnableService - // here we mimic that behaviour by running the da block cost service. - let mut da_source_watcher = StateWatcher::started(); - service - .da_source_adapter_handle - .run(&mut da_source_watcher) - .await; + // // the RunnableTask depends on the handle passed to it for the da block cost source to already be running, + // // which is the responsibility of the UninitializedTask in the `into_task` method of the RunnableService + // // here we mimic that behaviour by running the da block cost service. + // let mut da_source_watcher = StateWatcher::started(); + // service + // .da_source_adapter_handle + // .run(&mut da_source_watcher) + // .await; service.run(&mut watcher).await; tokio::time::sleep(Duration::from_millis(100)).await; @@ -623,6 +634,7 @@ mod tests { capped_range_size: 100, decrease_range_size: 4, block_activity_threshold: 20, + da_poll_interval: None, } } @@ -674,25 +686,27 @@ mod tests { Some(Duration::from_millis(1)), ); let mut watcher = StateWatcher::started(); + let da_service_runner = ServiceRunner::new(da_source); + da_service_runner.start_and_await().await.unwrap(); let mut service = GasPriceServiceV1::new( l2_block_source, shared_algo, algo_updater, - da_source, + da_service_runner, inner, ); let read_algo = service.next_block_algorithm(); let initial_price = read_algo.next_gas_price(); - // the RunnableTask depends on the handle passed to it for the da block cost source to already be running, - // which is the responsibility of the UninitializedTask in the `into_task` method of the RunnableService - // here we mimic that behaviour by running the da block cost service. - let mut da_source_watcher = StateWatcher::started(); - service - .da_source_adapter_handle - .run(&mut da_source_watcher) - .await; + // // the RunnableTask depends on the handle passed to it for the da block cost source to already be running, + // // which is the responsibility of the UninitializedTask in the `into_task` method of the RunnableService + // // here we mimic that behaviour by running the da block cost service. + // let mut da_source_watcher = StateWatcher::started(); + // service + // .da_source_adapter_handle + // .run(&mut da_source_watcher) + // .await; service.run(&mut watcher).await; tokio::time::sleep(Duration::from_millis(100)).await; diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 9be9e33be10..37d5d680890 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -61,6 +61,8 @@ use fuel_core_services::{ IntoBoxStream, }, RunnableTask, + Service, + ServiceRunner, StateWatcher, }; use fuel_core_storage::{ @@ -291,6 +293,7 @@ fn zero_threshold_arbitrary_config() -> V1AlgorithmConfig { capped_range_size: 0, decrease_range_size: 0, block_activity_threshold: 0, + da_poll_interval: None, } } @@ -324,6 +327,7 @@ fn different_arb_config() -> V1AlgorithmConfig { capped_range_size: 0, decrease_range_size: 0, block_activity_threshold: 0, + da_poll_interval: None, } } @@ -368,11 +372,14 @@ async fn next_gas_price__affected_by_new_l2_block() { initialize_algorithm(&config, height, &metadata_storage).unwrap(); let da_source = FakeDABlockCost::never_returns(); let da_source_service = DaSourceService::new(da_source, None); + let da_service_runner = ServiceRunner::new(da_source_service); + da_service_runner.start_and_await().await.unwrap(); + let mut service = GasPriceServiceV1::new( l2_block_source, shared_algo, algo_updater, - da_source_service, + da_service_runner, inner, ); @@ -412,11 +419,13 @@ async fn run__new_l2_block_saves_old_metadata() { let shared_algo = SharedV1Algorithm::new_with_algorithm(algo_updater.algorithm()); let da_source = FakeDABlockCost::never_returns(); let da_source_service = DaSourceService::new(da_source, None); + let da_service_runner = ServiceRunner::new(da_source_service); + da_service_runner.start_and_await().await.unwrap(); let mut service = GasPriceServiceV1::new( l2_block_source, shared_algo, algo_updater, - da_source_service, + da_service_runner, inner, ); let mut watcher = StateWatcher::started(); diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 4fa819ee195..199856bc26c 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -52,6 +52,7 @@ use anyhow::Error; use fuel_core_services::{ stream::BoxStream, RunnableService, + Service, ServiceRunner, StateWatcher, }; @@ -76,6 +77,7 @@ use fuel_gas_price_algorithm::v1::{ AlgorithmUpdaterV1, UnrecordedBlocks, }; +use std::time::Duration; pub mod fuel_storage_unrecorded_blocks; @@ -115,7 +117,7 @@ where PersistedData: TransactionableStorage, for<'a> PersistedData::Transaction<'a>: SetMetadataStorage + UnrecordedBlocks + SetDaSequenceNumber, - DA: DaBlockCostsSource, + DA: DaBlockCostsSource + 'static, SettingsProvider: GasPriceSettingsProvider, { #[allow(clippy::too_many_arguments)] @@ -180,16 +182,19 @@ where self.block_stream, ); - // TODO: Add to config - // https://github.com/FuelLabs/fuel-core/issues/2140 - let poll_interval = None; if let Some(sequence_number) = self .persisted_data .get_sequence_number(&metadata_height.into())? { self.da_source.set_last_value(sequence_number).await?; } - let da_service = DaSourceService::new(self.da_source, poll_interval); + let poll_duration = self + .config + .da_poll_interval + .map(|x| Duration::from_millis(x.into())); + let da_service = DaSourceService::new(self.da_source, poll_duration); + let da_service_runner = ServiceRunner::new(da_service); + da_service_runner.start_and_await().await?; if BlockHeight::from(latest_block_height) == self.genesis_block_height || first_run @@ -198,7 +203,7 @@ where l2_block_source, self.shared_algo, self.algo_updater, - da_service, + da_service_runner, self.persisted_data, ); Ok(service) @@ -218,7 +223,7 @@ where l2_block_source, self.shared_algo, self.algo_updater, - da_service, + da_service_runner, self.persisted_data, ); Ok(service) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index cd341948ff4..428f1a14ac4 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -451,17 +451,18 @@ fn produce_block__l1_committed_block_effects_gas_price() { let url = mock.url(); // add the da committer url to the args - args.extend(&["--da-committer-url", &url]); + args.extend(&["--da-committer-url", &url, "--da-poll-interval", "10"]); - rt.block_on(async { + let new_gas_price = rt.block_on(async { let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) .await .unwrap(); tokio::time::sleep(Duration::from_millis(100)).await; driver.client.produce_blocks(1, None).await.unwrap(); - let new_gas_price = driver.client.latest_gas_price().await.unwrap().gas_price; - assert_ne!(first_gas_price, new_gas_price); + tokio::time::sleep(Duration::from_millis(100)).await; + driver.client.latest_gas_price().await.unwrap().gas_price }); + assert_ne!(first_gas_price, new_gas_price); } struct FakeServer { From ae9605d9e9abcab7ac55221ae08e3f9294257b70 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 5 Dec 2024 21:15:22 -0700 Subject: [PATCH 068/131] WIP debugging test --- crates/fuel-gas-price-algorithm/src/v1.rs | 8 ++++++++ .../src/v1/da_source_service/service.rs | 4 +++- crates/services/gas_price_service/src/v1/service.rs | 1 + tests/tests/gas_price.rs | 13 +++++++++++-- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index bffec4934c4..f38a68da9e9 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -494,6 +494,14 @@ impl AlgorithmUpdaterV1 { 0u64 } }); + tracing::debug!("Profit: {}", self.last_profit); + tracing::debug!( + "DA gas price change: p: {}, d: {}, change: {}, new: {}", + p, + d, + da_change, + maybe_new_scaled_da_gas_price + ); self.new_scaled_da_gas_price = max( self.min_scaled_da_gas_price(), maybe_new_scaled_da_gas_price, diff --git a/crates/services/gas_price_service/src/v1/da_source_service/service.rs b/crates/services/gas_price_service/src/v1/da_source_service/service.rs index 207879adc83..5fb073de7eb 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/service.rs @@ -61,7 +61,9 @@ where } async fn process_block_costs(&mut self) -> Result<()> { - let da_block_costs = self.source.request_da_block_cost().await?; + let da_block_costs_res = self.source.request_da_block_cost().await; + tracing::debug!("Received block costs: {:?}", da_block_costs_res); + let da_block_costs = da_block_costs_res?; self.shared_state.0.send(da_block_costs)?; Ok(()) } diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 773a8a8a113..d9c0ff1cbc8 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -197,6 +197,7 @@ where .map_err(|err| anyhow!(err))?; StorageTxProvider::commit_transaction(storage_tx)?; let new_algo = self.algorithm_updater.algorithm(); + tracing::debug!("Updating gas price: {}", &new_algo.calculate()); self.shared_algo.update(new_algo).await; Ok(()) } diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 428f1a14ac4..cfe83a776df 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -451,7 +451,16 @@ fn produce_block__l1_committed_block_effects_gas_price() { let url = mock.url(); // add the da committer url to the args - args.extend(&["--da-committer-url", &url, "--da-poll-interval", "10"]); + args.extend(&[ + "--da-committer-url", + &url, + "--da-poll-interval", + "50", + "--da-p-component", + "1", + "--max-da-gas-price-change-percent", + "100", + ]); let new_gas_price = rt.block_on(async { let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) @@ -474,7 +483,7 @@ impl FakeServer { let mut server = mockito::Server::new(); let costs = RawDaBlockCosts { sequence_number: 1, - blocks_heights: vec![1, 2, 3], + blocks_heights: vec![1], da_block_height: DaBlockHeight(100), total_cost: 100, total_size_bytes: 100, From 51d41c73be8d78cc1a4e9e3bcdde6e73d83f54c5 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 6 Dec 2024 11:15:51 -0700 Subject: [PATCH 069/131] Correct typo --- crates/fuel-gas-price-algorithm/src/v1.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 09cddbb42ab..2464f196a09 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -16,7 +16,7 @@ pub enum Error { #[error("Could not calculate cost per byte: {bytes:?} bytes, {cost:?} cost")] CouldNotCalculateCostPerByte { bytes: u128, cost: u128 }, #[error("Failed to include L2 block data: {0}")] - FailedTooIncludeL2BlockData(String), + FailedToIncludeL2BlockData(String), #[error("L2 block expected but not found in unrecorded blocks: {height}")] L2BlockExpectedNotFound { height: u32 }, } From a642cb72a9e998bc6b1dcc37b6dd54d1c8400956 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 6 Dec 2024 11:20:58 -0700 Subject: [PATCH 070/131] Use relevant config for v1 constructor --- crates/services/gas_price_service/src/v1/uninitialized_task.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 60ed5184ef5..2f5699c41f7 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -324,7 +324,7 @@ pub fn new_gas_price_service_v1< DA, SettingsProvider, >( - config: GasPriceServiceConfig, + v1_config: V1AlgorithmConfig, genesis_block_height: BlockHeight, settings: SettingsProvider, block_stream: BoxStream, @@ -345,7 +345,6 @@ where Metadata: MetadataStorage, DA: DaBlockCostsSource, { - let v1_config = config.v1().ok_or(anyhow::anyhow!("Expected V1 config"))?; let gas_price_init = UninitializedTask::new( v1_config, genesis_block_height, From abf0cca485b22cfcad204d98d5c2ebcb2f47934a Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 6 Dec 2024 14:04:27 -0700 Subject: [PATCH 071/131] Add todo --- crates/services/gas_price_service/src/v1/metadata.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index bf33cca6ffd..ef2e88466f5 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -63,6 +63,8 @@ pub struct V1AlgorithmConfig { pub min_exec_gas_price: u64, pub exec_gas_price_change_percent: u16, pub l2_block_fullness_threshold_percent: u8, + // TODO:We don't need this after we implement + // https://github.com/FuelLabs/fuel-core/issues/2481 pub gas_price_factor: NonZeroU64, pub min_da_gas_price: u64, pub max_da_gas_price_change_percent: u16, From 288f51f5fbb1c75520ad31b26ebadbfdbc49e80a Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 6 Dec 2024 21:08:13 -0700 Subject: [PATCH 072/131] Add missed things --- crates/services/gas_price_service/src/v1/tests.rs | 13 ------------- .../gas_price_service/src/v1/uninitialized_task.rs | 14 -------------- 2 files changed, 27 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index b48e6351d52..66a32e80662 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -55,7 +55,6 @@ use anyhow::{ anyhow, Result, }; -use anyhow::anyhow; use fuel_core_services::{ stream::{ BoxStream, @@ -89,7 +88,6 @@ use fuel_core_types::{ SharedImportResult, }, }; -use fuel_gas_price_algorithm::v1::AlgorithmUpdaterV1; use fuel_gas_price_algorithm::v1::{ AlgorithmUpdaterV1, Bytes, @@ -100,11 +98,6 @@ use fuel_gas_price_algorithm::v1::{ use std::{ num::NonZeroU64, ops::Deref, - sync::Arc, - ops::{ - Deref, - Range, - }, sync::{ Arc, Mutex, @@ -261,13 +254,11 @@ impl FakeDABlockCost { sequence_number: sequence_number.clone(), }; (service, sequence_number) - Self { da_block_costs } } } #[async_trait::async_trait] impl DaBlockCostsSource for FakeDABlockCost { - async fn request_da_block_cost(&mut self) -> anyhow::Result { async fn request_da_block_cost(&mut self) -> Result { let costs = self.da_block_costs.recv().await.unwrap(); Ok(costs) @@ -376,7 +367,6 @@ async fn next_gas_price__affected_by_new_l2_block() { let da_source_service = DaSourceService::new(da_source, None); let mut service = GasPriceServiceV1::new( l2_block_source, - metadata_storage, shared_algo, algo_updater, da_source_service, @@ -425,7 +415,6 @@ async fn run__new_l2_block_saves_old_metadata() { let da_source_service = DaSourceService::new(da_source, None); let mut service = GasPriceServiceV1::new( l2_block_source, - metadata_storage, shared_algo, algo_updater, da_source_service, @@ -565,7 +554,6 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr settings, block_stream, gas_price_db, - metadata_storage, da_cost_source, on_chain_db, inner, @@ -672,7 +660,6 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { settings, block_stream, gas_price_db, - metadata_storage, da_cost_source, on_chain_db, erroring_persisted_data, diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index aaba8d2194c..a9a8eb2792a 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -81,7 +81,6 @@ pub mod fuel_storage_unrecorded_blocks; pub struct UninitializedTask< L2DataStoreView, GasPriceStore, - Metadata, DA, SettingsProvider, PersistedData, @@ -94,7 +93,6 @@ pub struct UninitializedTask< pub block_stream: BoxStream, pub(crate) shared_algo: SharedV1Algorithm, pub(crate) algo_updater: AlgorithmUpdaterV1, - pub(crate) metadata_storage: Metadata, pub(crate) da_source: DA, pub persisted_data: PersistedData, } @@ -126,7 +124,6 @@ where settings: SettingsProvider, block_stream: BoxStream, gas_price_db: GasPriceStore, - metadata_storage: Metadata, da_source: DA, on_chain_db: L2DataStoreView, persisted_data: PersistedData, @@ -149,7 +146,6 @@ where block_stream, algo_updater, shared_algo, - metadata_storage, da_source, persisted_data, }; @@ -199,7 +195,6 @@ where { let service = GasPriceServiceV1::new( l2_block_source, - self.metadata_storage, self.shared_algo, self.algo_updater, da_service, @@ -211,7 +206,6 @@ where sync_gas_price_db_with_on_chain_storage( &self.settings, &self.config, - &mut self.metadata_storage, &self.on_chain_db, metadata_height, latest_block_height, @@ -221,7 +215,6 @@ where let service = GasPriceServiceV1::new( l2_block_source, - self.metadata_storage, self.shared_algo, self.algo_updater, da_service, @@ -281,13 +274,11 @@ fn sync_gas_price_db_with_on_chain_storage< 'a, L2DataStore, L2DataStoreView, - Metadata, SettingsProvider, PersistedData, >( settings: &SettingsProvider, config: &V1AlgorithmConfig, - metadata_storage: &mut Metadata, on_chain_db: &L2DataStoreView, metadata_height: u32, latest_block_height: u32, @@ -296,7 +287,6 @@ fn sync_gas_price_db_with_on_chain_storage< where L2DataStore: L2Data, L2DataStoreView: AtomicView, - Metadata: MetadataStorage, SettingsProvider: GasPriceSettingsProvider, PersistedData: GetMetadataStorage + TransactionableStorage + 'a, ::Transaction<'a>: @@ -345,7 +335,6 @@ fn sync_v1_metadata< where L2DataStore: L2Data, L2DataStoreView: AtomicView, - Metadata: MetadataStorage, SettingsProvider: GasPriceSettingsProvider, StorageTxGenerator: TransactionableStorage + 'a, ::Transaction<'a>: @@ -407,7 +396,6 @@ pub fn new_gas_price_service_v1< settings: SettingsProvider, block_stream: BoxStream, gas_price_db: GasPriceStore, - metadata: Metadata, da_source: DA, on_chain_db: L2DataStore, persisted_data: PersistedData, @@ -427,7 +415,6 @@ where L2DataStore::LatestView: L2Data, GasPriceStore: GasPriceData, SettingsProvider: GasPriceSettingsProvider, - Metadata: MetadataStorage, DA: DaBlockCostsSource, PersistedData: GetMetadataStorage + GetDaSequenceNumber + TransactionableStorage + 'static, @@ -440,7 +427,6 @@ where settings, block_stream, gas_price_db, - metadata, da_source, on_chain_db, persisted_data, From 9fe913c9642f1e3f0be7645337dc9e85e2aed228 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 9 Dec 2024 11:13:04 -0700 Subject: [PATCH 073/131] Remove extra code from merge --- crates/services/gas_price_service/src/v1/tests.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 66a32e80662..976ed49c625 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -436,10 +436,6 @@ async fn run__new_l2_block_saves_old_metadata() { // cleanup service.shutdown().await.unwrap(); - - // then - let metadata_is_some = metadata_inner.lock().unwrap().is_some(); - assert!(metadata_is_some) } #[derive(Clone)] From 2de6505e5ab99285f8432b55cecfc692f2561c36 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 9 Dec 2024 11:20:49 -0700 Subject: [PATCH 074/131] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2a66d36128..52e5ccc6293 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [2233](https://github.com/FuelLabs/fuel-core/pull/2233): Introduce a new column `modification_history_v2` for storing the modification history in the historical rocksDB. Keys in this column are stored in big endian order. Changed the behaviour of the historical rocksDB to write changes for new block heights to the new column, and to perform lookup of values from the `modification_history_v2` table first, and then from the `modification_history` table, performing a migration upon access if necessary. #### Breaking +- [](https://github.com/FuelLabs/fuel-core/pull/2469): Move from `GasPriceServicev0` to `GasPriceServiceV1`. Include new config values. - [2389](https://github.com/FuelLabs/fuel-core/pull/2258): Updated the `messageProof` GraphQL schema to return a non-nullable `MessageProof`. - [2154](https://github.com/FuelLabs/fuel-core/pull/2154): Transaction graphql endpoints use `TransactionType` instead of `fuel_tx::Transaction`. - [2446](https://github.com/FuelLabs/fuel-core/pull/2446): Use graphiql instead of graphql-playground due to known vulnerability and stale development. From 368b939e62508d2604462b4d8cab78f12c9676da Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 9 Dec 2024 11:22:07 -0700 Subject: [PATCH 075/131] Add missing PR number --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52e5ccc6293..cd3116d21de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,7 +47,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [2233](https://github.com/FuelLabs/fuel-core/pull/2233): Introduce a new column `modification_history_v2` for storing the modification history in the historical rocksDB. Keys in this column are stored in big endian order. Changed the behaviour of the historical rocksDB to write changes for new block heights to the new column, and to perform lookup of values from the `modification_history_v2` table first, and then from the `modification_history` table, performing a migration upon access if necessary. #### Breaking -- [](https://github.com/FuelLabs/fuel-core/pull/2469): Move from `GasPriceServicev0` to `GasPriceServiceV1`. Include new config values. +- [2469](https://github.com/FuelLabs/fuel-core/pull/2469): Move from `GasPriceServicev0` to `GasPriceServiceV1`. Include new config values. - [2389](https://github.com/FuelLabs/fuel-core/pull/2258): Updated the `messageProof` GraphQL schema to return a non-nullable `MessageProof`. - [2154](https://github.com/FuelLabs/fuel-core/pull/2154): Transaction graphql endpoints use `TransactionType` instead of `fuel_tx::Transaction`. - [2446](https://github.com/FuelLabs/fuel-core/pull/2446): Use graphiql instead of graphql-playground due to known vulnerability and stale development. From 86c3e2e21adff41c94f91560e00a58f57bb3caad Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 9 Dec 2024 11:29:16 -0700 Subject: [PATCH 076/131] Lint cargo.toml --- tests/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 2e5555242ae..bae9c54fc2d 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -75,7 +75,7 @@ tokio = { workspace = true, features = [ "rt-multi-thread", "test-util", ] } -tracing-subscriber.workspace = true +tracing-subscriber = { workspace = true } [dev-dependencies] pretty_assertions = "1.4" From 6291999136a7d4b740fd71e22a083bb0c3e4a97d Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 9 Dec 2024 12:07:03 -0700 Subject: [PATCH 077/131] Use clearer types in storage definitions, fix init code --- crates/fuel-core/src/service/sub_services.rs | 23 +++++++++++++------ .../src/common/fuel_core_storage_adapter.rs | 6 +++-- .../fuel_core_storage_adapter/storage.rs | 12 ++++++---- .../src/v1/uninitialized_task.rs | 1 - .../fuel_storage_unrecorded_blocks.rs | 20 +++++++++++----- 5 files changed, 42 insertions(+), 20 deletions(-) diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index 50ef1cd3e17..1095754a769 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -38,13 +38,17 @@ use crate::{ SubServices, }, }; -use fuel_core_gas_price_service::v1::{ - algorithm::AlgorithmV1, - da_source_service::block_committer_costs::{ - BlockCommitterDaBlockCosts, - BlockCommitterHttpApi, +use anyhow::anyhow; +use fuel_core_gas_price_service::{ + ports::GasPriceServiceConfig, + v1::{ + algorithm::AlgorithmV1, + da_source_service::block_committer_costs::{ + BlockCommitterDaBlockCosts, + BlockCommitterHttpApi, + }, + uninitialized_task::new_gas_price_service_v1, }, - uninitialized_task::new_gas_price_service_v1, }; use fuel_core_poa::{ signer::SignMode, @@ -191,9 +195,14 @@ pub fn init_sub_services( tracing::debug!("da_committer_url: {:?}", config.da_committer_url); let committer_api = BlockCommitterHttpApi::new(config.da_committer_url.clone()); let da_source = BlockCommitterDaBlockCosts::new(committer_api, None); + let v1_config = GasPriceServiceConfig::from(config.clone()) + .v1() + .ok_or(anyhow!( + "Gas price service configuration is not compatible with V1 algorithm" + ))?; let gas_price_service = new_gas_price_service_v1( - config.clone().into(), + v1_config, genesis_block_height, settings, block_stream, diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs index f1c38d504d2..f28f4cd0e4a 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs @@ -134,9 +134,10 @@ where Storage: KeyValueInspect + Modifiable + Send + Sync, { fn insert(&mut self, height: Height, bytes: Bytes) -> Result<(), Error> { + let block_height = height.into(); self.inner .storage_as_mut::() - .insert(&height, &bytes) + .insert(&block_height, &bytes) .map_err(|err| { Error::CouldNotInsertUnrecordedBlock(format!("Error: {:?}", err)) })?; @@ -144,10 +145,11 @@ where } fn remove(&mut self, height: &Height) -> Result, Error> { + let block_height = BlockHeight::from(*height); let bytes = self .inner .storage_as_mut::() - .take(height) + .take(&block_height) .map_err(|err| { Error::CouldNotRemoveUnrecordedBlock(format!("Error: {:?}", err)) })?; diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs index e6cb745b36c..72f41de7a17 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs @@ -75,11 +75,13 @@ impl TableWithBlueprint for GasPriceMetadata { /// for future blocks to be recorded on the DA chain pub struct UnrecordedBlocksTable; +pub type BlockBytes = u64; + impl Mappable for UnrecordedBlocksTable { type Key = Self::OwnedKey; - type OwnedKey = u32; + type OwnedKey = BlockHeight; type Value = Self::OwnedValue; - type OwnedValue = u64; + type OwnedValue = BlockBytes; } impl TableWithBlueprint for UnrecordedBlocksTable { @@ -93,11 +95,13 @@ impl TableWithBlueprint for UnrecordedBlocksTable { pub struct SequenceNumberTable; +pub type SequenceNumber = u32; + impl Mappable for SequenceNumberTable { type Key = Self::OwnedKey; - type OwnedKey = u32; + type OwnedKey = BlockHeight; type Value = Self::OwnedValue; - type OwnedValue = u32; + type OwnedValue = SequenceNumber; } impl TableWithBlueprint for SequenceNumberTable { diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 9120a501e2d..a207e2dfe49 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -392,7 +392,6 @@ where pub fn new_gas_price_service_v1< L2DataStore, GasPriceStore, - Metadata, DA, SettingsProvider, PersistedData, diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs index 6e3985476ed..fc96819f38f 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs @@ -1,6 +1,9 @@ -use crate::common::fuel_core_storage_adapter::storage::{ - GasPriceColumn, - UnrecordedBlocksTable, +use crate::common::{ + fuel_core_storage_adapter::storage::{ + GasPriceColumn, + UnrecordedBlocksTable, + }, + utils::BlockInfo::Block, }; use fuel_core_storage::{ kv_store::{ @@ -14,7 +17,10 @@ use fuel_core_storage::{ StorageAsMut, StorageAsRef, }; -use fuel_core_types::fuel_merkle::storage::StorageMutateInfallible; +use fuel_core_types::{ + fuel_merkle::storage::StorageMutateInfallible, + fuel_types::BlockHeight, +}; use fuel_gas_price_algorithm::{ v1, v1::UnrecordedBlocks, @@ -38,8 +44,9 @@ where { fn insert(&mut self, height: v1::Height, bytes: v1::Bytes) -> Result<(), v1::Error> { let mut tx = self.inner.write_transaction(); + let block_height = BlockHeight::from(height); tx.storage_as_mut::() - .insert(&height, &bytes) + .insert(&block_height, &bytes) .and_then(|_| tx.commit()) .map_err(|err| { v1::Error::CouldNotInsertUnrecordedBlock(format!("Error: {:?}", err)) @@ -49,9 +56,10 @@ where fn remove(&mut self, height: &v1::Height) -> Result, v1::Error> { let mut tx = self.inner.write_transaction(); + let block_height = BlockHeight::from(*height); let bytes = tx .storage_as_mut::() - .take(height) + .take(&block_height) .map_err(|err| { v1::Error::CouldNotRemoveUnrecordedBlock(format!("Error: {:?}", err)) })?; From 2eba245e3fd7fc096b54894f4ef6234311426d3e Mon Sep 17 00:00:00 2001 From: Green Baneling Date: Mon, 9 Dec 2024 16:58:51 -0500 Subject: [PATCH 078/131] Suggestions to simplify the V1 storage requirements and initialization (#2483) Suggestions for the https://github.com/FuelLabs/fuel-core/pull/2468 PR. --------- Co-authored-by: Mitchell Turner --- crates/fuel-core/src/service/sub_services.rs | 4 +- crates/fuel-gas-price-algorithm/src/v1.rs | 45 +++--- .../src/common/fuel_core_storage_adapter.rs | 148 ++++-------------- .../services/gas_price_service/src/ports.rs | 19 ++- .../src/v1/da_source_service/service.rs | 5 +- .../gas_price_service/src/v1/service.rs | 70 ++++----- .../gas_price_service/src/v1/tests.rs | 44 +++--- .../src/v1/uninitialized_task.rs | 140 +++++------------ .../fuel_storage_unrecorded_blocks.rs | 52 +++--- 9 files changed, 196 insertions(+), 331 deletions(-) diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index 4fd45534b24..03003319235 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -182,7 +182,7 @@ pub fn init_sub_services( let genesis_block_height = *genesis_block.header().height(); let settings = consensus_parameters_provider.clone(); let block_stream = importer_adapter.events_shared_result(); - let metadata = StructuredStorage::new(database.gas_price().clone()); + let metadata = database.gas_price().clone(); let gas_price_service_v0 = new_gas_price_service_v0( config.clone().into(), @@ -190,7 +190,7 @@ pub fn init_sub_services( settings, block_stream, database.gas_price().clone(), - metadata, + StructuredStorage::new(metadata), database.on_chain().clone(), )?; diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 10f71d1d3d9..ac670d7b38f 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -65,18 +65,20 @@ impl AlgorithmV1 { pub type Height = u32; pub type Bytes = u64; + pub trait UnrecordedBlocks { - fn insert(&mut self, height: Height, bytes: Bytes) -> Result<(), Error>; - fn remove(&mut self, height: &Height) -> Result, Error>; + fn insert(&mut self, height: Height, bytes: Bytes) -> Result<(), String>; + + fn remove(&mut self, height: &Height) -> Result, String>; } impl UnrecordedBlocks for BTreeMap { - fn insert(&mut self, height: Height, bytes: Bytes) -> Result<(), Error> { + fn insert(&mut self, height: Height, bytes: Bytes) -> Result<(), String> { self.insert(height, bytes); Ok(()) } - fn remove(&mut self, height: &Height) -> Result, Error> { + fn remove(&mut self, height: &Height) -> Result, String> { let value = self.remove(height); Ok(value) } @@ -395,7 +397,9 @@ impl AlgorithmUpdaterV1 { self.update_da_gas_price(); // metadata - unrecorded_blocks.insert(height, block_bytes)?; + unrecorded_blocks + .insert(height, block_bytes) + .map_err(Error::CouldNotInsertUnrecordedBlock)?; self.unrecorded_blocks_bytes = self .unrecorded_blocks_bytes .saturating_add(block_bytes as u128); @@ -563,7 +567,7 @@ impl AlgorithmUpdaterV1 { recording_cost: u128, unrecorded_blocks: &mut U, ) -> Result<(), Error> { - self.update_unrecorded_block_bytes(heights, unrecorded_blocks); + self.update_unrecorded_block_bytes(heights, unrecorded_blocks)?; let new_da_block_cost = self .latest_known_total_da_cost_excess @@ -589,26 +593,25 @@ impl AlgorithmUpdaterV1 { &mut self, heights: &[u32], unrecorded_blocks: &mut U, - ) { + ) -> Result<(), Error> { let mut total: u128 = 0; for expected_height in heights { - let res = unrecorded_blocks.remove(expected_height); - match res { - Ok(Some(bytes)) => { - total = total.saturating_add(bytes as u128); - } - Ok(None) => { - tracing::warn!( - "L2 block expected but not found in unrecorded blocks: {}", - expected_height, - ); - } - Err(err) => { - tracing::error!("Could not remove unrecorded block: {}", err); - } + let maybe_bytes = unrecorded_blocks + .remove(expected_height) + .map_err(Error::CouldNotRemoveUnrecordedBlock)?; + + if let Some(bytes) = maybe_bytes { + total = total.saturating_add(bytes as u128); + } else { + tracing::warn!( + "L2 block expected but not found in unrecorded blocks: {}", + expected_height, + ); } } self.unrecorded_blocks_bytes = self.unrecorded_blocks_bytes.saturating_sub(total); + + Ok(()) } fn recalculate_projected_cost(&mut self) { diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs index f1c38d504d2..591707cb46b 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs @@ -1,29 +1,27 @@ -use crate::common::utils::{ - BlockInfo, - Error as GasPriceError, - Result as GasPriceResult, -}; -use anyhow::anyhow; -use fuel_core_types::fuel_types::BlockHeight; - use crate::{ common::{ fuel_core_storage_adapter::storage::{ GasPriceColumn, GasPriceMetadata, SequenceNumberTable, - UnrecordedBlocksTable, }, updater_metadata::UpdaterMetadata, + utils::{ + BlockInfo, + Error as GasPriceError, + Result as GasPriceResult, + }, }, ports::{ + GasPriceServiceAtomicStorage, GetDaSequenceNumber, GetMetadataStorage, SetDaSequenceNumber, SetMetadataStorage, - TransactionableStorage, }, }; +use anyhow::anyhow; +use core::cmp::min; use fuel_core_storage::{ codec::{ postcard::Postcard, @@ -36,14 +34,17 @@ use fuel_core_storage::{ StorageTransaction, WriteTransaction, }, + Error as StorageError, StorageAsMut, StorageAsRef, + StorageInspect, }; use fuel_core_types::{ blockchain::{ block::Block, header::ConsensusParametersVersion, }, + fuel_merkle::storage::StorageMutate, fuel_tx::{ field::{ MintAmount, @@ -51,14 +52,8 @@ use fuel_core_types::{ }, Transaction, }, + fuel_types::BlockHeight, }; -use fuel_gas_price_algorithm::v1::{ - Bytes, - Error, - Height, - UnrecordedBlocks, -}; -use std::cmp::min; #[cfg(test)] mod metadata_tests; @@ -67,8 +62,8 @@ pub mod storage; impl SetMetadataStorage for StructuredStorage where - Storage: KeyValueInspect + Modifiable, Storage: Send + Sync, + Storage: KeyValueInspect + Modifiable, { fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { let block_height = metadata.l2_block_height(); @@ -84,9 +79,10 @@ where } } -impl GetMetadataStorage for StructuredStorage +impl GetMetadataStorage for Storage where - Storage: KeyValueInspect + Send + Sync, + Storage: Send + Sync, + Storage: StorageInspect, { fn get_metadata( &self, @@ -102,9 +98,10 @@ where } } -impl GetDaSequenceNumber for StructuredStorage +impl GetDaSequenceNumber for Storage where - Storage: KeyValueInspect + Send + Sync, + Storage: Send + Sync, + Storage: StorageInspect, { fn get_sequence_number( &self, @@ -119,137 +116,50 @@ where } } -pub struct NewStorageTransaction<'a, Storage> { - inner: StorageTransaction<&'a mut StructuredStorage>, -} - -impl<'a, Storage> NewStorageTransaction<'a, Storage> { - fn wrap(inner: StorageTransaction<&'a mut StructuredStorage>) -> Self { - Self { inner } - } -} - -impl<'a, Storage> UnrecordedBlocks for NewStorageTransaction<'a, Storage> +impl GasPriceServiceAtomicStorage for Storage where + Storage: 'static, + Storage: GetMetadataStorage + GetDaSequenceNumber, Storage: KeyValueInspect + Modifiable + Send + Sync, { - fn insert(&mut self, height: Height, bytes: Bytes) -> Result<(), Error> { - self.inner - .storage_as_mut::() - .insert(&height, &bytes) - .map_err(|err| { - Error::CouldNotInsertUnrecordedBlock(format!("Error: {:?}", err)) - })?; - Ok(()) - } - - fn remove(&mut self, height: &Height) -> Result, Error> { - let bytes = self - .inner - .storage_as_mut::() - .take(height) - .map_err(|err| { - Error::CouldNotRemoveUnrecordedBlock(format!("Error: {:?}", err)) - })?; - Ok(bytes) - } -} - -impl TransactionableStorage for StructuredStorage -where - Storage: Modifiable + Send + Sync, -{ - type Transaction<'a> = NewStorageTransaction<'a, Storage> where Self: 'a; + type Transaction<'a> = StorageTransaction<&'a mut Storage> where Self: 'a; fn begin_transaction(&mut self) -> GasPriceResult> { let tx = self.write_transaction(); - let wrapped = NewStorageTransaction::wrap(tx); - Ok(wrapped) + Ok(tx) } fn commit_transaction(transaction: Self::Transaction<'_>) -> GasPriceResult<()> { transaction - .inner .commit() .map_err(|err| GasPriceError::CouldNotCommit(err.into()))?; Ok(()) } } -impl<'a, Storage> SetMetadataStorage for NewStorageTransaction<'a, Storage> -where - Storage: KeyValueInspect + Modifiable + Send + Sync, -{ - fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { - let block_height = metadata.l2_block_height(); - self.inner - .storage_as_mut::() - .insert(&block_height, metadata) - .map_err(|err| GasPriceError::CouldNotSetMetadata { - block_height, - source_error: err.into(), - })?; - Ok(()) - } -} -impl<'a, Storage> GetMetadataStorage for NewStorageTransaction<'a, Storage> -where - Storage: KeyValueInspect + Send + Sync, -{ - fn get_metadata( - &self, - block_height: &BlockHeight, - ) -> GasPriceResult> { - let metadata = self - .inner - .storage::() - .get(block_height) - .map_err(|err| GasPriceError::CouldNotFetchMetadata { - source_error: err.into(), - })?; - Ok(metadata.map(|inner| inner.into_owned())) - } -} - -impl<'a, Storage> SetDaSequenceNumber for NewStorageTransaction<'a, Storage> +impl SetDaSequenceNumber for Storage where - Storage: KeyValueInspect + Modifiable + Send + Sync, + Storage: Send + Sync, + Storage: StorageMutate, { fn set_sequence_number( &mut self, block_height: &BlockHeight, sequence_number: u32, ) -> GasPriceResult<()> { - self.inner - .storage_as_mut::() + self.storage_as_mut::() .insert(block_height, &sequence_number) .map_err(|err| GasPriceError::CouldNotFetchDARecord(err.into()))?; Ok(()) } } -impl<'a, Storage> GetDaSequenceNumber for NewStorageTransaction<'a, Storage> -where - Storage: KeyValueInspect + Modifiable + Send + Sync, -{ - fn get_sequence_number( - &self, - block_height: &BlockHeight, - ) -> GasPriceResult> { - let sequence_number = self - .inner - .storage::() - .get(block_height) - .map_err(|err| GasPriceError::CouldNotFetchDARecord(err.into()))? - .map(|no| *no); - Ok(sequence_number) - } -} #[derive(Debug, Clone, PartialEq)] pub struct GasPriceSettings { pub gas_price_factor: u64, pub block_gas_limit: u64, } + pub trait GasPriceSettingsProvider: Send + Sync + Clone { fn settings( &self, diff --git a/crates/services/gas_price_service/src/ports.rs b/crates/services/gas_price_service/src/ports.rs index 9b92156bb69..0bd601cf7a8 100644 --- a/crates/services/gas_price_service/src/ports.rs +++ b/crates/services/gas_price_service/src/ports.rs @@ -11,7 +11,10 @@ use crate::{ utils::Result, }, v0::metadata::V0AlgorithmConfig, - v1::metadata::V1AlgorithmConfig, + v1::{ + metadata::V1AlgorithmConfig, + uninitialized_task::fuel_storage_unrecorded_blocks::AsUnrecordedBlocks, + }, }; pub trait L2Data: Send + Sync { @@ -43,12 +46,22 @@ pub trait GetDaSequenceNumber: Send + Sync { fn get_sequence_number(&self, block_height: &BlockHeight) -> Result>; } -pub trait TransactionableStorage: Send + Sync { - type Transaction<'a> +pub trait GasPriceServiceAtomicStorage +where + Self: 'static, + Self: Send + Sync, + Self: GetMetadataStorage + GetDaSequenceNumber, +{ + type Transaction<'a>: AsUnrecordedBlocks + + SetMetadataStorage + + GetMetadataStorage + + SetDaSequenceNumber + + GetDaSequenceNumber where Self: 'a; fn begin_transaction(&mut self) -> Result>; + fn commit_transaction(transaction: Self::Transaction<'_>) -> Result<()>; } diff --git a/crates/services/gas_price_service/src/v1/da_source_service/service.rs b/crates/services/gas_price_service/src/v1/da_source_service/service.rs index 7800d80489c..108f50591e6 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/service.rs @@ -32,10 +32,7 @@ impl SharedState { /// This struct houses the shared_state, polling interval /// and a source, which does the actual fetching of the data -pub struct DaSourceService -where - Source: DaBlockCostsSource, -{ +pub struct DaSourceService { poll_interval: Interval, source: Source, shared_state: SharedState, diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index fbd4959e01b..1ba2462165d 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -11,10 +11,10 @@ use crate::{ }, }, ports::{ + GasPriceServiceAtomicStorage, GetMetadataStorage, SetDaSequenceNumber, SetMetadataStorage, - TransactionableStorage, }, v0::metadata::V0Metadata, v1::{ @@ -33,7 +33,10 @@ use crate::{ V1AlgorithmConfig, V1Metadata, }, - uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, + uninitialized_task::fuel_storage_unrecorded_blocks::{ + AsUnrecordedBlocks, + FuelStorageUnrecordedBlocks, + }, }, }; use anyhow::anyhow; @@ -57,11 +60,7 @@ use futures::FutureExt; use tokio::sync::broadcast::Receiver; /// The service that updates the gas price algorithm. -pub struct GasPriceServiceV1 -where - DA: DaBlockCostsSource, - StorageTxProvider: TransactionableStorage, -{ +pub struct GasPriceServiceV1 { /// The algorithm that can be used in the next block shared_algo: SharedV1Algorithm, /// The L2 block source @@ -78,20 +77,16 @@ where storage_tx_provider: StorageTxProvider, } -impl GasPriceServiceV1 +impl GasPriceServiceV1 where L2: L2BlockSource, DA: DaBlockCostsSource, - StorageTxProvider: TransactionableStorage, + AtomicStorage: GasPriceServiceAtomicStorage, { - async fn commit_block_data_to_algorithm<'a>( - &'a mut self, + async fn commit_block_data_to_algorithm( + &mut self, l2_block_res: GasPriceResult, - ) -> anyhow::Result<()> - where - StorageTxProvider::Transaction<'a>: - SetMetadataStorage + UnrecordedBlocks + SetDaSequenceNumber, - { + ) -> anyhow::Result<()> { tracing::info!("Received L2 block result: {:?}", l2_block_res); let block = l2_block_res?; @@ -101,17 +96,17 @@ where } } -impl GasPriceServiceV1 +impl GasPriceServiceV1 where DA: DaBlockCostsSource, - StorageTxProvider: TransactionableStorage, + AtomicStorage: GasPriceServiceAtomicStorage, { pub fn new( l2_block_source: L2, shared_algo: SharedV1Algorithm, algorithm_updater: AlgorithmUpdaterV1, da_source_adapter_handle: DaSourceService, - storage_tx_provider: StorageTxProvider, + storage_tx_provider: AtomicStorage, ) -> Self { let da_source_channel = da_source_adapter_handle.shared_data().clone().subscribe(); @@ -135,7 +130,7 @@ where } #[cfg(test)] - pub fn storage_tx_provider(&self) -> &StorageTxProvider { + pub fn storage_tx_provider(&self) -> &AtomicStorage { &self.storage_tx_provider } @@ -150,18 +145,14 @@ where .ok_or_else(|| anyhow!("Block gas capacity must be non-zero")) } - async fn handle_normal_block<'a>( - &'a mut self, + async fn handle_normal_block( + &mut self, height: u32, gas_used: u64, block_gas_capacity: u64, block_bytes: u64, block_fees: u64, - ) -> anyhow::Result<()> - where - StorageTxProvider::Transaction<'a>: - UnrecordedBlocks + SetMetadataStorage + SetDaSequenceNumber, - { + ) -> anyhow::Result<()> { let capacity = Self::validate_block_gas_capacity(block_gas_capacity)?; let mut storage_tx = self.storage_tx_provider.begin_transaction()?; @@ -171,7 +162,7 @@ where &da_block_costs.l2_blocks, da_block_costs.blob_size_bytes, da_block_costs.blob_cost_wei, - &mut storage_tx, + &mut storage_tx.as_unrecorded_blocks(), )?; storage_tx .set_sequence_number( @@ -187,33 +178,29 @@ where capacity, block_bytes, block_fees as u128, - &mut storage_tx, + &mut storage_tx.as_unrecorded_blocks(), )?; let metadata = self.algorithm_updater.clone().into(); storage_tx .set_metadata(&metadata) .map_err(|err| anyhow!(err))?; - StorageTxProvider::commit_transaction(storage_tx)?; + AtomicStorage::commit_transaction(storage_tx)?; let new_algo = self.algorithm_updater.algorithm(); self.shared_algo.update(new_algo).await; Ok(()) } - async fn apply_block_info_to_gas_algorithm<'a>( - &'a mut self, + async fn apply_block_info_to_gas_algorithm( + &mut self, l2_block: BlockInfo, - ) -> anyhow::Result<()> - where - StorageTxProvider::Transaction<'a>: - SetMetadataStorage + UnrecordedBlocks + SetDaSequenceNumber, - { + ) -> anyhow::Result<()> { match l2_block { BlockInfo::GenesisBlock => { let metadata: UpdaterMetadata = self.algorithm_updater.clone().into(); let mut tx = self.storage_tx_provider.begin_transaction()?; tx.set_metadata(&metadata).map_err(|err| anyhow!(err))?; - StorageTxProvider::commit_transaction(tx)?; + AtomicStorage::commit_transaction(tx)?; let new_algo = self.algorithm_updater.algorithm(); // self.update(new_algo).await; self.shared_algo.update(new_algo).await; @@ -241,14 +228,11 @@ where } #[async_trait] -impl RunnableTask - for GasPriceServiceV1 +impl RunnableTask for GasPriceServiceV1 where L2: L2BlockSource, DA: DaBlockCostsSource, - StorageTxProvider: TransactionableStorage, - for<'a> StorageTxProvider::Transaction<'a>: - UnrecordedBlocks + SetMetadataStorage + SetDaSequenceNumber, + AtomicStorage: GasPriceServiceAtomicStorage, { async fn run(&mut self, watcher: &mut StateWatcher) -> TaskNextAction { tokio::select! { diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 976ed49c625..92fd6757f05 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -8,7 +8,6 @@ use crate::{ }, GasPriceSettings, GasPriceSettingsProvider, - NewStorageTransaction, }, l2_block_source::L2BlockSource, updater_metadata::UpdaterMetadata, @@ -20,12 +19,12 @@ use crate::{ }, ports::{ GasPriceData, + GasPriceServiceAtomicStorage, GetDaSequenceNumber, GetMetadataStorage, L2Data, SetDaSequenceNumber, SetMetadataStorage, - TransactionableStorage, }, v1::{ algorithm::SharedV1Algorithm, @@ -46,7 +45,7 @@ use crate::{ GasPriceServiceV1, }, uninitialized_task::{ - fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, + fuel_storage_unrecorded_blocks::AsUnrecordedBlocks, UninitializedTask, }, }, @@ -135,6 +134,7 @@ impl SetMetadataStorage for FakeMetadata { Ok(()) } } + impl GetMetadataStorage for FakeMetadata { fn get_metadata(&self, _: &BlockHeight) -> GasPriceResult> { let metadata = self.inner.lock().unwrap().clone(); @@ -152,6 +152,7 @@ impl SetMetadataStorage for ErroringPersistedData { }) } } + impl GetMetadataStorage for ErroringPersistedData { fn get_metadata(&self, _: &BlockHeight) -> GasPriceResult> { Err(GasPriceError::CouldNotFetchMetadata { @@ -171,7 +172,7 @@ impl GetDaSequenceNumber for ErroringPersistedData { struct UnimplementedStorageTx; -impl TransactionableStorage for ErroringPersistedData { +impl GasPriceServiceAtomicStorage for ErroringPersistedData { type Transaction<'a> = UnimplementedStorageTx; fn begin_transaction(&mut self) -> GasPriceResult> { @@ -188,6 +189,7 @@ impl SetMetadataStorage for UnimplementedStorageTx { unimplemented!() } } + impl GetMetadataStorage for UnimplementedStorageTx { fn get_metadata( &self, @@ -198,11 +200,11 @@ impl GetMetadataStorage for UnimplementedStorageTx { } impl UnrecordedBlocks for UnimplementedStorageTx { - fn insert(&mut self, _height: Height, _bytes: Bytes) -> Result<(), Error> { + fn insert(&mut self, _height: Height, _bytes: Bytes) -> Result<(), String> { unimplemented!() } - fn remove(&mut self, _height: &Height) -> Result, Error> { + fn remove(&mut self, _height: &Height) -> Result, String> { unimplemented!() } } @@ -216,6 +218,7 @@ impl SetDaSequenceNumber for UnimplementedStorageTx { unimplemented!() } } + impl GetDaSequenceNumber for UnimplementedStorageTx { fn get_sequence_number( &self, @@ -225,6 +228,16 @@ impl GetDaSequenceNumber for UnimplementedStorageTx { } } +impl AsUnrecordedBlocks for UnimplementedStorageTx { + type Wrapper<'a> = UnimplementedStorageTx + where + Self: 'a; + + fn as_unrecorded_blocks(&mut self) -> Self::Wrapper<'_> { + UnimplementedStorageTx + } +} + struct FakeDABlockCost { da_block_costs: Receiver, sequence_number: Arc>>, @@ -402,10 +415,6 @@ async fn run__new_l2_block_saves_old_metadata() { let l2_block_source = FakeL2BlockSource { l2_block: l2_block_receiver, }; - let metadata_inner = Arc::new(std::sync::Mutex::new(None)); - let metadata_storage = FakeMetadata { - inner: metadata_inner.clone(), - }; let config = zero_threshold_arbitrary_config(); let inner = database(); @@ -539,20 +548,19 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr let different_l2_block = 0; let settings = FakeSettings; let block_stream = empty_block_stream(); - let gas_price_db = FakeGasPriceDb::empty(); let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); let inner = database_with_metadata(&original_metadata); // when let service = UninitializedTask::new( different_config.clone(), + None, 0.into(), settings, block_stream, - gas_price_db, + inner, da_cost_source, on_chain_db, - inner, ) .unwrap(); @@ -645,20 +653,19 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { let erroring_persisted_data = ErroringPersistedData; let settings = FakeSettings; let block_stream = empty_block_stream(); - let gas_price_db = FakeGasPriceDb::empty(); let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); // when let res = UninitializedTask::new( config, + None, 0.into(), settings, block_stream, - gas_price_db, + erroring_persisted_data, da_cost_source, on_chain_db, - erroring_persisted_data, ); // then @@ -680,7 +687,6 @@ async fn uninitialized_task__init__starts_da_service_with_sequence_number_in_sto let different_l2_block = 0; let settings = FakeSettings; let block_stream = empty_block_stream(); - let gas_price_db = FakeGasPriceDb::new(block_height); let on_chain_db = FakeOnChainDb::new(different_l2_block); let (da_cost_source, sequence_number_handle) = FakeDABlockCost::never_returns_with_handle_to_sequence_number(); @@ -691,13 +697,13 @@ async fn uninitialized_task__init__starts_da_service_with_sequence_number_in_sto StorageTransaction::commit_transaction(tx).unwrap(); let service = UninitializedTask::new( different_config.clone(), + Some(block_height.into()), 0.into(), settings, block_stream, - gas_price_db, + inner, da_cost_source, on_chain_db, - inner, ) .unwrap(); diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index a9a8eb2792a..ac0ce06da01 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -18,13 +18,13 @@ use crate::{ }, ports::{ GasPriceData, + GasPriceServiceAtomicStorage, GasPriceServiceConfig, GetDaSequenceNumber, GetMetadataStorage, L2Data, SetDaSequenceNumber, SetMetadataStorage, - TransactionableStorage, }, v1::{ algorithm::SharedV1Algorithm, @@ -44,7 +44,10 @@ use crate::{ initialize_algorithm, GasPriceServiceV1, }, - uninitialized_task::fuel_storage_unrecorded_blocks::FuelStorageUnrecordedBlocks, + uninitialized_task::fuel_storage_unrecorded_blocks::{ + AsUnrecordedBlocks, + FuelStorageUnrecordedBlocks, + }, }, }; use anyhow::Error; @@ -78,14 +81,9 @@ use fuel_gas_price_algorithm::v1::{ pub mod fuel_storage_unrecorded_blocks; -pub struct UninitializedTask< - L2DataStoreView, - GasPriceStore, - DA, - SettingsProvider, - PersistedData, -> { +pub struct UninitializedTask { pub config: V1AlgorithmConfig, + pub gas_metadata_height: Option, pub genesis_block_height: BlockHeight, pub settings: SettingsProvider, pub gas_price_db: GasPriceStore, @@ -94,39 +92,27 @@ pub struct UninitializedTask< pub(crate) shared_algo: SharedV1Algorithm, pub(crate) algo_updater: AlgorithmUpdaterV1, pub(crate) da_source: DA, - pub persisted_data: PersistedData, } -impl< - L2DataStore, - L2DataStoreView, - GasPriceStore, - DA, - SettingsProvider, - PersistedData, - > - UninitializedTask +impl + UninitializedTask where L2DataStore: L2Data, L2DataStoreView: AtomicView, - GasPriceStore: GasPriceData, - PersistedData: GetMetadataStorage + GetDaSequenceNumber, - PersistedData: TransactionableStorage, - for<'a> PersistedData::Transaction<'a>: - SetMetadataStorage + UnrecordedBlocks + SetDaSequenceNumber, + AtomicStorage: GasPriceServiceAtomicStorage, DA: DaBlockCostsSource, SettingsProvider: GasPriceSettingsProvider, { #[allow(clippy::too_many_arguments)] pub fn new( config: V1AlgorithmConfig, + gas_metadata_height: Option, genesis_block_height: BlockHeight, settings: SettingsProvider, block_stream: BoxStream, - gas_price_db: GasPriceStore, + gas_price_db: AtomicStorage, da_source: DA, on_chain_db: L2DataStoreView, - persisted_data: PersistedData, ) -> anyhow::Result { let latest_block_height: u32 = on_chain_db .latest_view()? @@ -135,10 +121,11 @@ where .into(); let (algo_updater, shared_algo) = - initialize_algorithm(&config, latest_block_height, &persisted_data)?; + initialize_algorithm(&config, latest_block_height, &gas_price_db)?; let task = Self { config, + gas_metadata_height, genesis_block_height, settings, gas_price_db, @@ -147,7 +134,6 @@ where algo_updater, shared_algo, da_source, - persisted_data, }; Ok(task) } @@ -155,7 +141,7 @@ where pub async fn init( mut self, ) -> anyhow::Result< - GasPriceServiceV1, DA, PersistedData>, + GasPriceServiceV1, DA, AtomicStorage>, > { let mut first_run = false; let latest_block_height: u32 = self @@ -165,7 +151,7 @@ where .unwrap_or(self.genesis_block_height) .into(); - let maybe_metadata_height = self.gas_price_db.latest_height(); + let maybe_metadata_height = self.gas_metadata_height; let metadata_height = if let Some(metadata_height) = maybe_metadata_height { metadata_height.into() } else { @@ -183,7 +169,7 @@ where // https://github.com/FuelLabs/fuel-core/issues/2140 let poll_interval = None; if let Some(sequence_number) = self - .persisted_data + .gas_price_db .get_sequence_number(&metadata_height.into())? { self.da_source.set_last_value(sequence_number).await?; @@ -198,7 +184,7 @@ where self.shared_algo, self.algo_updater, da_service, - self.persisted_data, + self.gas_price_db, ); Ok(service) } else { @@ -209,7 +195,7 @@ where &self.on_chain_db, metadata_height, latest_block_height, - &mut self.persisted_data, + &mut self.gas_price_db, )?; } @@ -218,7 +204,7 @@ where self.shared_algo, self.algo_updater, da_service, - self.persisted_data, + self.gas_price_db, ); Ok(service) } @@ -226,35 +212,18 @@ where } #[async_trait::async_trait] -impl< - L2DataStore, - L2DataStoreView, - GasPriceStore, - DA, - SettingsProvider, - PersistedData, - > RunnableService - for UninitializedTask< - L2DataStoreView, - GasPriceStore, - DA, - SettingsProvider, - PersistedData, - > +impl RunnableService + for UninitializedTask where L2DataStore: L2Data, L2DataStoreView: AtomicView, - GasPriceStore: GasPriceData, + AtomicStorage: GasPriceServiceAtomicStorage + GasPriceData, DA: DaBlockCostsSource + 'static, SettingsProvider: GasPriceSettingsProvider + 'static, - PersistedData: - GetMetadataStorage + GetDaSequenceNumber + TransactionableStorage + 'static, - for<'a> ::Transaction<'a>: - SetMetadataStorage + UnrecordedBlocks + SetDaSequenceNumber, { const NAME: &'static str = "GasPriceServiceV1"; type SharedData = SharedV1Algorithm; - type Task = GasPriceServiceV1, DA, PersistedData>; + type Task = GasPriceServiceV1, DA, AtomicStorage>; type TaskParams = (); fn shared_data(&self) -> Self::SharedData { @@ -271,26 +240,23 @@ where } fn sync_gas_price_db_with_on_chain_storage< - 'a, L2DataStore, L2DataStoreView, SettingsProvider, - PersistedData, + AtomicStorage, >( settings: &SettingsProvider, config: &V1AlgorithmConfig, on_chain_db: &L2DataStoreView, metadata_height: u32, latest_block_height: u32, - persisted_data: &'a mut PersistedData, + persisted_data: &mut AtomicStorage, ) -> anyhow::Result<()> where L2DataStore: L2Data, L2DataStoreView: AtomicView, SettingsProvider: GasPriceSettingsProvider, - PersistedData: GetMetadataStorage + TransactionableStorage + 'a, - ::Transaction<'a>: - SetMetadataStorage + UnrecordedBlocks, + AtomicStorage: GasPriceServiceAtomicStorage, { let metadata = persisted_data .get_metadata(&metadata_height.into())? @@ -318,31 +284,23 @@ where Ok(()) } -fn sync_v1_metadata< - 'a, - L2DataStore, - L2DataStoreView, - SettingsProvider, - StorageTxGenerator, ->( +fn sync_v1_metadata( settings: &SettingsProvider, on_chain_db: &L2DataStoreView, metadata_height: u32, latest_block_height: u32, updater: &mut AlgorithmUpdaterV1, - storage_tx_generator: &'a mut StorageTxGenerator, + da_storage: &mut AtomicStorage, ) -> anyhow::Result<()> where L2DataStore: L2Data, L2DataStoreView: AtomicView, SettingsProvider: GasPriceSettingsProvider, - StorageTxGenerator: TransactionableStorage + 'a, - ::Transaction<'a>: - SetMetadataStorage + UnrecordedBlocks, + AtomicStorage: GasPriceServiceAtomicStorage, { let first = metadata_height.saturating_add(1); let view = on_chain_db.latest_view()?; - let mut tx = storage_tx_generator.begin_transaction()?; + let mut tx = da_storage.begin_transaction()?; for height in first..=latest_block_height { let block = view .get_block(&height.into())? @@ -371,65 +329,45 @@ where block_gas_capacity, block_bytes, fee_wei.into(), - &mut tx, + &mut tx.as_unrecorded_blocks(), )?; let metadata: UpdaterMetadata = updater.clone().into(); tx.set_metadata(&metadata)?; } - StorageTxGenerator::commit_transaction(tx)?; + AtomicStorage::commit_transaction(tx)?; Ok(()) } #[allow(clippy::type_complexity)] -#[allow(clippy::too_many_arguments)] -pub fn new_gas_price_service_v1< - L2DataStore, - GasPriceStore, - Metadata, - DA, - SettingsProvider, - PersistedData, ->( +pub fn new_gas_price_service_v1( v1_config: V1AlgorithmConfig, genesis_block_height: BlockHeight, settings: SettingsProvider, block_stream: BoxStream, - gas_price_db: GasPriceStore, + gas_price_db: AtomicStorage, da_source: DA, on_chain_db: L2DataStore, - persisted_data: PersistedData, ) -> anyhow::Result< - ServiceRunner< - UninitializedTask< - L2DataStore, - GasPriceStore, - DA, - SettingsProvider, - PersistedData, - >, - >, + ServiceRunner>, > where L2DataStore: AtomicView, L2DataStore::LatestView: L2Data, - GasPriceStore: GasPriceData, + AtomicStorage: GasPriceServiceAtomicStorage + GasPriceData, SettingsProvider: GasPriceSettingsProvider, DA: DaBlockCostsSource, - PersistedData: - GetMetadataStorage + GetDaSequenceNumber + TransactionableStorage + 'static, - for<'a> PersistedData::Transaction<'a>: - SetMetadataStorage + UnrecordedBlocks + SetDaSequenceNumber, { + let metadata_height = gas_price_db.latest_height(); let gas_price_init = UninitializedTask::new( v1_config, + metadata_height, genesis_block_height, settings, block_stream, gas_price_db, da_source, on_chain_db, - persisted_data, )?; Ok(ServiceRunner::new(gas_price_init)) } diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs index 6e3985476ed..780479c2e04 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs @@ -11,8 +11,10 @@ use fuel_core_storage::{ Modifiable, WriteTransaction, }, + Error as StorageError, StorageAsMut, StorageAsRef, + StorageMutate, }; use fuel_core_types::fuel_merkle::storage::StorageMutateInfallible; use fuel_gas_price_algorithm::{ @@ -20,6 +22,27 @@ use fuel_gas_price_algorithm::{ v1::UnrecordedBlocks, }; +pub trait AsUnrecordedBlocks { + type Wrapper<'a>: UnrecordedBlocks + where + Self: 'a; + + fn as_unrecorded_blocks(&mut self) -> Self::Wrapper<'_>; +} + +impl AsUnrecordedBlocks for S +where + S: StorageMutate, +{ + type Wrapper<'a> = FuelStorageUnrecordedBlocks<&'a mut Self> + where + Self: 'a; + + fn as_unrecorded_blocks(&mut self) -> Self::Wrapper<'_> { + FuelStorageUnrecordedBlocks::new(self) + } +} + #[derive(Debug, Clone)] pub struct FuelStorageUnrecordedBlocks { inner: Storage, @@ -31,33 +54,24 @@ impl FuelStorageUnrecordedBlocks { } } -impl UnrecordedBlocks for FuelStorageUnrecordedBlocks +impl UnrecordedBlocks for FuelStorageUnrecordedBlocks where - Storage: KeyValueInspect + Modifiable, - Storage: Send + Sync, + S: StorageMutate, { - fn insert(&mut self, height: v1::Height, bytes: v1::Bytes) -> Result<(), v1::Error> { - let mut tx = self.inner.write_transaction(); - tx.storage_as_mut::() + fn insert(&mut self, height: v1::Height, bytes: v1::Bytes) -> Result<(), String> { + self.inner + .storage_as_mut::() .insert(&height, &bytes) - .and_then(|_| tx.commit()) - .map_err(|err| { - v1::Error::CouldNotInsertUnrecordedBlock(format!("Error: {:?}", err)) - })?; + .map_err(|err| format!("Error: {:?}", err))?; Ok(()) } - fn remove(&mut self, height: &v1::Height) -> Result, v1::Error> { - let mut tx = self.inner.write_transaction(); - let bytes = tx + fn remove(&mut self, height: &v1::Height) -> Result, String> { + let bytes = self + .inner .storage_as_mut::() .take(height) - .map_err(|err| { - v1::Error::CouldNotRemoveUnrecordedBlock(format!("Error: {:?}", err)) - })?; - tx.commit().map_err(|err| { - v1::Error::CouldNotRemoveUnrecordedBlock(format!("Error: {:?}", err)) - })?; + .map_err(|err| format!("Error: {:?}", err))?; Ok(bytes) } } From 33aa2912937b6a013661187bfde58cef00753062 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 9 Dec 2024 16:13:11 -0700 Subject: [PATCH 079/131] Additional merging changes --- crates/fuel-core/src/service/sub_services.rs | 3 --- .../gas_price_service/src/v1/service.rs | 8 ++++++-- .../fuel_storage_unrecorded_blocks.rs | 17 +++++++---------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index b3e5c37818d..0bb2bb38383 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -56,7 +56,6 @@ use fuel_core_poa::{ }; use fuel_core_storage::{ self, - structured_storage::StructuredStorage, transactional::AtomicView, }; #[cfg(feature = "relayer")] @@ -190,7 +189,6 @@ pub fn init_sub_services( let genesis_block_height = *genesis_block.header().height(); let settings = consensus_parameters_provider.clone(); let block_stream = importer_adapter.events_shared_result(); - let persisted_data = database.gas_price().clone(); tracing::debug!("da_committer_url: {:?}", config.da_committer_url); let committer_api = BlockCommitterHttpApi::new(config.da_committer_url.clone()); @@ -209,7 +207,6 @@ pub fn init_sub_services( database.gas_price().clone(), da_source, database.on_chain().clone(), - persisted_data, )?; let gas_price_provider = FuelGasPriceProvider::new(gas_price_service.shared.clone()); diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 0485533b71a..2def8a9507a 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -62,7 +62,11 @@ use futures::FutureExt; use tokio::sync::broadcast::Receiver; /// The service that updates the gas price algorithm. -pub struct GasPriceServiceV1 { +pub struct GasPriceServiceV1 +where + DA: DaBlockCostsSource + 'static, + AtomicStorage: GasPriceServiceAtomicStorage, +{ /// The algorithm that can be used in the next block shared_algo: SharedV1Algorithm, /// The L2 block source @@ -76,7 +80,7 @@ pub struct GasPriceServiceV1 { /// Buffer of block costs from the DA chain da_block_costs_buffer: Vec, /// Storage transaction provider for metadata and unrecorded blocks - storage_tx_provider: StorageTxProvider, + storage_tx_provider: AtomicStorage, } impl GasPriceServiceV1 diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs index 09d3360ffef..dbc15375927 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs @@ -64,22 +64,19 @@ impl UnrecordedBlocks for FuelStorageUnrecordedBlocks where S: StorageMutate, { - fn insert(&mut self, height: v1::Height, bytes: v1::Bytes) -> Result<(), v1::Error> { - let mut tx = self.inner.write_transaction(); + fn insert(&mut self, height: v1::Height, bytes: v1::Bytes) -> Result<(), String> { let block_height = BlockHeight::from(height); - tx.storage_as_mut::() + self.inner + .storage_as_mut::() .insert(&block_height, &bytes) - .and_then(|_| tx.commit()) - .map_err(|err| { - v1::Error::CouldNotInsertUnrecordedBlock(format!("Error: {:?}", err)) - })?; + .map_err(|err| format!("Error: {:?}", err))?; Ok(()) } - fn remove(&mut self, height: &v1::Height) -> Result, v1::Error> { - let mut tx = self.inner.write_transaction(); + fn remove(&mut self, height: &v1::Height) -> Result, String> { let block_height = BlockHeight::from(*height); - let bytes = tx + let bytes = self + .inner .storage_as_mut::() .take(&block_height) .map_err(|err| format!("Error: {:?}", err))?; From 498c2ad00fc4ea26e54ed41d4f06985fedea6270 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 10 Dec 2024 12:20:38 -0700 Subject: [PATCH 080/131] Get first test passing --- tests/tests/gas_price.rs | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index cfe83a776df..3cdc22750d8 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -422,9 +422,9 @@ use fuel_core_gas_price_service::v1::da_source_service::block_committer_costs::R #[test] fn produce_block__l1_committed_block_effects_gas_price() { - let _ = tracing_subscriber::fmt() - .with_max_level(tracing::Level::DEBUG) - .try_init(); + // let _ = tracing_subscriber::fmt() + // .with_max_level(tracing::Level::DEBUG) + // .try_init(); let rt = tokio::runtime::Runtime::new().unwrap(); // set up chain with single unrecorded block @@ -462,16 +462,21 @@ fn produce_block__l1_committed_block_effects_gas_price() { "100", ]); - let new_gas_price = rt.block_on(async { - let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) - .await - .unwrap(); - tokio::time::sleep(Duration::from_millis(100)).await; - driver.client.produce_blocks(1, None).await.unwrap(); - tokio::time::sleep(Duration::from_millis(100)).await; - driver.client.latest_gas_price().await.unwrap().gas_price - }); - assert_ne!(first_gas_price, new_gas_price); + // when + let new_gas_price = rt + .block_on(async { + let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + driver.client.produce_blocks(1, None).await.unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + driver.client.estimate_gas_price(0).await.unwrap().gas_price + }) + .into(); + + // then + assert!(first_gas_price < new_gas_price); } struct FakeServer { From 6f4634398cb7f58f975c2699afe59d62ccf3b4dd Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 10 Dec 2024 12:26:55 -0700 Subject: [PATCH 081/131] Simplify test --- tests/tests/gas_price.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 3cdc22750d8..833e60b699c 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -438,15 +438,20 @@ fn produce_block__l1_committed_block_effects_gas_price() { let (first_gas_price, temp_dir) = rt.block_on(async { let driver = FuelCoreDriver::spawn(&args).await.unwrap(); driver.client.produce_blocks(1, None).await.unwrap(); - let first_gas_price = driver.client.latest_gas_price().await.unwrap().gas_price; + let first_gas_price: u64 = driver + .client + .estimate_gas_price(0) + .await + .unwrap() + .gas_price + .into(); tokio::time::sleep(Duration::from_millis(100)).await; let temp_dir = driver.kill().await; (first_gas_price, temp_dir) }); - assert_eq!(100, first_gas_price); + assert_eq!(100u64, first_gas_price); - // set up chain with single recorded block let mock = FakeServer::new(); let url = mock.url(); @@ -455,7 +460,7 @@ fn produce_block__l1_committed_block_effects_gas_price() { "--da-committer-url", &url, "--da-poll-interval", - "50", + "5", "--da-p-component", "1", "--max-da-gas-price-change-percent", @@ -468,9 +473,9 @@ fn produce_block__l1_committed_block_effects_gas_price() { let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) .await .unwrap(); - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(10)).await; driver.client.produce_blocks(1, None).await.unwrap(); - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(10)).await; driver.client.estimate_gas_price(0).await.unwrap().gas_price }) .into(); From 3496adcac1d15c50ea34dff721fd17794f4c013a Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Tue, 10 Dec 2024 18:10:59 -0700 Subject: [PATCH 082/131] Add another messy test, fix bug in activity code --- crates/fuel-core/src/service.rs | 62 +++---- crates/fuel-gas-price-algorithm/src/v1.rs | 19 ++- crates/services/executor/src/executor.rs | 2 + .../src/common/updater_metadata.rs | 7 + .../gas_price_service/src/v1/service.rs | 6 +- .../services/producer/src/block_producer.rs | 7 +- tests/proptest-regressions/gas_price.txt | 7 + tests/tests/gas_price.rs | 155 ++++++++++++++++-- 8 files changed, 214 insertions(+), 51 deletions(-) create mode 100644 tests/proptest-regressions/gas_price.txt diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index 1f751f1de5e..676b3f506f6 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -1,17 +1,13 @@ -use self::adapters::BlockImporterAdapter; -use crate::{ - combined_database::{ - CombinedDatabase, - ShutdownListener, - }, - database::Database, - service::{ - adapters::{ - ExecutorAdapter, - PoAAdapter, - }, - sub_services::TxPoolSharedState, - }, +use std::{ + net::SocketAddr, + sync::Arc, +}; + +pub use config::{ + Config, + DbType, + RelayerConsensusConfig, + VMConfig, }; use fuel_core_chain_config::{ ConsensusConfig, @@ -21,6 +17,7 @@ use fuel_core_poa::{ ports::BlockImporter, verifier::verify_consensus, }; +pub use fuel_core_services::Service as ServiceTrait; use fuel_core_services::{ RunnableService, RunnableTask, @@ -40,18 +37,23 @@ use fuel_core_storage::{ StorageAsMut, }; use fuel_core_types::blockchain::consensus::Consensus; -use std::{ - net::SocketAddr, - sync::Arc, -}; -pub use config::{ - Config, - DbType, - RelayerConsensusConfig, - VMConfig, +use crate::{ + combined_database::{ + CombinedDatabase, + ShutdownListener, + }, + database::Database, + service::{ + adapters::{ + ExecutorAdapter, + PoAAdapter, + }, + sub_services::TxPoolSharedState, + }, }; -pub use fuel_core_services::Service as ServiceTrait; + +use self::adapters::BlockImporterAdapter; pub mod adapters; pub mod config; @@ -465,6 +467,13 @@ impl RunnableTask for Task { #[allow(non_snake_case)] #[cfg(test)] mod tests { + use std::{ + thread::sleep, + time::Duration, + }; + + use fuel_core_services::State; + use crate::{ service::{ Config, @@ -472,11 +481,6 @@ mod tests { }, ShutdownListener, }; - use fuel_core_services::State; - use std::{ - thread::sleep, - time::Duration, - }; #[tokio::test] async fn stop_sub_service_shutdown_all_services() { diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 0637880fc23..3ff71e980f2 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -269,9 +269,9 @@ impl L2ActivityTracker { } pub fn safety_mode(&self) -> DAGasPriceSafetyMode { - if self.chain_activity > self.capped_activity_threshold { + if self.chain_activity >= self.capped_activity_threshold { DAGasPriceSafetyMode::Normal - } else if self.chain_activity > self.decrease_activity_threshold { + } else if self.chain_activity >= self.decrease_activity_threshold { DAGasPriceSafetyMode::Capped } else { DAGasPriceSafetyMode::AlwaysDecrease @@ -279,7 +279,15 @@ impl L2ActivityTracker { } pub fn update(&mut self, block_usage: ClampedPercentage) { + tracing::info!("Block usage: {:?}", block_usage); + tracing::info!("Chain activity: {}", self.chain_activity); + tracing::info!("threshold: {:?}", self.block_activity_threshold); if block_usage < self.block_activity_threshold { + tracing::info!( + "Decreasing activity {:?} < {:?}", + block_usage, + self.block_activity_threshold + ); self.chain_activity = self.chain_activity.saturating_sub(1); } else { self.chain_activity = @@ -414,7 +422,9 @@ impl AlgorithmUpdaterV1 { } fn update_da_rewards(&mut self, fee_wei: u128) { + tracing::info!("Fee: {}", fee_wei); let block_da_reward = self.da_portion_of_fee(fee_wei); + tracing::info!("DA reward: {}", block_da_reward); self.total_da_rewards_excess = self.total_da_rewards_excess.saturating_add(block_da_reward); } @@ -498,8 +508,8 @@ impl AlgorithmUpdaterV1 { 0u64 } }); - tracing::debug!("Profit: {}", self.last_profit); - tracing::debug!( + tracing::info!("Profit: {}", self.last_profit); + tracing::info!( "DA gas price change: p: {}, d: {}, change: {}, new: {}", p, d, @@ -518,6 +528,7 @@ impl AlgorithmUpdaterV1 { DAGasPriceSafetyMode::Normal => maybe_da_change, DAGasPriceSafetyMode::Capped => 0, DAGasPriceSafetyMode::AlwaysDecrease => { + tracing::info!("Activity is low, decreasing DA gas price"); self.max_change().saturating_mul(-1) } } diff --git a/crates/services/executor/src/executor.rs b/crates/services/executor/src/executor.rs index bc690c52d3d..f5106d8048c 100644 --- a/crates/services/executor/src/executor.rs +++ b/crates/services/executor/src/executor.rs @@ -1626,7 +1626,9 @@ where .iter() .map(|input| input.predicate_gas_used()) .collect(); + tracing::info!("Executing transaction: {:?}", tx_id); let ready_tx = checked_tx.into_ready(gas_price, gas_costs, fee_params)?; + tracing::info!("Transaction ready: {:?}", tx_id); let mut vm = Interpreter::with_storage( memory, diff --git a/crates/services/gas_price_service/src/common/updater_metadata.rs b/crates/services/gas_price_service/src/common/updater_metadata.rs index 6ce274ea1a7..41093056c44 100644 --- a/crates/services/gas_price_service/src/common/updater_metadata.rs +++ b/crates/services/gas_price_service/src/common/updater_metadata.rs @@ -22,6 +22,13 @@ impl UpdaterMetadata { UpdaterMetadata::V1(v1) => v1.l2_block_height.into(), } } + + pub fn v1(&self) -> Option<&V1Metadata> { + match self { + UpdaterMetadata::V1(v1) => Some(v1), + _ => None, + } + } } impl From for UpdaterMetadata { diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 2def8a9507a..85364bfbb27 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -161,8 +161,10 @@ where let capacity = Self::validate_block_gas_capacity(block_gas_capacity)?; let mut storage_tx = self.storage_tx_provider.begin_transaction()?; - for da_block_costs in self.da_block_costs_buffer.drain(..) { - tracing::debug!("Updating DA block costs: {:?}", da_block_costs); + let drained = self.da_block_costs_buffer.drain(..); + tracing::info!("Drained DA block costs: {:?}", drained); + for da_block_costs in drained { + tracing::info!("Updating DA block costs: {:?}", da_block_costs); self.algorithm_updater.update_da_record_data( &da_block_costs.l2_blocks, da_block_costs.blob_size_bytes, diff --git a/crates/services/producer/src/block_producer.rs b/crates/services/producer/src/block_producer.rs index f5ad44be901..663aaf31ae8 100644 --- a/crates/services/producer/src/block_producer.rs +++ b/crates/services/producer/src/block_producer.rs @@ -245,10 +245,13 @@ where } async fn calculate_gas_price(&self) -> anyhow::Result { - self.gas_price_provider + let price = self + .gas_price_provider .next_gas_price() .await - .map_err(|e| anyhow!("No gas price found: {e:?}")) + .map_err(|e| anyhow!("No gas price found: {e:?}")); + tracing::info!("Gas price: {:?}", price); + price } } diff --git a/tests/proptest-regressions/gas_price.txt b/tests/proptest-regressions/gas_price.txt new file mode 100644 index 00000000000..e06f62efa45 --- /dev/null +++ b/tests/proptest-regressions/gas_price.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 76d053f5828c12ddd3090b4b076bb287e1b4dde50e69f7edd5b3a5b64ae3c743 # shrinks to block_delay = 103, blob_size = 70 diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 833e60b699c..75360516c5f 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -1,4 +1,7 @@ #![allow(non_snake_case)] +// TODO: REMOVE BEFORE MERGING +#![allow(dead_code)] +#![allow(unused_imports)] use crate::helpers::{ TestContext, @@ -21,6 +24,10 @@ use fuel_core_client::client::{ }; use fuel_core_gas_price_service::{ common::fuel_core_storage_adapter::storage::GasPriceMetadata, + ports::{ + GasPriceData, + GetMetadataStorage, + }, v0::metadata::V0Metadata, }; use fuel_core_poa::Trigger; @@ -123,6 +130,9 @@ async fn latest_gas_price__for_single_block_should_be_starting_gas_price() { #[tokio::test] async fn produce_block__raises_gas_price() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .try_init(); // given let block_gas_limit = 3_000_000; let chain_config = ChainConfig { @@ -452,15 +462,23 @@ fn produce_block__l1_committed_block_effects_gas_price() { assert_eq!(100u64, first_gas_price); - let mock = FakeServer::new(); + let mut mock = FakeServer::new(); let url = mock.url(); + let costs = RawDaBlockCosts { + sequence_number: 1, + blocks_heights: vec![1], + da_block_height: DaBlockHeight(100), + total_cost: 100, + total_size_bytes: 100, + }; + mock.add_response(costs); // add the da committer url to the args args.extend(&[ "--da-committer-url", &url, "--da-poll-interval", - "5", + "1", "--da-p-component", "1", "--max-da-gas-price-change-percent", @@ -473,9 +491,9 @@ fn produce_block__l1_committed_block_effects_gas_price() { let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) .await .unwrap(); - tokio::time::sleep(Duration::from_millis(10)).await; + tokio::time::sleep(Duration::from_millis(2)).await; driver.client.produce_blocks(1, None).await.unwrap(); - tokio::time::sleep(Duration::from_millis(10)).await; + tokio::time::sleep(Duration::from_millis(2)).await; driver.client.estimate_gas_price(0).await.unwrap().gas_price }) .into(); @@ -484,27 +502,136 @@ fn produce_block__l1_committed_block_effects_gas_price() { assert!(first_gas_price < new_gas_price); } +#[allow(unused_variables)] +mod prop_tests { + use super::*; + use proptest::prelude::*; + + // the blobs costs will never change more than 12.5% between blobs + + proptest! { + // This test will generate a many l2 blocks, and then after the block delay, it will + // start receiving costs for committing to DA in blobs. If the profit is positive after the + // first blob, we will continue to process blocks until the profit is negative (if it is + // negative, we will check the opposite). We will say the test passed if it can recover from + // the divergent profit after a set amount of l2 blocks. + #![proptest_config(ProptestConfig::with_cases(1))] + #[test] + fn produce_bock__algorithm_recovers_from_divergent_profit(block_delay in 11..12usize, blob_size in 50..100usize) { + produce_blocks__lolz(block_delay, blob_size); + } + } + + #[ignore] + #[test] + fn dummy() {} +} + +fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .try_init(); + // given + let mut mock = FakeServer::new(); + let url = mock.url(); + let rt = tokio::runtime::Runtime::new().unwrap(); + let block_gas_limit = 3_000_000; + let chain_config = ChainConfig { + consensus_parameters: ConsensusParameters::V1(ConsensusParametersV1 { + block_gas_limit, + ..Default::default() + }), + ..ChainConfig::local_testnet() + }; + let mut node_config = + Config::local_node_with_configs(chain_config, StateConfig::local_testnet()); + let starting_gas_price = 1_000_000_000; + node_config.block_producer.coinbase_recipient = Some([5; 32].into()); + node_config.min_da_gas_price = starting_gas_price; + node_config.max_da_gas_price_change_percent = 10; + node_config.block_production = Trigger::Never; + node_config.da_committer_url = Some(url.clone()); + node_config.da_poll_interval = Some(1); + node_config.da_p_component = 1; + node_config.block_activity_threshold = 0; + + let (srv, client) = rt.block_on(async { + let srv = FuelService::new_node(node_config.clone()).await.unwrap(); + let client = FuelClient::from(srv.bound_address); + let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); + + // when + for _b in 0..block_delay { + let arb_tx_count = 10; + for i in 0..arb_tx_count { + let tx = arb_large_tx(189028 + i as Word, &mut rng); + let _status = client.submit(&tx).await.unwrap(); + } + let _ = client.produce_blocks(1, None).await.unwrap(); + } + let _ = client.produce_blocks(1, None).await.unwrap(); + + let height = srv.shared.database.gas_price().latest_height().unwrap(); + let metadata = srv + .shared + .database + .gas_price() + .get_metadata(&height) + .unwrap() + .and_then(|x| x.v1().cloned()) + .unwrap(); + tracing::info!("metadata: {:?}", metadata); + assert_ne!(metadata.last_profit, 0); + (srv, client) + }); + + let half_of_blocks = block_delay as u32 / 2; + let blocks_heights = (1..half_of_blocks).collect(); + mock.add_response(RawDaBlockCosts { + sequence_number: 1, + blocks_heights, + da_block_height: DaBlockHeight(100), + total_cost: 9999999999999, + total_size_bytes: 100, + }); + + rt.block_on(async { + tokio::time::sleep(Duration::from_millis(10)).await; + client.produce_blocks(1, None).await.unwrap(); + client.produce_blocks(1, None).await.unwrap(); + client.produce_blocks(1, None).await.unwrap(); + client.produce_blocks(1, None).await.unwrap(); + let height = srv.shared.database.gas_price().latest_height().unwrap(); + let metadata = srv + .shared + .database + .gas_price() + .get_metadata(&height) + .unwrap() + .and_then(|x| x.v1().cloned()) + .unwrap(); + tracing::info!("metadata: {:?}", metadata); + }) +} + struct FakeServer { server: mockito::ServerGuard, } impl FakeServer { fn new() -> Self { - let mut server = mockito::Server::new(); - let costs = RawDaBlockCosts { - sequence_number: 1, - blocks_heights: vec![1], - da_block_height: DaBlockHeight(100), - total_cost: 100, - total_size_bytes: 100, - }; + let server = mockito::Server::new(); + Self { server } + } + + pub fn add_response(&mut self, costs: RawDaBlockCosts) { let body = serde_json::to_string(&costs).unwrap(); - let _mock = server + let _mock = self + .server .mock("GET", "/") .with_status(201) .with_body(body) .create(); - Self { server } } fn url(&self) -> String { From 0c5ab28dc43295d9c90f8ee9e0ad58142e44e082 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 11 Dec 2024 15:23:13 -0700 Subject: [PATCH 083/131] WIP add dynamic test --- .../adapters/fuel_gas_price_provider.rs | 28 ++++-- crates/fuel-gas-price-algorithm/src/v1.rs | 19 +++-- .../gas_price_service/src/v1/service.rs | 3 +- tests/tests/gas_price.rs | 85 ++++++++++++++++--- 4 files changed, 106 insertions(+), 29 deletions(-) diff --git a/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs b/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs index aeb17627ccf..cbd1f87c87e 100644 --- a/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs +++ b/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs @@ -30,12 +30,15 @@ mod tests; #[derive(Debug)] /// Receives the next gas price algorithm via a shared `BlockGasPriceAlgo` instance pub struct FuelGasPriceProvider { + // Scale the gas price down by this factor, for e.g. Wei to Gwei + scaling_factor: u64, algorithm: SharedGasPriceAlgo, } impl Clone for FuelGasPriceProvider { fn clone(&self) -> Self { Self { + scaling_factor: self.scaling_factor, algorithm: self.algorithm.clone(), } } @@ -43,7 +46,11 @@ impl Clone for FuelGasPriceProvider { impl FuelGasPriceProvider { pub fn new(algorithm: SharedGasPriceAlgo) -> Self { - Self { algorithm } + Self { + // Default scaling factor is 1_000_000_000, to convert Wei to Gwei + scaling_factor: 1_000_000_000, + algorithm, + } } } @@ -51,8 +58,17 @@ impl FuelGasPriceProvider where A: GasPriceAlgorithm + Send + Sync, { - fn next_gas_price(&self) -> u64 { - self.algorithm.next_gas_price() + fn get_scaled_gas_price(&self) -> u64 { + self.algorithm + .next_gas_price() + .saturating_div(self.scaling_factor) + } + + async fn get_scaled_worst_case_gas_price(&self, height: BlockHeight) -> u64 { + self.algorithm + .worst_case_gas_price(height) + .await + .saturating_div(self.scaling_factor) } } @@ -62,7 +78,7 @@ where A: GasPriceAlgorithm + Send + Sync, { async fn next_gas_price(&self) -> anyhow::Result { - Ok(self.next_gas_price()) + Ok(self.get_scaled_gas_price()) } } @@ -71,7 +87,7 @@ where A: GasPriceAlgorithm + Send + Sync + 'static, { fn next_gas_price(&self) -> u64 { - self.next_gas_price() + self.get_scaled_gas_price() } } @@ -81,6 +97,6 @@ where A: GasPriceAlgorithm + Send + Sync, { async fn worst_case_gas_price(&self, height: BlockHeight) -> Option { - Some(self.algorithm.worst_case_gas_price(height).await) + Some(self.get_scaled_worst_case_gas_price(height).await) } } diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 3ff71e980f2..a250a1aba5a 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -496,13 +496,14 @@ impl AlgorithmUpdaterV1 { fn update_da_gas_price(&mut self) { let p = self.p(); let d = self.d(); - let maybe_da_change = self.da_change(p, d); - let da_change = self.da_change_accounting_for_activity(maybe_da_change); + let maybe_scaled_da_change = self.da_change(p, d); + let scaled_da_change = + self.da_change_accounting_for_activity(maybe_scaled_da_change); let maybe_new_scaled_da_gas_price = i128::from(self.new_scaled_da_gas_price) - .checked_add(da_change) + .checked_add(scaled_da_change) .and_then(|x| u64::try_from(x).ok()) .unwrap_or_else(|| { - if da_change.is_positive() { + if scaled_da_change.is_positive() { u64::MAX } else { 0u64 @@ -513,7 +514,7 @@ impl AlgorithmUpdaterV1 { "DA gas price change: p: {}, d: {}, change: {}, new: {}", p, d, - da_change, + scaled_da_change, maybe_new_scaled_da_gas_price ); self.new_scaled_da_gas_price = max( @@ -558,10 +559,12 @@ impl AlgorithmUpdaterV1 { } fn da_change(&self, p: i128, d: i128) -> i128 { - let pd_change = p.saturating_add(d); + let scaled_pd_change = p + .saturating_add(d) + .saturating_mul(self.gas_price_factor.get() as i128); let max_change = self.max_change(); - let clamped_change = pd_change.saturating_abs().min(max_change); - pd_change.signum().saturating_mul(clamped_change) + let clamped_change = scaled_pd_change.saturating_abs().min(max_change); + scaled_pd_change.signum().saturating_mul(clamped_change) } // Should always be positive diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 85364bfbb27..b0eb79d63d6 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -179,12 +179,13 @@ where .map_err(|err| anyhow!(err))?; } + let fee_in_wei = block_fees as u128 * 1_000_000_000; self.algorithm_updater.update_l2_block_data( height, gas_used, capacity, block_bytes, - block_fees as u128, + fee_in_wei, &mut storage_tx.as_unrecorded_blocks(), )?; diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 75360516c5f..681149462c7 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -54,6 +54,7 @@ use fuel_core_types::{ use rand::Rng; use std::{ iter::repeat, + num::NonZero, ops::Deref, time::Duration, }; @@ -527,10 +528,13 @@ mod prop_tests { fn dummy() {} } +// P: 4,707,680, D: 114,760 fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { let _ = tracing_subscriber::fmt() .with_max_level(tracing::Level::INFO) .try_init(); + let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); + // given let mut mock = FakeServer::new(); let url = mock.url(); @@ -552,22 +556,17 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { node_config.block_production = Trigger::Never; node_config.da_committer_url = Some(url.clone()); node_config.da_poll_interval = Some(1); + // node_config.da_p_component = 4_707_680; + // node_config.da_d_component = 114_760; node_config.da_p_component = 1; node_config.block_activity_threshold = 0; let (srv, client) = rt.block_on(async { let srv = FuelService::new_node(node_config.clone()).await.unwrap(); let client = FuelClient::from(srv.bound_address); - let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); - // when for _b in 0..block_delay { - let arb_tx_count = 10; - for i in 0..arb_tx_count { - let tx = arb_large_tx(189028 + i as Word, &mut rng); - let _status = client.submit(&tx).await.unwrap(); - } - let _ = client.produce_blocks(1, None).await.unwrap(); + produce_a_block(&client, &mut rng).await; } let _ = client.produce_blocks(1, None).await.unwrap(); @@ -586,21 +585,24 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { }); let half_of_blocks = block_delay as u32 / 2; - let blocks_heights = (1..half_of_blocks).collect(); + let blocks_heights: Vec<_> = (1..half_of_blocks).collect(); + let count = blocks_heights.len() as u128; + let new_price = 2_000_000_000; + let cost = count * new_price; mock.add_response(RawDaBlockCosts { sequence_number: 1, blocks_heights, da_block_height: DaBlockHeight(100), - total_cost: 9999999999999, - total_size_bytes: 100, + total_cost: cost, + total_size_bytes: 128_000, }); + let mut profits = Vec::new(); + let mut gas_prices = Vec::new(); rt.block_on(async { tokio::time::sleep(Duration::from_millis(10)).await; client.produce_blocks(1, None).await.unwrap(); client.produce_blocks(1, None).await.unwrap(); - client.produce_blocks(1, None).await.unwrap(); - client.produce_blocks(1, None).await.unwrap(); let height = srv.shared.database.gas_price().latest_height().unwrap(); let metadata = srv .shared @@ -611,7 +613,61 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { .and_then(|x| x.v1().cloned()) .unwrap(); tracing::info!("metadata: {:?}", metadata); - }) + profits.push(metadata.last_profit); + gas_prices.push(metadata.new_scaled_da_gas_price / metadata.gas_price_factor); + }); + + let tries = 100; + + let mut success = false; + let mut success_iteration = i32::MAX; + rt.block_on(async { + for i in 0..tries { + produce_a_block(&client, &mut rng).await; + let metadata = srv + .shared + .database + .gas_price() + .get_metadata(&srv.shared.database.gas_price().latest_height().unwrap()) + .unwrap() + .and_then(|x| x.v1().cloned()) + .unwrap(); + let profit = metadata.last_profit; + profits.push(profit); + gas_prices.push(metadata.new_scaled_da_gas_price / metadata.gas_price_factor); + if profit > 0 && !success { + success = true; + success_iteration = i as i32; + } + } + }); + let changes = profits.windows(2).map(|x| x[1] - x[0]).collect::>(); + let gas_price_changes = gas_prices + .windows(2) + .map(|x| x[1] as i128 - x[0] as i128) + .collect::>(); + if !success { + panic!( + "Could not recover from divergent profit after {} tries.\n Profits: {:?}.\n Changes: {:?}.\n Gas prices: {:?}\n Gas price changes: {:?}", + tries, profits, changes, gas_prices, gas_price_changes + ); + } else { + tracing::info!("Success on try {}/{}", success_iteration, tries); + tracing::info!("Profits: {:?}", profits); + tracing::info!("Changes: {:?}", changes); + tracing::info!("Gas prices: {:?}", gas_prices); + tracing::info!("Gas price changes: {:?}", gas_price_changes); + } +} + +async fn produce_a_block(client: &FuelClient, rng: &mut R) { + let arb_tx_count = 10; + for i in 0..arb_tx_count { + let large_fee_limit = u32::MAX as u64 - i; + let tx = arb_large_tx(large_fee_limit, rng); + let _status = client.submit(&tx).await.unwrap(); + } + let _ = client.produce_blocks(1, None).await.unwrap(); } struct FakeServer { @@ -631,6 +687,7 @@ impl FakeServer { .mock("GET", "/") .with_status(201) .with_body(body) + .expect_at_least(1) .create(); } From f2abd4df237f29f278923157d54156bd7fe28869 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 11 Dec 2024 16:18:34 -0700 Subject: [PATCH 084/131] Make sane --- .../adapters/fuel_gas_price_provider.rs | 21 ++++++++----------- crates/fuel-gas-price-algorithm/src/v1.rs | 1 + tests/tests/gas_price.rs | 6 ++++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs b/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs index cbd1f87c87e..d42f14a5ea7 100644 --- a/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs +++ b/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs @@ -30,15 +30,15 @@ mod tests; #[derive(Debug)] /// Receives the next gas price algorithm via a shared `BlockGasPriceAlgo` instance pub struct FuelGasPriceProvider { - // Scale the gas price down by this factor, for e.g. Wei to Gwei - scaling_factor: u64, + // // Scale the gas price down by this factor, for e.g. Wei to Gwei + // scaling_factor: u64, algorithm: SharedGasPriceAlgo, } impl Clone for FuelGasPriceProvider { fn clone(&self) -> Self { Self { - scaling_factor: self.scaling_factor, + // scaling_factor: self.scaling_factor, algorithm: self.algorithm.clone(), } } @@ -47,8 +47,8 @@ impl Clone for FuelGasPriceProvider { impl FuelGasPriceProvider { pub fn new(algorithm: SharedGasPriceAlgo) -> Self { Self { - // Default scaling factor is 1_000_000_000, to convert Wei to Gwei - scaling_factor: 1_000_000_000, + // // Default scaling factor is 1_000_000_000, to convert Wei to Gwei + // scaling_factor: 1_000_000_000, algorithm, } } @@ -59,16 +59,13 @@ where A: GasPriceAlgorithm + Send + Sync, { fn get_scaled_gas_price(&self) -> u64 { - self.algorithm - .next_gas_price() - .saturating_div(self.scaling_factor) + self.algorithm.next_gas_price() + // .saturating_div(self.scaling_factor) } async fn get_scaled_worst_case_gas_price(&self, height: BlockHeight) -> u64 { - self.algorithm - .worst_case_gas_price(height) - .await - .saturating_div(self.scaling_factor) + self.algorithm.worst_case_gas_price(height).await + // .saturating_div(self.scaling_factor) } } diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index a250a1aba5a..8dbc280f7a1 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -390,6 +390,7 @@ impl AlgorithmUpdaterV1 { let rewards = self.clamped_rewards_as_i128(); // costs + tracing::info!("Block bytes: {}", block_bytes); self.update_projected_da_cost(block_bytes); let projected_total_da_cost = self.clamped_projected_cost_as_i128(); diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 681149462c7..46315f4b030 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -587,7 +587,8 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { let half_of_blocks = block_delay as u32 / 2; let blocks_heights: Vec<_> = (1..half_of_blocks).collect(); let count = blocks_heights.len() as u128; - let new_price = 2_000_000_000; + let new_price_gwei = 500_000; + let new_price = new_price_gwei * 1_000_000_000; // Wei let cost = count * new_price; mock.add_response(RawDaBlockCosts { sequence_number: 1, @@ -661,10 +662,11 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { } async fn produce_a_block(client: &FuelClient, rng: &mut R) { - let arb_tx_count = 10; + let arb_tx_count = 2; for i in 0..arb_tx_count { let large_fee_limit = u32::MAX as u64 - i; let tx = arb_large_tx(large_fee_limit, rng); + // let tx = arb_large_tx(189028 + i as Word, rng); let _status = client.submit(&tx).await.unwrap(); } let _ = client.produce_blocks(1, None).await.unwrap(); From 1b74ff531d7fede482429418c38c723d1411aa46 Mon Sep 17 00:00:00 2001 From: green Date: Wed, 11 Dec 2024 23:27:31 -0500 Subject: [PATCH 085/131] Small simplificaitons --- .../src/common/fuel_core_storage_adapter.rs | 7 ++++--- .../src/common/fuel_core_storage_adapter/storage.rs | 13 +++++++++---- .../fuel_storage_unrecorded_blocks.rs | 4 ++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs index 591707cb46b..083399480e8 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs @@ -28,7 +28,6 @@ use fuel_core_storage::{ Encode, }, kv_store::KeyValueInspect, - structured_storage::StructuredStorage, transactional::{ Modifiable, StorageTransaction, @@ -60,10 +59,12 @@ mod metadata_tests; pub mod storage; -impl SetMetadataStorage for StructuredStorage +impl SetMetadataStorage for Storage where Storage: Send + Sync, - Storage: KeyValueInspect + Modifiable, + Storage: Modifiable, + for<'a> StorageTransaction<&'a mut Storage>: + StorageMutate, { fn set_metadata(&mut self, metadata: &UpdaterMetadata) -> GasPriceResult<()> { let block_height = metadata.l2_block_height(); diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs index e6cb745b36c..d6c0a0aa8e0 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs @@ -75,11 +75,13 @@ impl TableWithBlueprint for GasPriceMetadata { /// for future blocks to be recorded on the DA chain pub struct UnrecordedBlocksTable; +type BlockSizeInBytes = u64; + impl Mappable for UnrecordedBlocksTable { type Key = Self::OwnedKey; - type OwnedKey = u32; + type OwnedKey = BlockHeight; type Value = Self::OwnedValue; - type OwnedValue = u64; + type OwnedValue = BlockSizeInBytes; } impl TableWithBlueprint for UnrecordedBlocksTable { @@ -93,11 +95,14 @@ impl TableWithBlueprint for UnrecordedBlocksTable { pub struct SequenceNumberTable; +/// The sequence number or bundle id of the posted blocks. +type BundleId = u32; + impl Mappable for SequenceNumberTable { type Key = Self::OwnedKey; - type OwnedKey = u32; + type OwnedKey = BlockHeight; type Value = Self::OwnedValue; - type OwnedValue = u32; + type OwnedValue = BundleId; } impl TableWithBlueprint for SequenceNumberTable { diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs index 780479c2e04..61d6bae5528 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs @@ -61,7 +61,7 @@ where fn insert(&mut self, height: v1::Height, bytes: v1::Bytes) -> Result<(), String> { self.inner .storage_as_mut::() - .insert(&height, &bytes) + .insert(&height.into(), &bytes) .map_err(|err| format!("Error: {:?}", err))?; Ok(()) } @@ -70,7 +70,7 @@ where let bytes = self .inner .storage_as_mut::() - .take(height) + .take(&(*height).into()) .map_err(|err| format!("Error: {:?}", err))?; Ok(bytes) } From 287f50fa547eb23709d167444c58862abbf9a0e3 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 11 Dec 2024 22:03:14 -0700 Subject: [PATCH 086/131] Remove comment --- tests/tests/gas_price.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 46315f4b030..e6e977b7f31 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -528,7 +528,6 @@ mod prop_tests { fn dummy() {} } -// P: 4,707,680, D: 114,760 fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { let _ = tracing_subscriber::fmt() .with_max_level(tracing::Level::INFO) From 986c6c6354b71fd605c232c7c0afa602112f6639 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 11 Dec 2024 22:37:46 -0700 Subject: [PATCH 087/131] Fix the setting of the sequence number --- crates/services/gas_price_service/src/v1/service.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 1ba2462165d..2273d475a5c 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -58,6 +58,7 @@ use fuel_gas_price_algorithm::{ }; use futures::FutureExt; use tokio::sync::broadcast::Receiver; +use crate::ports::GetDaSequenceNumber; /// The service that updates the gas price algorithm. pub struct GasPriceServiceV1 { @@ -155,6 +156,9 @@ where ) -> anyhow::Result<()> { let capacity = Self::validate_block_gas_capacity(block_gas_capacity)?; let mut storage_tx = self.storage_tx_provider.begin_transaction()?; + let mut sequence_number = storage_tx + .get_sequence_number(&BlockHeight::from(height -1)) + .map_err(|err| anyhow!(err))?; for da_block_costs in self.da_block_costs_buffer.drain(..) { tracing::debug!("Updating DA block costs: {:?}", da_block_costs); @@ -164,10 +168,14 @@ where da_block_costs.blob_cost_wei, &mut storage_tx.as_unrecorded_blocks(), )?; + sequence_number = Some(da_block_costs.sequence_number); + } + + if let Some(sequence_number) = sequence_number { storage_tx .set_sequence_number( &BlockHeight::from(height), - da_block_costs.sequence_number, + sequence_number, ) .map_err(|err| anyhow!(err))?; } From ffce7824b2cb025cfd081d226dce5cdd6d3019ff Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 11 Dec 2024 22:38:57 -0700 Subject: [PATCH 088/131] Format --- crates/services/gas_price_service/src/v1/service.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 2273d475a5c..c8f0900efa6 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -12,6 +12,7 @@ use crate::{ }, ports::{ GasPriceServiceAtomicStorage, + GetDaSequenceNumber, GetMetadataStorage, SetDaSequenceNumber, SetMetadataStorage, @@ -58,7 +59,6 @@ use fuel_gas_price_algorithm::{ }; use futures::FutureExt; use tokio::sync::broadcast::Receiver; -use crate::ports::GetDaSequenceNumber; /// The service that updates the gas price algorithm. pub struct GasPriceServiceV1 { @@ -157,7 +157,7 @@ where let capacity = Self::validate_block_gas_capacity(block_gas_capacity)?; let mut storage_tx = self.storage_tx_provider.begin_transaction()?; let mut sequence_number = storage_tx - .get_sequence_number(&BlockHeight::from(height -1)) + .get_sequence_number(&BlockHeight::from(height - 1)) .map_err(|err| anyhow!(err))?; for da_block_costs in self.da_block_costs_buffer.drain(..) { @@ -173,10 +173,7 @@ where if let Some(sequence_number) = sequence_number { storage_tx - .set_sequence_number( - &BlockHeight::from(height), - sequence_number, - ) + .set_sequence_number(&BlockHeight::from(height), sequence_number) .map_err(|err| anyhow!(err))?; } From 4fc718470a796cddd005a89c203b68bcc5dba29b Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 11 Dec 2024 22:44:24 -0700 Subject: [PATCH 089/131] Remove comment, move mutation, rename field --- .../src/v1/da_source_service.rs | 8 ++++---- .../da_source_service/block_committer_costs.rs | 8 ++++---- .../gas_price_service/src/v1/service.rs | 17 +++++++++-------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/da_source_service.rs b/crates/services/gas_price_service/src/v1/da_source_service.rs index 13f5e0c8045..799eb5b02b3 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service.rs @@ -8,9 +8,9 @@ pub mod service; #[derive(Debug, Default, Clone, Eq, Hash, PartialEq)] pub struct DaBlockCosts { - pub sequence_number: u32, + pub bundle_sequence_number: u32, pub l2_blocks: Vec, - pub blob_size_bytes: u32, + pub bundle_size_bytes: u32, pub blob_cost_wei: u128, } @@ -32,9 +32,9 @@ mod tests { async fn run__when_da_block_cost_source_gives_value_shared_state_is_updated() { // given let expected_da_cost = DaBlockCosts { - sequence_number: 1, + bundle_sequence_number: 1, l2_blocks: (0..10).collect(), - blob_size_bytes: 1024 * 128, + bundle_size_bytes: 1024 * 128, blob_cost_wei: 2, }; let notifier = Arc::new(tokio::sync::Notify::new()); diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index 5c9b37b7fb6..09cc5d2fea6 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -54,13 +54,13 @@ pub struct RawDaBlockCosts { impl From<&RawDaBlockCosts> for DaBlockCosts { fn from(raw_da_block_costs: &RawDaBlockCosts) -> Self { DaBlockCosts { - sequence_number: raw_da_block_costs.sequence_number, + bundle_sequence_number: raw_da_block_costs.sequence_number, l2_blocks: raw_da_block_costs .blocks_heights .clone() .into_iter() .collect(), - blob_size_bytes: raw_da_block_costs.total_size_bytes, + bundle_size_bytes: raw_da_block_costs.total_size_bytes, blob_cost_wei: raw_da_block_costs.total_cost, } } @@ -99,7 +99,7 @@ where |costs: DaBlockCostsResult, last_value| { let costs = costs.expect("Defined to be OK"); let blob_size_bytes = costs - .blob_size_bytes + .bundle_size_bytes .checked_sub(last_value.total_size_bytes) .ok_or(anyhow!("Blob size bytes underflow"))?; let blob_cost_wei = raw_da_block_costs @@ -107,7 +107,7 @@ where .checked_sub(last_value.total_cost) .ok_or(anyhow!("Blob cost wei underflow"))?; Ok(DaBlockCosts { - blob_size_bytes, + bundle_size_bytes: blob_size_bytes, blob_cost_wei, ..costs }) diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index c8f0900efa6..6fd8e749c0b 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -160,15 +160,15 @@ where .get_sequence_number(&BlockHeight::from(height - 1)) .map_err(|err| anyhow!(err))?; - for da_block_costs in self.da_block_costs_buffer.drain(..) { + for da_block_costs in self.da_block_costs_buffer { tracing::debug!("Updating DA block costs: {:?}", da_block_costs); self.algorithm_updater.update_da_record_data( &da_block_costs.l2_blocks, - da_block_costs.blob_size_bytes, + da_block_costs.bundle_size_bytes, da_block_costs.blob_cost_wei, &mut storage_tx.as_unrecorded_blocks(), )?; - sequence_number = Some(da_block_costs.sequence_number); + sequence_number = Some(da_block_costs.bundle_sequence_number); } if let Some(sequence_number) = sequence_number { @@ -193,6 +193,8 @@ where AtomicStorage::commit_transaction(storage_tx)?; let new_algo = self.algorithm_updater.algorithm(); self.shared_algo.update(new_algo).await; + // Clear the buffer after committing changes + self.da_block_costs_buffer.clear(); Ok(()) } @@ -207,7 +209,6 @@ where tx.set_metadata(&metadata).map_err(|err| anyhow!(err))?; AtomicStorage::commit_transaction(tx)?; let new_algo = self.algorithm_updater.algorithm(); - // self.update(new_algo).await; self.shared_algo.update(new_algo).await; } BlockInfo::Block { @@ -552,10 +553,10 @@ mod tests { let da_source = DaSourceService::new( DummyDaBlockCosts::new( Ok(DaBlockCosts { - sequence_number: 1, + bundle_sequence_number: 1, l2_blocks: (1..2).collect(), blob_cost_wei: 9000, - blob_size_bytes: 3000, + bundle_size_bytes: 3000, }), notifier.clone(), ), @@ -652,10 +653,10 @@ mod tests { let da_source = DaSourceService::new( DummyDaBlockCosts::new( Ok(DaBlockCosts { - sequence_number, + bundle_sequence_number: sequence_number, l2_blocks: (1..2).collect(), blob_cost_wei: 9000, - blob_size_bytes: 3000, + bundle_size_bytes: 3000, }), notifier.clone(), ), From ee1c8864df24cde376338df510930e734b1bd494 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 11 Dec 2024 22:44:51 -0700 Subject: [PATCH 090/131] Fix compilation --- crates/services/gas_price_service/src/v1/service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 6fd8e749c0b..70aa2728b78 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -160,7 +160,7 @@ where .get_sequence_number(&BlockHeight::from(height - 1)) .map_err(|err| anyhow!(err))?; - for da_block_costs in self.da_block_costs_buffer { + for da_block_costs in &self.da_block_costs_buffer { tracing::debug!("Updating DA block costs: {:?}", da_block_costs); self.algorithm_updater.update_da_record_data( &da_block_costs.l2_blocks, From 43e19f22c987e14314dfea1aa66440c27d0bb82d Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 11 Dec 2024 23:06:28 -0700 Subject: [PATCH 091/131] Appease Clippy-sama --- crates/services/gas_price_service/src/v1/service.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 70aa2728b78..4f2289fa6e7 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -156,8 +156,9 @@ where ) -> anyhow::Result<()> { let capacity = Self::validate_block_gas_capacity(block_gas_capacity)?; let mut storage_tx = self.storage_tx_provider.begin_transaction()?; + let prev_height = height.saturating_sub(1); let mut sequence_number = storage_tx - .get_sequence_number(&BlockHeight::from(height - 1)) + .get_sequence_number(&BlockHeight::from(prev_height)) .map_err(|err| anyhow!(err))?; for da_block_costs in &self.da_block_costs_buffer { From 198a07e285ce42bbe2306a6d9cf7b26e09885648 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 12 Dec 2024 12:06:27 -0700 Subject: [PATCH 092/131] Rewrite server to be more dynamic --- .../block_committer_costs.rs | 4 +- tests/tests/gas_price.rs | 49 +++++++++++++++---- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index f48035fc077..f06e47ed3ad 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -160,9 +160,9 @@ impl BlockCommitterApi for BlockCommitterHttpApi { .get(format!("{}/{}", url, number)) .send() .await? - .json::() + .json::>() .await?; - Ok(Some(response)) + Ok(response) } else { Ok(None) } diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index e6e977b7f31..ba68df89279 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -53,9 +53,14 @@ use fuel_core_types::{ }; use rand::Rng; use std::{ + collections::HashMap, iter::repeat, num::NonZero, ops::Deref, + sync::{ + Arc, + Mutex, + }, time::Duration, }; use test_helpers::fuel_core_driver::FuelCoreDriver; @@ -673,23 +678,49 @@ async fn produce_a_block(client: &FuelClient, rng: &mu struct FakeServer { server: mockito::ServerGuard, + responses: Arc, u32)>>, } impl FakeServer { fn new() -> Self { - let server = mockito::Server::new(); - Self { server } - } - - pub fn add_response(&mut self, costs: RawDaBlockCosts) { - let body = serde_json::to_string(&costs).unwrap(); - let _mock = self - .server + let mut server = mockito::Server::new(); + let responses = Arc::new(Mutex::new((HashMap::new(), 1))); + let shared_responses = responses.clone(); + server .mock("GET", "/") .with_status(201) - .with_body(body) + .with_body_from_request(move |request| { + // take the requested number and return the corresponding response from the `responses` hashmap + let path = request.path(); + let maybe_sequence_number = + path.split('/').last().and_then(|x| x.parse::().ok()); + match maybe_sequence_number { + Some(sequence_number) => { + let guard = shared_responses.lock().unwrap(); + let responses = &guard.0; + let response = responses.get(&sequence_number).cloned(); + serde_json::to_string(&response).unwrap().into() + } + None => { + let guard = shared_responses.lock().unwrap(); + let responses = &guard.0; + let latest = &guard.1; + let response = responses.get(latest).cloned(); + serde_json::to_string(&response).unwrap().into() + } + } + }) .expect_at_least(1) .create(); + Self { server, responses } + } + + pub fn add_response(&mut self, costs: RawDaBlockCosts) { + let mut guard = self.responses.lock().unwrap(); + let latest = guard.1; + let new_seq_no = latest + 1; + guard.0.insert(new_seq_no, costs); + guard.1 = new_seq_no; } fn url(&self) -> String { From eae3add9c7d989a619c8b5ddac4b3e48980c200c Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 12 Dec 2024 13:09:46 -0700 Subject: [PATCH 093/131] Cleanup test a lot --- .../block_committer_costs.rs | 24 +++++--- .../src/v1/da_source_service/service.rs | 7 ++- tests/tests/gas_price.rs | 56 +++++++++++-------- 3 files changed, 54 insertions(+), 33 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index f06e47ed3ad..c79e896eff7 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -81,7 +81,9 @@ impl DaBlockCostsSource for BlockCommitterDaBlockCosts DaBlockCostsResult { + async fn request_da_block_cost( + &mut self, + ) -> DaBlockCostsResult> { let raw_da_block_costs = match self.last_raw_da_block_costs { Some(ref last_value) => self .client @@ -91,7 +93,10 @@ where .await?; let Some(ref raw_da_block_costs) = raw_da_block_costs else { - return Err(anyhow!("No response from block committer")) + // TODO: This is really annoying if there haven't been any costs yet. Do we need this? + // Gonna return `Option::None` for now + // return Err(anyhow!("No response from block committer")) + return Ok(None) }; let da_block_costs = self.last_raw_da_block_costs.iter().fold( @@ -115,7 +120,7 @@ where )?; self.last_raw_da_block_costs = Some(raw_da_block_costs.clone()); - Ok(da_block_costs) + Ok(Some(da_block_costs)) } async fn set_last_value(&mut self, sequence_number: u32) -> DaBlockCostsResult<()> { self.last_raw_da_block_costs = @@ -143,8 +148,10 @@ impl BlockCommitterApi for BlockCommitterHttpApi { async fn get_latest_costs(&self) -> DaBlockCostsResult> { if let Some(url) = &self.url { let val = self.client.get(url).send().await?; - let response = val.json::().await?; - Ok(Some(response)) + tracing::warn!("val: {:?}", val); + let response = val.json::>().await?; + tracing::warn!("Response: {:?}", response); + Ok(response) } else { Ok(None) } @@ -155,13 +162,14 @@ impl BlockCommitterApi for BlockCommitterHttpApi { number: u32, ) -> DaBlockCostsResult> { if let Some(url) = &self.url { - let response = self + let val = self .client .get(format!("{}/{}", url, number)) .send() - .await? - .json::>() .await?; + tracing::warn!("val: {:?}", val); + let response = val.json::>().await?; + tracing::warn!("Response: {:?}", response); Ok(response) } else { Ok(None) diff --git a/crates/services/gas_price_service/src/v1/da_source_service/service.rs b/crates/services/gas_price_service/src/v1/da_source_service/service.rs index c7cd44e665c..08226a84d40 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/service.rs @@ -60,8 +60,9 @@ where async fn process_block_costs(&mut self) -> Result<()> { let da_block_costs_res = self.source.request_da_block_cost().await; tracing::debug!("Received block costs: {:?}", da_block_costs_res); - let da_block_costs = da_block_costs_res?; - self.shared_state.0.send(da_block_costs)?; + if let Some(da_block_costs) = da_block_costs_res? { + self.shared_state.0.send(da_block_costs)?; + } Ok(()) } } @@ -70,7 +71,7 @@ where /// da block costs in a way they see fit #[async_trait::async_trait] pub trait DaBlockCostsSource: Send + Sync { - async fn request_da_block_cost(&mut self) -> Result; + async fn request_da_block_cost(&mut self) -> Result>; async fn set_last_value(&mut self, sequence_number: u32) -> Result<()>; } diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index ba68df89279..ef0c0e940c8 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -51,6 +51,7 @@ use fuel_core_types::{ }, services::executor::TransactionExecutionResult, }; +use mockito::Matcher::Any; use rand::Rng; use std::{ collections::HashMap, @@ -515,17 +516,21 @@ mod prop_tests { // the blobs costs will never change more than 12.5% between blobs - proptest! { - // This test will generate a many l2 blocks, and then after the block delay, it will - // start receiving costs for committing to DA in blobs. If the profit is positive after the - // first blob, we will continue to process blocks until the profit is negative (if it is - // negative, we will check the opposite). We will say the test passed if it can recover from - // the divergent profit after a set amount of l2 blocks. - #![proptest_config(ProptestConfig::with_cases(1))] - #[test] - fn produce_bock__algorithm_recovers_from_divergent_profit(block_delay in 11..12usize, blob_size in 50..100usize) { - produce_blocks__lolz(block_delay, blob_size); - } + // proptest! { + // // This test will generate a many l2 blocks, and then after the block delay, it will + // // start receiving costs for committing to DA in blobs. If the profit is positive after the + // // first blob, we will continue to process blocks until the profit is negative (if it is + // // negative, we will check the opposite). We will say the test passed if it can recover from + // // the divergent profit after a set amount of l2 blocks. + // #![proptest_config(ProptestConfig::with_cases(1))] + // #[test] + // fn produce_bock__algorithm_recovers_from_divergent_profit(block_delay in 11..12usize, blob_size in 50..100usize) { + // produce_blocks__lolz(block_delay, blob_size); + // } + // } + #[test] + fn produce_block__algorithm_recovers_from_divergent_profit() { + produce_blocks__lolz(11, 50); } #[ignore] @@ -559,7 +564,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { node_config.max_da_gas_price_change_percent = 10; node_config.block_production = Trigger::Never; node_config.da_committer_url = Some(url.clone()); - node_config.da_poll_interval = Some(1); + node_config.da_poll_interval = Some(100); // node_config.da_p_component = 4_707_680; // node_config.da_d_component = 114_760; node_config.da_p_component = 1; @@ -591,7 +596,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { let half_of_blocks = block_delay as u32 / 2; let blocks_heights: Vec<_> = (1..half_of_blocks).collect(); let count = blocks_heights.len() as u128; - let new_price_gwei = 500_000; + let new_price_gwei = 100; let new_price = new_price_gwei * 1_000_000_000; // Wei let cost = count * new_price; mock.add_response(RawDaBlockCosts { @@ -605,7 +610,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { let mut profits = Vec::new(); let mut gas_prices = Vec::new(); rt.block_on(async { - tokio::time::sleep(Duration::from_millis(10)).await; + tokio::time::sleep(Duration::from_millis(200)).await; client.produce_blocks(1, None).await.unwrap(); client.produce_blocks(1, None).await.unwrap(); let height = srv.shared.database.gas_price().latest_height().unwrap(); @@ -622,7 +627,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { gas_prices.push(metadata.new_scaled_da_gas_price / metadata.gas_price_factor); }); - let tries = 100; + let tries = 200; let mut success = false; let mut success_iteration = i32::MAX; @@ -638,9 +643,10 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { .and_then(|x| x.v1().cloned()) .unwrap(); let profit = metadata.last_profit; + tracing::info!("metadata: {:?}", metadata); profits.push(profit); gas_prices.push(metadata.new_scaled_da_gas_price / metadata.gas_price_factor); - if profit > 0 && !success { + if profit > 0 { success = true; success_iteration = i as i32; } @@ -663,6 +669,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { tracing::info!("Gas prices: {:?}", gas_prices); tracing::info!("Gas price changes: {:?}", gas_price_changes); } + drop(mock); } async fn produce_a_block(client: &FuelClient, rng: &mut R) { @@ -683,11 +690,17 @@ struct FakeServer { impl FakeServer { fn new() -> Self { - let mut server = mockito::Server::new(); - let responses = Arc::new(Mutex::new((HashMap::new(), 1))); - let shared_responses = responses.clone(); - server - .mock("GET", "/") + let server = mockito::Server::new(); + let responses = Arc::new(Mutex::new((HashMap::new(), 0))); + let mut fake = Self { server, responses }; + fake.init(); + fake + } + + pub fn init(&mut self) { + let shared_responses = self.responses.clone(); + self.server + .mock("GET", Any) .with_status(201) .with_body_from_request(move |request| { // take the requested number and return the corresponding response from the `responses` hashmap @@ -712,7 +725,6 @@ impl FakeServer { }) .expect_at_least(1) .create(); - Self { server, responses } } pub fn add_response(&mut self, costs: RawDaBlockCosts) { From 8c6092b3bf857f57986c51e7ca74892ad6189ea9 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 12 Dec 2024 13:23:53 -0700 Subject: [PATCH 094/131] Fix annoying little bug --- crates/services/gas_price_service/src/v1/service.rs | 2 +- .../gas_price_service/src/v1/uninitialized_task.rs | 3 ++- tests/tests/gas_price.rs | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 9c7b9465b31..4f2d87329bb 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -183,7 +183,7 @@ where .map_err(|err| anyhow!(err))?; } - let fee_in_wei = u128::from(block_fees).saturating_add(1_000_000_000); + let fee_in_wei = u128::from(block_fees).saturating_mul(1_000_000_000); self.algorithm_updater.update_l2_block_data( height, gas_used, diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 98c02625eb3..c56096b88ad 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -328,7 +328,8 @@ where }; let block_bytes = block_bytes(&block); - let (fee_wei, _) = mint_values(&block)?; + let (fee_gwei, _) = mint_values(&block)?; + let fee_wei = fee_gwei.saturating_mul(1_000_000_000); updater.update_l2_block_data( height, block_gas_used, diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index ef0c0e940c8..c5cffd7d3d6 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -558,7 +558,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { }; let mut node_config = Config::local_node_with_configs(chain_config, StateConfig::local_testnet()); - let starting_gas_price = 1_000_000_000; + let starting_gas_price = 10_000_000; node_config.block_producer.coinbase_recipient = Some([5; 32].into()); node_config.min_da_gas_price = starting_gas_price; node_config.max_da_gas_price_change_percent = 10; @@ -596,7 +596,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { let half_of_blocks = block_delay as u32 / 2; let blocks_heights: Vec<_> = (1..half_of_blocks).collect(); let count = blocks_heights.len() as u128; - let new_price_gwei = 100; + let new_price_gwei = 500_000; let new_price = new_price_gwei * 1_000_000_000; // Wei let cost = count * new_price; mock.add_response(RawDaBlockCosts { @@ -627,7 +627,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { gas_prices.push(metadata.new_scaled_da_gas_price / metadata.gas_price_factor); }); - let tries = 200; + let tries = 100; let mut success = false; let mut success_iteration = i32::MAX; @@ -646,7 +646,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { tracing::info!("metadata: {:?}", metadata); profits.push(profit); gas_prices.push(metadata.new_scaled_da_gas_price / metadata.gas_price_factor); - if profit > 0 { + if profit > 0 && !success { success = true; success_iteration = i as i32; } From b82f354a1b235974a913f0eb7f97fb7a67cd145f Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 12 Dec 2024 14:41:19 -0700 Subject: [PATCH 095/131] Add more helpful messages --- tests/tests/gas_price.rs | 67 ++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index c5cffd7d3d6..98653953452 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -509,33 +509,9 @@ fn produce_block__l1_committed_block_effects_gas_price() { assert!(first_gas_price < new_gas_price); } -#[allow(unused_variables)] -mod prop_tests { - use super::*; - use proptest::prelude::*; - - // the blobs costs will never change more than 12.5% between blobs - - // proptest! { - // // This test will generate a many l2 blocks, and then after the block delay, it will - // // start receiving costs for committing to DA in blobs. If the profit is positive after the - // // first blob, we will continue to process blocks until the profit is negative (if it is - // // negative, we will check the opposite). We will say the test passed if it can recover from - // // the divergent profit after a set amount of l2 blocks. - // #![proptest_config(ProptestConfig::with_cases(1))] - // #[test] - // fn produce_bock__algorithm_recovers_from_divergent_profit(block_delay in 11..12usize, blob_size in 50..100usize) { - // produce_blocks__lolz(block_delay, blob_size); - // } - // } - #[test] - fn produce_block__algorithm_recovers_from_divergent_profit() { - produce_blocks__lolz(11, 50); - } - - #[ignore] - #[test] - fn dummy() {} +#[test] +fn produce_block__algorithm_recovers_from_divergent_profit() { + produce_blocks__lolz(11, 50); } fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { @@ -558,7 +534,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { }; let mut node_config = Config::local_node_with_configs(chain_config, StateConfig::local_testnet()); - let starting_gas_price = 10_000_000; + let starting_gas_price = 1_000_000_000; node_config.block_producer.coinbase_recipient = Some([5; 32].into()); node_config.min_da_gas_price = starting_gas_price; node_config.max_da_gas_price_change_percent = 10; @@ -567,7 +543,8 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { node_config.da_poll_interval = Some(100); // node_config.da_p_component = 4_707_680; // node_config.da_d_component = 114_760; - node_config.da_p_component = 1; + node_config.da_p_component = 10; + node_config.da_d_component = 1; node_config.block_activity_threshold = 0; let (srv, client) = rt.block_on(async { @@ -627,7 +604,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { gas_prices.push(metadata.new_scaled_da_gas_price / metadata.gas_price_factor); }); - let tries = 100; + let tries = 1000; let mut success = false; let mut success_iteration = i32::MAX; @@ -649,6 +626,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { if profit > 0 && !success { success = true; success_iteration = i as i32; + // break; } } }); @@ -664,10 +642,31 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { ); } else { tracing::info!("Success on try {}/{}", success_iteration, tries); - tracing::info!("Profits: {:?}", profits); - tracing::info!("Changes: {:?}", changes); - tracing::info!("Gas prices: {:?}", gas_prices); - tracing::info!("Gas price changes: {:?}", gas_price_changes); + // tracing::info!("Profits: {:?}", profits); + // tracing::info!("Changes: {:?}", changes); + // tracing::info!("Gas prices: {:?}", gas_prices); + // tracing::info!("Gas price changes: {:?}", gas_price_changes); + // trace the same as f64 and converted into gwei + let profits_as_gwei = profits + .iter() + .map(|x| *x as f64 / 1_000_000_000.0) + .collect::>(); + let changes_as_gwei = changes + .iter() + .map(|x| *x as f64 / 1_000_000_000.0) + .collect::>(); + let gas_prices_as_gwei = gas_prices + .iter() + .map(|x| *x as f64 / 1_000_000_000.0) + .collect::>(); + let gas_price_changes_as_gwei = gas_price_changes + .iter() + .map(|x| *x as f64 / 1_000_000_000.0) + .collect::>(); + tracing::info!("Profits as gwei: {:?}", profits_as_gwei); + tracing::info!("Changes as gwei: {:?}", changes_as_gwei); + tracing::info!("Gas prices as gwei: {:?}", gas_prices_as_gwei); + tracing::info!("Gas price changes as gwei: {:?}", gas_price_changes_as_gwei); } drop(mock); } From 93876fa98f10ded0575cf79b4b1bc67a35af23f9 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 12 Dec 2024 14:52:32 -0700 Subject: [PATCH 096/131] Rename sequence number to bundle id --- .../v1/tests/update_da_record_data_tests.rs | 4 -- .../src/common/fuel_core_storage_adapter.rs | 35 +++++------ .../fuel_core_storage_adapter/storage.rs | 10 ++-- .../services/gas_price_service/src/ports.rs | 19 +++--- .../src/v1/da_source_service.rs | 4 +- .../block_committer_costs.rs | 21 ++++--- .../src/v1/da_source_service/dummy_costs.rs | 2 +- .../src/v1/da_source_service/service.rs | 2 +- .../gas_price_service/src/v1/service.rs | 30 +++++----- .../gas_price_service/src/v1/tests.rs | 60 ++++++++----------- .../src/v1/uninitialized_task.rs | 11 ++-- 11 files changed, 88 insertions(+), 110 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index c262352a311..6d1feb220b4 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -366,10 +366,6 @@ fn update_da_record_data__da_block_lowers_da_gas_price() { let da_cost_per_byte = 40; let l2_block_height = 11; let original_known_total_cost = 150; - // let unrecorded_blocks = vec![BlockBytes { - // height: 11, - // block_bytes: 3000, - // }]; let mut unrecorded_blocks: BTreeMap<_, _> = [(11, 3000)].into_iter().collect(); let da_p_component = 2; let guessed_cost: u64 = unrecorded_blocks diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs index 083399480e8..74bec1207d2 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs @@ -1,9 +1,9 @@ use crate::{ common::{ fuel_core_storage_adapter::storage::{ + BundleIdTable, GasPriceColumn, GasPriceMetadata, - SequenceNumberTable, }, updater_metadata::UpdaterMetadata, utils::{ @@ -14,9 +14,9 @@ use crate::{ }, ports::{ GasPriceServiceAtomicStorage, - GetDaSequenceNumber, + GetDaBundleId, GetMetadataStorage, - SetDaSequenceNumber, + SetDaBundleId, SetMetadataStorage, }, }; @@ -99,28 +99,25 @@ where } } -impl GetDaSequenceNumber for Storage +impl GetDaBundleId for Storage where Storage: Send + Sync, - Storage: StorageInspect, + Storage: StorageInspect, { - fn get_sequence_number( - &self, - block_height: &BlockHeight, - ) -> GasPriceResult> { - let sequence_number = self - .storage::() + fn get_bundle_id(&self, block_height: &BlockHeight) -> GasPriceResult> { + let bundle_id = self + .storage::() .get(block_height) .map_err(|err| GasPriceError::CouldNotFetchDARecord(err.into()))? .map(|no| *no); - Ok(sequence_number) + Ok(bundle_id) } } impl GasPriceServiceAtomicStorage for Storage where Storage: 'static, - Storage: GetMetadataStorage + GetDaSequenceNumber, + Storage: GetMetadataStorage + GetDaBundleId, Storage: KeyValueInspect + Modifiable + Send + Sync, { type Transaction<'a> = StorageTransaction<&'a mut Storage> where Self: 'a; @@ -138,18 +135,18 @@ where } } -impl SetDaSequenceNumber for Storage +impl SetDaBundleId for Storage where Storage: Send + Sync, - Storage: StorageMutate, + Storage: StorageMutate, { - fn set_sequence_number( + fn set_bundle_id( &mut self, block_height: &BlockHeight, - sequence_number: u32, + bundle_id: u32, ) -> GasPriceResult<()> { - self.storage_as_mut::() - .insert(block_height, &sequence_number) + self.storage_as_mut::() + .insert(block_height, &bundle_id) .map_err(|err| GasPriceError::CouldNotFetchDARecord(err.into()))?; Ok(()) } diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs index d6c0a0aa8e0..e9f20b411d8 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs @@ -28,7 +28,7 @@ pub enum GasPriceColumn { Metadata = 0, State = 1, UnrecordedBlocks = 2, - SequenceNumber = 3, + BundleId = 3, } impl GasPriceColumn { @@ -93,23 +93,23 @@ impl TableWithBlueprint for UnrecordedBlocksTable { } } -pub struct SequenceNumberTable; +pub struct BundleIdTable; /// The sequence number or bundle id of the posted blocks. type BundleId = u32; -impl Mappable for SequenceNumberTable { +impl Mappable for BundleIdTable { type Key = Self::OwnedKey; type OwnedKey = BlockHeight; type Value = Self::OwnedValue; type OwnedValue = BundleId; } -impl TableWithBlueprint for SequenceNumberTable { +impl TableWithBlueprint for BundleIdTable { type Blueprint = Plain, Postcard>; type Column = GasPriceColumn; fn column() -> Self::Column { - GasPriceColumn::SequenceNumber + GasPriceColumn::BundleId } } diff --git a/crates/services/gas_price_service/src/ports.rs b/crates/services/gas_price_service/src/ports.rs index 0bd601cf7a8..0ccd77cb01a 100644 --- a/crates/services/gas_price_service/src/ports.rs +++ b/crates/services/gas_price_service/src/ports.rs @@ -34,29 +34,26 @@ pub trait GetMetadataStorage: Send + Sync { -> Result>; } -pub trait SetDaSequenceNumber: Send + Sync { - fn set_sequence_number( - &mut self, - block_height: &BlockHeight, - sequence_number: u32, - ) -> Result<()>; +pub trait SetDaBundleId: Send + Sync { + fn set_bundle_id(&mut self, block_height: &BlockHeight, bundle_id: u32) + -> Result<()>; } -pub trait GetDaSequenceNumber: Send + Sync { - fn get_sequence_number(&self, block_height: &BlockHeight) -> Result>; +pub trait GetDaBundleId: Send + Sync { + fn get_bundle_id(&self, block_height: &BlockHeight) -> Result>; } pub trait GasPriceServiceAtomicStorage where Self: 'static, Self: Send + Sync, - Self: GetMetadataStorage + GetDaSequenceNumber, + Self: GetMetadataStorage + GetDaBundleId, { type Transaction<'a>: AsUnrecordedBlocks + SetMetadataStorage + GetMetadataStorage - + SetDaSequenceNumber - + GetDaSequenceNumber + + SetDaBundleId + + GetDaBundleId where Self: 'a; diff --git a/crates/services/gas_price_service/src/v1/da_source_service.rs b/crates/services/gas_price_service/src/v1/da_source_service.rs index 799eb5b02b3..bae8d3d46bc 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service.rs @@ -8,7 +8,7 @@ pub mod service; #[derive(Debug, Default, Clone, Eq, Hash, PartialEq)] pub struct DaBlockCosts { - pub bundle_sequence_number: u32, + pub bundle_id: u32, pub l2_blocks: Vec, pub bundle_size_bytes: u32, pub blob_cost_wei: u128, @@ -32,7 +32,7 @@ mod tests { async fn run__when_da_block_cost_source_gives_value_shared_state_is_updated() { // given let expected_da_cost = DaBlockCosts { - bundle_sequence_number: 1, + bundle_id: 1, l2_blocks: (0..10).collect(), bundle_size_bytes: 1024 * 128, blob_cost_wei: 2, diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index 09cc5d2fea6..98c024933c8 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -40,7 +40,7 @@ pub struct BlockCommitterDaBlockCosts { #[derive(Debug, Deserialize, Serialize, Clone, Default, PartialEq)] pub struct RawDaBlockCosts { /// Sequence number (Monotonically increasing nonce) - pub sequence_number: u32, + pub bundle_id: u32, /// The range of blocks that the costs apply to pub blocks_heights: Vec, /// The DA block height of the last transaction for the range of blocks @@ -54,7 +54,7 @@ pub struct RawDaBlockCosts { impl From<&RawDaBlockCosts> for DaBlockCosts { fn from(raw_da_block_costs: &RawDaBlockCosts) -> Self { DaBlockCosts { - bundle_sequence_number: raw_da_block_costs.sequence_number, + bundle_id: raw_da_block_costs.bundle_id, l2_blocks: raw_da_block_costs .blocks_heights .clone() @@ -83,9 +83,9 @@ where { async fn request_da_block_cost(&mut self) -> DaBlockCostsResult { let raw_da_block_costs = match self.last_raw_da_block_costs { - Some(ref last_value) => self - .client - .get_costs_by_seqno(last_value.sequence_number + 1), + Some(ref last_value) => { + self.client.get_costs_by_seqno(last_value.bundle_id + 1) + } _ => self.client.get_latest_costs(), } .await?; @@ -117,9 +117,8 @@ where self.last_raw_da_block_costs = Some(raw_da_block_costs.clone()); Ok(da_block_costs) } - async fn set_last_value(&mut self, sequence_number: u32) -> DaBlockCostsResult<()> { - self.last_raw_da_block_costs = - self.client.get_costs_by_seqno(sequence_number).await?; + async fn set_last_value(&mut self, bundle_id: u32) -> DaBlockCostsResult<()> { + self.last_raw_da_block_costs = self.client.get_costs_by_seqno(bundle_id).await?; Ok(()) } } @@ -207,7 +206,7 @@ mod tests { // arbitrary logic to generate a new value let mut value = self.value.clone(); if let Some(value) = &mut value { - value.sequence_number = seq_no; + value.bundle_id = seq_no; value.blocks_heights = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] .to_vec() .iter() @@ -230,7 +229,7 @@ mod tests { fn test_da_block_costs() -> RawDaBlockCosts { RawDaBlockCosts { - sequence_number: 1, + bundle_id: 1, blocks_heights: (0..10).collect(), da_block_height: 1u64.into(), total_cost: 1, @@ -305,7 +304,7 @@ mod tests { // arbitrary logic to generate a new value let mut value = self.value.clone(); if let Some(value) = &mut value { - value.sequence_number = seq_no; + value.bundle_id = seq_no; value.blocks_heights = value.blocks_heights.iter().map(|x| x + seq_no).collect(); value.da_block_height = value.da_block_height + 1u64.into(); diff --git a/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs index 234c253383f..5204ea5fba0 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs @@ -35,7 +35,7 @@ impl DaBlockCostsSource for DummyDaBlockCosts { } } - async fn set_last_value(&mut self, sequence_number: u32) -> DaBlockCostsResult<()> { + async fn set_last_value(&mut self, _bundle_id: u32) -> DaBlockCostsResult<()> { unimplemented!("This is a dummy implementation"); } } diff --git a/crates/services/gas_price_service/src/v1/da_source_service/service.rs b/crates/services/gas_price_service/src/v1/da_source_service/service.rs index 108f50591e6..d7dfca30a20 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/service.rs @@ -69,7 +69,7 @@ where #[async_trait::async_trait] pub trait DaBlockCostsSource: Send + Sync { async fn request_da_block_cost(&mut self) -> Result; - async fn set_last_value(&mut self, sequence_number: u32) -> Result<()>; + async fn set_last_value(&mut self, bundle_id: u32) -> Result<()>; } #[async_trait::async_trait] diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 4f2289fa6e7..df0c91a5c93 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -12,9 +12,9 @@ use crate::{ }, ports::{ GasPriceServiceAtomicStorage, - GetDaSequenceNumber, + GetDaBundleId, GetMetadataStorage, - SetDaSequenceNumber, + SetDaBundleId, SetMetadataStorage, }, v0::metadata::V0Metadata, @@ -157,8 +157,8 @@ where let capacity = Self::validate_block_gas_capacity(block_gas_capacity)?; let mut storage_tx = self.storage_tx_provider.begin_transaction()?; let prev_height = height.saturating_sub(1); - let mut sequence_number = storage_tx - .get_sequence_number(&BlockHeight::from(prev_height)) + let mut bundle_id = storage_tx + .get_bundle_id(&BlockHeight::from(prev_height)) .map_err(|err| anyhow!(err))?; for da_block_costs in &self.da_block_costs_buffer { @@ -169,12 +169,12 @@ where da_block_costs.blob_cost_wei, &mut storage_tx.as_unrecorded_blocks(), )?; - sequence_number = Some(da_block_costs.bundle_sequence_number); + bundle_id = Some(da_block_costs.bundle_id); } - if let Some(sequence_number) = sequence_number { + if let Some(bundle_id) = bundle_id { storage_tx - .set_sequence_number(&BlockHeight::from(height), sequence_number) + .set_bundle_id(&BlockHeight::from(height), bundle_id) .map_err(|err| anyhow!(err))?; } @@ -356,9 +356,9 @@ mod tests { use crate::{ common::{ fuel_core_storage_adapter::storage::{ + BundleIdTable, GasPriceColumn, GasPriceColumn::UnrecordedBlocks, - SequenceNumberTable, UnrecordedBlocksTable, }, gas_price_algorithm::SharedGasPriceAlgo, @@ -554,7 +554,7 @@ mod tests { let da_source = DaSourceService::new( DummyDaBlockCosts::new( Ok(DaBlockCosts { - bundle_sequence_number: 1, + bundle_id: 1, l2_blocks: (1..2).collect(), blob_cost_wei: 9000, bundle_size_bytes: 3000, @@ -617,9 +617,9 @@ mod tests { } #[tokio::test] - async fn run__responses_from_da_service_update_sequence_number_in_storage() { + async fn run__responses_from_da_service_update_bundle_id_in_storage() { // given - let sequence_number = 1234; + let bundle_id = 1234; let block_height = 2; let l2_block_2 = BlockInfo::Block { height: block_height, @@ -654,7 +654,7 @@ mod tests { let da_source = DaSourceService::new( DummyDaBlockCosts::new( Ok(DaBlockCosts { - bundle_sequence_number: sequence_number, + bundle_id, l2_blocks: (1..2).collect(), blob_cost_wei: 9000, bundle_size_bytes: 3000, @@ -693,13 +693,13 @@ mod tests { tokio::time::sleep(Duration::from_millis(100)).await; // then - let latest_sequence_number = service + let latest_bundle_id = service .storage_tx_provider - .storage::() + .storage::() .get(&BlockHeight::from(block_height)) .unwrap() .unwrap(); - assert_eq!(*latest_sequence_number, sequence_number); + assert_eq!(*latest_bundle_id, bundle_id); service.shutdown().await.unwrap(); } diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 92fd6757f05..f5524508edc 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -20,10 +20,10 @@ use crate::{ ports::{ GasPriceData, GasPriceServiceAtomicStorage, - GetDaSequenceNumber, + GetDaBundleId, GetMetadataStorage, L2Data, - SetDaSequenceNumber, + SetDaBundleId, SetMetadataStorage, }, v1::{ @@ -161,11 +161,8 @@ impl GetMetadataStorage for ErroringPersistedData { } } -impl GetDaSequenceNumber for ErroringPersistedData { - fn get_sequence_number( - &self, - _block_height: &BlockHeight, - ) -> GasPriceResult> { +impl GetDaBundleId for ErroringPersistedData { + fn get_bundle_id(&self, _block_height: &BlockHeight) -> GasPriceResult> { Err(GasPriceError::CouldNotFetchDARecord(anyhow!("boo!"))) } } @@ -209,21 +206,18 @@ impl UnrecordedBlocks for UnimplementedStorageTx { } } -impl SetDaSequenceNumber for UnimplementedStorageTx { - fn set_sequence_number( +impl SetDaBundleId for UnimplementedStorageTx { + fn set_bundle_id( &mut self, _block_height: &BlockHeight, - _sequence_number: u32, + _bundle_id: u32, ) -> GasPriceResult<()> { unimplemented!() } } -impl GetDaSequenceNumber for UnimplementedStorageTx { - fn get_sequence_number( - &self, - _block_height: &BlockHeight, - ) -> GasPriceResult> { +impl GetDaBundleId for UnimplementedStorageTx { + fn get_bundle_id(&self, _block_height: &BlockHeight) -> GasPriceResult> { unimplemented!() } } @@ -240,7 +234,7 @@ impl AsUnrecordedBlocks for UnimplementedStorageTx { struct FakeDABlockCost { da_block_costs: Receiver, - sequence_number: Arc>>, + bundle_id: Arc>>, } impl FakeDABlockCost { @@ -248,25 +242,25 @@ impl FakeDABlockCost { let (_sender, receiver) = tokio::sync::mpsc::channel(1); Self { da_block_costs: receiver, - sequence_number: Arc::new(Mutex::new(None)), + bundle_id: Arc::new(Mutex::new(None)), } } fn new(da_block_costs: Receiver) -> Self { Self { da_block_costs, - sequence_number: Arc::new(Mutex::new(None)), + bundle_id: Arc::new(Mutex::new(None)), } } - fn never_returns_with_handle_to_sequence_number() -> (Self, Arc>>) { + fn never_returns_with_handle_to_bundle_id() -> (Self, Arc>>) { let (_sender, receiver) = tokio::sync::mpsc::channel(1); - let sequence_number = Arc::new(Mutex::new(None)); + let bundle_id = Arc::new(Mutex::new(None)); let service = Self { da_block_costs: receiver, - sequence_number: sequence_number.clone(), + bundle_id: bundle_id.clone(), }; - (service, sequence_number) + (service, bundle_id) } } @@ -277,11 +271,8 @@ impl DaBlockCostsSource for FakeDABlockCost { Ok(costs) } - async fn set_last_value(&mut self, sequence_number: u32) -> Result<()> { - self.sequence_number - .lock() - .unwrap() - .replace(sequence_number); + async fn set_last_value(&mut self, bundle_id: u32) -> Result<()> { + self.bundle_id.lock().unwrap().replace(bundle_id); Ok(()) } } @@ -674,10 +665,10 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { } #[tokio::test] -async fn uninitialized_task__init__starts_da_service_with_sequence_number_in_storage() { +async fn uninitialized_task__init__starts_da_service_with_bundle_id_in_storage() { // given let block_height = 1; - let sequence_number: u32 = 123; + let bundle_id: u32 = 123; let original_metadata = arbitrary_metadata(); let different_config = different_arb_config(); @@ -688,12 +679,11 @@ async fn uninitialized_task__init__starts_da_service_with_sequence_number_in_sto let settings = FakeSettings; let block_stream = empty_block_stream(); let on_chain_db = FakeOnChainDb::new(different_l2_block); - let (da_cost_source, sequence_number_handle) = - FakeDABlockCost::never_returns_with_handle_to_sequence_number(); + let (da_cost_source, bundle_id_handle) = + FakeDABlockCost::never_returns_with_handle_to_bundle_id(); let mut inner = database_with_metadata(&original_metadata); let mut tx = inner.begin_transaction().unwrap(); - tx.set_sequence_number(&block_height.into(), sequence_number) - .unwrap(); + tx.set_bundle_id(&block_height.into(), bundle_id).unwrap(); StorageTransaction::commit_transaction(tx).unwrap(); let service = UninitializedTask::new( different_config.clone(), @@ -711,7 +701,7 @@ async fn uninitialized_task__init__starts_da_service_with_sequence_number_in_sto service.init().await.unwrap(); // then - let actual = sequence_number_handle.lock().unwrap(); - let expected = Some(sequence_number); + let actual = bundle_id_handle.lock().unwrap(); + let expected = Some(bundle_id); assert_eq!(*actual, expected); } diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index ac0ce06da01..96da25427d4 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -20,10 +20,10 @@ use crate::{ GasPriceData, GasPriceServiceAtomicStorage, GasPriceServiceConfig, - GetDaSequenceNumber, + GetDaBundleId, GetMetadataStorage, L2Data, - SetDaSequenceNumber, + SetDaBundleId, SetMetadataStorage, }, v1::{ @@ -168,11 +168,10 @@ where // TODO: Add to config // https://github.com/FuelLabs/fuel-core/issues/2140 let poll_interval = None; - if let Some(sequence_number) = self - .gas_price_db - .get_sequence_number(&metadata_height.into())? + if let Some(bundle_id) = + self.gas_price_db.get_bundle_id(&metadata_height.into())? { - self.da_source.set_last_value(sequence_number).await?; + self.da_source.set_last_value(bundle_id).await?; } let da_service = DaSourceService::new(self.da_source, poll_interval); From 9218c32f35b62cdc133f49eb55cd105a62d247e5 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 13 Dec 2024 13:03:50 -0700 Subject: [PATCH 097/131] Fix simulation --- .../gas-price-analysis/src/charts.rs | 8 +- .../gas-price-analysis/src/main.rs | 8 +- .../gas-price-analysis/src/simulation.rs | 97 ++++++++++++------- .../src/simulation/da_cost_per_byte.rs | 3 +- tests/tests/gas_price.rs | 14 +-- 5 files changed, 77 insertions(+), 53 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs index 41f0fe8fab6..aa3f401ddcc 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/charts.rs @@ -191,16 +191,16 @@ pub fn draw_fullness( pub fn draw_bytes_and_cost_per_block( drawing_area: &DrawingArea, - bytes_and_costs_per_block: &[(u64, u64)], + bytes_and_costs_per_block: &[(u32, u64)], title: &str, ) -> anyhow::Result<()> { const BYTES_PER_BLOCK_COLOR: RGBColor = BLACK; - let (bytes, costs): (Vec, Vec) = + let (bytes, costs): (Vec, Vec) = bytes_and_costs_per_block.iter().cloned().unzip(); let costs_gwei: Vec<_> = costs.into_iter().map(|x| x / ONE_GWEI).collect(); let min = 0; - let max_left = *bytes.iter().max().unwrap(); + let max_left = *bytes.iter().max().unwrap() as u64; let max_right = *costs_gwei.iter().max().unwrap(); let mut chart = ChartBuilder::on(drawing_area) @@ -228,7 +228,7 @@ pub fn draw_bytes_and_cost_per_block( chart .draw_series(LineSeries::new( - bytes.iter().enumerate().map(|(x, y)| (x, *y)), + bytes.iter().enumerate().map(|(x, y)| (x, *y as u64)), BYTES_PER_BLOCK_COLOR, )) .unwrap() diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/main.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/main.rs index ae15233f37f..998f508151b 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/main.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/main.rs @@ -186,9 +186,9 @@ fn print_info(results: &SimulationResults) { .enumerate() .max_by(|(_, a), (_, b)| a.cmp(b)) .unwrap(); - let eth = *max_da_gas_price as f64 / (10_f64).powf(18.); + let eth = *max_da_gas_price as f64 / (10_f64).powf(9.); println!( - "max DA gas price: {} Wei ({} ETH) at {}", + "max DA gas price: {} Wei ({} Gwei) at {}", max_da_gas_price, eth, index ); @@ -198,9 +198,9 @@ fn print_info(results: &SimulationResults) { .enumerate() .min_by(|(_, a), (_, b)| a.cmp(b)) .unwrap(); - let eth = *min_da_gas_price as f64 / (10_f64).powf(18.); + let eth = *min_da_gas_price as f64 / (10_f64).powf(9.); println!( - "min DA gas price: {} Wei ({} ETH) at {}", + "min DA gas price: {} Wei ({} Gwei) at {}", min_da_gas_price, eth, index ); } diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs index 92003cebb19..763d5a307fe 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs @@ -1,14 +1,15 @@ -use super::*; -use fuel_gas_price_algorithm::v1::{ - AlgorithmUpdaterV1, - L2ActivityTracker, -}; use std::{ collections::BTreeMap, num::NonZeroU64, - ops::Range, }; +use fuel_gas_price_algorithm::v1::{ + AlgorithmUpdaterV1, + L2ActivityTracker, +}; + +use super::*; + pub mod da_cost_per_byte; pub struct SimulationResults { @@ -16,7 +17,7 @@ pub struct SimulationResults { pub exec_gas_prices: Vec, pub da_gas_prices: Vec, pub fullness: Vec<(u64, u64)>, - pub bytes_and_costs: Vec<(u64, u64)>, + pub bytes_and_costs: Vec<(u32, u64)>, pub actual_profit: Vec, pub projected_profit: Vec, pub pessimistic_costs: Vec, @@ -30,8 +31,15 @@ pub struct Simulator { // (usize, ((u64, u64), &'a Option<(Range, u128) struct BlockData { fullness: u64, - bytes: u64, - maybe_da_block: Option<(Range, u128)>, + bytes: u32, + maybe_da_bundle: Option, +} + +#[derive(Clone, Debug)] +struct Bundle { + heights: Vec, + bytes: u32, + cost: u128, } impl Simulator { @@ -51,7 +59,6 @@ impl Simulator { let max_block_bytes = capacity / gas_per_byte; let size = self.da_cost_per_byte.len(); let fullness_and_bytes = fullness_and_bytes_per_block(size, capacity); - let l2_blocks = fullness_and_bytes.clone().into_iter(); let da_blocks = self.calculate_da_blocks( update_period, @@ -64,7 +71,7 @@ impl Simulator { .map(|((fullness, bytes), maybe_da_block)| BlockData { fullness, bytes, - maybe_da_block: maybe_da_block.clone(), + maybe_da_bundle: maybe_da_block.clone(), }) .enumerate(); @@ -88,12 +95,12 @@ impl Simulator { let gas_price_factor = 100; let always_normal_activity = L2ActivityTracker::new_always_normal(); AlgorithmUpdaterV1 { - min_exec_gas_price: 10, - min_da_gas_price: 10, + min_exec_gas_price: 1_000_000_000, + min_da_gas_price: 1_000_000_000, // Change to adjust where the exec gas price starts on block 0 - new_scaled_exec_price: 10 * gas_price_factor, + new_scaled_exec_price: 10_000_000 * gas_price_factor, // Change to adjust where the da gas price starts on block 0 - new_scaled_da_gas_price: 100, + new_scaled_da_gas_price: 10_000_000 * gas_price_factor, gas_price_factor: NonZeroU64::new(gas_price_factor).unwrap(), l2_block_height: 0, // Choose the ideal fullness percentage for the L2 block @@ -101,13 +108,12 @@ impl Simulator { // Increase to make the exec price change faster exec_gas_price_change_percent: 2, // Increase to make the da price change faster - max_da_gas_price_change_percent: 10, + max_da_gas_price_change_percent: 15, total_da_rewards_excess: 0, // Change to adjust the cost per byte of the DA on block 0 latest_da_cost_per_byte: 0, projected_total_da_cost: 0, latest_known_total_da_cost_excess: 0, - unrecorded_blocks: BTreeMap::new(), da_p_component, da_d_component, last_profit: 0, @@ -121,7 +127,7 @@ impl Simulator { &self, capacity: u64, max_block_bytes: u64, - fullness_and_bytes: Vec<(u64, u64)>, + fullness_and_bytes: Vec<(u64, u32)>, blocks: impl Iterator, mut updater: AlgorithmUpdaterV1, ) -> SimulationResults { @@ -132,11 +138,12 @@ impl Simulator { let mut projected_cost_totals = vec![]; let mut actual_costs = vec![]; let mut pessimistic_costs = vec![]; + let mut unrecorded_blocks = BTreeMap::new(); for (index, block_data) in blocks { let BlockData { fullness, bytes, - maybe_da_block: da_block, + maybe_da_bundle, } = block_data; let height = index as u32 + 1; exec_gas_prices.push(updater.new_scaled_exec_price); @@ -144,13 +151,15 @@ impl Simulator { let gas_price = updater.algorithm().calculate(); gas_prices.push(gas_price); let total_fee = gas_price as u128 * fullness as u128; + println!("used: {}, capacity: {}", fullness, capacity); updater .update_l2_block_data( height, fullness, capacity.try_into().unwrap(), - bytes, + bytes as u64, total_fee, + &mut unrecorded_blocks, ) .unwrap(); pessimistic_costs @@ -159,10 +168,22 @@ impl Simulator { projected_cost_totals.push(updater.projected_total_da_cost); // Update DA blocks on the occasion there is one - if let Some((range, cost)) = da_block { - for height in range.to_owned() { + if let Some(bundle) = maybe_da_bundle { + let Bundle { + heights, + bytes, + cost, + } = bundle; + for height in heights { let block_heights: Vec = (height..(height) + 1).collect(); - updater.update_da_record_data(&block_heights, cost).unwrap(); + updater + .update_da_record_data( + &block_heights, + bytes, + cost, + &mut unrecorded_blocks, + ) + .unwrap(); actual_costs.push(updater.latest_known_total_da_cost_excess) } } @@ -176,7 +197,7 @@ impl Simulator { let bytes_and_costs: Vec<_> = bytes .iter() .zip(self.da_cost_per_byte.iter()) - .map(|(bytes, da_cost_per_byte)| (*bytes, *bytes * da_cost_per_byte)) + .map(|(bytes, da_cost_per_byte)| (*bytes, *bytes as u64 * da_cost_per_byte)) .collect(); let actual_profit: Vec = actual_costs @@ -207,8 +228,8 @@ impl Simulator { &self, da_recording_rate: usize, da_finalization_rate: usize, - fullness_and_bytes: &[(u64, u64)], - ) -> Vec, u128)>> { + fullness_and_bytes: &[(u64, u32)], + ) -> Vec> { let l2_blocks_with_no_da_blocks = std::iter::repeat(None).take(da_finalization_rate); let (_, da_blocks) = fullness_and_bytes @@ -219,8 +240,7 @@ impl Simulator { (vec![], vec![]), |(mut delayed, mut recorded), (index, ((_fullness, bytes), cost_per_byte))| { - let total_cost = *bytes * cost_per_byte; - + let total_cost = *bytes as u64 * cost_per_byte; let height = index as u32 + 1; let converted = (height, bytes, total_cost); delayed.push(converted); @@ -236,10 +256,13 @@ impl Simulator { let da_block_ranges = da_blocks.into_iter().map(|maybe_recorded_blocks| { maybe_recorded_blocks.map(|list| { let heights_iter = list.iter().map(|(height, _, _)| *height); - let min = heights_iter.clone().min().unwrap(); - let max = heights_iter.max().unwrap(); - let cost: u128 = list.iter().map(|(_, _, cost)| *cost as u128).sum(); - (min..(max + 1), cost) + let bytes = list.iter().map(|(_, bytes, _)| *bytes).sum(); + let cost = list.iter().map(|(_, _, cost)| *cost as u128).sum(); + Bundle { + heights: heights_iter.collect(), + bytes, + cost, + } }) }); l2_blocks_with_no_da_blocks.chain(da_block_ranges).collect() @@ -263,13 +286,13 @@ where gen_noisy_signal(input, COMPONENTS) } -fn fullness_and_bytes_per_block(size: usize, capacity: u64) -> Vec<(u64, u64)> { +fn fullness_and_bytes_per_block(size: usize, capacity: u64) -> Vec<(u64, u32)> { let mut rng = StdRng::seed_from_u64(888); let fullness_noise: Vec<_> = std::iter::repeat(()) .take(size) - .map(|_| rng.gen_range(-0.25..0.25)) - .map(|val| val * capacity as f64) + .map(|_| rng.gen_range(-0.5..0.5)) + // .map(|val| val * capacity as f64) .collect(); const ROUGH_GAS_TO_BYTE_RATIO: f64 = 0.01; @@ -282,7 +305,7 @@ fn fullness_and_bytes_per_block(size: usize, capacity: u64) -> Vec<(u64, u64)> { (0usize..size) .map(|val| val as f64) .map(noisy_fullness) - .map(|signal| (0.5 * signal + 0.5) * capacity as f64) // Scale and shift so it's between 0 and capacity + .map(|signal| (0.01 * signal + 0.01) * capacity as f64) // Scale and shift so it's between 0 and capacity .zip(fullness_noise) .map(|(fullness, noise)| fullness + noise) .map(|x| f64::min(x, capacity as f64)) @@ -292,6 +315,6 @@ fn fullness_and_bytes_per_block(size: usize, capacity: u64) -> Vec<(u64, u64)> { let bytes = fullness * bytes_scale; (fullness, bytes) }) - .map(|(fullness, bytes)| (fullness as u64, std::cmp::max(bytes as u64, 1))) + .map(|(fullness, bytes)| (fullness as u64, std::cmp::max(bytes as u32, 1))) .collect() } diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation/da_cost_per_byte.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation/da_cost_per_byte.rs index 8d4353300be..911dcf713d8 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation/da_cost_per_byte.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation/da_cost_per_byte.rs @@ -53,8 +53,9 @@ fn get_costs_from_csv_file(file_path: &str, sample_size: Option) -> Vec= size { + if costs.len() >= size / L2_BLOCKS_PER_L1_BLOCK { break; } }; diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 98653953452..304bd979028 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -472,7 +472,7 @@ fn produce_block__l1_committed_block_effects_gas_price() { let mut mock = FakeServer::new(); let url = mock.url(); let costs = RawDaBlockCosts { - sequence_number: 1, + bundle_id: 1, blocks_heights: vec![1], da_block_height: DaBlockHeight(100), total_cost: 100, @@ -511,13 +511,13 @@ fn produce_block__l1_committed_block_effects_gas_price() { #[test] fn produce_block__algorithm_recovers_from_divergent_profit() { - produce_blocks__lolz(11, 50); + produce_blocks__lolz(110, 50); } fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { - let _ = tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) - .try_init(); + // let _ = tracing_subscriber::fmt() + // .with_max_level(tracing::Level::INFO) + // .try_init(); let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); // given @@ -577,7 +577,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { let new_price = new_price_gwei * 1_000_000_000; // Wei let cost = count * new_price; mock.add_response(RawDaBlockCosts { - sequence_number: 1, + bundle_id: 1, blocks_heights, da_block_height: DaBlockHeight(100), total_cost: cost, @@ -626,7 +626,7 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { if profit > 0 && !success { success = true; success_iteration = i as i32; - // break; + break; } } }); From 599e358986e26a2cbc737dde8e8244a9c0abe58e Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 13 Dec 2024 13:19:06 -0700 Subject: [PATCH 098/131] Remove logging --- .../gas-price-analysis/src/simulation.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs index 763d5a307fe..65210000800 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs @@ -151,7 +151,6 @@ impl Simulator { let gas_price = updater.algorithm().calculate(); gas_prices.push(gas_price); let total_fee = gas_price as u128 * fullness as u128; - println!("used: {}, capacity: {}", fullness, capacity); updater .update_l2_block_data( height, From 558473185f194357289a14a115fa2ddf9e172b52 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 13 Dec 2024 13:36:30 -0700 Subject: [PATCH 099/131] Clean up code --- crates/services/gas_price_service/src/v1/service.rs | 5 ----- crates/services/gas_price_service/src/v1/tests.rs | 7 ------- 2 files changed, 12 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 42875096d74..3cfb0ee73dd 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -507,7 +507,6 @@ mod tests { algo_updater, da_service_runner, inner, - inner, ); let read_algo = service.next_block_algorithm(); let mut watcher = StateWatcher::default(); @@ -611,10 +610,6 @@ mod tests { tokio::time::sleep(Duration::from_millis(100)).await; l2_block_sender.send(l2_block_2).await.unwrap(); - service.run(&mut watcher).await; - tokio::time::sleep(Duration::from_millis(100)).await; - l2_block_sender.send(l2_block_2).await.unwrap(); - // when service.run(&mut watcher).await; tokio::time::sleep(Duration::from_millis(100)).await; diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 1fbbaecb4fb..9b6389c0e21 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -277,11 +277,6 @@ impl DaBlockCostsSource for FakeDABlockCost { self.bundle_id.lock().unwrap().replace(bundle_id); Ok(()) } - - async fn set_last_value(&mut self, bundle_id: u32) -> Result<()> { - self.bundle_id.lock().unwrap().replace(bundle_id); - Ok(()) - } } fn zero_threshold_arbitrary_config() -> V1AlgorithmConfig { @@ -387,7 +382,6 @@ async fn next_gas_price__affected_by_new_l2_block() { algo_updater, da_service_runner, inner, - inner, ); let read_algo = service.next_block_algorithm(); @@ -434,7 +428,6 @@ async fn run__new_l2_block_saves_old_metadata() { algo_updater, da_service_runner, inner, - inner, ); let mut watcher = StateWatcher::started(); From f5884858ca4d655526f28b562fa6fd46c38c4d2f Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 13 Dec 2024 15:13:39 -0700 Subject: [PATCH 100/131] Fix _bug_ that was really just a high min gas price --- .../gas-price-analysis/src/simulation.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs index 65210000800..803a946c980 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs @@ -96,7 +96,7 @@ impl Simulator { let always_normal_activity = L2ActivityTracker::new_always_normal(); AlgorithmUpdaterV1 { min_exec_gas_price: 1_000_000_000, - min_da_gas_price: 1_000_000_000, + min_da_gas_price: 100, // Change to adjust where the exec gas price starts on block 0 new_scaled_exec_price: 10_000_000 * gas_price_factor, // Change to adjust where the da gas price starts on block 0 From 3f6584d7c74b39fef08e4a4225ae4a7e7f0af946 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 13 Dec 2024 16:55:20 -0700 Subject: [PATCH 101/131] fmt --- .../gas-price-analysis/src/simulation.rs | 2 +- crates/services/gas_price_service/src/ports.rs | 1 - crates/services/gas_price_service/src/v1/metadata.rs | 2 +- .../fuel_storage_unrecorded_blocks.rs | 10 ++++------ 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs index 803a946c980..9befabf2a9c 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs @@ -95,7 +95,7 @@ impl Simulator { let gas_price_factor = 100; let always_normal_activity = L2ActivityTracker::new_always_normal(); AlgorithmUpdaterV1 { - min_exec_gas_price: 1_000_000_000, + min_exec_gas_price: 10_000_000, min_da_gas_price: 100, // Change to adjust where the exec gas price starts on block 0 new_scaled_exec_price: 10_000_000 * gas_price_factor, diff --git a/crates/services/gas_price_service/src/ports.rs b/crates/services/gas_price_service/src/ports.rs index 63586e381e1..094f781f170 100644 --- a/crates/services/gas_price_service/src/ports.rs +++ b/crates/services/gas_price_service/src/ports.rs @@ -18,7 +18,6 @@ use crate::{ }; use std::num::NonZeroU64; - pub trait L2Data: Send + Sync { fn latest_height(&self) -> StorageResult; fn get_block( diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index deb9fc56d94..76f886dde39 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -93,7 +93,7 @@ pub fn updater_from_config(value: &V1AlgorithmConfig) -> AlgorithmUpdaterV1 { l2_block_height: 0, new_scaled_da_gas_price: value .min_da_gas_price - .saturating_mul(value.gas_price_factor.get()), + .saturating_mul(value.gas_price_factor.get()), gas_price_factor: value.gas_price_factor, total_da_rewards_excess: 0, latest_known_total_da_cost_excess: 0, diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs index 3983d145695..62c9f3ff39c 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task/fuel_storage_unrecorded_blocks.rs @@ -1,8 +1,6 @@ -use crate::common::{ - fuel_core_storage_adapter::storage::{ - GasPriceColumn, - UnrecordedBlocksTable, - }, +use crate::common::fuel_core_storage_adapter::storage::{ + GasPriceColumn, + UnrecordedBlocksTable, }; use fuel_core_storage::{ kv_store::{ @@ -19,9 +17,9 @@ use fuel_core_storage::{ StorageMutate, }; use fuel_core_types::{ + fuel_merkle::storage::StorageMutateInfallible, fuel_types::BlockHeight, }; -use fuel_core_types::fuel_merkle::storage::StorageMutateInfallible; use fuel_gas_price_algorithm::{ v1, v1::UnrecordedBlocks, From 9876ebb48d8ed566751f4941ef375660c6b1a373 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 13 Dec 2024 17:19:34 -0700 Subject: [PATCH 102/131] Remove test --- .../block_committer_costs.rs | 25 ++++++++++--------- .../gas_price_service/src/v1/service.rs | 18 ------------- tests/tests/gas_price.rs | 16 +++--------- 3 files changed, 17 insertions(+), 42 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index a70cb1c9e7d..ac888a0a70d 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -284,18 +284,19 @@ mod tests { assert_ne!(da_block_costs.blocks_heights, actual.unwrap().l2_blocks); } - #[tokio::test] - async fn request_da_block_cost__when_response_is_none__then_error() { - // given - let mock_api = MockBlockCommitterApi::new(None); - let mut block_committer = BlockCommitterDaBlockCosts::new(mock_api, None); - - // when - let result = block_committer.request_da_block_cost().await; - - // then - assert!(result.is_err()); - } + // TODO: Do we need this? + // #[tokio::test] + // async fn request_da_block_cost__when_response_is_none__then_error() { + // // given + // let mock_api = MockBlockCommitterApi::new(None); + // let mut block_committer = BlockCommitterDaBlockCosts::new(mock_api, None); + // + // // when + // let result = block_committer.request_da_block_cost().await; + // + // // then + // assert!(result.is_err()); + // } struct UnderflowingMockBlockCommitterApi { value: Option, diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 3cfb0ee73dd..7eb31d0adfd 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -597,15 +597,6 @@ mod tests { let read_algo = service.next_block_algorithm(); let initial_price = read_algo.next_gas_price(); - // // the RunnableTask depends on the handle passed to it for the da block cost source to already be running, - // // which is the responsibility of the UninitializedTask in the `into_task` method of the RunnableService - // // here we mimic that behaviour by running the da block cost service. - // let mut da_source_watcher = StateWatcher::started(); - // service - // .da_source_adapter_handle - // .run(&mut da_source_watcher) - // .await; - service.run(&mut watcher).await; tokio::time::sleep(Duration::from_millis(100)).await; l2_block_sender.send(l2_block_2).await.unwrap(); @@ -700,15 +691,6 @@ mod tests { let read_algo = service.next_block_algorithm(); let initial_price = read_algo.next_gas_price(); - // // the RunnableTask depends on the handle passed to it for the da block cost source to already be running, - // // which is the responsibility of the UninitializedTask in the `into_task` method of the RunnableService - // // here we mimic that behaviour by running the da block cost service. - // let mut da_source_watcher = StateWatcher::started(); - // service - // .da_source_adapter_handle - // .run(&mut da_source_watcher) - // .await; - service.run(&mut watcher).await; tokio::time::sleep(Duration::from_millis(100)).await; l2_block_sender.send(l2_block_2).await.unwrap(); diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 304bd979028..a6dfd8d8d49 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -511,13 +511,10 @@ fn produce_block__l1_committed_block_effects_gas_price() { #[test] fn produce_block__algorithm_recovers_from_divergent_profit() { - produce_blocks__lolz(110, 50); + _produce_block__algorithm_recovers_from_divergent_profit(110); } -fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { - // let _ = tracing_subscriber::fmt() - // .with_max_level(tracing::Level::INFO) - // .try_init(); +fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) { let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); // given @@ -541,8 +538,8 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { node_config.block_production = Trigger::Never; node_config.da_committer_url = Some(url.clone()); node_config.da_poll_interval = Some(100); - // node_config.da_p_component = 4_707_680; - // node_config.da_d_component = 114_760; + node_config.da_p_component = 224_000; + node_config.da_d_component = 2_690_000; node_config.da_p_component = 10; node_config.da_d_component = 1; node_config.block_activity_threshold = 0; @@ -642,11 +639,6 @@ fn produce_blocks__lolz(block_delay: usize, _blob_size: usize) { ); } else { tracing::info!("Success on try {}/{}", success_iteration, tries); - // tracing::info!("Profits: {:?}", profits); - // tracing::info!("Changes: {:?}", changes); - // tracing::info!("Gas prices: {:?}", gas_prices); - // tracing::info!("Gas price changes: {:?}", gas_price_changes); - // trace the same as f64 and converted into gwei let profits_as_gwei = profits .iter() .map(|x| *x as f64 / 1_000_000_000.0) From de1aaa71ca7f2d60d551198efc42883a0d55aa9b Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 13 Dec 2024 17:33:29 -0700 Subject: [PATCH 103/131] Remove drop --- tests/tests/gas_price.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index a6dfd8d8d49..a722ed23e34 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -660,7 +660,6 @@ fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) tracing::info!("Gas prices as gwei: {:?}", gas_prices_as_gwei); tracing::info!("Gas price changes as gwei: {:?}", gas_price_changes_as_gwei); } - drop(mock); } async fn produce_a_block(client: &FuelClient, rng: &mut R) { From 30339bb297734250ff7b648db906d065152bc827 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 13 Dec 2024 18:37:34 -0700 Subject: [PATCH 104/131] Fix test to use smaller txs and be more precise about costs --- tests/tests/gas_price.rs | 50 +++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index a722ed23e34..5001a50dd05 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -97,6 +97,29 @@ fn arb_large_tx( .into() } +fn arb_small_tx( + max_fee_limit: Word, + rng: &mut R, +) -> Transaction { + let mut script: Vec<_> = repeat(op::noop()).take(10).collect(); + script.push(op::ret(RegId::ONE)); + let script_bytes = script.iter().flat_map(|op| op.to_bytes()).collect(); + let mut builder = TransactionBuilder::script(script_bytes, vec![]); + let asset_id = *builder.get_params().base_asset_id(); + builder + .max_fee_limit(max_fee_limit) + .script_gas_limit(22430) + .add_unsigned_coin_input( + SecretKey::random(rng), + rng.gen(), + u32::MAX as u64, + asset_id, + Default::default(), + ) + .finalize() + .into() +} + #[tokio::test] async fn latest_gas_price__if_no_mint_tx_in_previous_block_gas_price_is_zero() { // given @@ -515,6 +538,9 @@ fn produce_block__algorithm_recovers_from_divergent_profit() { } fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .try_init(); let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); // given @@ -538,10 +564,10 @@ fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) node_config.block_production = Trigger::Never; node_config.da_committer_url = Some(url.clone()); node_config.da_poll_interval = Some(100); - node_config.da_p_component = 224_000; - node_config.da_d_component = 2_690_000; - node_config.da_p_component = 10; - node_config.da_d_component = 1; + // node_config.da_p_component = 224_000; + // node_config.da_d_component = 2_690_000; + node_config.da_p_component = 1; + node_config.da_d_component = 10; node_config.block_activity_threshold = 0; let (srv, client) = rt.block_on(async { @@ -570,15 +596,17 @@ fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) let half_of_blocks = block_delay as u32 / 2; let blocks_heights: Vec<_> = (1..half_of_blocks).collect(); let count = blocks_heights.len() as u128; - let new_price_gwei = 500_000; - let new_price = new_price_gwei * 1_000_000_000; // Wei - let cost = count * new_price; + let block_bytes = 1000; + let total_size_bytes = block_bytes * count as u32; + let gas = 16 * total_size_bytes as u128; + let cost_gwei = gas * 1; // blob gas price 1 gwei + let cost = cost_gwei * 1_000_000_000; // Wei mock.add_response(RawDaBlockCosts { bundle_id: 1, blocks_heights, da_block_height: DaBlockHeight(100), total_cost: cost, - total_size_bytes: 128_000, + total_size_bytes, }); let mut profits = Vec::new(); @@ -601,7 +629,7 @@ fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) gas_prices.push(metadata.new_scaled_da_gas_price / metadata.gas_price_factor); }); - let tries = 1000; + let tries = 300; let mut success = false; let mut success_iteration = i32::MAX; @@ -623,7 +651,6 @@ fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) if profit > 0 && !success { success = true; success_iteration = i as i32; - break; } } }); @@ -666,7 +693,8 @@ async fn produce_a_block(client: &FuelClient, rng: &mu let arb_tx_count = 2; for i in 0..arb_tx_count { let large_fee_limit = u32::MAX as u64 - i; - let tx = arb_large_tx(large_fee_limit, rng); + // let tx = arb_large_tx(large_fee_limit, rng); + let tx = arb_small_tx(large_fee_limit, rng); // let tx = arb_large_tx(189028 + i as Word, rng); let _status = client.submit(&tx).await.unwrap(); } From 9f3f5fcf6f58665c692bf9182046b1a8ee7d87d2 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 13 Dec 2024 18:39:38 -0700 Subject: [PATCH 105/131] More fiddling --- tests/tests/gas_price.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 5001a50dd05..ea2f70cd727 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -557,7 +557,7 @@ fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) }; let mut node_config = Config::local_node_with_configs(chain_config, StateConfig::local_testnet()); - let starting_gas_price = 1_000_000_000; + let starting_gas_price = 10_000_000; node_config.block_producer.coinbase_recipient = Some([5; 32].into()); node_config.min_da_gas_price = starting_gas_price; node_config.max_da_gas_price_change_percent = 10; @@ -629,7 +629,7 @@ fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) gas_prices.push(metadata.new_scaled_da_gas_price / metadata.gas_price_factor); }); - let tries = 300; + let tries = 100; let mut success = false; let mut success_iteration = i32::MAX; From 9f855c97cf77719c640ae369c11c5724e075b4e2 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Fri, 13 Dec 2024 18:44:24 -0700 Subject: [PATCH 106/131] More fiddling --- tests/tests/gas_price.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index ea2f70cd727..43de21b4d97 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -557,17 +557,17 @@ fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) }; let mut node_config = Config::local_node_with_configs(chain_config, StateConfig::local_testnet()); - let starting_gas_price = 10_000_000; + let starting_gas_price = 1_000_000_000; node_config.block_producer.coinbase_recipient = Some([5; 32].into()); node_config.min_da_gas_price = starting_gas_price; node_config.max_da_gas_price_change_percent = 10; node_config.block_production = Trigger::Never; node_config.da_committer_url = Some(url.clone()); node_config.da_poll_interval = Some(100); - // node_config.da_p_component = 224_000; - // node_config.da_d_component = 2_690_000; - node_config.da_p_component = 1; - node_config.da_d_component = 10; + node_config.da_p_component = 224_000; + node_config.da_d_component = 2_690_000; + // node_config.da_p_component = 1; + // node_config.da_d_component = 10; node_config.block_activity_threshold = 0; let (srv, client) = rt.block_on(async { @@ -629,7 +629,7 @@ fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) gas_prices.push(metadata.new_scaled_da_gas_price / metadata.gas_price_factor); }); - let tries = 100; + let tries = 1000; let mut success = false; let mut success_iteration = i32::MAX; From c770613452ef4f0e138b914f3acd86fb07c1096f Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Sat, 14 Dec 2024 12:16:42 -0700 Subject: [PATCH 107/131] Fix test --- crates/services/gas_price_service/src/v1/service.rs | 6 +++--- tests/tests/gas_price.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 7eb31d0adfd..6ea66448d83 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -576,7 +576,7 @@ mod tests { Ok(DaBlockCosts { bundle_id: 1, l2_blocks: (1..2).collect(), - blob_cost_wei: 9000, + blob_cost_wei: u128::MAX, // Very expensive to trigger a change bundle_size_bytes: 3000, }), notifier.clone(), @@ -598,12 +598,12 @@ mod tests { let initial_price = read_algo.next_gas_price(); service.run(&mut watcher).await; - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(3)).await; l2_block_sender.send(l2_block_2).await.unwrap(); // when service.run(&mut watcher).await; - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(3)).await; service.shutdown().await.unwrap(); // then diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 43de21b4d97..4efeb7078c6 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -557,10 +557,10 @@ fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) }; let mut node_config = Config::local_node_with_configs(chain_config, StateConfig::local_testnet()); - let starting_gas_price = 1_000_000_000; + let starting_gas_price = 10_000_000; node_config.block_producer.coinbase_recipient = Some([5; 32].into()); node_config.min_da_gas_price = starting_gas_price; - node_config.max_da_gas_price_change_percent = 10; + node_config.max_da_gas_price_change_percent = 15; node_config.block_production = Trigger::Never; node_config.da_committer_url = Some(url.clone()); node_config.da_poll_interval = Some(100); From 39e9e97264f8540777958dfbfc77c7d57494eed4 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Sun, 15 Dec 2024 14:21:36 -0700 Subject: [PATCH 108/131] Add some tests, fix the storage tx in init task --- Cargo.lock | 1 + .../gas-price-analysis/src/simulation.rs | 2 +- crates/services/gas_price_service/Cargo.toml | 1 + .../gas_price_service/src/v1/tests.rs | 118 ++++++++++++++---- .../src/v1/uninitialized_task.rs | 10 +- tests/tests/gas_price.rs | 73 +++++++++++ 6 files changed, 181 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index acbba1a584e..7eb0dd52f43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3475,6 +3475,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", + "tracing-subscriber", ] [[package]] diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs index 9befabf2a9c..4fbc8df09f6 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs @@ -96,7 +96,7 @@ impl Simulator { let always_normal_activity = L2ActivityTracker::new_always_normal(); AlgorithmUpdaterV1 { min_exec_gas_price: 10_000_000, - min_da_gas_price: 100, + min_da_gas_price: 1_000_000_000_000, // Change to adjust where the exec gas price starts on block 0 new_scaled_exec_price: 10_000_000 * gas_price_factor, // Change to adjust where the da gas price starts on block 0 diff --git a/crates/services/gas_price_service/Cargo.toml b/crates/services/gas_price_service/Cargo.toml index 9903bba7d03..4eab33d503b 100644 --- a/crates/services/gas_price_service/Cargo.toml +++ b/crates/services/gas_price_service/Cargo.toml @@ -32,3 +32,4 @@ tracing = { workspace = true } fuel-core-services = { workspace = true, features = ["test-helpers"] } fuel-core-storage = { workspace = true, features = ["test-helpers"] } fuel-core-types = { path = "./../../types", features = ["test-helpers"] } +tracing-subscriber = { workspace = true } diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 9b6389c0e21..fdbdab3d0df 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -78,11 +78,18 @@ use fuel_core_storage::{ }; use fuel_core_types::{ blockchain::{ - block::Block, + block::{ + Block, + BlockV1, + PartialFuelBlock, + }, header::ConsensusParametersVersion, }, fuel_asm::op::exp, - fuel_tx::Transaction, + fuel_tx::{ + Mint, + Transaction, + }, fuel_types::BlockHeight, services::block_importer::{ ImportResult, @@ -97,6 +104,7 @@ use fuel_gas_price_algorithm::v1::{ UnrecordedBlocks, }; use std::{ + collections::HashMap, num::NonZeroU64, ops::Deref, sync::{ @@ -336,7 +344,7 @@ fn database() -> StorageTransaction> { InMemoryStorage::default().into_transaction() } -fn database_with_metadata( +fn gas_price_database_with_metadata( metadata: &V1Metadata, ) -> StorageTransaction> { let mut db = database(); @@ -448,14 +456,27 @@ async fn run__new_l2_block_saves_old_metadata() { } #[derive(Clone)] -struct FakeSettings; +struct FakeSettings { + settings: GasPriceSettings, +} + +impl Default for FakeSettings { + fn default() -> Self { + Self { + settings: GasPriceSettings { + gas_price_factor: 100, + block_gas_limit: u64::MAX, + }, + } + } +} impl GasPriceSettingsProvider for FakeSettings { fn settings( &self, _param_version: &ConsensusParametersVersion, ) -> GasPriceResult { - unimplemented!() + Ok(self.settings.clone()) } } @@ -486,23 +507,29 @@ impl GasPriceData for FakeGasPriceDb { #[derive(Clone)] struct FakeOnChainDb { height: BlockHeight, + blocks: HashMap>, } impl FakeOnChainDb { fn new(height: u32) -> Self { Self { height: height.into(), + blocks: HashMap::new(), } } } struct FakeL2Data { height: BlockHeight, + blocks: HashMap>, } impl FakeL2Data { - fn new(height: BlockHeight) -> Self { - Self { height } + fn new( + height: BlockHeight, + blocks: HashMap>, + ) -> Self { + Self { height, blocks } } } @@ -513,16 +540,16 @@ impl L2Data for FakeL2Data { fn get_block( &self, - _height: &BlockHeight, + height: &BlockHeight, ) -> StorageResult>> { - unimplemented!() + Ok(self.blocks.get(height).cloned()) } } impl AtomicView for FakeOnChainDb { type LatestView = FakeL2Data; fn latest_view(&self) -> StorageResult { - Ok(FakeL2Data::new(self.height)) + Ok(FakeL2Data::new(self.height, self.blocks.clone())) } } @@ -535,22 +562,16 @@ fn empty_block_stream() -> BoxStream { async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overrides() { // given let original_metadata = arbitrary_metadata(); - let original = UpdaterMetadata::V1(original_metadata.clone()); - let metadata_inner = Arc::new(std::sync::Mutex::new(Some(original.clone()))); - let metadata_storage = FakeMetadata { - inner: metadata_inner, - }; - let different_config = different_arb_config(); let descaleed_exec_price = original_metadata.new_scaled_exec_price / original_metadata.gas_price_factor; assert_ne!(different_config.new_exec_gas_price, descaleed_exec_price); let different_l2_block = 0; - let settings = FakeSettings; + let settings = FakeSettings::default(); let block_stream = empty_block_stream(); let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); - let inner = database_with_metadata(&original_metadata); + let inner = gas_price_database_with_metadata(&original_metadata); // when let service = UninitializedTask::new( different_config.clone(), @@ -651,7 +672,7 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { let config = zero_threshold_arbitrary_config(); let different_l2_block = 1231; let erroring_persisted_data = ErroringPersistedData; - let settings = FakeSettings; + let settings = FakeSettings::default(); let block_stream = empty_block_stream(); let on_chain_db = FakeOnChainDb::new(different_l2_block); let da_cost_source = FakeDABlockCost::never_returns(); @@ -685,12 +706,12 @@ async fn uninitialized_task__init__starts_da_service_with_bundle_id_in_storage() original_metadata.new_scaled_exec_price / original_metadata.gas_price_factor; assert_ne!(different_config.new_exec_gas_price, descaleed_exec_price); let different_l2_block = 0; - let settings = FakeSettings; + let settings = FakeSettings::default(); let block_stream = empty_block_stream(); let on_chain_db = FakeOnChainDb::new(different_l2_block); let (da_cost_source, bundle_id_handle) = FakeDABlockCost::never_returns_with_handle_to_bundle_id(); - let mut inner = database_with_metadata(&original_metadata); + let mut inner = gas_price_database_with_metadata(&original_metadata); let mut tx = inner.begin_transaction().unwrap(); tx.set_bundle_id(&block_height.into(), bundle_id).unwrap(); StorageTransaction::commit_transaction(tx).unwrap(); @@ -714,3 +735,58 @@ async fn uninitialized_task__init__starts_da_service_with_bundle_id_in_storage() let expected = Some(bundle_id); assert_eq!(*actual, expected); } + +fn arb_block() -> Block { + let mut block = Block::default(); + let mint = Mint::default(); + block.transactions_mut().push(mint.into()); + block +} + +#[tokio::test] +async fn uninitialized_task__init__if_metadata_behind_l2_height_then_sync() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::DEBUG) + .try_init(); + // given + let metadata_height = 100; + let l2_height = 200; + let config = zero_threshold_arbitrary_config(); + + let metadata = V1Metadata { + new_scaled_exec_price: 100, + l2_block_height: metadata_height, + new_scaled_da_gas_price: 0, + gas_price_factor: NonZeroU64::new(100).unwrap(), + total_da_rewards_excess: 0, + latest_known_total_da_cost_excess: 0, + last_profit: 0, + second_to_last_profit: 0, + latest_da_cost_per_byte: 0, + unrecorded_block_bytes: 0, + }; + let gas_price_db = gas_price_database_with_metadata(&metadata); + let mut onchain_db = FakeOnChainDb::new(l2_height); + for height in 1..=l2_height { + let block = arb_block(); + onchain_db.blocks.insert(BlockHeight::from(height), block); + } + + let service = UninitializedTask::new( + config, + Some(metadata_height.into()), + 0.into(), + FakeSettings::default(), + empty_block_stream(), + gas_price_db, + FakeDABlockCost::never_returns(), + onchain_db, + ) + .unwrap(); + + // when + service.init().await.unwrap(); + + // then + // no panic +} diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 3dd17cc5bc7..ff183f97355 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -177,6 +177,7 @@ where .config .da_poll_interval .map(|x| Duration::from_millis(x.into())); + // TODO: Dupe code if let Some(bundle_id) = self.gas_price_db.get_bundle_id(&metadata_height.into())? { @@ -310,8 +311,13 @@ where { let first = metadata_height.saturating_add(1); let view = on_chain_db.latest_view()?; - let mut tx = da_storage.begin_transaction()?; + tracing::info!( + "Syncing gas price metadata from {} to {}", + first, + latest_block_height + ); for height in first..=latest_block_height { + let mut tx = da_storage.begin_transaction()?; let block = view .get_block(&height.into())? .ok_or(not_found!("FullBlock"))?; @@ -344,8 +350,8 @@ where )?; let metadata: UpdaterMetadata = updater.clone().into(); tx.set_metadata(&metadata)?; + AtomicStorage::commit_transaction(tx)?; } - AtomicStorage::commit_transaction(tx)?; Ok(()) } diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 4efeb7078c6..10deaa2e44a 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -532,6 +532,79 @@ fn produce_block__l1_committed_block_effects_gas_price() { assert!(first_gas_price < new_gas_price); } +#[test] +fn run__if_metadata_is_behind_l2_then_will_catch_up() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::DEBUG) + .try_init(); + + // given + // produce 100 blocks + let args = vec![ + "--debug", + "--poa-instant", + "true", + "--min-da-gas-price", + "100", + ]; + let rt = tokio::runtime::Runtime::new().unwrap(); + let temp_dir = rt.block_on(async { + let driver = FuelCoreDriver::spawn(&args).await.unwrap(); + driver.client.produce_blocks(100, None).await.unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + driver.kill().await + }); + + // rollback 50 blocks + let temp_dir = rt.block_on(async { + let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) + .await + .unwrap(); + for _ in 0..50 { + driver + .node + .shared + .database + .gas_price() + .rollback_last_block() + .unwrap(); + let gas_price_db_height = driver + .node + .shared + .database + .gas_price() + .latest_height() + .unwrap(); + tracing::info!("gas price db height: {:?}", gas_price_db_height); + } + driver.kill().await + }); + + // when + // restart node + rt.block_on(async { + let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) + .await + .unwrap(); + let onchain_db_height = driver + .node + .shared + .database + .on_chain() + .latest_height_from_metadata() + .unwrap() + .unwrap(); + let gas_price_db_height = driver + .node + .shared + .database + .gas_price() + .latest_height() + .unwrap(); + assert_eq!(onchain_db_height, gas_price_db_height); + }); +} + #[test] fn produce_block__algorithm_recovers_from_divergent_profit() { _produce_block__algorithm_recovers_from_divergent_profit(110); From 5c06f5fa27e50d4a14972c35b9df0ea02c426e74 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Sun, 15 Dec 2024 19:49:39 -0700 Subject: [PATCH 109/131] Change base values --- .../gas-price-analysis/src/simulation.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs index 4fbc8df09f6..0080aa1a837 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs @@ -96,7 +96,7 @@ impl Simulator { let always_normal_activity = L2ActivityTracker::new_always_normal(); AlgorithmUpdaterV1 { min_exec_gas_price: 10_000_000, - min_da_gas_price: 1_000_000_000_000, + min_da_gas_price: 100_000_000, // Change to adjust where the exec gas price starts on block 0 new_scaled_exec_price: 10_000_000 * gas_price_factor, // Change to adjust where the da gas price starts on block 0 From b958ebc3e5a93d52c2cd86ed9d41b7d851607d53 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 16 Dec 2024 12:40:03 -0700 Subject: [PATCH 110/131] Fix syncing bug by removing redundant updater --- .gitignore | 2 +- .../gas_price_service/src/v1/service.rs | 5 +- .../src/v1/uninitialized_task.rs | 60 +++--- tests/tests/gas_price.rs | 187 +++++++++++++----- 4 files changed, 164 insertions(+), 90 deletions(-) diff --git a/.gitignore b/.gitignore index f62b3ff7167..a35bee1be51 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,4 @@ node_modules/ package-lock.json package.json bin/fuel-core/chainspec/local-testnet/state_transition_bytecode.wasm - +.DS_Store diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 6ea66448d83..679d6372646 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -200,12 +200,13 @@ where )?; let metadata = self.algorithm_updater.clone().into(); + tracing::info!("Setting metadata: {:?}", metadata); storage_tx .set_metadata(&metadata) .map_err(|err| anyhow!(err))?; AtomicStorage::commit_transaction(storage_tx)?; let new_algo = self.algorithm_updater.algorithm(); - tracing::debug!("Updating gas price: {}", &new_algo.calculate()); + tracing::info!("Updating gas price: {}", &new_algo.calculate()); self.shared_algo.update(new_algo).await; // Clear the buffer after committing changes self.da_block_costs_buffer.clear(); @@ -262,7 +263,7 @@ where TaskNextAction::Stop } l2_block_res = self.l2_block_source.get_l2_block() => { - tracing::debug!("Received L2 block result: {:?}", l2_block_res); + tracing::info!("Received L2 block result: {:?}", l2_block_res); let res = self.commit_block_data_to_algorithm(l2_block_res).await; TaskNextAction::always_continue(res) } diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index ff183f97355..e75e8d11d97 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -86,7 +86,7 @@ pub mod fuel_storage_unrecorded_blocks; pub struct UninitializedTask { pub config: V1AlgorithmConfig, - pub gas_metadata_height: Option, + pub gas_metadata_height: BlockHeight, pub genesis_block_height: BlockHeight, pub settings: SettingsProvider, pub gas_price_db: GasPriceStore, @@ -117,18 +117,14 @@ where da_source: DA, on_chain_db: L2DataStoreView, ) -> anyhow::Result { - let latest_block_height: u32 = on_chain_db - .latest_view()? - .latest_height() - .unwrap_or(genesis_block_height) - .into(); + let gas_price_db_height = gas_metadata_height.unwrap_or(genesis_block_height); let (algo_updater, shared_algo) = - initialize_algorithm(&config, latest_block_height, &gas_price_db)?; + initialize_algorithm(&config, gas_price_db_height.into(), &gas_price_db)?; let task = Self { config, - gas_metadata_height, + gas_metadata_height: gas_price_db_height, genesis_block_height, settings, gas_price_db, @@ -146,7 +142,6 @@ where ) -> anyhow::Result< GasPriceServiceV1, DA, AtomicStorage>, > { - let mut first_run = false; let latest_block_height: u32 = self .on_chain_db .latest_view()? @@ -154,22 +149,15 @@ where .unwrap_or(self.genesis_block_height) .into(); - let maybe_metadata_height = self.gas_metadata_height; - let metadata_height = if let Some(metadata_height) = maybe_metadata_height { - metadata_height.into() - } else { - first_run = true; - latest_block_height - }; - let l2_block_source = FuelL2BlockSource::new( self.genesis_block_height, self.settings.clone(), self.block_stream, ); - if let Some(bundle_id) = - self.gas_price_db.get_bundle_id(&metadata_height.into())? + if let Some(bundle_id) = self + .gas_price_db + .get_bundle_id(&self.gas_metadata_height.into())? { self.da_source.set_last_value(bundle_id).await?; } @@ -178,8 +166,9 @@ where .da_poll_interval .map(|x| Duration::from_millis(x.into())); // TODO: Dupe code - if let Some(bundle_id) = - self.gas_price_db.get_bundle_id(&metadata_height.into())? + if let Some(bundle_id) = self + .gas_price_db + .get_bundle_id(&self.gas_metadata_height.into())? { self.da_source.set_last_value(bundle_id).await?; } @@ -187,9 +176,7 @@ where let da_service_runner = ServiceRunner::new(da_service); da_service_runner.start_and_await().await?; - if BlockHeight::from(latest_block_height) == self.genesis_block_height - || first_run - { + if BlockHeight::from(latest_block_height) == self.genesis_block_height { let service = GasPriceServiceV1::new( l2_block_source, self.shared_algo, @@ -199,12 +186,12 @@ where ); Ok(service) } else { - if latest_block_height > metadata_height { + if latest_block_height > *self.gas_metadata_height { sync_gas_price_db_with_on_chain_storage( &self.settings, - &self.config, + &mut self.algo_updater, &self.on_chain_db, - metadata_height, + *self.gas_metadata_height, latest_block_height, &mut self.gas_price_db, )?; @@ -257,7 +244,7 @@ fn sync_gas_price_db_with_on_chain_storage< AtomicStorage, >( settings: &SettingsProvider, - config: &V1AlgorithmConfig, + updater: &mut AlgorithmUpdaterV1, on_chain_db: &L2DataStoreView, metadata_height: u32, latest_block_height: u32, @@ -275,20 +262,20 @@ where "Expected metadata to exist for height: {metadata_height}" ))?; - let metadata = match metadata { - UpdaterMetadata::V1(metadata) => metadata, - UpdaterMetadata::V0(metadata) => { - V1Metadata::construct_from_v0_metadata(metadata, config)? - } - }; - let mut algo_updater = v1_algorithm_from_metadata(metadata, config); + // let metadata = match metadata { + // UpdaterMetadata::V1(metadata) => metadata, + // UpdaterMetadata::V0(metadata) => { + // V1Metadata::construct_from_v0_metadata(metadata, config)? + // } + // }; + // let mut algo_updater = v1_algorithm_from_metadata(metadata, config); sync_v1_metadata( settings, on_chain_db, metadata_height, latest_block_height, - &mut algo_updater, + updater, persisted_data, )?; @@ -317,6 +304,7 @@ where latest_block_height ); for height in first..=latest_block_height { + tracing::info!("Syncing gas price metadata for block {}", height); let mut tx = da_storage.begin_transaction()?; let block = view .get_block(&height.into())? diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 10deaa2e44a..53e8e67d79d 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -12,11 +12,14 @@ use fuel_core::{ ChainConfig, StateConfig, }, + combined_database::CombinedDatabase, database::Database, + fuel_core_graphql_api::ports::worker::OnChainDatabase, service::{ Config, FuelService, }, + state::historical_rocksdb::StateRewindPolicy, }; use fuel_core_client::client::{ types::gas_price::LatestGasPrice, @@ -28,7 +31,7 @@ use fuel_core_gas_price_service::{ GasPriceData, GetMetadataStorage, }, - v0::metadata::V0Metadata, + v1::metadata::V1Metadata, }; use fuel_core_poa::Trigger; use fuel_core_storage::{ @@ -311,7 +314,7 @@ async fn latest_gas_price__if_node_restarts_gets_latest_value() { "--starting-exec-gas-price", "1000", "--gas-price-change-percent", - "10", + "100", "--gas-price-threshold-percent", "0", ]; @@ -439,6 +442,7 @@ async fn startup__can_override_gas_price_values_by_changing_config() { .produce_blocks(1, None) .await .unwrap(); + tokio::time::sleep(Duration::from_millis(10)).await; let new_height = 2; let recovered_database = &recovered_driver.node.shared.database; @@ -451,7 +455,7 @@ async fn startup__can_override_gas_price_values_by_changing_config() { .deref() .clone(); - let V0Metadata { + let V1Metadata { l2_block_height, .. } = new_metadata.try_into().unwrap(); assert_eq!(l2_block_height, new_height); @@ -548,61 +552,142 @@ fn run__if_metadata_is_behind_l2_then_will_catch_up() { "100", ]; let rt = tokio::runtime::Runtime::new().unwrap(); - let temp_dir = rt.block_on(async { + let _temp_dir = rt.block_on(async { let driver = FuelCoreDriver::spawn(&args).await.unwrap(); driver.client.produce_blocks(100, None).await.unwrap(); tokio::time::sleep(Duration::from_millis(100)).await; driver.kill().await }); - // rollback 50 blocks - let temp_dir = rt.block_on(async { - let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) - .await - .unwrap(); - for _ in 0..50 { - driver - .node - .shared - .database - .gas_price() - .rollback_last_block() - .unwrap(); - let gas_price_db_height = driver - .node - .shared - .database - .gas_price() - .latest_height() - .unwrap(); - tracing::info!("gas price db height: {:?}", gas_price_db_height); - } - driver.kill().await - }); + // // rollback 50 blocks + // let temp_dir = rt.block_on(async { + // let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) + // .await + // .unwrap(); + // for _ in 0..50 { + // driver + // .node + // .shared + // .database + // .gas_price() + // .rollback_last_block() + // .unwrap(); + // let gas_price_db_height = driver + // .node + // .shared + // .database + // .gas_price() + // .latest_height() + // .unwrap(); + // tracing::info!("gas price db height: {:?}", gas_price_db_height); + // } + // driver.kill().await + // }); + // + // // when + // // restart node + // rt.block_on(async { + // let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) + // .await + // .unwrap(); + // let onchain_db_height = driver + // .node + // .shared + // .database + // .on_chain() + // .latest_height_from_metadata() + // .unwrap() + // .unwrap(); + // let gas_price_db_height = driver + // .node + // .shared + // .database + // .gas_price() + // .latest_height() + // .unwrap(); + // assert_eq!(onchain_db_height, gas_price_db_height); + // }); +} - // when - // restart node - rt.block_on(async { - let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) - .await - .unwrap(); - let onchain_db_height = driver - .node - .shared - .database - .on_chain() - .latest_height_from_metadata() - .unwrap() - .unwrap(); - let gas_price_db_height = driver - .node - .shared - .database - .gas_price() - .latest_height() - .unwrap(); - assert_eq!(onchain_db_height, gas_price_db_height); - }); +#[test] +fn get_heights() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::DEBUG) + .try_init(); + let db = CombinedDatabase::open( + &std::path::Path::new("/Users/jamesturner/fuel/dev-net/.dev-net-db"), + 0, + StateRewindPolicy::RewindFullRange, + -1, + ) + .unwrap(); + + let on_chain_height: u32 = db.on_chain().latest_height().unwrap().unwrap().into(); + let gas_price_height: u32 = db.gas_price().latest_height().unwrap().into(); + + let difference: u32 = on_chain_height - gas_price_height; + tracing::info!("{}", on_chain_height); + tracing::info!("{}", gas_price_height); + tracing::info!("{}", difference); + + // rollback metadata database + // let rollback_blocks = 5; + // for _ in 0..rollback_blocks { + // db.gas_price().rollback_last_block().unwrap(); + // } + // + // let on_chain_height: u32 = db.on_chain().latest_height().unwrap().unwrap().into(); + // let gas_price_height: u32 = db.gas_price().latest_height().unwrap().into(); + // + // let difference: u32 = on_chain_height - gas_price_height; + // tracing::info!("onchain {}", on_chain_height); + // tracing::info!("gas price {}", gas_price_height); + // tracing::info!("{}", difference); + + // Skipped L2 block update: expected 19106879, got 19107223 + // 19108096 + + // +} +#[test] +fn rollback() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::DEBUG) + .try_init(); + let db = CombinedDatabase::open( + &std::path::Path::new("/Users/jamesturner/fuel/dev-net/.dev-net-db"), + 0, + StateRewindPolicy::RewindFullRange, + -1, + ) + .unwrap(); + + let on_chain_height: u32 = db.on_chain().latest_height().unwrap().unwrap().into(); + let gas_price_height: u32 = db.gas_price().latest_height().unwrap().into(); + + let difference: u32 = on_chain_height - gas_price_height; + tracing::info!("{}", on_chain_height); + tracing::info!("{}", gas_price_height); + tracing::info!("{}", difference); + + // rollback metadata database + let rollback_blocks = 5; + for _ in 0..rollback_blocks { + db.gas_price().rollback_last_block().unwrap(); + } + + let on_chain_height: u32 = db.on_chain().latest_height().unwrap().unwrap().into(); + let gas_price_height: u32 = db.gas_price().latest_height().unwrap().into(); + + let difference: u32 = on_chain_height - gas_price_height; + tracing::info!("onchain {}", on_chain_height); + tracing::info!("gas price {}", gas_price_height); + tracing::info!("{}", difference); + + // Skipped L2 block update: expected 19106879, got 19107223 + // 19108096 + + // } #[test] From 6d881d34bda08f364128a4d7c1ce5e12fdf7039a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Mon, 16 Dec 2024 21:30:45 +0100 Subject: [PATCH 111/131] chore: killed stale dead tests --- tests/tests/gas_price.rs | 81 ---------------------------------------- 1 file changed, 81 deletions(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 53e8e67d79d..6e14fdf52fd 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -609,87 +609,6 @@ fn run__if_metadata_is_behind_l2_then_will_catch_up() { // }); } -#[test] -fn get_heights() { - let _ = tracing_subscriber::fmt() - .with_max_level(tracing::Level::DEBUG) - .try_init(); - let db = CombinedDatabase::open( - &std::path::Path::new("/Users/jamesturner/fuel/dev-net/.dev-net-db"), - 0, - StateRewindPolicy::RewindFullRange, - -1, - ) - .unwrap(); - - let on_chain_height: u32 = db.on_chain().latest_height().unwrap().unwrap().into(); - let gas_price_height: u32 = db.gas_price().latest_height().unwrap().into(); - - let difference: u32 = on_chain_height - gas_price_height; - tracing::info!("{}", on_chain_height); - tracing::info!("{}", gas_price_height); - tracing::info!("{}", difference); - - // rollback metadata database - // let rollback_blocks = 5; - // for _ in 0..rollback_blocks { - // db.gas_price().rollback_last_block().unwrap(); - // } - // - // let on_chain_height: u32 = db.on_chain().latest_height().unwrap().unwrap().into(); - // let gas_price_height: u32 = db.gas_price().latest_height().unwrap().into(); - // - // let difference: u32 = on_chain_height - gas_price_height; - // tracing::info!("onchain {}", on_chain_height); - // tracing::info!("gas price {}", gas_price_height); - // tracing::info!("{}", difference); - - // Skipped L2 block update: expected 19106879, got 19107223 - // 19108096 - - // -} -#[test] -fn rollback() { - let _ = tracing_subscriber::fmt() - .with_max_level(tracing::Level::DEBUG) - .try_init(); - let db = CombinedDatabase::open( - &std::path::Path::new("/Users/jamesturner/fuel/dev-net/.dev-net-db"), - 0, - StateRewindPolicy::RewindFullRange, - -1, - ) - .unwrap(); - - let on_chain_height: u32 = db.on_chain().latest_height().unwrap().unwrap().into(); - let gas_price_height: u32 = db.gas_price().latest_height().unwrap().into(); - - let difference: u32 = on_chain_height - gas_price_height; - tracing::info!("{}", on_chain_height); - tracing::info!("{}", gas_price_height); - tracing::info!("{}", difference); - - // rollback metadata database - let rollback_blocks = 5; - for _ in 0..rollback_blocks { - db.gas_price().rollback_last_block().unwrap(); - } - - let on_chain_height: u32 = db.on_chain().latest_height().unwrap().unwrap().into(); - let gas_price_height: u32 = db.gas_price().latest_height().unwrap().into(); - - let difference: u32 = on_chain_height - gas_price_height; - tracing::info!("onchain {}", on_chain_height); - tracing::info!("gas price {}", gas_price_height); - tracing::info!("{}", difference); - - // Skipped L2 block update: expected 19106879, got 19107223 - // 19108096 - - // -} - #[test] fn produce_block__algorithm_recovers_from_divergent_profit() { _produce_block__algorithm_recovers_from_divergent_profit(110); From 64108cec248d4051047be5fca8126beecb837963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Mon, 16 Dec 2024 22:20:28 +0100 Subject: [PATCH 112/131] fix: Increase exec gas price if threshold is met --- crates/fuel-gas-price-algorithm/src/v1.rs | 3 +-- .../src/v1/tests/update_l2_block_data_tests.rs | 9 +++++---- tests/tests/gas_price.rs | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 8dbc280f7a1..ad3718a89bb 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -473,7 +473,7 @@ impl AlgorithmUpdaterV1 { .unwrap_or(threshold); match fullness_percent.cmp(&threshold) { - std::cmp::Ordering::Greater => { + std::cmp::Ordering::Greater | std::cmp::Ordering::Equal => { let change_amount = self.exec_change(scaled_exec_gas_price); scaled_exec_gas_price = scaled_exec_gas_price.saturating_add(change_amount); @@ -483,7 +483,6 @@ impl AlgorithmUpdaterV1 { scaled_exec_gas_price = scaled_exec_gas_price.saturating_sub(change_amount); } - std::cmp::Ordering::Equal => {} } self.new_scaled_exec_price = max(self.min_scaled_exec_gas_price(), scaled_exec_gas_price); diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs index 6f0dbc4debe..f4555551579 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs @@ -199,12 +199,12 @@ fn update_l2_block_data__updates_the_total_reward_value() { } #[test] -fn update_l2_block_data__even_threshold_will_not_change_exec_gas_price() { +fn update_l2_block_data__even_threshold_will_increase_exec_gas_price() { // given - let starting_gas_price = 100; + let starting_exec_gas_price = 100; let unused_percent = 11; let mut updater = UpdaterBuilder::new() - .with_starting_exec_gas_price(starting_gas_price) + .with_starting_exec_gas_price(starting_exec_gas_price) .with_exec_gas_price_change_percent(unused_percent) .build(); @@ -221,7 +221,8 @@ fn update_l2_block_data__even_threshold_will_not_change_exec_gas_price() { .unwrap(); // then - let expected = starting_gas_price; + let expected_change = starting_exec_gas_price * unused_percent as u64 / 100; + let expected = starting_exec_gas_price + expected_change; let actual = updater.new_scaled_exec_price; assert_eq!(actual, expected); } diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 6e14fdf52fd..6ac0cf772cd 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -304,6 +304,7 @@ async fn estimate_gas_price__returns_min_gas_price_if_starting_gas_price_is_zero assert_eq!(MIN_GAS_PRICE, actual) } +// This test passed before this PR, but doesn't now #[tokio::test(flavor = "multi_thread")] async fn latest_gas_price__if_node_restarts_gets_latest_value() { // given From 2f8100ff83a5a7ecf2a1a76a943a72cc2ad532ac Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 16 Dec 2024 19:10:37 -0700 Subject: [PATCH 113/131] Add failing backward compatibility test --- .../forkless-upgrade/Cargo.toml | 2 + .../src/gas_price_algo_compatibility.rs | 120 ++++++++++++++++++ .../forkless-upgrade/src/lib.rs | 4 + 3 files changed, 126 insertions(+) create mode 100644 version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs diff --git a/version-compatibility/forkless-upgrade/Cargo.toml b/version-compatibility/forkless-upgrade/Cargo.toml index 0792f3c6fc7..509880e0157 100644 --- a/version-compatibility/forkless-upgrade/Cargo.toml +++ b/version-compatibility/forkless-upgrade/Cargo.toml @@ -29,6 +29,8 @@ latest-fuel-core-type = { path = "../../crates/types", package = "fuel-core-type "test-helpers", ] } latest-fuel-core-client = { path = "../../crates/client", package = "fuel-core-client" } +latest-fuel-core-gas-price-service = { path = "../../crates/services/gas_price_service", package = "fuel-core-gas-price-service" } +latest-fuel-core-storage = { path = "../../crates/storage", package = "fuel-core-storage" } latest-fuel-core-upgradable-executor = { path = "../../crates/services/upgradable-executor", package = "fuel-core-upgradable-executor", features = [ "wasm-executor", ] } diff --git a/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs b/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs new file mode 100644 index 00000000000..11de191507d --- /dev/null +++ b/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs @@ -0,0 +1,120 @@ +#![allow(unused_imports)] + +use crate::tests_helper::{ + default_multiaddr, + LatestFuelCoreDriver, + Version36FuelCoreDriver, + IGNITION_TESTNET_SNAPSHOT, + POA_SECRET_KEY, +}; +use latest_fuel_core_gas_price_service::{ + common::{ + fuel_core_storage_adapter::storage::GasPriceMetadata, + updater_metadata::UpdaterMetadata, + }, + ports::GasPriceData, + v1::metadata::V1Metadata, +}; +use latest_fuel_core_storage::{ + transactional::AtomicView, + StorageAsRef, +}; +use libp2p::{ + identity::{ + secp256k1::Keypair as SecpKeypair, + Keypair, + }, + PeerId, +}; +use std::ops::Deref; + +#[tokio::test] +async fn latest_gas_price_algorithm_is_compatible_with_ignition() { + // Given + let genesis_keypair = SecpKeypair::generate(); + let hexed_secret = hex::encode(genesis_keypair.secret().to_bytes()); + let genesis_port = "30333"; + let starting_gas_price = 987; + let _ = Version36FuelCoreDriver::spawn(&[ + "--service-name", + "GenesisProducer", + "--debug", + "--poa-instant", + "true", + "--consensus-key", + POA_SECRET_KEY, + "--snapshot", + IGNITION_TESTNET_SNAPSHOT, + "--enable-p2p", + "--keypair", + hexed_secret.as_str(), + "--peering-port", + genesis_port, + "--starting-gas-price", + starting_gas_price.to_string().as_str(), + ]) + .await + .unwrap(); + let public_key = Keypair::from(genesis_keypair).public(); + let genesis_peer_id = PeerId::from_public_key(&public_key); + let genesis_multiaddr = default_multiaddr(genesis_port, genesis_peer_id); + + // Starting node that uses latest fuel core. + // It will connect to the genesis node and sync blocks. + let latest_keypair = SecpKeypair::generate(); + let hexed_secret = hex::encode(latest_keypair.secret().to_bytes()); + let latest_node = LatestFuelCoreDriver::spawn(&[ + "--service-name", + "LatestValidator", + "--debug", + "--poa-instant", + "false", + "--snapshot", + IGNITION_TESTNET_SNAPSHOT, + "--enable-p2p", + "--keypair", + hexed_secret.as_str(), + "--reserved-nodes", + genesis_multiaddr.as_str(), + "--peering-port", + "0", + ]) + .await + .unwrap(); + + // When + const BLOCKS_TO_PRODUCE: u32 = 10; + latest_node + .client + .produce_blocks(BLOCKS_TO_PRODUCE, None) + .await + .unwrap(); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + // Then + let db = &latest_node.node.shared.database; + let latest_height = db.gas_price().latest_height().unwrap(); + let view = db.gas_price().latest_view().unwrap(); + let metadata = view + .storage::() + .get(&latest_height.into()) + .unwrap() + .unwrap() + .deref() + .clone(); + let UpdaterMetadata::V1(V1Metadata { + new_scaled_exec_price, + gas_price_factor, + .. + }) = metadata + else { + panic!("Expected V1Metadata, got {:?}", metadata); + }; + // assert!( + // matches!(metadata, UpdaterMetadata::V1(V1Metadata { new_scaled_exec_price, gas_price_factor, .. }) if new_scaled_exec_price == starting_gas_price * gas_price_factor.get()) + // ); + assert_eq!( + new_scaled_exec_price, + starting_gas_price * gas_price_factor.get() + ); +} diff --git a/version-compatibility/forkless-upgrade/src/lib.rs b/version-compatibility/forkless-upgrade/src/lib.rs index ae7415e13d0..f2170f0043b 100644 --- a/version-compatibility/forkless-upgrade/src/lib.rs +++ b/version-compatibility/forkless-upgrade/src/lib.rs @@ -5,6 +5,10 @@ mod backward_compatibility; #[cfg(test)] mod forward_compatibility; + +#[cfg(test)] +mod gas_price_algo_compatibility; + #[cfg(test)] mod genesis; #[cfg(test)] From 78f5b94c0cffb2614a97e995b1b00b8a97e5a988 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 16 Dec 2024 21:29:15 -0700 Subject: [PATCH 114/131] Fix algo compatibility test --- .../src/gas_price_algo_compatibility.rs | 41 ++++++++++--------- .../forkless-upgrade/src/tests_helper.rs | 31 ++++++++++++-- 2 files changed, 49 insertions(+), 23 deletions(-) diff --git a/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs b/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs index 11de191507d..a19a6ffdb30 100644 --- a/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs +++ b/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs @@ -35,7 +35,7 @@ async fn latest_gas_price_algorithm_is_compatible_with_ignition() { let hexed_secret = hex::encode(genesis_keypair.secret().to_bytes()); let genesis_port = "30333"; let starting_gas_price = 987; - let _ = Version36FuelCoreDriver::spawn(&[ + let old_driver = Version36FuelCoreDriver::spawn(&[ "--service-name", "GenesisProducer", "--debug", @@ -58,27 +58,31 @@ async fn latest_gas_price_algorithm_is_compatible_with_ignition() { let public_key = Keypair::from(genesis_keypair).public(); let genesis_peer_id = PeerId::from_public_key(&public_key); let genesis_multiaddr = default_multiaddr(genesis_port, genesis_peer_id); + let temp_dir = old_driver.kill().await; // Starting node that uses latest fuel core. // It will connect to the genesis node and sync blocks. let latest_keypair = SecpKeypair::generate(); let hexed_secret = hex::encode(latest_keypair.secret().to_bytes()); - let latest_node = LatestFuelCoreDriver::spawn(&[ - "--service-name", - "LatestValidator", - "--debug", - "--poa-instant", - "false", - "--snapshot", - IGNITION_TESTNET_SNAPSHOT, - "--enable-p2p", - "--keypair", - hexed_secret.as_str(), - "--reserved-nodes", - genesis_multiaddr.as_str(), - "--peering-port", - "0", - ]) + let latest_node = LatestFuelCoreDriver::spawn_with_directory( + temp_dir, + &[ + "--service-name", + "LatestValidator", + "--debug", + "--poa-instant", + "false", + "--snapshot", + IGNITION_TESTNET_SNAPSHOT, + "--enable-p2p", + "--keypair", + hexed_secret.as_str(), + "--reserved-nodes", + genesis_multiaddr.as_str(), + "--peering-port", + "0", + ], + ) .await .unwrap(); @@ -110,9 +114,6 @@ async fn latest_gas_price_algorithm_is_compatible_with_ignition() { else { panic!("Expected V1Metadata, got {:?}", metadata); }; - // assert!( - // matches!(metadata, UpdaterMetadata::V1(V1Metadata { new_scaled_exec_price, gas_price_factor, .. }) if new_scaled_exec_price == starting_gas_price * gas_price_factor.get()) - // ); assert_eq!( new_scaled_exec_price, starting_gas_price * gas_price_factor.get() diff --git a/version-compatibility/forkless-upgrade/src/tests_helper.rs b/version-compatibility/forkless-upgrade/src/tests_helper.rs index c48a1b512a1..d60e47e53e8 100644 --- a/version-compatibility/forkless-upgrade/src/tests_helper.rs +++ b/version-compatibility/forkless-upgrade/src/tests_helper.rs @@ -52,12 +52,17 @@ macro_rules! define_core_driver { impl $name { pub async fn spawn(extra_args: &[&str]) -> anyhow::Result { - use clap::Parser; use tempfile::tempdir; - - // Generate temp params let db_dir = tempdir()?; + Self::spawn_with_directory(db_dir, extra_args).await + } + pub async fn spawn_with_directory( + db_dir: tempfile::TempDir, + extra_args: &[&str], + ) -> anyhow::Result { + use clap::Parser; + let mut args = vec![ "_IGNORED_", "--db-path", @@ -103,6 +108,16 @@ define_core_driver!( true ); +impl Version36FuelCoreDriver { + pub async fn kill(self) -> tempfile::TempDir { + self.node + .send_stop_signal_and_await_shutdown() + .await + .expect("Failed to stop the node"); + self._db_dir + } +} + define_core_driver!( latest_fuel_core_bin, LatestFuelService, @@ -111,6 +126,16 @@ define_core_driver!( true ); +impl LatestFuelCoreDriver { + pub async fn kill(self) -> tempfile::TempDir { + self.node + .send_stop_signal_and_await_shutdown() + .await + .expect("Failed to stop the node"); + self._db_dir + } +} + pub const IGNITION_TESTNET_SNAPSHOT: &str = "./chain-configurations/ignition"; pub const V36_TESTNET_SNAPSHOT: &str = "./chain-configurations/v36"; pub const POA_SECRET_KEY: &str = From 084f1b7463ffa66ed599468aa8e5e41209d8a690 Mon Sep 17 00:00:00 2001 From: Aaryamann Challani <43716372+rymnc@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:20:44 +0530 Subject: [PATCH 115/131] chore(gas_price_service_v1): allow stopping the service during sync with sigint/etc (#2503) ## Linked Issues/PRs - from the notion doc ## Description allows stopping the node when the gas price db is syncing with onchain db. ## Checklist - [ ] Breaking changes are clearly marked as such in the PR description and changelog - [ ] New behavior is reflected in tests - [ ] [The specification](https://github.com/FuelLabs/fuel-specs/) matches the implemented behavior (link update PR if changes are needed) ### Before requesting review - [x] I have reviewed the code myself - [ ] I have created follow-up issues caused by this PR and linked them here ### After merging, notify other teams [Add or remove entries as needed] - [ ] [Rust SDK](https://github.com/FuelLabs/fuels-rs/) - [ ] [Sway compiler](https://github.com/FuelLabs/sway/) - [ ] [Platform documentation](https://github.com/FuelLabs/devrel-requests/issues/new?assignees=&labels=new+request&projects=&template=NEW-REQUEST.yml&title=%5BRequest%5D%3A+) (for out-of-organization contributors, the person merging the PR will do this) - [ ] Someone else? --- .../gas_price_service/src/v1/tests.rs | 5 +- .../src/v1/uninitialized_task.rs | 73 +++++-------------- 2 files changed, 20 insertions(+), 58 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index fdbdab3d0df..349a090e97c 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -65,7 +65,6 @@ use fuel_core_services::{ StateWatcher, }; use fuel_core_storage::{ - iter::IteratorOverTable, structured_storage::test::InMemoryStorage, transactional::{ AtomicView, @@ -728,7 +727,7 @@ async fn uninitialized_task__init__starts_da_service_with_bundle_id_in_storage() .unwrap(); // when - service.init().await.unwrap(); + service.init(&StateWatcher::started()).await.unwrap(); // then let actual = bundle_id_handle.lock().unwrap(); @@ -785,7 +784,7 @@ async fn uninitialized_task__init__if_metadata_behind_l2_height_then_sync() { .unwrap(); // when - service.init().await.unwrap(); + service.init(&StateWatcher::started()).await.unwrap(); // then // no panic diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index e75e8d11d97..10ca224fdb4 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -57,6 +57,7 @@ use fuel_core_services::{ RunnableService, Service, ServiceRunner, + State, StateWatcher, }; use fuel_core_storage::{ @@ -139,6 +140,7 @@ where pub async fn init( mut self, + state_watcher: &StateWatcher, ) -> anyhow::Result< GasPriceServiceV1, DA, AtomicStorage>, > { @@ -155,9 +157,8 @@ where self.block_stream, ); - if let Some(bundle_id) = self - .gas_price_db - .get_bundle_id(&self.gas_metadata_height.into())? + if let Some(bundle_id) = + self.gas_price_db.get_bundle_id(&self.gas_metadata_height)? { self.da_source.set_last_value(bundle_id).await?; } @@ -166,9 +167,8 @@ where .da_poll_interval .map(|x| Duration::from_millis(x.into())); // TODO: Dupe code - if let Some(bundle_id) = self - .gas_price_db - .get_bundle_id(&self.gas_metadata_height.into())? + if let Some(bundle_id) = + self.gas_price_db.get_bundle_id(&self.gas_metadata_height)? { self.da_source.set_last_value(bundle_id).await?; } @@ -187,13 +187,14 @@ where Ok(service) } else { if latest_block_height > *self.gas_metadata_height { - sync_gas_price_db_with_on_chain_storage( + sync_v1_metadata( &self.settings, - &mut self.algo_updater, &self.on_chain_db, *self.gas_metadata_height, latest_block_height, + &mut self.algo_updater, &mut self.gas_price_db, + state_watcher, )?; } @@ -230,58 +231,13 @@ where async fn into_task( self, - _state_watcher: &StateWatcher, + state_watcher: &StateWatcher, _params: Self::TaskParams, ) -> anyhow::Result { - UninitializedTask::init(self).await + UninitializedTask::init(self, state_watcher).await } } -fn sync_gas_price_db_with_on_chain_storage< - L2DataStore, - L2DataStoreView, - SettingsProvider, - AtomicStorage, ->( - settings: &SettingsProvider, - updater: &mut AlgorithmUpdaterV1, - on_chain_db: &L2DataStoreView, - metadata_height: u32, - latest_block_height: u32, - persisted_data: &mut AtomicStorage, -) -> anyhow::Result<()> -where - L2DataStore: L2Data, - L2DataStoreView: AtomicView, - SettingsProvider: GasPriceSettingsProvider, - AtomicStorage: GasPriceServiceAtomicStorage, -{ - let metadata = persisted_data - .get_metadata(&metadata_height.into())? - .ok_or(anyhow::anyhow!( - "Expected metadata to exist for height: {metadata_height}" - ))?; - - // let metadata = match metadata { - // UpdaterMetadata::V1(metadata) => metadata, - // UpdaterMetadata::V0(metadata) => { - // V1Metadata::construct_from_v0_metadata(metadata, config)? - // } - // }; - // let mut algo_updater = v1_algorithm_from_metadata(metadata, config); - - sync_v1_metadata( - settings, - on_chain_db, - metadata_height, - latest_block_height, - updater, - persisted_data, - )?; - - Ok(()) -} - fn sync_v1_metadata( settings: &SettingsProvider, on_chain_db: &L2DataStoreView, @@ -289,6 +245,7 @@ fn sync_v1_metadata anyhow::Result<()> where L2DataStore: L2Data, @@ -304,6 +261,12 @@ where latest_block_height ); for height in first..=latest_block_height { + // allows early exit if the service is stopping + let state = state_watcher.borrow(); + if state.stopping() || state.stopped() { + return Ok(()); + } + tracing::info!("Syncing gas price metadata for block {}", height); let mut tx = da_storage.begin_transaction()?; let block = view From 108aa0d247e27b3f77d9f9d1eb3d446f93668858 Mon Sep 17 00:00:00 2001 From: Aaryamann Challani <43716372+rymnc@users.noreply.github.com> Date: Thu, 19 Dec 2024 00:37:01 +0530 Subject: [PATCH 116/131] fix(gas_price_service_v1): block committer api format (#2506) > [!NOTE] > This PR is wip, we need to decide if we should sunset bundleId's or not ## Linked Issues/PRs ## Description the block committer deviated from the spec on the api format, this PR updates the code to match the implementation. ## Checklist - [ ] Breaking changes are clearly marked as such in the PR description and changelog - [ ] New behavior is reflected in tests - [ ] [The specification](https://github.com/FuelLabs/fuel-specs/) matches the implemented behavior (link update PR if changes are needed) ### Before requesting review - [ ] I have reviewed the code myself - [ ] I have created follow-up issues caused by this PR and linked them here ### After merging, notify other teams [Add or remove entries as needed] - [ ] [Rust SDK](https://github.com/FuelLabs/fuels-rs/) - [ ] [Sway compiler](https://github.com/FuelLabs/sway/) - [ ] [Platform documentation](https://github.com/FuelLabs/devrel-requests/issues/new?assignees=&labels=new+request&projects=&template=NEW-REQUEST.yml&title=%5BRequest%5D%3A+) (for out-of-organization contributors, the person merging the PR will do this) - [ ] Someone else? --------- Co-authored-by: Mitchell Turner --- Cargo.lock | 9 +- crates/fuel-gas-price-algorithm/src/v1.rs | 13 +- .../v1/tests/update_da_record_data_tests.rs | 154 +---- crates/services/gas_price_service/Cargo.toml | 7 + .../src/common/fuel_core_storage_adapter.rs | 37 +- .../fuel_core_storage_adapter/storage.rs | 16 +- .../services/gas_price_service/src/ports.rs | 16 +- .../src/v1/da_source_service.rs | 11 +- .../block_committer_costs.rs | 587 ++++++++++++------ .../src/v1/da_source_service/dummy_costs.rs | 9 +- .../src/v1/da_source_service/service.rs | 10 +- .../gas_price_service/src/v1/service.rs | 54 +- .../gas_price_service/src/v1/tests.rs | 58 +- .../src/v1/uninitialized_task.rs | 17 +- tests/Cargo.toml | 5 +- tests/tests/gas_price.rs | 148 ++--- 16 files changed, 597 insertions(+), 554 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7eb0dd52f43..3fedf8bdb41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1609,12 +1609,12 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "colored" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -3465,10 +3465,12 @@ dependencies = [ "fuel-core-types 0.40.0", "fuel-gas-price-algorithm", "futures", + "mockito", "num_enum", "parking_lot", "reqwest 0.12.9", "serde", + "serde_json", "strum 0.25.0", "strum_macros 0.25.3", "thiserror 1.0.69", @@ -3745,7 +3747,6 @@ dependencies = [ "insta", "itertools 0.12.1", "k256", - "mockito", "postcard", "pretty_assertions", "primitive-types", diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index ad3718a89bb..7fe26ce10e3 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -3,7 +3,10 @@ use std::{ cmp::max, collections::BTreeMap, num::NonZeroU64, - ops::Div, + ops::{ + Div, + RangeInclusive, + }, }; #[cfg(test)] @@ -349,7 +352,7 @@ impl core::ops::Deref for ClampedPercentage { impl AlgorithmUpdaterV1 { pub fn update_da_record_data( &mut self, - heights: &[u32], + heights: RangeInclusive, recorded_bytes: u32, recording_cost: u128, unrecorded_blocks: &mut U, @@ -584,7 +587,7 @@ impl AlgorithmUpdaterV1 { fn da_block_update( &mut self, - heights: &[u32], + heights: RangeInclusive, recorded_bytes: u128, recording_cost: u128, unrecorded_blocks: &mut U, @@ -613,13 +616,13 @@ impl AlgorithmUpdaterV1 { // Always remove the blocks from the unrecorded blocks so they don't build up indefinitely fn update_unrecorded_block_bytes( &mut self, - heights: &[u32], + heights: RangeInclusive, unrecorded_blocks: &mut U, ) -> Result<(), Error> { let mut total: u128 = 0; for expected_height in heights { let maybe_bytes = unrecorded_blocks - .remove(expected_height) + .remove(&expected_height) .map_err(Error::CouldNotRemoveUnrecordedBlock)?; if let Some(bytes) = maybe_bytes { diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index 6d1feb220b4..da6812cce2f 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -5,7 +5,7 @@ use std::collections::BTreeMap; fn update_da_record_data__if_receives_batch_with_unknown_blocks_will_include_known_blocks_with_previous_cost( ) { // given - let recorded_heights: Vec = (1u32..3).collect(); + let recorded_heights = 1u32..=2; let recorded_cost = 1_000_000; let recorded_bytes = 500; let block_bytes = 1000; @@ -22,7 +22,7 @@ fn update_da_record_data__if_receives_batch_with_unknown_blocks_will_include_kno // when updater .update_da_record_data( - &recorded_heights, + recorded_heights, recorded_bytes, recorded_cost, &mut unrecorded_blocks, @@ -41,7 +41,7 @@ fn update_da_record_data__if_receives_batch_with_unknown_blocks_will_include_kno fn update_da_record_data__if_receives_batch_with_unknown_blocks_will_never_increase_cost_more_than_recorded_cost( ) { // given - let recorded_heights: Vec = (1u32..3).collect(); + let recorded_heights = 1u32..=2; let recorded_cost = 200; let block_bytes = 1000; let recorded_bytes = 500; @@ -58,7 +58,7 @@ fn update_da_record_data__if_receives_batch_with_unknown_blocks_will_never_incre // when updater .update_da_record_data( - &recorded_heights, + recorded_heights, recorded_bytes, recorded_cost, &mut unrecorded_blocks, @@ -87,11 +87,11 @@ fn update_da_record_data__updates_cost_per_byte() { let new_cost_per_byte = 100; let recorded_bytes = 500; let recorded_cost = (recorded_bytes * new_cost_per_byte) as u128; - let recorded_heights: Vec = (1u32..2).collect(); + let recorded_heights = 1u32..=1; // when updater .update_da_record_data( - &recorded_heights, + recorded_heights, recorded_bytes, recorded_cost, &mut unrecorded_blocks, @@ -121,13 +121,13 @@ fn update_da_record_data__updates_known_total_cost() { .with_unrecorded_blocks(&unrecorded_blocks) .build(); - let recorded_heights: Vec = (11u32..14).collect(); + let recorded_heights = 11u32..=13; let recorded_bytes = 500; let recorded_cost = 300; // when updater .update_da_record_data( - &recorded_heights, + recorded_heights, recorded_bytes, recorded_cost, &mut unrecorded_blocks, @@ -166,13 +166,13 @@ fn update_da_record_data__if_da_height_matches_l2_height_projected_and_known_mat let new_cost_per_byte = 100; let block_cost = block_bytes * new_cost_per_byte; - let recorded_heights: Vec = (11u32..14).collect(); + let recorded_heights = 11u32..=13; let recorded_bytes = 500; let recorded_cost = block_cost * 3; // when updater .update_da_record_data( - &recorded_heights, + recorded_heights, recorded_bytes, recorded_cost, &mut unrecorded_blocks, @@ -216,7 +216,7 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g .build(); let new_cost_per_byte = 100; - let recorded_heights = vec![11, 12, 13]; + let recorded_heights = 11u32..=13; let recorded_bytes = 500; let recorded_cost = recorded_bytes * new_cost_per_byte; let recorded_bytes = 500; @@ -224,7 +224,7 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g // when updater .update_da_record_data( - &recorded_heights, + recorded_heights, recorded_bytes, recorded_cost, &mut unrecorded_blocks, @@ -242,124 +242,6 @@ fn update_da_record_data__da_block_updates_projected_total_cost_with_known_and_g assert_eq!(actual, expected); } -#[test] -fn update_da_record_data__updates_known_total_cost_if_blocks_are_out_of_order() { - // given - let da_cost_per_byte = 20; - let block_bytes = 1000; - let mut unrecorded_blocks: BTreeMap<_, _> = - [(1, block_bytes), (2, block_bytes), (3, block_bytes)] - .into_iter() - .collect(); - let old_known_total_cost = 500; - let old_projected_total_cost = - old_known_total_cost + (block_bytes as u128 * da_cost_per_byte * 3); - let old_da_cost_per_byte = 20; - let mut updater = UpdaterBuilder::new() - .with_da_cost_per_byte(da_cost_per_byte) - .with_unrecorded_blocks(&unrecorded_blocks) - .with_da_cost_per_byte(old_da_cost_per_byte) - .with_known_total_cost(old_known_total_cost) - .with_projected_total_cost(old_projected_total_cost) - .build(); - let new_cost_per_byte = 100; - let recorded_bytes = 500; - let recorded_cost = recorded_bytes * new_cost_per_byte; - let recorded_heights: Vec = vec![3, 2]; - - // when - updater - .update_da_record_data( - &recorded_heights, - recorded_bytes, - recorded_cost as u128, - &mut unrecorded_blocks, - ) - .unwrap(); - - // then - let expected = updater.latest_known_total_da_cost_excess - + (block_bytes as u128 * new_cost_per_byte as u128); - let actual = updater.projected_total_da_cost; - assert_eq!(actual, expected); -} - -#[test] -fn update_da_record_data__updates_projected_total_cost_if_blocks_are_out_of_order() { - // given - let da_cost_per_byte = 20; - let block_bytes = 1000; - let mut unrecorded_blocks: BTreeMap<_, _> = - [(1, block_bytes), (2, block_bytes), (3, block_bytes)] - .into_iter() - .collect(); - let old_known_total_cost = 500; - let old_projected_total_cost = - old_known_total_cost + (block_bytes as u128 * da_cost_per_byte * 3); - let old_da_cost_per_byte = 20; - let mut updater = UpdaterBuilder::new() - .with_da_cost_per_byte(da_cost_per_byte) - .with_unrecorded_blocks(&unrecorded_blocks) - .with_da_cost_per_byte(old_da_cost_per_byte) - .with_known_total_cost(old_known_total_cost) - .with_projected_total_cost(old_projected_total_cost) - .build(); - let new_cost_per_byte = 100; - let recorded_bytes = 500; - let recorded_cost = recorded_bytes * new_cost_per_byte; - let recorded_heights: Vec = vec![3, 2]; - - // when - updater - .update_da_record_data( - &recorded_heights, - recorded_bytes, - recorded_cost as u128, - &mut unrecorded_blocks, - ) - .unwrap(); - - // then - let expected = updater.latest_known_total_da_cost_excess - + (block_bytes as u128 * new_cost_per_byte as u128); - let actual = updater.projected_total_da_cost; - assert_eq!(actual, expected); -} - -#[test] -fn update_da_record_data__updates_unrecorded_blocks() { - // given - let da_cost_per_byte = 20; - let block_bytes = 1000; - let mut unrecorded_blocks: BTreeMap<_, _> = - [(1, block_bytes), (2, block_bytes), (3, block_bytes)] - .into_iter() - .collect(); - let mut updater = UpdaterBuilder::new() - .with_da_cost_per_byte(da_cost_per_byte) - .with_unrecorded_blocks(&unrecorded_blocks) - .build(); - let new_cost_per_byte = 100; - let recorded_bytes = 500; - let recorded_cost = 2 * (recorded_bytes * new_cost_per_byte) as u128; - let recorded_heights: Vec = vec![3, 2]; - - // when - updater - .update_da_record_data( - &recorded_heights, - recorded_bytes, - recorded_cost, - &mut unrecorded_blocks, - ) - .unwrap(); - - // then - let expected = vec![(1, block_bytes)]; - let actual: Vec<_> = unrecorded_blocks.into_iter().collect(); - assert_eq!(actual, expected); -} - #[test] fn update_da_record_data__da_block_lowers_da_gas_price() { // given @@ -394,7 +276,7 @@ fn update_da_record_data__da_block_lowers_da_gas_price() { ); let min = *recorded_heights.iter().min().unwrap(); let max = *recorded_heights.iter().max().unwrap(); - let recorded_range: Vec = (*min..(max + 1)).collect(); + let recorded_range = *min..=*max; let recorded_bytes = 500; let old_da_gas_price = updater.new_scaled_da_gas_price; @@ -402,7 +284,7 @@ fn update_da_record_data__da_block_lowers_da_gas_price() { // when updater .update_da_record_data( - &recorded_range, + recorded_range, recorded_bytes, recorded_cost as u128, &mut unrecorded_blocks, @@ -451,7 +333,7 @@ fn update_da_record_data__da_block_increases_da_gas_price() { let min = *recorded_heights.iter().min().unwrap(); let max = *recorded_heights.iter().max().unwrap(); - let recorded_range: Vec = (*min..(max + 1)).collect(); + let recorded_range = *min..=*max; let recorded_bytes = 500; let old_da_gas_price = updater.new_scaled_da_gas_price; @@ -459,7 +341,7 @@ fn update_da_record_data__da_block_increases_da_gas_price() { // when updater .update_da_record_data( - &recorded_range, + recorded_range, recorded_bytes, recorded_cost as u128, &mut unrecorded_blocks, @@ -507,7 +389,7 @@ fn update_da_record_data__da_block_will_not_change_da_gas_price() { ); let min = *recorded_heights.iter().min().unwrap(); let max = *recorded_heights.iter().max().unwrap(); - let recorded_range: Vec = (*min..(max + 1)).collect(); + let recorded_range = *min..=*max; let recorded_bytes = 500; let old_da_gas_price = updater.new_scaled_da_gas_price; @@ -515,7 +397,7 @@ fn update_da_record_data__da_block_will_not_change_da_gas_price() { // when updater .update_da_record_data( - &recorded_range, + recorded_range, recorded_bytes, recorded_cost as u128, &mut unrecorded_blocks, diff --git a/crates/services/gas_price_service/Cargo.toml b/crates/services/gas_price_service/Cargo.toml index 4eab33d503b..efde85a485f 100644 --- a/crates/services/gas_price_service/Cargo.toml +++ b/crates/services/gas_price_service/Cargo.toml @@ -17,10 +17,12 @@ fuel-core-storage = { workspace = true, features = ["std"] } fuel-core-types = { workspace = true, features = ["std"] } fuel-gas-price-algorithm = { workspace = true } futures = { workspace = true } +mockito = { version = "1.6.1", optional = true } num_enum = { workspace = true } parking_lot = { workspace = true } reqwest = { workspace = true, features = ["json"] } serde = { workspace = true } +serde_json = { workspace = true, optional = true } strum = { workspace = true, features = ["derive"] } strum_macros = { workspace = true } thiserror = { workspace = true } @@ -32,4 +34,9 @@ tracing = { workspace = true } fuel-core-services = { workspace = true, features = ["test-helpers"] } fuel-core-storage = { workspace = true, features = ["test-helpers"] } fuel-core-types = { path = "./../../types", features = ["test-helpers"] } +mockito = { version = "1.6.1" } +serde_json = { workspace = true } tracing-subscriber = { workspace = true } + +[features] +test-helpers = ["dep:mockito", "dep:serde_json"] diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs index 74bec1207d2..14b0675d762 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter.rs @@ -1,9 +1,9 @@ use crate::{ common::{ fuel_core_storage_adapter::storage::{ - BundleIdTable, GasPriceColumn, GasPriceMetadata, + RecordedHeights, }, updater_metadata::UpdaterMetadata, utils::{ @@ -14,9 +14,9 @@ use crate::{ }, ports::{ GasPriceServiceAtomicStorage, - GetDaBundleId, + GetLatestRecordedHeight, GetMetadataStorage, - SetDaBundleId, + SetLatestRecordedHeight, SetMetadataStorage, }, }; @@ -99,25 +99,26 @@ where } } -impl GetDaBundleId for Storage +impl GetLatestRecordedHeight for Storage where Storage: Send + Sync, - Storage: StorageInspect, + Storage: StorageInspect, { - fn get_bundle_id(&self, block_height: &BlockHeight) -> GasPriceResult> { - let bundle_id = self - .storage::() - .get(block_height) + fn get_recorded_height(&self) -> GasPriceResult> { + const KEY: &() = &(); + let recorded_height = self + .storage::() + .get(KEY) .map_err(|err| GasPriceError::CouldNotFetchDARecord(err.into()))? .map(|no| *no); - Ok(bundle_id) + Ok(recorded_height) } } impl GasPriceServiceAtomicStorage for Storage where Storage: 'static, - Storage: GetMetadataStorage + GetDaBundleId, + Storage: GetMetadataStorage + GetLatestRecordedHeight, Storage: KeyValueInspect + Modifiable + Send + Sync, { type Transaction<'a> = StorageTransaction<&'a mut Storage> where Self: 'a; @@ -135,18 +136,18 @@ where } } -impl SetDaBundleId for Storage +impl SetLatestRecordedHeight for Storage where Storage: Send + Sync, - Storage: StorageMutate, + Storage: StorageMutate, { - fn set_bundle_id( + fn set_recorded_height( &mut self, - block_height: &BlockHeight, - bundle_id: u32, + recorded_height: BlockHeight, ) -> GasPriceResult<()> { - self.storage_as_mut::() - .insert(block_height, &bundle_id) + const KEY: &() = &(); + self.storage_as_mut::() + .insert(KEY, &recorded_height) .map_err(|err| GasPriceError::CouldNotFetchDARecord(err.into()))?; Ok(()) } diff --git a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs index e9f20b411d8..61532696565 100644 --- a/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs +++ b/crates/services/gas_price_service/src/common/fuel_core_storage_adapter/storage.rs @@ -93,20 +93,18 @@ impl TableWithBlueprint for UnrecordedBlocksTable { } } -pub struct BundleIdTable; +/// Used to store the latest L2 block that has been recorded on the DA chain +pub struct RecordedHeights; -/// The sequence number or bundle id of the posted blocks. -type BundleId = u32; - -impl Mappable for BundleIdTable { +impl Mappable for RecordedHeights { type Key = Self::OwnedKey; - type OwnedKey = BlockHeight; + type OwnedKey = (); type Value = Self::OwnedValue; - type OwnedValue = BundleId; + type OwnedValue = BlockHeight; } -impl TableWithBlueprint for BundleIdTable { - type Blueprint = Plain, Postcard>; +impl TableWithBlueprint for RecordedHeights { + type Blueprint = Plain; type Column = GasPriceColumn; fn column() -> Self::Column { diff --git a/crates/services/gas_price_service/src/ports.rs b/crates/services/gas_price_service/src/ports.rs index 094f781f170..7670b3e0eab 100644 --- a/crates/services/gas_price_service/src/ports.rs +++ b/crates/services/gas_price_service/src/ports.rs @@ -35,26 +35,26 @@ pub trait GetMetadataStorage: Send + Sync { -> Result>; } -pub trait SetDaBundleId: Send + Sync { - fn set_bundle_id(&mut self, block_height: &BlockHeight, bundle_id: u32) - -> Result<()>; +pub trait SetLatestRecordedHeight: Send + Sync { + /// For any given L2 block produced, the DA will have committed some + fn set_recorded_height(&mut self, recorded_height: BlockHeight) -> Result<()>; } -pub trait GetDaBundleId: Send + Sync { - fn get_bundle_id(&self, block_height: &BlockHeight) -> Result>; +pub trait GetLatestRecordedHeight: Send + Sync { + fn get_recorded_height(&self) -> Result>; } pub trait GasPriceServiceAtomicStorage where Self: 'static, Self: Send + Sync, - Self: GetMetadataStorage + GetDaBundleId, + Self: GetMetadataStorage + GetLatestRecordedHeight, { type Transaction<'a>: AsUnrecordedBlocks + SetMetadataStorage + GetMetadataStorage - + SetDaBundleId - + GetDaBundleId + + SetLatestRecordedHeight + + GetLatestRecordedHeight where Self: 'a; diff --git a/crates/services/gas_price_service/src/v1/da_source_service.rs b/crates/services/gas_price_service/src/v1/da_source_service.rs index bae8d3d46bc..0c90b9a36d3 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service.rs @@ -1,15 +1,18 @@ use crate::v1::da_source_service::service::DaBlockCostsSource; -use std::time::Duration; +use std::{ + ops::RangeInclusive, + time::Duration, +}; pub mod block_committer_costs; #[cfg(test)] pub mod dummy_costs; pub mod service; -#[derive(Debug, Default, Clone, Eq, Hash, PartialEq)] +#[derive(Debug, Clone, Eq, Hash, PartialEq)] pub struct DaBlockCosts { pub bundle_id: u32, - pub l2_blocks: Vec, + pub l2_blocks: RangeInclusive, pub bundle_size_bytes: u32, pub blob_cost_wei: u128, } @@ -33,7 +36,7 @@ mod tests { // given let expected_da_cost = DaBlockCosts { bundle_id: 1, - l2_blocks: (0..10).collect(), + l2_blocks: 0..=9, bundle_size_bytes: 1024 * 128, blob_cost_wei: 2, }; diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index ac888a0a70d..fb6077472c4 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -8,70 +8,84 @@ use crate::v1::da_source_service::{ DaBlockCosts, }; use anyhow::anyhow; -use fuel_core_types::blockchain::primitives::DaBlockHeight; +use fuel_core_types::{ + blockchain::primitives::DaBlockHeight, + fuel_types::BlockHeight, +}; use serde::{ Deserialize, Serialize, }; +use std::ops::Deref; #[async_trait::async_trait] pub trait BlockCommitterApi: Send + Sync { /// Used on first run to get the latest costs and seqno async fn get_latest_costs(&self) -> DaBlockCostsResult>; /// Used to get the costs for a specific seqno - async fn get_costs_by_seqno( - &self, - number: u32, - ) -> DaBlockCostsResult>; - /// Used to get the costs for a range of blocks (inclusive) - async fn get_cost_bundles_by_range( + async fn get_costs_by_l2_block_number( &self, - range: core::ops::Range, - ) -> DaBlockCostsResult>>; + l2_block_number: u32, + ) -> DaBlockCostsResult>; } /// This struct is used to denote the block committer da block costs source /// which receives data from the block committer (only http api for now) pub struct BlockCommitterDaBlockCosts { client: BlockCommitter, - last_raw_da_block_costs: Option, + last_recorded_height: Option, } #[derive(Debug, Deserialize, Serialize, Clone, Default, PartialEq)] pub struct RawDaBlockCosts { - /// Sequence number (Monotonically increasing nonce) - pub bundle_id: u32, - /// The range of blocks that the costs apply to - pub blocks_heights: Vec, + pub id: u32, + /// The beginning of the range of blocks that the costs apply to + pub start_height: u32, + /// The end of the range of blocks that the costs apply to + pub end_height: u32, /// The DA block height of the last transaction for the range of blocks pub da_block_height: DaBlockHeight, - /// Rolling sum cost of posting blobs (wei) - pub total_cost: u128, - /// Rolling sum size of blobs (bytes) - pub total_size_bytes: u32, + /// cost of posting this blob (wei) + pub cost: u128, + /// size of this blob (bytes) + pub size: u32, } impl From<&RawDaBlockCosts> for DaBlockCosts { fn from(raw_da_block_costs: &RawDaBlockCosts) -> Self { + let RawDaBlockCosts { + start_height, + end_height, + cost: cost_wei, + size: size_bytes, + id: bundle_id, + .. + } = *raw_da_block_costs; DaBlockCosts { - bundle_id: raw_da_block_costs.bundle_id, - l2_blocks: raw_da_block_costs - .blocks_heights - .clone() - .into_iter() - .collect(), - bundle_size_bytes: raw_da_block_costs.total_size_bytes, - blob_cost_wei: raw_da_block_costs.total_cost, + bundle_id, + // construct a vec of l2 blocks from the start_height to the end_height + l2_blocks: start_height..=end_height, + bundle_size_bytes: size_bytes, + blob_cost_wei: cost_wei, } } } +impl From for DaBlockCosts { + fn from(value: RawDaBlockCosts) -> Self { + Self::from(&value) + } +} + impl BlockCommitterDaBlockCosts { /// Create a new instance of the block committer da block costs source - pub fn new(client: BlockCommitter, last_value: Option) -> Self { + pub fn new( + client: BlockCommitter, + last_recorded_height: Option, + ) -> Self { Self { client, - last_raw_da_block_costs: last_value, + last_recorded_height, } } } @@ -81,49 +95,30 @@ impl DaBlockCostsSource for BlockCommitterDaBlockCosts DaBlockCostsResult> { - let raw_da_block_costs = match self.last_raw_da_block_costs { - Some(ref last_value) => { - self.client.get_costs_by_seqno(last_value.bundle_id + 1) - } - _ => self.client.get_latest_costs(), + async fn request_da_block_costs(&mut self) -> DaBlockCostsResult> { + let raw_da_block_costs: Vec<_> = + match self.last_recorded_height.and_then(|x| x.succ()) { + Some(ref next_height) => { + self.client + .get_costs_by_l2_block_number(*next_height.deref()) + .await? + } + None => self.client.get_latest_costs().await?.into_iter().collect(), + }; + + tracing::info!("raw_da_block_costs: {:?}", raw_da_block_costs); + let da_block_costs: Vec<_> = + raw_da_block_costs.iter().map(DaBlockCosts::from).collect(); + tracing::info!("da_block_costs: {:?}", da_block_costs); + if let Some(cost) = raw_da_block_costs.last() { + self.last_recorded_height = Some(BlockHeight::from(cost.end_height)); } - .await?; - - let Some(ref raw_da_block_costs) = raw_da_block_costs else { - // TODO: This is really annoying if there haven't been any costs yet. Do we need this? - // Gonna return `Option::None` for now - // return Err(anyhow!("No response from block committer")) - return Ok(None) - }; - - let da_block_costs = self.last_raw_da_block_costs.iter().fold( - Ok(raw_da_block_costs.into()), - |costs: DaBlockCostsResult, last_value| { - let costs = costs.expect("Defined to be OK"); - let blob_size_bytes = costs - .bundle_size_bytes - .checked_sub(last_value.total_size_bytes) - .ok_or(anyhow!("Blob size bytes underflow"))?; - let blob_cost_wei = raw_da_block_costs - .total_cost - .checked_sub(last_value.total_cost) - .ok_or(anyhow!("Blob cost wei underflow"))?; - Ok(DaBlockCosts { - bundle_size_bytes: blob_size_bytes, - blob_cost_wei, - ..costs - }) - }, - )?; - self.last_raw_da_block_costs = Some(raw_da_block_costs.clone()); - Ok(Some(da_block_costs)) + Ok(da_block_costs) } - async fn set_last_value(&mut self, bundle_id: u32) -> DaBlockCostsResult<()> { - self.last_raw_da_block_costs = self.client.get_costs_by_seqno(bundle_id).await?; + + async fn set_last_value(&mut self, height: BlockHeight) -> DaBlockCostsResult<()> { + self.last_recorded_height = Some(height); Ok(()) } } @@ -142,54 +137,319 @@ impl BlockCommitterHttpApi { } } +const PAGE_SIZE: u32 = 10; #[async_trait::async_trait] impl BlockCommitterApi for BlockCommitterHttpApi { - async fn get_latest_costs(&self) -> DaBlockCostsResult> { + async fn get_costs_by_l2_block_number( + &self, + l2_block_number: u32, + ) -> DaBlockCostsResult> { + // Specific: http://localhost:8080/v1/costs?variant=specific&value=19098935&limit=5 if let Some(url) = &self.url { - let val = self.client.get(url).send().await?; - tracing::warn!("val: {:?}", val); - let response = val.json::>().await?; - tracing::warn!("Response: {:?}", response); - Ok(response) + tracing::info!("getting costs by l2 block number"); + let formatted_url = format!("{url}/v1/costs?variant=specific&value={l2_block_number}&limit={PAGE_SIZE}"); + tracing::info!("Formatted URL: {:?}", formatted_url); + let response = self.client.get(formatted_url).send().await?; + tracing::info!("response: {:?}", response); + let parsed = response.json::>().await?; + tracing::info!("parse: {:?}", parsed); + Ok(parsed) } else { - Ok(None) + Ok(vec![]) } } - async fn get_costs_by_seqno( - &self, - number: u32, - ) -> DaBlockCostsResult> { + async fn get_latest_costs(&self) -> DaBlockCostsResult> { + // Latest: http://localhost:8080/v1/costs?variant=latest&limit=5 if let Some(url) = &self.url { - let val = self - .client - .get(format!("{}/{}", url, number)) - .send() - .await?; - tracing::warn!("val: {:?}", val); - let response = val.json::>().await?; - tracing::warn!("Response: {:?}", response); - Ok(response) + tracing::info!("getting latest costs"); + let formatted_url = format!("{url}/v1/costs?variant=latest&limit=1"); + tracing::info!("Formatted URL: {:?}", formatted_url); + let response = self.client.get(formatted_url).send().await?; + tracing::info!("response: {:?}", response); + let raw_da_block_costs = response.json::>().await?; + tracing::info!("Parsed: {:?}", raw_da_block_costs); + // only take the first element, since we are only looking for the most recent + Ok(raw_da_block_costs.first().cloned()) } else { Ok(None) } } +} - async fn get_cost_bundles_by_range( - &self, - range: core::ops::Range, - ) -> DaBlockCostsResult>> { - if let Some(url) = &self.url { - let response = self - .client - .get(format!("{}/{}-{}", url, range.start, range.end)) - .send() - .await? - .json::>() - .await?; - Ok(response.into_iter().map(Some).collect()) - } else { - Ok(vec![]) +#[cfg(test)] +mod test_block_committer_http_api { + #![allow(non_snake_case)] + + use super::*; + use fake_server::FakeServer; + + #[test] + fn get_costs_by_l2_block_number__when_url_is_none__then_returns_empty_vec() { + let rt = tokio::runtime::Runtime::new().unwrap(); + + // given + let block_committer = BlockCommitterHttpApi::new(None); + let l2_block_number = 1; + + // when + let actual = rt.block_on(async { + block_committer + .get_costs_by_l2_block_number(l2_block_number) + .await + .unwrap() + }); + + // then + assert_eq!(actual.len(), 0); + } + #[test] + fn get_costs_by_l2_block_number__when_url_is_some__then_returns_expected_costs() { + let rt = tokio::runtime::Runtime::new().unwrap(); + let mut mock = FakeServer::new(); + let url = mock.url(); + + // given + let l2_block_number = 51; + let block_committer = BlockCommitterHttpApi::new(Some(url)); + + let too_early_count = 5; + let do_not_fit_count = 5; + + let mut current_height = 0; + let mut bundle_id = 0; + let mut da_block_height: u64 = 0; + + // shouldn't return + for _ in 0..too_early_count { + bundle_id += 1; + da_block_height += 1; + current_height += 1; + let start_height = current_height; + current_height += 9; + let end_height = current_height; + let costs = RawDaBlockCosts { + id: bundle_id, + start_height, + end_height, + da_block_height: DaBlockHeight::from(da_block_height), + cost: 1, + size: 1, + }; + mock.add_response(costs); + } + let mut expected = Vec::new(); + + // should return + for _ in 0..PAGE_SIZE { + bundle_id += 1; + da_block_height += 1; + current_height += 1; + let start_height = current_height; + current_height += 9; + let end_height = current_height; + let costs = RawDaBlockCosts { + id: bundle_id, + start_height, + end_height, + da_block_height: DaBlockHeight::from(da_block_height), + cost: 1, + size: 1, + }; + mock.add_response(costs.clone()); + expected.push(costs); + } + // don't fit + for _ in 0..do_not_fit_count { + bundle_id += 1; + da_block_height += 1; + current_height += 1; + let start_height = current_height; + current_height += 9; + let end_height = current_height; + let costs = RawDaBlockCosts { + id: bundle_id, + start_height, + end_height, + da_block_height: DaBlockHeight::from(da_block_height), + cost: 1, + size: 1, + }; + mock.add_response(costs); + } + + // when + mock.init(); + let actual = rt.block_on(async { + block_committer + .get_costs_by_l2_block_number(l2_block_number) + .await + .unwrap() + }); + + // then + assert_eq!(actual, expected); + } + + #[test] + fn get_latest_costs__when_url_is_none__then_returns_none() { + let rt = tokio::runtime::Runtime::new().unwrap(); + + // given + let block_committer = BlockCommitterHttpApi::new(None); + + // when + let actual = + rt.block_on(async { block_committer.get_latest_costs().await.unwrap() }); + + // then + assert_eq!(actual, None); + } + + #[test] + fn get_latest_costs__when_url_is_some__then_returns_expected_costs() { + let rt = tokio::runtime::Runtime::new().unwrap(); + let mut mock = FakeServer::new(); + let url = mock.url(); + + // given + let block_committer = BlockCommitterHttpApi::new(Some(url)); + let not_expected = RawDaBlockCosts { + id: 1, + start_height: 1, + end_height: 10, + da_block_height: 1u64.into(), + cost: 1, + size: 1, + }; + mock.add_response(not_expected); + let expected = RawDaBlockCosts { + id: 2, + start_height: 11, + end_height: 20, + da_block_height: 2u64.into(), + cost: 2, + size: 2, + }; + mock.add_response(expected.clone()); + + // when + let actual = + rt.block_on(async { block_committer.get_latest_costs().await.unwrap() }); + + // then + assert_eq!(actual, Some(expected)); + } +} +#[cfg(any(test, feature = "test-helpers"))] +pub mod fake_server { + use crate::v1::da_source_service::block_committer_costs::RawDaBlockCosts; + use mockito::Matcher::Any; + use std::{ + collections::HashMap, + sync::{ + Arc, + Mutex, + }, + }; + + pub struct FakeServer { + server: mockito::ServerGuard, + responses: Arc>>, + } + + impl FakeServer { + pub fn new() -> Self { + let server = mockito::Server::new(); + let responses = Arc::new(Mutex::new(Vec::new())); + let mut fake = Self { server, responses }; + fake.init(); + fake + } + + #[allow(unused_variables)] + pub fn init(&mut self) { + let shared_responses = self.responses.clone(); + self.server + .mock("GET", Any) + .with_status(201) + .with_body_from_request(move |request| { + // take the requested number and return the corresponding response from the `responses` hashmap + let path = request.path_and_query(); + tracing::info!("Path: {:?}", path); + let query = path.split("variant=").last().unwrap(); + tracing::info!("Query: {:?}", query); + let mut values = query.split('&'); + let variant = values.next().unwrap(); + tracing::info!("Variant: {:?}", variant); + match variant { + // Latest: http://localhost:8080/v1/costs?variant=latest&limit=5 + // We don't support `limit` yet!!!! + "latest" => { + let args = values.next().unwrap(); + let limit = + args.split('=').last().unwrap().parse::().unwrap(); + assert!(limit == 1); + let guard = shared_responses.lock().unwrap(); + let most_recent = guard + .iter() + .fold(None, |acc, x| match acc { + None => Some(x), + Some(acc) => { + if x.end_height > acc.end_height { + Some(x) + } else { + Some(acc) + } + } + }) + .cloned(); + let response: Vec = + most_recent.into_iter().collect(); + serde_json::to_string(&response).unwrap().into() + } + // Specific: http://localhost:8080/v1/costs?variant=specific&value=19098935&limit=5 + "specific" => { + let args = values.next().unwrap(); + let mut specific_values = args.split('='); + let height = + specific_values.last().unwrap().parse::().unwrap(); + tracing::info!("Height: {:?}", height); + let maybe_limit = values + .next() + .and_then(|x| x.split('=').last()) + .and_then(|x| x.parse::().ok()); + tracing::info!("Limit: {:?}", maybe_limit); + let guard = shared_responses.lock().unwrap(); + let response = guard + .iter() + .filter(|costs| costs.end_height >= height) + .take(maybe_limit.unwrap_or(usize::MAX)) + .cloned() + .collect::>(); + serde_json::to_string(&response).unwrap().into() + } + _ => { + panic!("Invalid variant: {}", variant); + } + } + }) + .expect_at_least(1) + .create(); + } + + pub fn add_response(&mut self, costs: RawDaBlockCosts) { + let mut guard = self.responses.lock().unwrap(); + guard.push(costs); + } + + pub fn url(&self) -> String { + self.server.url() + } + } + impl Default for FakeServer { + fn default() -> Self { + Self::new() } } } @@ -214,41 +474,32 @@ mod tests { async fn get_latest_costs(&self) -> DaBlockCostsResult> { Ok(self.value.clone()) } - async fn get_costs_by_seqno( + async fn get_costs_by_l2_block_number( &self, - seq_no: u32, - ) -> DaBlockCostsResult> { + l2_block_number: u32, + ) -> DaBlockCostsResult> { // arbitrary logic to generate a new value let mut value = self.value.clone(); if let Some(value) = &mut value { - value.bundle_id = seq_no; - value.blocks_heights = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - .to_vec() - .iter() - .map(|x| x * seq_no) - .collect(); + value.start_height = l2_block_number; + value.end_height = value.end_height + l2_block_number + 10; value.da_block_height = - value.da_block_height + ((seq_no + 1) as u64).into(); - value.total_cost += 1; - value.total_size_bytes += 1; + value.da_block_height + ((l2_block_number + 1) as u64).into(); + value.cost += 1; + value.size += 1; } - Ok(value) - } - async fn get_cost_bundles_by_range( - &self, - _: core::ops::Range, - ) -> DaBlockCostsResult>> { - Ok(vec![self.value.clone()]) + Ok(value.into_iter().collect()) } } fn test_da_block_costs() -> RawDaBlockCosts { RawDaBlockCosts { - bundle_id: 1, - blocks_heights: (0..10).collect(), + id: 1, + start_height: 1, + end_height: 10, da_block_height: 1u64.into(), - total_cost: 1, - total_size_bytes: 1, + cost: 1, + size: 1, } } @@ -257,98 +508,34 @@ mod tests { ) { // given let da_block_costs = test_da_block_costs(); - let expected = Some((&da_block_costs).into()); + let expected = vec![(&da_block_costs).into()]; let mock_api = MockBlockCommitterApi::new(Some(da_block_costs)); let mut block_committer = BlockCommitterDaBlockCosts::new(mock_api, None); // when - let actual = block_committer.request_da_block_cost().await.unwrap(); + let actual = block_committer.request_da_block_costs().await.unwrap(); // then assert_eq!(actual, expected); } #[tokio::test] - async fn request_da_block_cost__when_last_value_is_some__then_get_costs_by_seqno_is_called( + async fn request_da_block_cost__when_last_value_is_some__then_get_costs_by_l2_block_number_is_called( ) { // given let mut da_block_costs = test_da_block_costs(); + let da_block_costs_len = da_block_costs.end_height - da_block_costs.start_height; let mock_api = MockBlockCommitterApi::new(Some(da_block_costs.clone())); + let latest_height = BlockHeight::new(da_block_costs.end_height); let mut block_committer = - BlockCommitterDaBlockCosts::new(mock_api, Some(da_block_costs.clone())); - - // when - let actual = block_committer.request_da_block_cost().await.unwrap(); - - // then - assert_ne!(da_block_costs.blocks_heights, actual.unwrap().l2_blocks); - } - - // TODO: Do we need this? - // #[tokio::test] - // async fn request_da_block_cost__when_response_is_none__then_error() { - // // given - // let mock_api = MockBlockCommitterApi::new(None); - // let mut block_committer = BlockCommitterDaBlockCosts::new(mock_api, None); - // - // // when - // let result = block_committer.request_da_block_cost().await; - // - // // then - // assert!(result.is_err()); - // } - - struct UnderflowingMockBlockCommitterApi { - value: Option, - } - - impl UnderflowingMockBlockCommitterApi { - fn new(value: Option) -> Self { - Self { value } - } - } - - #[async_trait::async_trait] - impl BlockCommitterApi for UnderflowingMockBlockCommitterApi { - async fn get_latest_costs(&self) -> DaBlockCostsResult> { - Ok(self.value.clone()) - } - async fn get_costs_by_seqno( - &self, - seq_no: u32, - ) -> DaBlockCostsResult> { - // arbitrary logic to generate a new value - let mut value = self.value.clone(); - if let Some(value) = &mut value { - value.bundle_id = seq_no; - value.blocks_heights = - value.blocks_heights.iter().map(|x| x + seq_no).collect(); - value.da_block_height = value.da_block_height + 1u64.into(); - value.total_cost -= 1; - value.total_size_bytes -= 1; - } - Ok(value) - } - async fn get_cost_bundles_by_range( - &self, - _: core::ops::Range, - ) -> DaBlockCostsResult>> { - Ok(vec![self.value.clone()]) - } - } - - #[tokio::test] - async fn request_da_block_cost__when_underflow__then_error() { - // given - let da_block_costs = test_da_block_costs(); - let mock_api = UnderflowingMockBlockCommitterApi::new(Some(da_block_costs)); - let mut block_committer = BlockCommitterDaBlockCosts::new(mock_api, None); - let _ = block_committer.request_da_block_cost().await.unwrap(); + BlockCommitterDaBlockCosts::new(mock_api, Some(latest_height)); // when - let result = block_committer.request_da_block_cost().await; + let actual = block_committer.request_da_block_costs().await.unwrap(); // then - assert!(result.is_err()); + let l2_blocks = actual.first().unwrap().l2_blocks.clone(); + let range_len = l2_blocks.end() - l2_blocks.start(); + assert_ne!(da_block_costs_len, range_len); } } diff --git a/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs index 93c02f583f4..8b5094d12e4 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs @@ -5,6 +5,7 @@ use crate::v1::da_source_service::{ }, DaBlockCosts, }; +use fuel_core_types::fuel_types::BlockHeight; use std::sync::Arc; use tokio::sync::Notify; @@ -22,13 +23,11 @@ impl DummyDaBlockCosts { #[async_trait::async_trait] impl DaBlockCostsSource for DummyDaBlockCosts { - async fn request_da_block_cost( - &mut self, - ) -> DaBlockCostsResult> { + async fn request_da_block_costs(&mut self) -> DaBlockCostsResult> { match &self.value { Ok(da_block_costs) => { self.notifier.notify_waiters(); - Ok(Some(da_block_costs.clone())) + Ok(vec![da_block_costs.clone()]) } Err(err) => { self.notifier.notify_waiters(); @@ -37,7 +36,7 @@ impl DaBlockCostsSource for DummyDaBlockCosts { } } - async fn set_last_value(&mut self, _bundle_id: u32) -> DaBlockCostsResult<()> { + async fn set_last_value(&mut self, _height: BlockHeight) -> DaBlockCostsResult<()> { unimplemented!("This is a dummy implementation"); } } diff --git a/crates/services/gas_price_service/src/v1/da_source_service/service.rs b/crates/services/gas_price_service/src/v1/da_source_service/service.rs index 0c36c3012fc..675d82cda55 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/service.rs @@ -16,6 +16,7 @@ use tokio::{ use crate::v1::da_source_service::DaBlockCosts; pub use anyhow::Result; +use fuel_core_types::fuel_types::BlockHeight; #[derive(Clone)] pub struct SharedState(Sender); @@ -58,9 +59,10 @@ where } async fn process_block_costs(&mut self) -> Result<()> { - let da_block_costs_res = self.source.request_da_block_cost().await; + let da_block_costs_res = self.source.request_da_block_costs().await; tracing::debug!("Received block costs: {:?}", da_block_costs_res); - if let Some(da_block_costs) = da_block_costs_res? { + let da_block_costs = da_block_costs_res?; + for da_block_costs in da_block_costs { self.shared_state.0.send(da_block_costs)?; } Ok(()) @@ -71,8 +73,8 @@ where /// da block costs in a way they see fit #[async_trait::async_trait] pub trait DaBlockCostsSource: Send + Sync { - async fn request_da_block_cost(&mut self) -> Result>; - async fn set_last_value(&mut self, bundle_id: u32) -> Result<()>; + async fn request_da_block_costs(&mut self) -> Result>; + async fn set_last_value(&mut self, block_height: BlockHeight) -> Result<()>; } #[async_trait::async_trait] diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 679d6372646..1c773c3a7e0 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -12,9 +12,9 @@ use crate::{ }, ports::{ GasPriceServiceAtomicStorage, - GetDaBundleId, + GetLatestRecordedHeight, GetMetadataStorage, - SetDaBundleId, + SetLatestRecordedHeight, SetMetadataStorage, }, v0::metadata::V0Metadata, @@ -161,31 +161,26 @@ where ) -> anyhow::Result<()> { let capacity = Self::validate_block_gas_capacity(block_gas_capacity)?; let mut storage_tx = self.storage_tx_provider.begin_transaction()?; - let prev_height = height.saturating_sub(1); - let mut bundle_id = storage_tx - .get_bundle_id(&BlockHeight::from(prev_height)) + let mut latest_recorded_height = storage_tx + .get_recorded_height() .map_err(|err| anyhow!(err))?; for da_block_costs in &self.da_block_costs_buffer { tracing::debug!("Updating DA block costs: {:?}", da_block_costs); + let l2_blocks = da_block_costs.l2_blocks.clone(); + let end = *l2_blocks.end(); self.algorithm_updater.update_da_record_data( - &da_block_costs.l2_blocks, + l2_blocks, da_block_costs.bundle_size_bytes, da_block_costs.blob_cost_wei, &mut storage_tx.as_unrecorded_blocks(), )?; - bundle_id = Some(da_block_costs.bundle_id); + latest_recorded_height = Some(BlockHeight::from(end)); } - if let Some(bundle_id) = bundle_id { + if let Some(recorded_height) = latest_recorded_height { storage_tx - .set_bundle_id(&BlockHeight::from(height), bundle_id) - .map_err(|err| anyhow!(err))?; - } - - if let Some(bundle_id) = bundle_id { - storage_tx - .set_bundle_id(&BlockHeight::from(height), bundle_id) + .set_recorded_height(recorded_height) .map_err(|err| anyhow!(err))?; } @@ -372,9 +367,9 @@ mod tests { use crate::{ common::{ fuel_core_storage_adapter::storage::{ - BundleIdTable, GasPriceColumn, GasPriceColumn::UnrecordedBlocks, + RecordedHeights, UnrecordedBlocksTable, }, gas_price_algorithm::SharedGasPriceAlgo, @@ -576,7 +571,7 @@ mod tests { DummyDaBlockCosts::new( Ok(DaBlockCosts { bundle_id: 1, - l2_blocks: (1..2).collect(), + l2_blocks: 1..=1, blob_cost_wei: u128::MAX, // Very expensive to trigger a change bundle_size_bytes: 3000, }), @@ -632,11 +627,11 @@ mod tests { } #[tokio::test] - async fn run__responses_from_da_service_update_bundle_id_in_storage() { + async fn run__responses_from_da_service_update_recorded_height_in_storage() { // given - let bundle_id = 1234; - let block_height = 2; - let l2_block_2 = BlockInfo::Block { + let recorded_block_height = 100; + let block_height = 200; + let l2_block = BlockInfo::Block { height: block_height, gas_used: 60, block_gas_capacity: 100, @@ -669,8 +664,8 @@ mod tests { let da_source = DaSourceService::new( DummyDaBlockCosts::new( Ok(DaBlockCosts { - bundle_id, - l2_blocks: (1..2).collect(), + bundle_id: 8765, + l2_blocks: 1..=recorded_block_height, blob_cost_wei: 9000, bundle_size_bytes: 3000, }), @@ -694,20 +689,23 @@ mod tests { service.run(&mut watcher).await; tokio::time::sleep(Duration::from_millis(100)).await; - l2_block_sender.send(l2_block_2).await.unwrap(); + l2_block_sender.send(l2_block).await.unwrap(); // when service.run(&mut watcher).await; tokio::time::sleep(Duration::from_millis(100)).await; // then - let latest_bundle_id = service + let latest_recorded_block_height = service .storage_tx_provider - .storage::() - .get(&BlockHeight::from(block_height)) + .storage::() + .get(&()) .unwrap() .unwrap(); - assert_eq!(*latest_bundle_id, bundle_id); + assert_eq!( + *latest_recorded_block_height, + BlockHeight::from(recorded_block_height) + ); service.shutdown().await.unwrap(); } diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 349a090e97c..1a20b98fb5c 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -20,10 +20,10 @@ use crate::{ ports::{ GasPriceData, GasPriceServiceAtomicStorage, - GetDaBundleId, + GetLatestRecordedHeight, GetMetadataStorage, L2Data, - SetDaBundleId, + SetLatestRecordedHeight, SetMetadataStorage, }, v1::{ @@ -170,8 +170,8 @@ impl GetMetadataStorage for ErroringPersistedData { } } -impl GetDaBundleId for ErroringPersistedData { - fn get_bundle_id(&self, _block_height: &BlockHeight) -> GasPriceResult> { +impl GetLatestRecordedHeight for ErroringPersistedData { + fn get_recorded_height(&self) -> GasPriceResult> { Err(GasPriceError::CouldNotFetchDARecord(anyhow!("boo!"))) } } @@ -215,18 +215,14 @@ impl UnrecordedBlocks for UnimplementedStorageTx { } } -impl SetDaBundleId for UnimplementedStorageTx { - fn set_bundle_id( - &mut self, - _block_height: &BlockHeight, - _bundle_id: u32, - ) -> GasPriceResult<()> { +impl SetLatestRecordedHeight for UnimplementedStorageTx { + fn set_recorded_height(&mut self, _bundle_id: BlockHeight) -> GasPriceResult<()> { unimplemented!() } } -impl GetDaBundleId for UnimplementedStorageTx { - fn get_bundle_id(&self, _block_height: &BlockHeight) -> GasPriceResult> { +impl GetLatestRecordedHeight for UnimplementedStorageTx { + fn get_recorded_height(&self) -> GasPriceResult> { unimplemented!() } } @@ -243,7 +239,7 @@ impl AsUnrecordedBlocks for UnimplementedStorageTx { struct FakeDABlockCost { da_block_costs: Receiver, - bundle_id: Arc>>, + latest_received_height: Arc>>, } impl FakeDABlockCost { @@ -251,37 +247,38 @@ impl FakeDABlockCost { let (_sender, receiver) = tokio::sync::mpsc::channel(1); Self { da_block_costs: receiver, - bundle_id: Arc::new(Mutex::new(None)), + latest_received_height: Arc::new(Mutex::new(None)), } } fn new(da_block_costs: Receiver) -> Self { Self { da_block_costs, - bundle_id: Arc::new(Mutex::new(None)), + latest_received_height: Arc::new(Mutex::new(None)), } } - fn never_returns_with_handle_to_bundle_id() -> (Self, Arc>>) { + fn never_returns_with_handle_to_last_height( + ) -> (Self, Arc>>) { let (_sender, receiver) = tokio::sync::mpsc::channel(1); - let bundle_id = Arc::new(Mutex::new(None)); + let height = Arc::new(Mutex::new(None)); let service = Self { da_block_costs: receiver, - bundle_id: bundle_id.clone(), + latest_received_height: height.clone(), }; - (service, bundle_id) + (service, height) } } #[async_trait::async_trait] impl DaBlockCostsSource for FakeDABlockCost { - async fn request_da_block_cost(&mut self) -> Result> { + async fn request_da_block_costs(&mut self) -> Result> { let costs = self.da_block_costs.recv().await.unwrap(); - Ok(Some(costs)) + Ok(vec![costs]) } - async fn set_last_value(&mut self, bundle_id: u32) -> Result<()> { - self.bundle_id.lock().unwrap().replace(bundle_id); + async fn set_last_value(&mut self, height: BlockHeight) -> Result<()> { + self.latest_received_height.lock().unwrap().replace(height); Ok(()) } } @@ -696,8 +693,8 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { #[tokio::test] async fn uninitialized_task__init__starts_da_service_with_bundle_id_in_storage() { // given - let block_height = 1; - let bundle_id: u32 = 123; + let block_height = 100; + let recorded_height: u32 = 200; let original_metadata = arbitrary_metadata(); let different_config = different_arb_config(); @@ -708,11 +705,12 @@ async fn uninitialized_task__init__starts_da_service_with_bundle_id_in_storage() let settings = FakeSettings::default(); let block_stream = empty_block_stream(); let on_chain_db = FakeOnChainDb::new(different_l2_block); - let (da_cost_source, bundle_id_handle) = - FakeDABlockCost::never_returns_with_handle_to_bundle_id(); + let (da_cost_source, recorded_block_height_handle) = + FakeDABlockCost::never_returns_with_handle_to_last_height(); let mut inner = gas_price_database_with_metadata(&original_metadata); let mut tx = inner.begin_transaction().unwrap(); - tx.set_bundle_id(&block_height.into(), bundle_id).unwrap(); + tx.set_recorded_height(BlockHeight::from(recorded_height)) + .unwrap(); StorageTransaction::commit_transaction(tx).unwrap(); let service = UninitializedTask::new( different_config.clone(), @@ -730,8 +728,8 @@ async fn uninitialized_task__init__starts_da_service_with_bundle_id_in_storage() service.init(&StateWatcher::started()).await.unwrap(); // then - let actual = bundle_id_handle.lock().unwrap(); - let expected = Some(bundle_id); + let actual = recorded_block_height_handle.lock().unwrap(); + let expected = Some(BlockHeight::new(recorded_height)); assert_eq!(*actual, expected); } diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 10ca224fdb4..db77274ccca 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -21,10 +21,10 @@ use crate::{ GasPriceData, GasPriceServiceAtomicStorage, GasPriceServiceConfig, - GetDaBundleId, + GetLatestRecordedHeight, GetMetadataStorage, L2Data, - SetDaBundleId, + SetLatestRecordedHeight, SetMetadataStorage, }, v1::{ @@ -157,21 +157,14 @@ where self.block_stream, ); - if let Some(bundle_id) = - self.gas_price_db.get_bundle_id(&self.gas_metadata_height)? - { - self.da_source.set_last_value(bundle_id).await?; + if let Some(last_recorded_height) = self.gas_price_db.get_recorded_height()? { + self.da_source.set_last_value(last_recorded_height).await?; + tracing::info!("Set last recorded height to {}", last_recorded_height); } let poll_duration = self .config .da_poll_interval .map(|x| Duration::from_millis(x.into())); - // TODO: Dupe code - if let Some(bundle_id) = - self.gas_price_db.get_bundle_id(&self.gas_metadata_height)? - { - self.da_source.set_last_value(bundle_id).await?; - } let da_service = DaSourceService::new(self.da_source, poll_duration); let da_service_runner = ServiceRunner::new(da_service); da_service_runner.start_and_await().await?; diff --git a/tests/Cargo.toml b/tests/Cargo.toml index bae9c54fc2d..6e9a03ce96d 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -38,7 +38,9 @@ fuel-core-bin = { path = "../bin/fuel-core", features = ["parquet", "p2p"] } fuel-core-client = { path = "../crates/client", features = ["test-helpers"] } fuel-core-compression = { path = "../crates/compression" } fuel-core-executor = { workspace = true, features = ["test-helpers"] } -fuel-core-gas-price-service = { path = "../crates/services/gas_price_service" } +fuel-core-gas-price-service = { path = "../crates/services/gas_price_service", features = [ + "test-helpers", +] } fuel-core-p2p = { path = "../crates/services/p2p", features = [ "test-helpers", ], optional = true } @@ -59,7 +61,6 @@ hyper = { workspace = true, features = ["server"] } insta = { workspace = true } itertools = { workspace = true } k256 = { version = "0.13.3", features = ["ecdsa-core"] } -mockito = "1.6.1" postcard = { workspace = true } primitive-types = { workspace = true, default-features = false } rand = { workspace = true } diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 6ac0cf772cd..9d4ab9bff72 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -26,12 +26,19 @@ use fuel_core_client::client::{ FuelClient, }; use fuel_core_gas_price_service::{ - common::fuel_core_storage_adapter::storage::GasPriceMetadata, + common::fuel_core_storage_adapter::storage::{ + GasPriceMetadata, + RecordedHeights, + }, ports::{ GasPriceData, + GetLatestRecordedHeight, GetMetadataStorage, }, - v1::metadata::V1Metadata, + v1::{ + da_source_service::block_committer_costs::fake_server::FakeServer, + metadata::V1Metadata, + }, }; use fuel_core_poa::Trigger; use fuel_core_storage::{ @@ -54,7 +61,6 @@ use fuel_core_types::{ }, services::executor::TransactionExecutionResult, }; -use mockito::Matcher::Any; use rand::Rng; use std::{ collections::HashMap, @@ -285,24 +291,25 @@ async fn estimate_gas_price__is_greater_than_actual_price_at_desired_height() { assert!(estimated >= real); } -#[tokio::test] -async fn estimate_gas_price__returns_min_gas_price_if_starting_gas_price_is_zero() { - const MIN_GAS_PRICE: u64 = 1; - - // Given - let mut node_config = Config::local_node(); - node_config.min_exec_gas_price = MIN_GAS_PRICE; - node_config.starting_exec_gas_price = 0; - let srv = FuelService::new_node(node_config.clone()).await.unwrap(); - let client = FuelClient::from(srv.bound_address); - - // When - let result = client.estimate_gas_price(10).await.unwrap(); - - // Then - let actual = result.gas_price.0; - assert_eq!(MIN_GAS_PRICE, actual) -} +// TODO: this behavior is changing with https://github.com/FuelLabs/fuel-core/pull/2501 +// #[tokio::test] +// async fn estimate_gas_price__returns_min_gas_price_if_starting_gas_price_is_zero() { +// const MIN_GAS_PRICE: u64 = 1; +// +// // Given +// let mut node_config = Config::local_node(); +// node_config.min_exec_gas_price = MIN_GAS_PRICE; +// node_config.starting_exec_gas_price = 0; +// let srv = FuelService::new_node(node_config.clone()).await.unwrap(); +// let client = FuelClient::from(srv.bound_address); +// +// // When +// let result = client.estimate_gas_price(10).await.unwrap(); +// +// // Then +// let actual = result.gas_price.0; +// assert_eq!(MIN_GAS_PRICE, actual) +// } // This test passed before this PR, but doesn't now #[tokio::test(flavor = "multi_thread")] @@ -464,13 +471,10 @@ async fn startup__can_override_gas_price_values_by_changing_config() { } use fuel_core_gas_price_service::v1::da_source_service::block_committer_costs::RawDaBlockCosts; +use fuel_core_storage::iter::IterDirection; #[test] fn produce_block__l1_committed_block_effects_gas_price() { - // let _ = tracing_subscriber::fmt() - // .with_max_level(tracing::Level::DEBUG) - // .try_init(); - let rt = tokio::runtime::Runtime::new().unwrap(); // set up chain with single unrecorded block let mut args = vec![ @@ -500,11 +504,12 @@ fn produce_block__l1_committed_block_effects_gas_price() { let mut mock = FakeServer::new(); let url = mock.url(); let costs = RawDaBlockCosts { - bundle_id: 1, - blocks_heights: vec![1], + id: 1, + start_height: 1, + end_height: 1, da_block_height: DaBlockHeight(100), - total_cost: 100, - total_size_bytes: 100, + cost: 100, + size: 100, }; mock.add_response(costs); @@ -672,19 +677,19 @@ fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) }); let half_of_blocks = block_delay as u32 / 2; - let blocks_heights: Vec<_> = (1..half_of_blocks).collect(); - let count = blocks_heights.len() as u128; + let count = half_of_blocks; let block_bytes = 1000; let total_size_bytes = block_bytes * count as u32; let gas = 16 * total_size_bytes as u128; let cost_gwei = gas * 1; // blob gas price 1 gwei let cost = cost_gwei * 1_000_000_000; // Wei mock.add_response(RawDaBlockCosts { - bundle_id: 1, - blocks_heights, + id: 1, + start_height: 1, + end_height: half_of_blocks, da_block_height: DaBlockHeight(100), - total_cost: cost, - total_size_bytes, + cost, + size: total_size_bytes, }); let mut profits = Vec::new(); @@ -779,59 +784,24 @@ async fn produce_a_block(client: &FuelClient, rng: &mu let _ = client.produce_blocks(1, None).await.unwrap(); } -struct FakeServer { - server: mockito::ServerGuard, - responses: Arc, u32)>>, -} - -impl FakeServer { - fn new() -> Self { - let server = mockito::Server::new(); - let responses = Arc::new(Mutex::new((HashMap::new(), 0))); - let mut fake = Self { server, responses }; - fake.init(); - fake - } - - pub fn init(&mut self) { - let shared_responses = self.responses.clone(); - self.server - .mock("GET", Any) - .with_status(201) - .with_body_from_request(move |request| { - // take the requested number and return the corresponding response from the `responses` hashmap - let path = request.path(); - let maybe_sequence_number = - path.split('/').last().and_then(|x| x.parse::().ok()); - match maybe_sequence_number { - Some(sequence_number) => { - let guard = shared_responses.lock().unwrap(); - let responses = &guard.0; - let response = responses.get(&sequence_number).cloned(); - serde_json::to_string(&response).unwrap().into() - } - None => { - let guard = shared_responses.lock().unwrap(); - let responses = &guard.0; - let latest = &guard.1; - let response = responses.get(latest).cloned(); - serde_json::to_string(&response).unwrap().into() - } - } - }) - .expect_at_least(1) - .create(); - } - - pub fn add_response(&mut self, costs: RawDaBlockCosts) { - let mut guard = self.responses.lock().unwrap(); - let latest = guard.1; - let new_seq_no = latest + 1; - guard.0.insert(new_seq_no, costs); - guard.1 = new_seq_no; - } - - fn url(&self) -> String { - self.server.url() +#[test] +fn inspect_dbs() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .try_init(); + use fuel_core_storage::iter::IteratorOverTable; + + let db_path = "/Users/jamesturner/fuel/dev-net/.dev-net-db"; + let path = std::path::Path::new(db_path); + let db = + CombinedDatabase::open(path, 0, StateRewindPolicy::RewindFullRange, -1).unwrap(); + + let latest_recorded_blocks = db + .gas_price() + .iter_all::(Some(IterDirection::Reverse)); + tracing::info!("latest recorded blocks:"); + for block in latest_recorded_blocks { + let (_, recorded_height) = block.unwrap(); + tracing::info!("recorded height: {}", recorded_height); } } From 15c26f0815092f748c61d46cf2844977ebed16a9 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Wed, 18 Dec 2024 13:45:32 -0700 Subject: [PATCH 117/131] Remove test used for debugging --- tests/tests/gas_price.rs | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 9d4ab9bff72..dd5392e0c86 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -783,25 +783,3 @@ async fn produce_a_block(client: &FuelClient, rng: &mu } let _ = client.produce_blocks(1, None).await.unwrap(); } - -#[test] -fn inspect_dbs() { - let _ = tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) - .try_init(); - use fuel_core_storage::iter::IteratorOverTable; - - let db_path = "/Users/jamesturner/fuel/dev-net/.dev-net-db"; - let path = std::path::Path::new(db_path); - let db = - CombinedDatabase::open(path, 0, StateRewindPolicy::RewindFullRange, -1).unwrap(); - - let latest_recorded_blocks = db - .gas_price() - .iter_all::(Some(IterDirection::Reverse)); - tracing::info!("latest recorded blocks:"); - for block in latest_recorded_blocks { - let (_, recorded_height) = block.unwrap(); - tracing::info!("recorded height: {}", recorded_height); - } -} From 0dc2dc772461cd31219bc38c3c0b25b6b2055c15 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Thu, 19 Dec 2024 16:44:30 -0700 Subject: [PATCH 118/131] Use backward-compatible V0 metadata --- .../services/gas_price_service/src/v0/metadata.rs | 13 +++++++++++++ crates/services/gas_price_service/src/v0/tests.rs | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/crates/services/gas_price_service/src/v0/metadata.rs b/crates/services/gas_price_service/src/v0/metadata.rs index f42e33e041c..eff224becff 100644 --- a/crates/services/gas_price_service/src/v0/metadata.rs +++ b/crates/services/gas_price_service/src/v0/metadata.rs @@ -4,8 +4,17 @@ use fuel_gas_price_algorithm::v0::AlgorithmUpdaterV0; pub struct V0Metadata { /// The gas price to cover the execution of the next block pub new_exec_price: u64, + // Execution + /// The lowest the algorithm allows the exec gas price to go + pub min_exec_gas_price: u64, + /// The Percentage the execution gas price will change in a single block, either increase or decrease + /// based on the fullness of the last L2 block + pub exec_gas_price_change_percent: u64, /// The height for which the `new_exec_price` is calculated, which should be the _next_ block pub l2_block_height: u32, + /// The threshold of gas usage above and below which the gas price will increase or decrease + /// This is a percentage of the total capacity of the L2 block + pub l2_block_fullness_threshold_percent: u64, } pub struct V0AlgorithmConfig { @@ -19,7 +28,11 @@ impl From for V0Metadata { fn from(updater: AlgorithmUpdaterV0) -> Self { Self { new_exec_price: updater.new_exec_price, + min_exec_gas_price: updater.min_exec_gas_price, + exec_gas_price_change_percent: updater.exec_gas_price_change_percent, l2_block_height: updater.l2_block_height, + l2_block_fullness_threshold_percent: updater + .l2_block_fullness_threshold_percent, } } } diff --git a/crates/services/gas_price_service/src/v0/tests.rs b/crates/services/gas_price_service/src/v0/tests.rs index d29116e59d8..16c090edf6c 100644 --- a/crates/services/gas_price_service/src/v0/tests.rs +++ b/crates/services/gas_price_service/src/v0/tests.rs @@ -132,7 +132,10 @@ fn arbitrary_config() -> V0AlgorithmConfig { fn arbitrary_metadata() -> V0Metadata { V0Metadata { new_exec_price: 100, + min_exec_gas_price: 0, + exec_gas_price_change_percent: 0, l2_block_height: 0, + l2_block_fullness_threshold_percent: 0, } } @@ -339,6 +342,7 @@ async fn uninitialized_task__new__if_exists_already_reload_old_values_with_overr let V0Metadata { new_exec_price, l2_block_height, + .. } = original_metadata; let UninitializedTask { algo_updater, .. } = service; assert_eq!(algo_updater.new_exec_price, new_exec_price); From ea02c59f140905f5406d9fb08d2bcb368f1809a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Sat, 21 Dec 2024 00:29:04 +0100 Subject: [PATCH 119/131] Chore/non breaking cli arguments with sensible defaults (#2507) ### Context Based on the description in the [DA gas price push doc](https://www.notion.so/fuellabs/DA-Gas-Price-Final-Final-Push-15d2f2293f31805ab4d1fac196b83b38). ### PR description This PR reverts the `starting_gas_price` => `starting_exec_gas_price` CLI parameter name change to maintain backwards compatibility for existing configurations. In addition, a few sensible default values are introduced for parameters to ensure the gas price algo is operational for a new node even if these parameters are unset. --------- Co-authored-by: Mitchell Turner Co-authored-by: Aaryamann Challani <43716372+rymnc@users.noreply.github.com> --- .github/workflows/ci.yml | 12 +- bin/fuel-core/src/cli/run.rs | 22 +-- tests/test-helpers/src/fuel_core_driver.rs | 12 +- tests/tests/gas_price.rs | 127 ++++++++++++++++-- tests/tests/regenesis.rs | 16 +++ .../src/gas_price_algo_compatibility.rs | 24 +--- 6 files changed, 170 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ca3d379b5a..a94554842d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,19 +119,19 @@ jobs: - command: make args: check --locked - command: nextest - args: run --workspace + args: run --workspace --retries 3 --no-fail-fast - command: nextest - args: run --all-features --workspace + args: run --all-features --workspace --retries 3 --no-fail-fast - command: nextest - args: run -p fuel-core --no-default-features + args: run -p fuel-core --no-default-features --retries 3 --no-fail-fast - command: nextest - args: run -p fuel-core --lib executor --features wasm-executor + args: run -p fuel-core --lib executor --features wasm-executor --retries 3 --no-fail-fast env: FUEL_ALWAYS_USE_WASM=true - command: nextest - args: run -p fuel-core-client --no-default-features + args: run -p fuel-core-client --no-default-features --retries 3 --no-fail-fast - command: nextest - args: run -p fuel-core-chain-config --no-default-features + args: run -p fuel-core-chain-config --no-default-features --retries 3 --no-fail-fast # Don't split this command; this is a workaround. # We need to run `cargo check` first to fetch the locked dependencies # for `fuel-core 0.26.0`(because of the bug with `--offline` diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index a2bf44b3b4d..bd19f9b1415 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -189,12 +189,12 @@ pub struct Command { #[arg(long = "native-executor-version", env)] pub native_executor_version: Option, - /// The starting gas price for the network - #[arg(long = "starting-exec-gas-price", default_value = "0", env)] - pub starting_exec_gas_price: u64, + /// The starting execution gas price for the network + #[arg(long = "starting-gas-price", default_value = "1000", env)] + pub starting_gas_price: u64, /// The percentage change in gas price per block - #[arg(long = "gas-price-change-percent", default_value = "0", env)] + #[arg(long = "gas-price-change-percent", default_value = "10", env)] pub gas_price_change_percent: u16, /// The minimum allowed gas price @@ -206,19 +206,23 @@ pub struct Command { pub gas_price_threshold_percent: u8, /// Minimum DA gas price - #[arg(long = "min-da-gas-price", default_value = "0", env)] + #[arg(long = "min-da-gas-price", default_value = "10000000", env)] pub min_da_gas_price: u64, /// P component of DA gas price calculation - #[arg(long = "da-p-component", default_value = "0", env)] + /// **NOTE**: This is the **inverse** gain of a typical P controller. + /// Increasing this value will reduce gas price fluctuations. + #[arg(long = "da-p-component", default_value = "620090", env)] pub da_p_component: i64, /// D component of DA gas price calculation - #[arg(long = "da-d-component", default_value = "0", env)] + /// **NOTE**: This is the **inverse** anticipatory control factor of a typical PD controller. + /// Increasing this value will reduce the dampening effect of quick algorithm changes. + #[arg(long = "da-d-component", default_value = "3528576", env)] pub da_d_component: i64, /// Maximum DA gas price change percent - #[arg(long = "max-da-gas-price-change-percent", default_value = "0", env)] + #[arg(long = "max-da-gas-price-change-percent", default_value = "10", env)] pub max_da_gas_price_change_percent: u16, /// The URL for the DA Block Committer info @@ -323,7 +327,7 @@ impl Command { debug, utxo_validation, native_executor_version, - starting_exec_gas_price: starting_gas_price, + starting_gas_price, gas_price_change_percent, min_gas_price, gas_price_threshold_percent, diff --git a/tests/test-helpers/src/fuel_core_driver.rs b/tests/test-helpers/src/fuel_core_driver.rs index 4f2f3f8363b..127ce955d30 100644 --- a/tests/test-helpers/src/fuel_core_driver.rs +++ b/tests/test-helpers/src/fuel_core_driver.rs @@ -22,10 +22,18 @@ impl FuelCoreDriver { pub async fn spawn_feeless(extra_args: &[&str]) -> anyhow::Result { let mut args = vec![ - "--starting-exec-gas-price", + "--starting-gas-price", "0", "--gas-price-change-percent", "0", + "--min-da-gas-price", + "0", + "--da-p-component", + "0", + "--da-d-component", + "0", + "--max-da-gas-price-change-percent", + "0", ]; args.extend(extra_args); Self::spawn_with_directory(tempdir()?, &args).await @@ -36,7 +44,7 @@ impl FuelCoreDriver { extra_args: &[&str], ) -> anyhow::Result { let mut args = vec![ - "--starting-exec-gas-price", + "--starting-gas-price", "0", "--gas-price-change-percent", "0", diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index dd5392e0c86..0092cc19823 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -7,6 +7,7 @@ use crate::helpers::{ TestContext, TestSetupBuilder, }; +use ethers::types::Opcode; use fuel_core::{ chain_config::{ ChainConfig, @@ -54,6 +55,7 @@ use fuel_core_types::{ }, fuel_tx::{ consensus_parameters::ConsensusParametersV1, + AssetId, ConsensusParameters, Finalizable, Transaction, @@ -74,6 +76,7 @@ use std::{ time::Duration, }; use test_helpers::fuel_core_driver::FuelCoreDriver; +use wideint::MathOp; fn tx_for_gas_limit(max_fee_limit: Word) -> Transaction { TransactionBuilder::script(vec![], vec![]) @@ -83,18 +86,42 @@ fn tx_for_gas_limit(max_fee_limit: Word) -> Transaction { .into() } +fn infinite_loop_tx( + max_fee_limit: Word, + rng: &mut R, + asset_id: Option, +) -> Transaction { + let script = vec![op::jmp(RegId::ZERO)]; + let script_bytes = script.iter().flat_map(|op| op.to_bytes()).collect(); + let mut builder = TransactionBuilder::script(script_bytes, vec![]); + let asset_id = asset_id.unwrap_or_else(|| *builder.get_params().base_asset_id()); + builder + .max_fee_limit(max_fee_limit) + .script_gas_limit(800_000) + .add_unsigned_coin_input( + SecretKey::random(rng), + rng.gen(), + u32::MAX as u64, + asset_id, + Default::default(), + ) + .finalize() + .into() +} + fn arb_large_tx( max_fee_limit: Word, rng: &mut R, + asset_id: Option, ) -> Transaction { let mut script: Vec<_> = repeat(op::noop()).take(10_000).collect(); script.push(op::ret(RegId::ONE)); let script_bytes = script.iter().flat_map(|op| op.to_bytes()).collect(); let mut builder = TransactionBuilder::script(script_bytes, vec![]); - let asset_id = *builder.get_params().base_asset_id(); + let asset_id = asset_id.unwrap_or_else(|| *builder.get_params().base_asset_id()); builder .max_fee_limit(max_fee_limit) - .script_gas_limit(22430) + .script_gas_limit(600_000) .add_unsigned_coin_input( SecretKey::random(rng), rng.gen(), @@ -191,6 +218,10 @@ async fn produce_block__raises_gas_price() { node_config.exec_gas_price_change_percent = percent; node_config.exec_gas_price_threshold_percent = threshold; node_config.block_production = Trigger::Never; + node_config.da_p_component = 0; + node_config.da_d_component = 0; + node_config.max_da_gas_price_change_percent = 0; + node_config.min_da_gas_price = 0; let srv = FuelService::new_node(node_config.clone()).await.unwrap(); let client = FuelClient::from(srv.bound_address); @@ -199,7 +230,7 @@ async fn produce_block__raises_gas_price() { // when let arb_tx_count = 10; for i in 0..arb_tx_count { - let tx = arb_large_tx(189028 + i as Word, &mut rng); + let tx = arb_large_tx(18902800 + i as Word, &mut rng, None); let _status = client.submit(&tx).await.unwrap(); } // starting gas price @@ -236,6 +267,10 @@ async fn produce_block__lowers_gas_price() { node_config.exec_gas_price_change_percent = percent; node_config.exec_gas_price_threshold_percent = threshold; node_config.block_production = Trigger::Never; + node_config.da_p_component = 0; + node_config.da_d_component = 0; + node_config.max_da_gas_price_change_percent = 0; + node_config.min_da_gas_price = 0; let srv = FuelService::new_node(node_config.clone()).await.unwrap(); let client = FuelClient::from(srv.bound_address); @@ -244,7 +279,7 @@ async fn produce_block__lowers_gas_price() { // when let arb_tx_count = 5; for i in 0..arb_tx_count { - let tx = arb_large_tx(189028 + i as Word, &mut rng); + let tx = arb_large_tx(18902800 + i as Word, &mut rng, None); let _status = client.submit(&tx).await.unwrap(); } // starting gas price @@ -260,6 +295,64 @@ async fn produce_block__lowers_gas_price() { assert_eq!(expected, actual); } +#[tokio::test] +async fn produce_block__raises_gas_price_with_default_parameters() { + // given + let args = vec![ + "--debug", + "--poa-instant", + "false", + "--coinbase-recipient", + "0x1111111111111111111111111111111111111111111111111111111111111111", + "--min-da-gas-price", + "0", + "--da-p-component", + "0", + "--da-d-component", + "0", + "--starting-gas-price", + "1000", + "--gas-price-change-percent", + "10", + "--max-da-gas-price-change-percent", + "0", + ]; + let driver = FuelCoreDriver::spawn(&args).await.unwrap(); + + let starting_gas_price = 1000; + let expected_default_percentage_increase = 10; + + let expected_gas_price = + starting_gas_price * (100 + expected_default_percentage_increase) / 100; + + let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); + + let base_asset_id = driver + .client + .consensus_parameters(0) + .await + .unwrap() + .unwrap() + .base_asset_id() + .clone(); + + // when + let arb_tx_count = 20; + for _ in 0..arb_tx_count { + let tx = infinite_loop_tx(200_000_000, &mut rng, Some(base_asset_id)); + let _status = driver.client.submit(&tx).await.unwrap(); + } + + // starting gas price + let _ = driver.client.produce_blocks(1, None).await.unwrap(); + + // updated gas price + let _ = driver.client.produce_blocks(1, None).await.unwrap(); + let latest_gas_price = driver.client.latest_gas_price().await.unwrap().gas_price; + + assert_eq!(expected_gas_price, latest_gas_price); +} + #[tokio::test] async fn estimate_gas_price__is_greater_than_actual_price_at_desired_height() { // given @@ -319,7 +412,7 @@ async fn latest_gas_price__if_node_restarts_gets_latest_value() { "--debug", "--poa-instant", "true", - "--starting-exec-gas-price", + "--starting-gas-price", "1000", "--gas-price-change-percent", "100", @@ -474,7 +567,7 @@ use fuel_core_gas_price_service::v1::da_source_service::block_committer_costs::R use fuel_core_storage::iter::IterDirection; #[test] -fn produce_block__l1_committed_block_effects_gas_price() { +fn produce_block__l1_committed_block_affects_gas_price() { let rt = tokio::runtime::Runtime::new().unwrap(); // set up chain with single unrecorded block let mut args = vec![ @@ -484,8 +577,23 @@ fn produce_block__l1_committed_block_effects_gas_price() { "--min-da-gas-price", "100", ]; + + let mut default_args = args.clone(); + default_args.extend([ + "--da-p-component", + "0", + "--da-d-component", + "0", + "--starting-gas-price", + "0", + "--gas-price-change-percent", + "0", + "--max-da-gas-price-change-percent", + "0", + ]); + let (first_gas_price, temp_dir) = rt.block_on(async { - let driver = FuelCoreDriver::spawn(&args).await.unwrap(); + let driver = FuelCoreDriver::spawn(&default_args).await.unwrap(); driver.client.produce_blocks(1, None).await.unwrap(); let first_gas_price: u64 = driver .client @@ -531,15 +639,16 @@ fn produce_block__l1_committed_block_effects_gas_price() { let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) .await .unwrap(); - tokio::time::sleep(Duration::from_millis(2)).await; + tokio::time::sleep(Duration::from_millis(20)).await; driver.client.produce_blocks(1, None).await.unwrap(); - tokio::time::sleep(Duration::from_millis(2)).await; + tokio::time::sleep(Duration::from_millis(20)).await; driver.client.estimate_gas_price(0).await.unwrap().gas_price }) .into(); // then assert!(first_gas_price < new_gas_price); + rt.shutdown_timeout(tokio::time::Duration::from_millis(100)); } #[test] diff --git a/tests/tests/regenesis.rs b/tests/tests/regenesis.rs index 67e62fda911..3a3df24f7b8 100644 --- a/tests/tests/regenesis.rs +++ b/tests/tests/regenesis.rs @@ -448,6 +448,10 @@ async fn test_regenesis_message_proofs_are_preserved() -> anyhow::Result<()> { snapshot_dir.path().to_str().unwrap(), "--native-executor-version", latest_state_transition_version.as_str(), + "--starting-gas-price", + "0", + "--gas-price-change-percent", + "0", ]) .await?; @@ -729,6 +733,18 @@ async fn starting_empty_node_with_overwritten_poa_works() -> anyhow::Result<()> tmp_path.to_str().unwrap(), "--consensus-key", original_secret_key.to_string().as_str(), + "--min-da-gas-price", + "0", + "--da-p-component", + "0", + "--da-d-component", + "0", + "--max-da-gas-price-change-percent", + "0", + "--starting-gas-price", + "0", + "--gas-price-change-percent", + "0", ], ) .await; diff --git a/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs b/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs index a19a6ffdb30..4b4cd454268 100644 --- a/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs +++ b/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs @@ -29,7 +29,7 @@ use libp2p::{ use std::ops::Deref; #[tokio::test] -async fn latest_gas_price_algorithm_is_compatible_with_ignition() { +async fn v1_gas_price_metadata_updates_successfully_from_v0() { // Given let genesis_keypair = SecpKeypair::generate(); let hexed_secret = hex::encode(genesis_keypair.secret().to_bytes()); @@ -53,8 +53,8 @@ async fn latest_gas_price_algorithm_is_compatible_with_ignition() { "--starting-gas-price", starting_gas_price.to_string().as_str(), ]) - .await - .unwrap(); + .await + .unwrap(); let public_key = Keypair::from(genesis_keypair).public(); let genesis_peer_id = PeerId::from_public_key(&public_key); let genesis_multiaddr = default_multiaddr(genesis_port, genesis_peer_id); @@ -83,8 +83,8 @@ async fn latest_gas_price_algorithm_is_compatible_with_ignition() { "0", ], ) - .await - .unwrap(); + .await + .unwrap(); // When const BLOCKS_TO_PRODUCE: u32 = 10; @@ -106,16 +106,6 @@ async fn latest_gas_price_algorithm_is_compatible_with_ignition() { .unwrap() .deref() .clone(); - let UpdaterMetadata::V1(V1Metadata { - new_scaled_exec_price, - gas_price_factor, - .. - }) = metadata - else { - panic!("Expected V1Metadata, got {:?}", metadata); - }; - assert_eq!( - new_scaled_exec_price, - starting_gas_price * gas_price_factor.get() - ); + + assert!(matches!(metadata, UpdaterMetadata::V1(_))); } From c4a42dc01abb4b70cb46af907d4c9a26ef1f3108 Mon Sep 17 00:00:00 2001 From: Aaryamann Challani <43716372+rymnc@users.noreply.github.com> Date: Wed, 25 Dec 2024 05:10:22 +0530 Subject: [PATCH 120/131] fix(gas_price_service_v1): ensure that the algo updater is synced to l2 block height (#2504) ## Linked Issues/PRs - none ## Description We didn't have this check in the `uninitialized_task__init__if_metadata_behind_l2_height_then_sync` test, which caused us to miss the duped algo updater. ## Checklist - [ ] Breaking changes are clearly marked as such in the PR description and changelog - [ ] New behavior is reflected in tests - [ ] [The specification](https://github.com/FuelLabs/fuel-specs/) matches the implemented behavior (link update PR if changes are needed) ### Before requesting review - [x] I have reviewed the code myself - [ ] I have created follow-up issues caused by this PR and linked them here ### After merging, notify other teams [Add or remove entries as needed] - [ ] [Rust SDK](https://github.com/FuelLabs/fuels-rs/) - [ ] [Sway compiler](https://github.com/FuelLabs/sway/) - [ ] [Platform documentation](https://github.com/FuelLabs/devrel-requests/issues/new?assignees=&labels=new+request&projects=&template=NEW-REQUEST.yml&title=%5BRequest%5D%3A+) (for out-of-organization contributors, the person merging the PR will do this) - [ ] Someone else? --- crates/services/gas_price_service/src/v1/tests.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 1a20b98fb5c..f43671b3420 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -777,13 +777,19 @@ async fn uninitialized_task__init__if_metadata_behind_l2_height_then_sync() { empty_block_stream(), gas_price_db, FakeDABlockCost::never_returns(), - onchain_db, + onchain_db.clone(), ) .unwrap(); // when - service.init(&StateWatcher::started()).await.unwrap(); + let gas_price_service = service.init(&StateWatcher::started()).await.unwrap(); // then - // no panic + // sleep to allow the service to sync + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + let on_chain_height = u32::from(onchain_db.height); + let algo_updater_height = gas_price_service.algorithm_updater().l2_block_height; + + assert_eq!(on_chain_height, algo_updater_height); } From b6a29aeb3ad9a47d846cb35317c92fc13797e36c Mon Sep 17 00:00:00 2001 From: rymnc <43716372+rymnc@users.noreply.github.com> Date: Wed, 25 Dec 2024 06:47:15 +0530 Subject: [PATCH 121/131] fix: fmt --- crates/services/gas_price_service/src/v0/metadata.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/services/gas_price_service/src/v0/metadata.rs b/crates/services/gas_price_service/src/v0/metadata.rs index 9868f6b8050..d16f0c078ae 100644 --- a/crates/services/gas_price_service/src/v0/metadata.rs +++ b/crates/services/gas_price_service/src/v0/metadata.rs @@ -38,4 +38,4 @@ impl From for V0Metadata { .l2_block_fullness_threshold_percent, } } -} \ No newline at end of file +} From bb007d395b009d63fc00f70b363966edbec6a790 Mon Sep 17 00:00:00 2001 From: rymnc <43716372+rymnc@users.noreply.github.com> Date: Wed, 25 Dec 2024 08:41:49 +0530 Subject: [PATCH 122/131] chore: add backward compatibility test for db storage fix: remove block production --- .../forkless-upgrade/Cargo.toml | 2 + .../src/gas_price_algo_compatibility.rs | 77 ++++++++++++------- .../forkless-upgrade/src/genesis.rs | 2 +- 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/version-compatibility/forkless-upgrade/Cargo.toml b/version-compatibility/forkless-upgrade/Cargo.toml index 509880e0157..ec5c4feb2c2 100644 --- a/version-compatibility/forkless-upgrade/Cargo.toml +++ b/version-compatibility/forkless-upgrade/Cargo.toml @@ -50,3 +50,5 @@ version-36-fuel-core-bin = { version = "0.36.0", package = "fuel-core-bin", feat ] } version-36-fuel-core-client = { version = "0.36.0", package = "fuel-core-client" } version-36-fuel-core-services = { version = "0.36.0", package = "fuel-core-services" } +version-36-fuel-core-gas-price-service = { version = "0.36.0", package = "fuel-core-gas-price-service" } +version-36-fuel-core-storage = { version = "0.36.0", package = "fuel-core-storage" } diff --git a/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs b/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs index 4b4cd454268..ebdd1b1473c 100644 --- a/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs +++ b/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs @@ -9,15 +9,15 @@ use crate::tests_helper::{ }; use latest_fuel_core_gas_price_service::{ common::{ - fuel_core_storage_adapter::storage::GasPriceMetadata, - updater_metadata::UpdaterMetadata, + fuel_core_storage_adapter::storage::GasPriceMetadata as NewGasPriceMetadata, + updater_metadata::UpdaterMetadata as NewUpdaterMetadata, }, - ports::GasPriceData, + ports::GasPriceData as NewGasPriceData, v1::metadata::V1Metadata, }; use latest_fuel_core_storage::{ - transactional::AtomicView, - StorageAsRef, + transactional::AtomicView as NewAtomicView, + StorageAsRef as NewStorageAsRef, }; use libp2p::{ identity::{ @@ -27,6 +27,18 @@ use libp2p::{ PeerId, }; use std::ops::Deref; +use version_36_fuel_core_gas_price_service::fuel_gas_price_updater::{ + fuel_core_storage_adapter::storage::GasPriceMetadata as OldGasPriceMetadata, + UpdaterMetadata as OldUpdaterMetadata, + V0Metadata, +}; +use version_36_fuel_core_storage::{ + transactional::{ + AtomicView as OldAtomicView, + HistoricalView as OldHistoricalView, + }, + StorageAsRef as OldStorageAsRef, +}; #[tokio::test] async fn v1_gas_price_metadata_updates_successfully_from_v0() { @@ -53,15 +65,29 @@ async fn v1_gas_price_metadata_updates_successfully_from_v0() { "--starting-gas-price", starting_gas_price.to_string().as_str(), ]) - .await - .unwrap(); + .await + .unwrap(); + + let db = &old_driver.node.shared.database; + let latest_height = db.gas_price().latest_height().unwrap(); + let view = db.gas_price().latest_view().unwrap(); + let v0_metadata = match OldStorageAsRef::storage::(&view) + .get(&latest_height) + .unwrap() + .unwrap() + .deref() + .clone() + { + OldUpdaterMetadata::V0(v0) => v0, + }; + let public_key = Keypair::from(genesis_keypair).public(); let genesis_peer_id = PeerId::from_public_key(&public_key); let genesis_multiaddr = default_multiaddr(genesis_port, genesis_peer_id); let temp_dir = old_driver.kill().await; // Starting node that uses latest fuel core. - // It will connect to the genesis node and sync blocks. + // When let latest_keypair = SecpKeypair::generate(); let hexed_secret = hex::encode(latest_keypair.secret().to_bytes()); let latest_node = LatestFuelCoreDriver::spawn_with_directory( @@ -83,29 +109,26 @@ async fn v1_gas_price_metadata_updates_successfully_from_v0() { "0", ], ) - .await - .unwrap(); - - // When - const BLOCKS_TO_PRODUCE: u32 = 10; - latest_node - .client - .produce_blocks(BLOCKS_TO_PRODUCE, None) - .await - .unwrap(); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + .await + .unwrap(); // Then let db = &latest_node.node.shared.database; let latest_height = db.gas_price().latest_height().unwrap(); let view = db.gas_price().latest_view().unwrap(); - let metadata = view - .storage::() - .get(&latest_height.into()) - .unwrap() - .unwrap() - .deref() - .clone(); + let v1_metadata = V1Metadata::try_from( + NewStorageAsRef::storage::(&view) + .get(&latest_height.into()) + .unwrap() + .unwrap() + .deref() + .clone(), + ) + .unwrap(); - assert!(matches!(metadata, UpdaterMetadata::V1(_))); + assert_eq!(v0_metadata.l2_block_height, v1_metadata.l2_block_height); + assert_eq!( + v0_metadata.new_exec_price, + v1_metadata.new_scaled_exec_price * v1_metadata.gas_price_factor.get() + ); } diff --git a/version-compatibility/forkless-upgrade/src/genesis.rs b/version-compatibility/forkless-upgrade/src/genesis.rs index 702eaf0e075..6374e069f79 100644 --- a/version-compatibility/forkless-upgrade/src/genesis.rs +++ b/version-compatibility/forkless-upgrade/src/genesis.rs @@ -44,4 +44,4 @@ async fn test__genesis_block__hash() { ) .unwrap() ) -} \ No newline at end of file +} From eb202cf44b4933c2998ede76972cc7d69d3deba0 Mon Sep 17 00:00:00 2001 From: rymnc <43716372+rymnc@users.noreply.github.com> Date: Wed, 25 Dec 2024 23:23:38 +0530 Subject: [PATCH 123/131] test: produce blocks so that the metadata is upgraded --- .../src/gas_price_algo_compatibility.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs b/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs index ebdd1b1473c..8b7d04fadaa 100644 --- a/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs +++ b/version-compatibility/forkless-upgrade/src/gas_price_algo_compatibility.rs @@ -84,10 +84,10 @@ async fn v1_gas_price_metadata_updates_successfully_from_v0() { let public_key = Keypair::from(genesis_keypair).public(); let genesis_peer_id = PeerId::from_public_key(&public_key); let genesis_multiaddr = default_multiaddr(genesis_port, genesis_peer_id); + drop(view); let temp_dir = old_driver.kill().await; // Starting node that uses latest fuel core. - // When let latest_keypair = SecpKeypair::generate(); let hexed_secret = hex::encode(latest_keypair.secret().to_bytes()); let latest_node = LatestFuelCoreDriver::spawn_with_directory( @@ -112,6 +112,15 @@ async fn v1_gas_price_metadata_updates_successfully_from_v0() { .await .unwrap(); + // When + const BLOCKS_TO_PRODUCE: u32 = 1; + latest_node + .client + .produce_blocks(BLOCKS_TO_PRODUCE, None) + .await + .unwrap(); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + // Then let db = &latest_node.node.shared.database; let latest_height = db.gas_price().latest_height().unwrap(); @@ -126,8 +135,12 @@ async fn v1_gas_price_metadata_updates_successfully_from_v0() { ) .unwrap(); - assert_eq!(v0_metadata.l2_block_height, v1_metadata.l2_block_height); assert_eq!( + v0_metadata.l2_block_height + BLOCKS_TO_PRODUCE, + v1_metadata.l2_block_height + ); + // Assert that v1 behaves differently from v0. + assert_ne!( v0_metadata.new_exec_price, v1_metadata.new_scaled_exec_price * v1_metadata.gas_price_factor.get() ); From c29dae7226262a6b255ca1cb5bd9000fd4886d3c Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Sun, 29 Dec 2024 11:48:04 -0700 Subject: [PATCH 124/131] Add tests for cost recording --- crates/fuel-core/src/service.rs | 12 +- .../gas_price_service/src/v1/service.rs | 87 +++++++++++++- tests/tests/gas_price.rs | 111 +++++++++++++++--- 3 files changed, 186 insertions(+), 24 deletions(-) diff --git a/crates/fuel-core/src/service.rs b/crates/fuel-core/src/service.rs index 676b3f506f6..6b4f82aee8d 100644 --- a/crates/fuel-core/src/service.rs +++ b/crates/fuel-core/src/service.rs @@ -467,13 +467,6 @@ impl RunnableTask for Task { #[allow(non_snake_case)] #[cfg(test)] mod tests { - use std::{ - thread::sleep, - time::Duration, - }; - - use fuel_core_services::State; - use crate::{ service::{ Config, @@ -481,6 +474,11 @@ mod tests { }, ShutdownListener, }; + use fuel_core_services::State; + use std::{ + thread::sleep, + time::Duration, + }; #[tokio::test] async fn stop_sub_service_shutdown_all_services() { diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 1c773c3a7e0..602b086539b 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -263,7 +263,7 @@ where TaskNextAction::always_continue(res) } da_block_costs_res = self.da_source_channel.recv() => { - tracing::debug!("Received DA block costs: {:?}", da_block_costs_res); + tracing::error!("Received DA block costs: {:?}", da_block_costs_res); match da_block_costs_res { Ok(da_block_costs) => { self.da_block_costs_buffer.push(da_block_costs); @@ -369,6 +369,7 @@ mod tests { fuel_core_storage_adapter::storage::{ GasPriceColumn, GasPriceColumn::UnrecordedBlocks, + GasPriceMetadata, RecordedHeights, UnrecordedBlocksTable, }, @@ -393,6 +394,7 @@ mod tests { metadata::{ updater_from_config, V1AlgorithmConfig, + V1Metadata, }, service::{ initialize_algorithm, @@ -709,4 +711,87 @@ mod tests { service.shutdown().await.unwrap(); } + + #[tokio::test] + async fn run__stores_correct_amount_for_costs() { + // given + let recorded_block_height = 100; + let block_height = 200; + let l2_block = BlockInfo::Block { + height: block_height, + gas_used: 60, + block_gas_capacity: 100, + block_bytes: 100, + block_fees: 100, + }; + + let (l2_block_sender, l2_block_receiver) = mpsc::channel(1); + let l2_block_source = FakeL2BlockSource { + l2_block: l2_block_receiver, + }; + + let metadata_storage = FakeMetadata::empty(); + // Configured so exec gas price doesn't change, only da gas price + let config = arbitrary_v1_algorithm_config(); + let mut inner = database(); + let mut tx = inner.write_transaction(); + tx.storage_as_mut::() + .insert(&BlockHeight::from(1), &100) + .unwrap(); + tx.commit().unwrap(); + let mut algo_updater = updater_from_config(&config); + let shared_algo = + SharedGasPriceAlgo::new_with_algorithm(algo_updater.algorithm()); + algo_updater.l2_block_height = block_height - 1; + algo_updater.last_profit = 10_000; + algo_updater.new_scaled_da_gas_price = 10_000_000; + + let notifier = Arc::new(tokio::sync::Notify::new()); + let blob_cost_wei = 9000; + let da_source = DaSourceService::new( + DummyDaBlockCosts::new( + Ok(DaBlockCosts { + bundle_id: 8765, + l2_blocks: 1..=recorded_block_height, + blob_cost_wei, + bundle_size_bytes: 3000, + }), + notifier.clone(), + ), + Some(Duration::from_millis(1)), + ); + let mut watcher = StateWatcher::started(); + let da_service_runner = ServiceRunner::new(da_source); + da_service_runner.start_and_await().await.unwrap(); + + let mut service = GasPriceServiceV1::new( + l2_block_source, + shared_algo, + algo_updater, + da_service_runner, + inner, + ); + let read_algo = service.next_block_algorithm(); + let initial_price = read_algo.next_gas_price(); + + service.run(&mut watcher).await; + tokio::time::sleep(Duration::from_millis(100)).await; + l2_block_sender.send(l2_block).await.unwrap(); + + // when + service.run(&mut watcher).await; + tokio::time::sleep(Duration::from_millis(100)).await; + + // then + let metadata: V1Metadata = service + .storage_tx_provider + .storage::() + .get(&block_height.into()) + .unwrap() + .and_then(|x| x.v1().cloned()) + .unwrap(); + assert_eq!(metadata.latest_known_total_da_cost_excess, blob_cost_wei); + + service.shutdown().await.unwrap(); + } } diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 0092cc19823..a86c5d5a67d 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -724,21 +724,7 @@ fn run__if_metadata_is_behind_l2_then_will_catch_up() { // }); } -#[test] -fn produce_block__algorithm_recovers_from_divergent_profit() { - _produce_block__algorithm_recovers_from_divergent_profit(110); -} - -fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) { - let _ = tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) - .try_init(); - let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); - - // given - let mut mock = FakeServer::new(); - let url = mock.url(); - let rt = tokio::runtime::Runtime::new().unwrap(); +fn node_config_with_da_committer_url(url: &str) -> Config { let block_gas_limit = 3_000_000; let chain_config = ChainConfig { consensus_parameters: ConsensusParameters::V1(ConsensusParametersV1 { @@ -754,13 +740,29 @@ fn _produce_block__algorithm_recovers_from_divergent_profit(block_delay: usize) node_config.min_da_gas_price = starting_gas_price; node_config.max_da_gas_price_change_percent = 15; node_config.block_production = Trigger::Never; - node_config.da_committer_url = Some(url.clone()); + node_config.da_committer_url = Some(url.to_string()); node_config.da_poll_interval = Some(100); node_config.da_p_component = 224_000; node_config.da_d_component = 2_690_000; // node_config.da_p_component = 1; // node_config.da_d_component = 10; node_config.block_activity_threshold = 0; + node_config +} + +#[test] +fn produce_block__algorithm_recovers_from_divergent_profit() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .try_init(); + let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); + + // given + let mut mock = FakeServer::new(); + let url = mock.url(); + let rt = tokio::runtime::Runtime::new().unwrap(); + let node_config = node_config_with_da_committer_url(&url); + let block_delay = 110; let (srv, client) = rt.block_on(async { let srv = FuelService::new_node(node_config.clone()).await.unwrap(); @@ -892,3 +894,80 @@ async fn produce_a_block(client: &FuelClient, rng: &mu } let _ = client.produce_blocks(1, None).await.unwrap(); } + +#[test] +fn produce_block__costs_from_da_are_properly_recorded_in_metadata() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::ERROR) + .try_init(); + let mut rng = rand::rngs::StdRng::seed_from_u64(2322u64); + + // given + let mut mock = FakeServer::new(); + let url = mock.url(); + let rt = tokio::runtime::Runtime::new().unwrap(); + let node_config = node_config_with_da_committer_url(&url); + let l2_blocks = 1000; + let da_blocks = l2_blocks / 2; + + let (srv, client) = rt.block_on(async { + let srv = FuelService::new_node(node_config.clone()).await.unwrap(); + let client = FuelClient::from(srv.bound_address); + + for _b in 0..l2_blocks { + produce_a_block(&client, &mut rng).await; + } + let _ = client.produce_blocks(1, None).await.unwrap(); + + let height = srv.shared.database.gas_price().latest_height().unwrap(); + let metadata = srv + .shared + .database + .gas_price() + .get_metadata(&height) + .unwrap() + .and_then(|x| x.v1().cloned()) + .unwrap(); + tracing::info!("metadata: {:?}", metadata); + assert_eq!(metadata.latest_known_total_da_cost_excess, 0); + (srv, client) + }); + + // Add multiple cost responses that add up to `da_cost` + let blob_count = 5; + let mut total_cost = 0; + for i in 0..blob_count { + let blob_size = da_blocks / blob_count; + let cost = rng.gen_range(10_000_000..100_000_000); + let costs = RawDaBlockCosts { + id: i + 1, + start_height: blob_size * i + 1, + end_height: blob_size * i + blob_size, + da_block_height: DaBlockHeight(999999999 + i as u64), + cost, + size: 100, + }; + total_cost += cost; + mock.add_response(costs); + + rt.block_on(async { + tokio::time::sleep(Duration::from_millis(200)).await; + // when + let _ = client.produce_blocks(1, None).await.unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + + let height = srv.shared.database.gas_price().latest_height().unwrap(); + let metadata = srv + .shared + .database + .gas_price() + .get_metadata(&height) + .unwrap() + .and_then(|x| x.v1().cloned()) + .unwrap(); + + // then + assert_eq!(metadata.latest_known_total_da_cost_excess, total_cost); + }); + } +} From b380e999344ea7f44825e4ffe281025d4316bce2 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 6 Jan 2025 11:48:58 -0700 Subject: [PATCH 125/131] Fix test --- crates/services/gas_price_service/src/v1/service.rs | 1 + .../services/gas_price_service/src/v1/uninitialized_task.rs | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index e60a6f3947a..47bbde96504 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -375,6 +375,7 @@ where let v1_metadata = convert_to_v1_metadata(updater_metadata, config)?; v1_algorithm_from_metadata(v1_metadata, config) } else { + // TODO: Shouldn't we be using the `latest_block_height` here for `l2_block_height`? updater_from_config(config) }; diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index 83b4beebf85..a987d7603d3 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -136,9 +136,12 @@ where Some(gas_price) }) .unwrap_or(0); + let gas_price_metadata_height = gas_metadata_height + .map(|x| x.into()) + .unwrap_or(latest_block_height); let (algo_updater, shared_algo) = - initialize_algorithm(&config, latest_block_height, &gas_price_db)?; + initialize_algorithm(&config, gas_price_metadata_height, &gas_price_db)?; let latest_gas_price = LatestGasPrice::new(latest_block_height, latest_gas_price); From bc983cb7804233bb48ae2369bc059ef76f264bcb Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 6 Jan 2025 17:03:41 -0700 Subject: [PATCH 126/131] Cleanup mess from debugging --- .../adapters/fuel_gas_price_provider.rs | 9 +---- crates/fuel-core/src/service/sub_services.rs | 9 +++-- .../gas-price-analysis/src/simulation.rs | 1 - crates/fuel-gas-price-algorithm/src/v1.rs | 12 ++----- crates/services/gas_price_service/Cargo.toml | 2 +- .../block_committer_costs.rs | 23 ++++-------- .../src/v1/da_source_service/service.rs | 1 - .../gas_price_service/src/v1/service.rs | 9 +++-- .../src/v1/uninitialized_task.rs | 2 +- tests/tests/gas_price.rs | 35 +++---------------- 10 files changed, 25 insertions(+), 78 deletions(-) diff --git a/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs b/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs index fa8c84ca35e..3acc97c9519 100644 --- a/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs +++ b/crates/fuel-core/src/service/adapters/fuel_gas_price_provider.rs @@ -30,15 +30,12 @@ mod tests; #[derive(Debug)] /// Receives the next gas price algorithm via a shared `BlockGasPriceAlgo` instance pub struct FuelGasPriceProvider { - // // Scale the gas price down by this factor, for e.g. Wei to Gwei - // scaling_factor: u64, algorithm: SharedGasPriceAlgo, } impl Clone for FuelGasPriceProvider { fn clone(&self) -> Self { Self { - // scaling_factor: self.scaling_factor, algorithm: self.algorithm.clone(), } } @@ -46,11 +43,7 @@ impl Clone for FuelGasPriceProvider { impl FuelGasPriceProvider { pub fn new(algorithm: SharedGasPriceAlgo) -> Self { - Self { - // // Default scaling factor is 1_000_000_000, to convert Wei to Gwei - // scaling_factor: 1_000_000_000, - algorithm, - } + Self { algorithm } } } diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index f43e77f80f1..2e22b7e98ce 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -190,16 +190,15 @@ pub fn init_sub_services( let settings = consensus_parameters_provider.clone(); let block_stream = importer_adapter.events_shared_result(); - tracing::debug!("da_committer_url: {:?}", config.da_committer_url); let committer_api = BlockCommitterHttpApi::new(config.da_committer_url.clone()); - let da_source = BlockCommitterDaBlockCosts::new(committer_api, None); + let da_source = BlockCommitterDaBlockCosts::new(committer_api); let v1_config = GasPriceServiceConfig::from(config.clone()) .v1() .ok_or(anyhow!( "Gas price service configuration is not compatible with V1 algorithm" ))?; - let gas_price_service = new_gas_price_service_v1( + let gas_price_service_v1 = new_gas_price_service_v1( v1_config, genesis_block_height, settings, @@ -208,7 +207,7 @@ pub fn init_sub_services( da_source, database.on_chain().clone(), )?; - let (gas_price_algo, _) = &gas_price_service.shared; + let (gas_price_algo, _) = &gas_price_service_v1.shared; let gas_price_provider = FuelGasPriceProvider::new(gas_price_algo.clone()); let txpool = fuel_core_txpool::new_service( chain_id, @@ -346,7 +345,7 @@ pub fn init_sub_services( #[allow(unused_mut)] // `FuelService` starts and shutdowns all sub-services in the `services` order let mut services: SubServices = vec![ - Box::new(gas_price_service), + Box::new(gas_price_service_v1), Box::new(txpool), Box::new(consensus_parameters_provider_service), ]; diff --git a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs index 0080aa1a837..c4095d05f2c 100644 --- a/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs +++ b/crates/fuel-gas-price-algorithm/gas-price-analysis/src/simulation.rs @@ -291,7 +291,6 @@ fn fullness_and_bytes_per_block(size: usize, capacity: u64) -> Vec<(u64, u32)> { let fullness_noise: Vec<_> = std::iter::repeat(()) .take(size) .map(|_| rng.gen_range(-0.5..0.5)) - // .map(|val| val * capacity as f64) .collect(); const ROUGH_GAS_TO_BYTE_RATIO: f64 = 0.01; diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 7fe26ce10e3..628ef7d0254 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -282,11 +282,8 @@ impl L2ActivityTracker { } pub fn update(&mut self, block_usage: ClampedPercentage) { - tracing::info!("Block usage: {:?}", block_usage); - tracing::info!("Chain activity: {}", self.chain_activity); - tracing::info!("threshold: {:?}", self.block_activity_threshold); if block_usage < self.block_activity_threshold { - tracing::info!( + tracing::debug!( "Decreasing activity {:?} < {:?}", block_usage, self.block_activity_threshold @@ -393,7 +390,6 @@ impl AlgorithmUpdaterV1 { let rewards = self.clamped_rewards_as_i128(); // costs - tracing::info!("Block bytes: {}", block_bytes); self.update_projected_da_cost(block_bytes); let projected_total_da_cost = self.clamped_projected_cost_as_i128(); @@ -426,9 +422,7 @@ impl AlgorithmUpdaterV1 { } fn update_da_rewards(&mut self, fee_wei: u128) { - tracing::info!("Fee: {}", fee_wei); let block_da_reward = self.da_portion_of_fee(fee_wei); - tracing::info!("DA reward: {}", block_da_reward); self.total_da_rewards_excess = self.total_da_rewards_excess.saturating_add(block_da_reward); } @@ -512,8 +506,8 @@ impl AlgorithmUpdaterV1 { 0u64 } }); - tracing::info!("Profit: {}", self.last_profit); - tracing::info!( + tracing::debug!("Profit: {}", self.last_profit); + tracing::debug!( "DA gas price change: p: {}, d: {}, change: {}, new: {}", p, d, diff --git a/crates/services/gas_price_service/Cargo.toml b/crates/services/gas_price_service/Cargo.toml index efde85a485f..3a769368c01 100644 --- a/crates/services/gas_price_service/Cargo.toml +++ b/crates/services/gas_price_service/Cargo.toml @@ -34,7 +34,7 @@ tracing = { workspace = true } fuel-core-services = { workspace = true, features = ["test-helpers"] } fuel-core-storage = { workspace = true, features = ["test-helpers"] } fuel-core-types = { path = "./../../types", features = ["test-helpers"] } -mockito = { version = "1.6.1" } +mockito = { version = "1.6.1" } serde_json = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index fb6077472c4..3e8d3a3722b 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -79,10 +79,8 @@ impl From for DaBlockCosts { impl BlockCommitterDaBlockCosts { /// Create a new instance of the block committer da block costs source - pub fn new( - client: BlockCommitter, - last_recorded_height: Option, - ) -> Self { + pub fn new(client: BlockCommitter) -> Self { + let last_recorded_height = None; Self { client, last_recorded_height, @@ -106,10 +104,8 @@ where None => self.client.get_latest_costs().await?.into_iter().collect(), }; - tracing::info!("raw_da_block_costs: {:?}", raw_da_block_costs); let da_block_costs: Vec<_> = raw_da_block_costs.iter().map(DaBlockCosts::from).collect(); - tracing::info!("da_block_costs: {:?}", da_block_costs); if let Some(cost) = raw_da_block_costs.last() { self.last_recorded_height = Some(BlockHeight::from(cost.end_height)); } @@ -146,13 +142,10 @@ impl BlockCommitterApi for BlockCommitterHttpApi { ) -> DaBlockCostsResult> { // Specific: http://localhost:8080/v1/costs?variant=specific&value=19098935&limit=5 if let Some(url) = &self.url { - tracing::info!("getting costs by l2 block number"); + tracing::debug!("getting da costs by l2 block number: {l2_block_number}"); let formatted_url = format!("{url}/v1/costs?variant=specific&value={l2_block_number}&limit={PAGE_SIZE}"); - tracing::info!("Formatted URL: {:?}", formatted_url); let response = self.client.get(formatted_url).send().await?; - tracing::info!("response: {:?}", response); let parsed = response.json::>().await?; - tracing::info!("parse: {:?}", parsed); Ok(parsed) } else { Ok(vec![]) @@ -162,13 +155,9 @@ impl BlockCommitterApi for BlockCommitterHttpApi { async fn get_latest_costs(&self) -> DaBlockCostsResult> { // Latest: http://localhost:8080/v1/costs?variant=latest&limit=5 if let Some(url) = &self.url { - tracing::info!("getting latest costs"); let formatted_url = format!("{url}/v1/costs?variant=latest&limit=1"); - tracing::info!("Formatted URL: {:?}", formatted_url); let response = self.client.get(formatted_url).send().await?; - tracing::info!("response: {:?}", response); let raw_da_block_costs = response.json::>().await?; - tracing::info!("Parsed: {:?}", raw_da_block_costs); // only take the first element, since we are only looking for the most recent Ok(raw_da_block_costs.first().cloned()) } else { @@ -510,7 +499,7 @@ mod tests { let da_block_costs = test_da_block_costs(); let expected = vec![(&da_block_costs).into()]; let mock_api = MockBlockCommitterApi::new(Some(da_block_costs)); - let mut block_committer = BlockCommitterDaBlockCosts::new(mock_api, None); + let mut block_committer = BlockCommitterDaBlockCosts::new(mock_api); // when let actual = block_committer.request_da_block_costs().await.unwrap(); @@ -527,8 +516,8 @@ mod tests { let da_block_costs_len = da_block_costs.end_height - da_block_costs.start_height; let mock_api = MockBlockCommitterApi::new(Some(da_block_costs.clone())); let latest_height = BlockHeight::new(da_block_costs.end_height); - let mut block_committer = - BlockCommitterDaBlockCosts::new(mock_api, Some(latest_height)); + let mut block_committer = BlockCommitterDaBlockCosts::new(mock_api); + block_committer.set_last_value(latest_height).await.unwrap(); // when let actual = block_committer.request_da_block_costs().await.unwrap(); diff --git a/crates/services/gas_price_service/src/v1/da_source_service/service.rs b/crates/services/gas_price_service/src/v1/da_source_service/service.rs index 675d82cda55..90e5acdb7ed 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/service.rs @@ -112,7 +112,6 @@ where /// This function polls the source according to a polling interval /// described by the DaBlockCostsService async fn run(&mut self, state_watcher: &mut StateWatcher) -> TaskNextAction { - tracing::debug!("111111111111111111111111111111111"); tokio::select! { biased; _ = state_watcher.while_started() => { diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 47bbde96504..5ddffa0cc11 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -91,7 +91,6 @@ impl LatestGasPrice { pub struct GasPriceServiceV1 where DA: DaBlockCostsSource + 'static, - // AtomicStorage: GasPriceServiceAtomicStorage, { /// The algorithm that can be used in the next block shared_algo: SharedV1Algorithm, @@ -243,13 +242,13 @@ where )?; let metadata = self.algorithm_updater.clone().into(); - tracing::info!("Setting metadata: {:?}", metadata); + tracing::debug!("Setting metadata: {:?}", metadata); storage_tx .set_metadata(&metadata) .map_err(|err| anyhow!(err))?; AtomicStorage::commit_transaction(storage_tx)?; let new_algo = self.algorithm_updater.algorithm(); - tracing::info!("Updating gas price: {}", &new_algo.calculate()); + tracing::debug!("Updating gas price: {}", &new_algo.calculate()); self.shared_algo.update(new_algo).await; // Clear the buffer after committing changes self.da_block_costs_buffer.clear(); @@ -307,12 +306,12 @@ where TaskNextAction::Stop } l2_block_res = self.l2_block_source.get_l2_block() => { - tracing::info!("Received L2 block result: {:?}", l2_block_res); + tracing::debug!("Received L2 block result: {:?}", l2_block_res); let res = self.commit_block_data_to_algorithm(l2_block_res).await; TaskNextAction::always_continue(res) } da_block_costs_res = self.da_source_channel.recv() => { - tracing::error!("Received DA block costs: {:?}", da_block_costs_res); + tracing::debug!("Received DA block costs: {:?}", da_block_costs_res); match da_block_costs_res { Ok(da_block_costs) => { self.da_block_costs_buffer.push(da_block_costs); diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index a987d7603d3..cfcbfb21604 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -282,7 +282,7 @@ where { let first = metadata_height.saturating_add(1); let view = on_chain_db.latest_view()?; - tracing::info!( + tracing::debug!( "Syncing gas price metadata from {} to {}", first, latest_block_height diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index a86c5d5a67d..3fb687bd422 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -1,7 +1,4 @@ #![allow(non_snake_case)] -// TODO: REMOVE BEFORE MERGING -#![allow(dead_code)] -#![allow(unused_imports)] use crate::helpers::{ TestContext, @@ -37,12 +34,16 @@ use fuel_core_gas_price_service::{ GetMetadataStorage, }, v1::{ - da_source_service::block_committer_costs::fake_server::FakeServer, + da_source_service::block_committer_costs::{ + fake_server::FakeServer, + RawDaBlockCosts, + }, metadata::V1Metadata, }, }; use fuel_core_poa::Trigger; use fuel_core_storage::{ + iter::IterDirection, transactional::AtomicView, StorageAsRef, }; @@ -384,27 +385,6 @@ async fn estimate_gas_price__is_greater_than_actual_price_at_desired_height() { assert!(estimated >= real); } -// TODO: this behavior is changing with https://github.com/FuelLabs/fuel-core/pull/2501 -// #[tokio::test] -// async fn estimate_gas_price__returns_min_gas_price_if_starting_gas_price_is_zero() { -// const MIN_GAS_PRICE: u64 = 1; -// -// // Given -// let mut node_config = Config::local_node(); -// node_config.min_exec_gas_price = MIN_GAS_PRICE; -// node_config.starting_exec_gas_price = 0; -// let srv = FuelService::new_node(node_config.clone()).await.unwrap(); -// let client = FuelClient::from(srv.bound_address); -// -// // When -// let result = client.estimate_gas_price(10).await.unwrap(); -// -// // Then -// let actual = result.gas_price.0; -// assert_eq!(MIN_GAS_PRICE, actual) -// } - -// This test passed before this PR, but doesn't now #[tokio::test(flavor = "multi_thread")] async fn latest_gas_price__if_node_restarts_gets_latest_value() { // given @@ -563,9 +543,6 @@ async fn startup__can_override_gas_price_values_by_changing_config() { recovered_driver.kill().await; } -use fuel_core_gas_price_service::v1::da_source_service::block_committer_costs::RawDaBlockCosts; -use fuel_core_storage::iter::IterDirection; - #[test] fn produce_block__l1_committed_block_affects_gas_price() { let rt = tokio::runtime::Runtime::new().unwrap(); @@ -744,8 +721,6 @@ fn node_config_with_da_committer_url(url: &str) -> Config { node_config.da_poll_interval = Some(100); node_config.da_p_component = 224_000; node_config.da_d_component = 2_690_000; - // node_config.da_p_component = 1; - // node_config.da_d_component = 10; node_config.block_activity_threshold = 0; node_config } From 4a21da4f64797aa828889ac7987014e46f238b7d Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 6 Jan 2025 17:11:02 -0700 Subject: [PATCH 127/131] Remove unused imports, reconstitute test --- tests/tests/gas_price.rs | 117 +++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 66 deletions(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 3fb687bd422..aa8a4631c40 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -4,33 +4,25 @@ use crate::helpers::{ TestContext, TestSetupBuilder, }; -use ethers::types::Opcode; use fuel_core::{ chain_config::{ ChainConfig, StateConfig, }, - combined_database::CombinedDatabase, database::Database, - fuel_core_graphql_api::ports::worker::OnChainDatabase, service::{ Config, FuelService, }, - state::historical_rocksdb::StateRewindPolicy, }; use fuel_core_client::client::{ types::gas_price::LatestGasPrice, FuelClient, }; use fuel_core_gas_price_service::{ - common::fuel_core_storage_adapter::storage::{ - GasPriceMetadata, - RecordedHeights, - }, + common::fuel_core_storage_adapter::storage::GasPriceMetadata, ports::{ GasPriceData, - GetLatestRecordedHeight, GetMetadataStorage, }, v1::{ @@ -43,7 +35,6 @@ use fuel_core_gas_price_service::{ }; use fuel_core_poa::Trigger; use fuel_core_storage::{ - iter::IterDirection, transactional::AtomicView, StorageAsRef, }; @@ -66,18 +57,12 @@ use fuel_core_types::{ }; use rand::Rng; use std::{ - collections::HashMap, + self, iter::repeat, - num::NonZero, ops::Deref, - sync::{ - Arc, - Mutex, - }, time::Duration, }; use test_helpers::fuel_core_driver::FuelCoreDriver; -use wideint::MathOp; fn tx_for_gas_limit(max_fee_limit: Word) -> Transaction { TransactionBuilder::script(vec![], vec![]) @@ -644,61 +629,61 @@ fn run__if_metadata_is_behind_l2_then_will_catch_up() { "100", ]; let rt = tokio::runtime::Runtime::new().unwrap(); - let _temp_dir = rt.block_on(async { + let temp_dir = rt.block_on(async { let driver = FuelCoreDriver::spawn(&args).await.unwrap(); driver.client.produce_blocks(100, None).await.unwrap(); tokio::time::sleep(Duration::from_millis(100)).await; driver.kill().await }); - // // rollback 50 blocks - // let temp_dir = rt.block_on(async { - // let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) - // .await - // .unwrap(); - // for _ in 0..50 { - // driver - // .node - // .shared - // .database - // .gas_price() - // .rollback_last_block() - // .unwrap(); - // let gas_price_db_height = driver - // .node - // .shared - // .database - // .gas_price() - // .latest_height() - // .unwrap(); - // tracing::info!("gas price db height: {:?}", gas_price_db_height); - // } - // driver.kill().await - // }); - // - // // when - // // restart node - // rt.block_on(async { - // let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) - // .await - // .unwrap(); - // let onchain_db_height = driver - // .node - // .shared - // .database - // .on_chain() - // .latest_height_from_metadata() - // .unwrap() - // .unwrap(); - // let gas_price_db_height = driver - // .node - // .shared - // .database - // .gas_price() - // .latest_height() - // .unwrap(); - // assert_eq!(onchain_db_height, gas_price_db_height); - // }); + // rollback 50 blocks + let temp_dir = rt.block_on(async { + let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) + .await + .unwrap(); + for _ in 0..50 { + driver + .node + .shared + .database + .gas_price() + .rollback_last_block() + .unwrap(); + let gas_price_db_height = driver + .node + .shared + .database + .gas_price() + .latest_height() + .unwrap(); + tracing::info!("gas price db height: {:?}", gas_price_db_height); + } + driver.kill().await + }); + + // when + // restart node + rt.block_on(async { + let driver = FuelCoreDriver::spawn_with_directory(temp_dir, &args) + .await + .unwrap(); + let onchain_db_height = driver + .node + .shared + .database + .on_chain() + .latest_height_from_metadata() + .unwrap() + .unwrap(); + let gas_price_db_height = driver + .node + .shared + .database + .gas_price() + .latest_height() + .unwrap(); + assert_eq!(onchain_db_height, gas_price_db_height); + }); } fn node_config_with_da_committer_url(url: &str) -> Config { From a7eaa32077f7c9b5350752b2cf1ede677f2b1af2 Mon Sep 17 00:00:00 2001 From: Mitch Turner Date: Mon, 6 Jan 2025 18:25:10 -0700 Subject: [PATCH 128/131] Lint toml file --- crates/services/gas_price_service/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/services/gas_price_service/Cargo.toml b/crates/services/gas_price_service/Cargo.toml index 3a769368c01..efde85a485f 100644 --- a/crates/services/gas_price_service/Cargo.toml +++ b/crates/services/gas_price_service/Cargo.toml @@ -34,7 +34,7 @@ tracing = { workspace = true } fuel-core-services = { workspace = true, features = ["test-helpers"] } fuel-core-storage = { workspace = true, features = ["test-helpers"] } fuel-core-types = { path = "./../../types", features = ["test-helpers"] } -mockito = { version = "1.6.1" } +mockito = { version = "1.6.1" } serde_json = { workspace = true } tracing-subscriber = { workspace = true } From 31fd51fa799d9537ffaf8578a1b15d5462f354b6 Mon Sep 17 00:00:00 2001 From: Mitchell Turner Date: Tue, 7 Jan 2025 10:19:02 -0700 Subject: [PATCH 129/131] Fix for DA L2 sync race condition (#2512) ## Linked Issues/PRs ## Description In the case we are syncing from an existing network, it is possible for the L2 blocks to sync slower than the DA costs, in which case you will try to apply DA costs to an algorithm that is missing the corresponding unrecorded L2 blocks. This PR - introduces a `latest_l2_height` to the `DaSourceService` which will filter out any DA bundles that include L2 blocks after the current height - moves the recorded height concept into the `DaSourceService` where it probably should have been to begin with ## Checklist - [ ] Breaking changes are clearly marked as such in the PR description and changelog - [ ] New behavior is reflected in tests - [ ] [The specification](https://github.com/FuelLabs/fuel-specs/) matches the implemented behavior (link update PR if changes are needed) ### Before requesting review - [ ] I have reviewed the code myself - [ ] I have created follow-up issues caused by this PR and linked them here ### After merging, notify other teams [Add or remove entries as needed] - [ ] [Rust SDK](https://github.com/FuelLabs/fuels-rs/) - [ ] [Sway compiler](https://github.com/FuelLabs/sway/) - [ ] [Platform documentation](https://github.com/FuelLabs/devrel-requests/issues/new?assignees=&labels=new+request&projects=&template=NEW-REQUEST.yml&title=%5BRequest%5D%3A+) (for out-of-organization contributors, the person merging the PR will do this) - [ ] Someone else? --------- Co-authored-by: Aaryamann Challani <43716372+rymnc@users.noreply.github.com> --- .../gas_price_service/src/common/utils.rs | 2 +- .../src/v1/da_source_service.rs | 144 +++++++++++++++++- .../block_committer_costs.rs | 46 +++--- .../src/v1/da_source_service/dummy_costs.rs | 9 +- .../src/v1/da_source_service/service.rs | 98 ++++++++++-- .../gas_price_service/src/v1/service.rs | 60 +++++++- .../gas_price_service/src/v1/tests.rs | 102 ++++++++++--- .../src/v1/uninitialized_task.rs | 15 +- crates/services/src/service.rs | 1 + tests/tests/gas_price.rs | 5 + 10 files changed, 401 insertions(+), 81 deletions(-) diff --git a/crates/services/gas_price_service/src/common/utils.rs b/crates/services/gas_price_service/src/common/utils.rs index 5c944bd9fb1..09fd4c4b424 100644 --- a/crates/services/gas_price_service/src/common/utils.rs +++ b/crates/services/gas_price_service/src/common/utils.rs @@ -26,7 +26,7 @@ pub enum Error { pub type Result = core::result::Result; // Info required about the l2 block for the gas price algorithm -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum BlockInfo { // The genesis block of the L2 chain GenesisBlock, diff --git a/crates/services/gas_price_service/src/v1/da_source_service.rs b/crates/services/gas_price_service/src/v1/da_source_service.rs index 0c90b9a36d3..ad4bd1debfe 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service.rs @@ -23,14 +23,30 @@ mod tests { use super::*; use crate::v1::da_source_service::{ dummy_costs::DummyDaBlockCosts, - service::new_service, + service::{ + new_da_service, + DaSourceService, + DA_BLOCK_COSTS_CHANNEL_SIZE, + }, }; - use fuel_core_services::Service; + use fuel_core_services::{ + RunnableTask, + Service, + StateWatcher, + }; + use fuel_core_types::fuel_types::BlockHeight; use std::{ - sync::Arc, + sync::{ + Arc, + Mutex, + }, time::Duration, }; + fn latest_l2_height(height: u32) -> Arc> { + Arc::new(Mutex::new(BlockHeight::new(height))) + } + #[tokio::test] async fn run__when_da_block_cost_source_gives_value_shared_state_is_updated() { // given @@ -43,7 +59,12 @@ mod tests { let notifier = Arc::new(tokio::sync::Notify::new()); let da_block_costs_source = DummyDaBlockCosts::new(Ok(expected_da_cost.clone()), notifier.clone()); - let service = new_service(da_block_costs_source, Some(Duration::from_millis(1))); + let latest_l2_height = Arc::new(Mutex::new(BlockHeight::new(10u32))); + let service = new_da_service( + da_block_costs_source, + Some(Duration::from_millis(1)), + latest_l2_height, + ); let mut shared_state = &mut service.shared.subscribe(); // when @@ -62,7 +83,12 @@ mod tests { let notifier = Arc::new(tokio::sync::Notify::new()); let da_block_costs_source = DummyDaBlockCosts::new(Err(anyhow::anyhow!("boo!")), notifier.clone()); - let service = new_service(da_block_costs_source, Some(Duration::from_millis(1))); + let latest_l2_height = latest_l2_height(0); + let service = new_da_service( + da_block_costs_source, + Some(Duration::from_millis(1)), + latest_l2_height, + ); let mut shared_state = &mut service.shared.subscribe(); // when @@ -74,4 +100,112 @@ mod tests { assert!(da_block_costs_res.is_err()); service.stop_and_await().await.unwrap(); } + + #[tokio::test] + async fn run__will_not_return_cost_bundles_for_bundles_that_are_greater_than_l2_height( + ) { + // given + let l2_height = 4; + let unexpected_costs = DaBlockCosts { + bundle_id: 1, + l2_blocks: 0..=9, + bundle_size_bytes: 1024 * 128, + blob_cost_wei: 2, + }; + assert!(unexpected_costs.l2_blocks.end() > &l2_height); + let notifier = Arc::new(tokio::sync::Notify::new()); + let da_block_costs_source = + DummyDaBlockCosts::new(Ok(unexpected_costs.clone()), notifier.clone()); + let latest_l2_height = latest_l2_height(l2_height); + let service = new_da_service( + da_block_costs_source, + Some(Duration::from_millis(1)), + latest_l2_height, + ); + let mut shared_state = &mut service.shared.subscribe(); + + // when + service.start_and_await().await.unwrap(); + notifier.notified().await; + + // then + let err = shared_state.try_recv(); + tracing::info!("err: {:?}", err); + assert!(err.is_err()); + } + + #[tokio::test] + async fn run__filtered_da_block_costs_do_not_update_latest_recorded_block() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::DEBUG) + .try_init(); + + // given + let l2_height = 4; + let unexpected_costs = DaBlockCosts { + bundle_id: 1, + l2_blocks: 2..=9, + bundle_size_bytes: 1024 * 128, + blob_cost_wei: 2, + }; + assert!(unexpected_costs.l2_blocks.end() > &l2_height); + let notifier = Arc::new(tokio::sync::Notify::new()); + let da_block_costs_source = + DummyDaBlockCosts::new(Ok(unexpected_costs.clone()), notifier.clone()); + let latest_l2_height = latest_l2_height(l2_height); + let mut service = DaSourceService::new( + da_block_costs_source, + Some(Duration::from_millis(1)), + latest_l2_height, + None, + ); + let mut watcher = StateWatcher::started(); + + // when + let _ = service.run(&mut watcher).await; + + // then + let recorded_height = service.recorded_height(); + let expected = 1; + assert!(recorded_height.is_none()) + } + + #[tokio::test] + async fn run__recorded_height_updated_by_da_costs() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::DEBUG) + .try_init(); + + // given + let l2_height = 10; + let recorded_height = 9; + let unexpected_costs = DaBlockCosts { + bundle_id: 1, + l2_blocks: 2..=recorded_height, + bundle_size_bytes: 1024 * 128, + blob_cost_wei: 2, + }; + let notifier = Arc::new(tokio::sync::Notify::new()); + let da_block_costs_source = + DummyDaBlockCosts::new(Ok(unexpected_costs.clone()), notifier.clone()); + let latest_l2_height = latest_l2_height(l2_height); + let (sender, mut receiver) = + tokio::sync::broadcast::channel(DA_BLOCK_COSTS_CHANNEL_SIZE); + let mut service = DaSourceService::new_with_sender( + da_block_costs_source, + Some(Duration::from_millis(1)), + latest_l2_height, + None, + sender, + ); + let mut watcher = StateWatcher::started(); + + // when + let next = service.run(&mut watcher).await; + + // then + let actual = service.recorded_height().unwrap(); + let expected = BlockHeight::from(recorded_height); + assert_eq!(expected, actual); + } } diff --git a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs index 3e8d3a3722b..0df59f45bab 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/block_committer_costs.rs @@ -33,7 +33,6 @@ pub trait BlockCommitterApi: Send + Sync { /// which receives data from the block committer (only http api for now) pub struct BlockCommitterDaBlockCosts { client: BlockCommitter, - last_recorded_height: Option, } #[derive(Debug, Deserialize, Serialize, Clone, Default, PartialEq)] @@ -80,11 +79,7 @@ impl From for DaBlockCosts { impl BlockCommitterDaBlockCosts { /// Create a new instance of the block committer da block costs source pub fn new(client: BlockCommitter) -> Self { - let last_recorded_height = None; - Self { - client, - last_recorded_height, - } + Self { client } } } @@ -93,30 +88,25 @@ impl DaBlockCostsSource for BlockCommitterDaBlockCosts DaBlockCostsResult> { - let raw_da_block_costs: Vec<_> = - match self.last_recorded_height.and_then(|x| x.succ()) { - Some(ref next_height) => { - self.client - .get_costs_by_l2_block_number(*next_height.deref()) - .await? - } - None => self.client.get_latest_costs().await?.into_iter().collect(), - }; + async fn request_da_block_costs( + &mut self, + last_recorded_height: &Option, + ) -> DaBlockCostsResult> { + let raw_da_block_costs: Vec<_> = match last_recorded_height.and_then(|x| x.succ()) + { + Some(ref next_height) => { + self.client + .get_costs_by_l2_block_number(*next_height.deref()) + .await? + } + None => self.client.get_latest_costs().await?.into_iter().collect(), + }; let da_block_costs: Vec<_> = raw_da_block_costs.iter().map(DaBlockCosts::from).collect(); - if let Some(cost) = raw_da_block_costs.last() { - self.last_recorded_height = Some(BlockHeight::from(cost.end_height)); - } Ok(da_block_costs) } - - async fn set_last_value(&mut self, height: BlockHeight) -> DaBlockCostsResult<()> { - self.last_recorded_height = Some(height); - Ok(()) - } } pub struct BlockCommitterHttpApi { @@ -502,7 +492,7 @@ mod tests { let mut block_committer = BlockCommitterDaBlockCosts::new(mock_api); // when - let actual = block_committer.request_da_block_costs().await.unwrap(); + let actual = block_committer.request_da_block_costs(&None).await.unwrap(); // then assert_eq!(actual, expected); @@ -517,10 +507,12 @@ mod tests { let mock_api = MockBlockCommitterApi::new(Some(da_block_costs.clone())); let latest_height = BlockHeight::new(da_block_costs.end_height); let mut block_committer = BlockCommitterDaBlockCosts::new(mock_api); - block_committer.set_last_value(latest_height).await.unwrap(); // when - let actual = block_committer.request_da_block_costs().await.unwrap(); + let actual = block_committer + .request_da_block_costs(&Some(latest_height)) + .await + .unwrap(); // then let l2_blocks = actual.first().unwrap().l2_blocks.clone(); diff --git a/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs b/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs index 8b5094d12e4..82c352d975e 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/dummy_costs.rs @@ -23,7 +23,10 @@ impl DummyDaBlockCosts { #[async_trait::async_trait] impl DaBlockCostsSource for DummyDaBlockCosts { - async fn request_da_block_costs(&mut self) -> DaBlockCostsResult> { + async fn request_da_block_costs( + &mut self, + _latest_recorded_height: &Option, + ) -> DaBlockCostsResult> { match &self.value { Ok(da_block_costs) => { self.notifier.notify_waiters(); @@ -35,8 +38,4 @@ impl DaBlockCostsSource for DummyDaBlockCosts { } } } - - async fn set_last_value(&mut self, _height: BlockHeight) -> DaBlockCostsResult<()> { - unimplemented!("This is a dummy implementation"); - } } diff --git a/crates/services/gas_price_service/src/v1/da_source_service/service.rs b/crates/services/gas_price_service/src/v1/da_source_service/service.rs index 90e5acdb7ed..e60b22907cb 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/service.rs @@ -5,7 +5,13 @@ use fuel_core_services::{ StateWatcher, TaskNextAction, }; -use std::time::Duration; +use std::{ + sync::{ + Arc, + Mutex, + }, + time::Duration, +}; use tokio::{ sync::broadcast::Sender, time::{ @@ -37,16 +43,23 @@ pub struct DaSourceService { poll_interval: Interval, source: Source, shared_state: SharedState, + latest_l2_height: Arc>, + recorded_height: Option, } -const DA_BLOCK_COSTS_CHANNEL_SIZE: usize = 16 * 1024; +pub(crate) const DA_BLOCK_COSTS_CHANNEL_SIZE: usize = 16 * 1024; const POLLING_INTERVAL_MS: u64 = 10_000; impl DaSourceService where Source: DaBlockCostsSource, { - pub fn new(source: Source, poll_interval: Option) -> Self { + pub fn new( + source: Source, + poll_interval: Option, + latest_l2_height: Arc>, + recorded_height: Option, + ) -> Self { let (sender, _) = tokio::sync::broadcast::channel(DA_BLOCK_COSTS_CHANNEL_SIZE); #[allow(clippy::arithmetic_side_effects)] Self { @@ -55,26 +68,87 @@ where poll_interval.unwrap_or(Duration::from_millis(POLLING_INTERVAL_MS)), ), source, + latest_l2_height, + recorded_height, + } + } + + #[cfg(test)] + pub fn new_with_sender( + source: Source, + poll_interval: Option, + latest_l2_height: Arc>, + recorded_height: Option, + sender: Sender, + ) -> Self { + Self { + shared_state: SharedState::new(sender), + poll_interval: interval( + poll_interval.unwrap_or(Duration::from_millis(POLLING_INTERVAL_MS)), + ), + source, + latest_l2_height, + recorded_height, } } async fn process_block_costs(&mut self) -> Result<()> { - let da_block_costs_res = self.source.request_da_block_costs().await; + let da_block_costs_res = self + .source + .request_da_block_costs(&self.recorded_height) + .await; tracing::debug!("Received block costs: {:?}", da_block_costs_res); let da_block_costs = da_block_costs_res?; - for da_block_costs in da_block_costs { + let filtered_block_costs = self + .filter_costs_that_have_values_greater_than_l2_block_height(da_block_costs)?; + tracing::debug!( + "the latest l2 height is: {:?}", + *self.latest_l2_height.lock().unwrap() + ); + for da_block_costs in filtered_block_costs { + tracing::debug!("Sending block costs: {:?}", da_block_costs); + let end = BlockHeight::from(*da_block_costs.l2_blocks.end()); self.shared_state.0.send(da_block_costs)?; + if let Some(recorded_height) = self.recorded_height { + if end > recorded_height { + self.recorded_height = Some(end) + } + } else { + self.recorded_height = Some(end) + } } Ok(()) } + + fn filter_costs_that_have_values_greater_than_l2_block_height( + &self, + da_block_costs: Vec, + ) -> Result> { + let latest_l2_height = *self + .latest_l2_height + .lock() + .map_err(|err| anyhow::anyhow!("lock error: {:?}", err))?; + let iter = da_block_costs.into_iter().filter(move |da_block_costs| { + let end = BlockHeight::from(*da_block_costs.l2_blocks.end()); + end < latest_l2_height + }); + Ok(iter) + } + + #[cfg(test)] + pub fn recorded_height(&self) -> Option { + self.recorded_height + } } /// This trait is implemented by the sources to obtain the /// da block costs in a way they see fit #[async_trait::async_trait] pub trait DaBlockCostsSource: Send + Sync { - async fn request_da_block_costs(&mut self) -> Result>; - async fn set_last_value(&mut self, block_height: BlockHeight) -> Result<()>; + async fn request_da_block_costs( + &mut self, + recorded_height: &Option, + ) -> Result>; } #[async_trait::async_trait] @@ -132,9 +206,15 @@ where } } -pub fn new_service( +pub fn new_da_service( da_source: S, poll_interval: Option, + latest_l2_height: Arc>, ) -> ServiceRunner> { - ServiceRunner::new(DaSourceService::new(da_source, poll_interval)) + ServiceRunner::new(DaSourceService::new( + da_source, + poll_interval, + latest_l2_height, + None, + )) } diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 5ddffa0cc11..87bb46445df 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -60,7 +60,10 @@ use fuel_gas_price_algorithm::{ use futures::FutureExt; use std::{ num::NonZeroU64, - sync::Arc, + sync::{ + Arc, + Mutex, + }, }; use tokio::sync::broadcast::Receiver; @@ -108,6 +111,8 @@ where da_block_costs_buffer: Vec, /// Storage transaction provider for metadata and unrecorded blocks storage_tx_provider: AtomicStorage, + /// communicates to the Da source service what the latest L2 block was + latest_l2_block: Arc>, } impl GasPriceServiceV1 @@ -144,6 +149,23 @@ where self.update_latest_gas_price(&block); tracing::debug!("Updating gas price algorithm"); self.apply_block_info_to_gas_algorithm(block).await?; + + self.notify_da_source_service_l2_block(block)?; + Ok(()) + } + + fn notify_da_source_service_l2_block(&self, block: BlockInfo) -> anyhow::Result<()> { + tracing::debug!("Notifying the Da source service of the latest L2 block"); + match block { + BlockInfo::GenesisBlock => {} + BlockInfo::Block { height, .. } => { + let mut latest_l2_block = self + .latest_l2_block + .lock() + .map_err(|err| anyhow!("Error locking latest L2 block: {:?}", err))?; + *latest_l2_block = BlockHeight::from(height); + } + } Ok(()) } } @@ -160,6 +182,7 @@ where algorithm_updater: AlgorithmUpdaterV1, da_source_adapter_handle: ServiceRunner>, storage_tx_provider: AtomicStorage, + latest_l2_block: Arc>, ) -> Self { let da_source_channel = da_source_adapter_handle.shared.clone().subscribe(); Self { @@ -171,6 +194,7 @@ where da_source_channel, da_block_costs_buffer: Vec::new(), storage_tx_provider, + latest_l2_block, } } @@ -390,10 +414,12 @@ where mod tests { use std::{ num::NonZeroU64, - sync::Arc, + sync::{ + Arc, + Mutex, + }, time::Duration, }; - use tokio::sync::mpsc; use fuel_core_services::{ @@ -499,6 +525,9 @@ mod tests { fn database() -> StorageTransaction> { InMemoryStorage::default().into_transaction() } + fn latest_l2_height(height: u32) -> Arc> { + Arc::new(Mutex::new(BlockHeight::new(height))) + } #[tokio::test] async fn run__updates_gas_price_with_l2_block_source() { @@ -541,16 +570,20 @@ mod tests { initialize_algorithm(&config, l2_block_height, &metadata_storage).unwrap(); let notifier = Arc::new(tokio::sync::Notify::new()); + let latest_l2_block = Arc::new(Mutex::new(BlockHeight::new(0))); let dummy_da_source = DaSourceService::new( DummyDaBlockCosts::new( Err(anyhow::anyhow!("unused at the moment")), notifier.clone(), ), None, + latest_l2_block, + None, ); let da_service_runner = ServiceRunner::new(dummy_da_source); da_service_runner.start_and_await().await.unwrap(); let latest_gas_price = LatestGasPrice::new(0, 0); + let latest_l2_height = latest_l2_height(0); let mut service = GasPriceServiceV1::new( l2_block_source, @@ -559,6 +592,7 @@ mod tests { algo_updater, da_service_runner, inner, + latest_l2_height, ); let read_algo = service.next_block_algorithm(); let mut watcher = StateWatcher::default(); @@ -577,7 +611,7 @@ mod tests { #[tokio::test] async fn run__updates_gas_price_with_da_block_cost_source() { // given - let block_height = 2; + let block_height = 3; let l2_block_2 = BlockInfo::Block { height: block_height, gas_used: 60, @@ -623,6 +657,7 @@ mod tests { algo_updater.last_profit = 10_000; algo_updater.new_scaled_da_gas_price = 10_000_000; + let latest_l2_block = latest_l2_height(block_height - 1); let notifier = Arc::new(tokio::sync::Notify::new()); let da_source = DaSourceService::new( DummyDaBlockCosts::new( @@ -635,6 +670,8 @@ mod tests { notifier.clone(), ), Some(Duration::from_millis(1)), + latest_l2_block.clone(), + None, ); let mut watcher = StateWatcher::started(); let da_service_runner = ServiceRunner::new(da_source); @@ -648,16 +685,19 @@ mod tests { algo_updater, da_service_runner, inner, + latest_l2_block, ); let read_algo = service.next_block_algorithm(); let initial_price = read_algo.next_gas_price(); - service.run(&mut watcher).await; + let next = service.run(&mut watcher).await; + tracing::info!("Next action: {:?}", next); tokio::time::sleep(Duration::from_millis(3)).await; l2_block_sender.send(l2_block_2).await.unwrap(); // when - service.run(&mut watcher).await; + let next = service.run(&mut watcher).await; + tracing::info!("Next action 2: {:?}", next); tokio::time::sleep(Duration::from_millis(3)).await; service.shutdown().await.unwrap(); @@ -720,6 +760,7 @@ mod tests { algo_updater.last_profit = 10_000; algo_updater.new_scaled_da_gas_price = 10_000_000; + let latest_l2_height = latest_l2_height(block_height - 1); let notifier = Arc::new(tokio::sync::Notify::new()); let da_source = DaSourceService::new( DummyDaBlockCosts::new( @@ -732,6 +773,8 @@ mod tests { notifier.clone(), ), Some(Duration::from_millis(1)), + latest_l2_height.clone(), + None, ); let mut watcher = StateWatcher::started(); let da_service_runner = ServiceRunner::new(da_source); @@ -745,6 +788,7 @@ mod tests { algo_updater, da_service_runner, inner, + latest_l2_height, ); let read_algo = service.next_block_algorithm(); let initial_price = read_algo.next_gas_price(); @@ -809,6 +853,7 @@ mod tests { let notifier = Arc::new(tokio::sync::Notify::new()); let blob_cost_wei = 9000; + let latest_l2_height = latest_l2_height(block_height - 1); let da_source = DaSourceService::new( DummyDaBlockCosts::new( Ok(DaBlockCosts { @@ -820,6 +865,8 @@ mod tests { notifier.clone(), ), Some(Duration::from_millis(1)), + latest_l2_height.clone(), + None, ); let mut watcher = StateWatcher::started(); let da_service_runner = ServiceRunner::new(da_source); @@ -833,6 +880,7 @@ mod tests { algo_updater, da_service_runner, inner, + latest_l2_height, ); let read_algo = service.next_block_algorithm(); let initial_price = read_algo.next_gas_price(); diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index 00412e962d8..fd29fb9e1b1 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -28,9 +28,10 @@ use crate::{ }, v1::{ algorithm::SharedV1Algorithm, + da_source_service, da_source_service::{ service::{ - new_service, + new_da_service, DaBlockCostsSource, DaSourceService, }, @@ -128,13 +129,13 @@ impl L2BlockSource for FakeL2BlockSource { } struct FakeMetadata { - inner: Arc>>, + inner: Arc>>, } impl FakeMetadata { fn empty() -> Self { Self { - inner: Arc::new(std::sync::Mutex::new(None)), + inner: Arc::new(Mutex::new(None)), } } } @@ -241,7 +242,7 @@ impl AsUnrecordedBlocks for UnimplementedStorageTx { struct FakeDABlockCost { da_block_costs: Receiver, - latest_received_height: Arc>>, + latest_requested_height: Arc>>, } impl FakeDABlockCost { @@ -249,14 +250,14 @@ impl FakeDABlockCost { let (_sender, receiver) = tokio::sync::mpsc::channel(1); Self { da_block_costs: receiver, - latest_received_height: Arc::new(Mutex::new(None)), + latest_requested_height: Arc::new(Mutex::new(None)), } } fn new(da_block_costs: Receiver) -> Self { Self { da_block_costs, - latest_received_height: Arc::new(Mutex::new(None)), + latest_requested_height: Arc::new(Mutex::new(None)), } } @@ -266,7 +267,7 @@ impl FakeDABlockCost { let height = Arc::new(Mutex::new(None)); let service = Self { da_block_costs: receiver, - latest_received_height: height.clone(), + latest_requested_height: height.clone(), }; (service, height) } @@ -274,15 +275,14 @@ impl FakeDABlockCost { #[async_trait::async_trait] impl DaBlockCostsSource for FakeDABlockCost { - async fn request_da_block_costs(&mut self) -> Result> { + async fn request_da_block_costs( + &mut self, + latest_recorded_height: &Option, + ) -> Result> { + *self.latest_requested_height.lock().unwrap() = *latest_recorded_height; let costs = self.da_block_costs.recv().await.unwrap(); Ok(vec![costs]) } - - async fn set_last_value(&mut self, height: BlockHeight) -> Result<()> { - self.latest_received_height.lock().unwrap().replace(height); - Ok(()) - } } fn zero_threshold_arbitrary_config() -> V1AlgorithmConfig { @@ -355,6 +355,9 @@ fn gas_price_database_with_metadata( tx.commit().unwrap(); db } +fn latest_l2_height(height: u32) -> Arc> { + Arc::new(Mutex::new(BlockHeight::new(height))) +} #[tokio::test] async fn next_gas_price__affected_by_new_l2_block() { @@ -379,8 +382,8 @@ async fn next_gas_price__affected_by_new_l2_block() { let (algo_updater, shared_algo) = initialize_algorithm(&config, height, &metadata_storage).unwrap(); let da_source = FakeDABlockCost::never_returns(); - let da_source_service = DaSourceService::new(da_source, None); - let da_service_runner = ServiceRunner::new(da_source_service); + let latest_l2_height = latest_l2_height(0); + let da_service_runner = new_da_service(da_source, None, latest_l2_height.clone()); da_service_runner.start_and_await().await.unwrap(); let latest_gas_price = LatestGasPrice::new(0, 0); @@ -391,6 +394,7 @@ async fn next_gas_price__affected_by_new_l2_block() { algo_updater, da_service_runner, inner, + latest_l2_height, ); let read_algo = service.next_block_algorithm(); @@ -429,8 +433,8 @@ async fn run__new_l2_block_saves_old_metadata() { let algo_updater = updater_from_config(&config); let shared_algo = SharedV1Algorithm::new_with_algorithm(algo_updater.algorithm()); let da_source = FakeDABlockCost::never_returns(); - let da_source_service = DaSourceService::new(da_source, None); - let da_service_runner = ServiceRunner::new(da_source_service); + let latest_l2_height = latest_l2_height(0); + let da_service_runner = new_da_service(da_source, None, latest_l2_height.clone()); da_service_runner.start_and_await().await.unwrap(); let latest_gas_price = LatestGasPrice::new(0, 0); let mut service = GasPriceServiceV1::new( @@ -440,6 +444,7 @@ async fn run__new_l2_block_saves_old_metadata() { algo_updater, da_service_runner, inner, + latest_l2_height, ); let mut watcher = StateWatcher::started(); @@ -482,7 +487,8 @@ async fn run__new_l2_block_updates_latest_gas_price_arc() { let algo_updater = updater_from_config(&config); let shared_algo = SharedV1Algorithm::new_with_algorithm(algo_updater.algorithm()); let da_source = FakeDABlockCost::never_returns(); - let da_service_runner = new_service(da_source, None); + let latest_l2_height = latest_l2_height(0); + let da_service_runner = new_da_service(da_source, None, latest_l2_height.clone()); let latest_gas_price = LatestGasPrice::new(0, 0); let mut service = GasPriceServiceV1::new( l2_block_source, @@ -491,6 +497,7 @@ async fn run__new_l2_block_updates_latest_gas_price_arc() { algo_updater, da_service_runner, inner, + latest_l2_height, ); let mut watcher = StateWatcher::started(); @@ -507,6 +514,53 @@ async fn run__new_l2_block_updates_latest_gas_price_arc() { service.shutdown().await.unwrap(); } +#[tokio::test] +async fn run__updates_da_service_latest_l2_height() { + // given + let l2_height = 10; + let l2_block = BlockInfo::Block { + height: l2_height, + gas_used: 60, + block_gas_capacity: 100, + block_bytes: 100, + block_fees: 100, + gas_price: 100, + }; + let (l2_block_sender, l2_block_receiver) = tokio::sync::mpsc::channel(1); + let l2_block_source = FakeL2BlockSource { + l2_block: l2_block_receiver, + }; + + let config = zero_threshold_arbitrary_config(); + let inner = database(); + let mut algo_updater = updater_from_config(&config); + algo_updater.l2_block_height = l2_height - 1; + let shared_algo = SharedV1Algorithm::new_with_algorithm(algo_updater.algorithm()); + let da_source = FakeDABlockCost::never_returns(); + let latest_l2_height = latest_l2_height(0); + let latest_gas_price = LatestGasPrice::new(0, 0); + let da_service_runner = new_da_service(da_source, None, latest_l2_height.clone()); + da_service_runner.start_and_await().await.unwrap(); + let mut service = GasPriceServiceV1::new( + l2_block_source, + shared_algo, + latest_gas_price, + algo_updater, + da_service_runner, + inner, + latest_l2_height.clone(), + ); + let mut watcher = StateWatcher::started(); + + // when + l2_block_sender.send(l2_block).await.unwrap(); + let _ = service.run(&mut watcher).await; + + // then + let latest_value = *latest_l2_height.lock().unwrap(); + assert_eq!(*latest_value, l2_height); +} + #[derive(Clone)] struct FakeSettings { settings: GasPriceSettings, @@ -747,13 +801,13 @@ async fn uninitialized_task__new__should_fail_if_cannot_fetch_metadata() { } #[tokio::test] -async fn uninitialized_task__init__starts_da_service_with_bundle_id_in_storage() { +async fn uninitialized_task__init__starts_da_service_with_recorded_height_in_storage() { // given let block_height = 100; let recorded_height: u32 = 200; let original_metadata = arbitrary_metadata(); - let different_config = different_arb_config(); + let mut different_config = different_arb_config(); let descaleed_exec_price = original_metadata.new_scaled_exec_price / original_metadata.gas_price_factor; assert_ne!(different_config.new_exec_gas_price, descaleed_exec_price); @@ -761,13 +815,14 @@ async fn uninitialized_task__init__starts_da_service_with_bundle_id_in_storage() let settings = FakeSettings::default(); let block_stream = empty_block_stream(); let on_chain_db = FakeOnChainDb::new(different_l2_block); - let (da_cost_source, recorded_block_height_handle) = + let (da_cost_source, latest_requested_recorded_height) = FakeDABlockCost::never_returns_with_handle_to_last_height(); let mut inner = gas_price_database_with_metadata(&original_metadata); let mut tx = inner.begin_transaction().unwrap(); tx.set_recorded_height(BlockHeight::from(recorded_height)) .unwrap(); StorageTransaction::commit_transaction(tx).unwrap(); + different_config.da_poll_interval = Some(1); let service = UninitializedTask::new( different_config.clone(), Some(block_height.into()), @@ -781,10 +836,11 @@ async fn uninitialized_task__init__starts_da_service_with_bundle_id_in_storage() .unwrap(); // when - service.init(&StateWatcher::started()).await.unwrap(); + let _gas_price_service = service.init(&StateWatcher::started()).await.unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(3)).await; // then - let actual = recorded_block_height_handle.lock().unwrap(); + let actual = latest_requested_recorded_height.lock().unwrap(); let expected = Some(BlockHeight::new(recorded_height)); assert_eq!(*actual, expected); } diff --git a/crates/services/gas_price_service/src/v1/uninitialized_task.rs b/crates/services/gas_price_service/src/v1/uninitialized_task.rs index cfcbfb21604..3b10a570f86 100644 --- a/crates/services/gas_price_service/src/v1/uninitialized_task.rs +++ b/crates/services/gas_price_service/src/v1/uninitialized_task.rs @@ -189,15 +189,18 @@ where self.block_stream, ); - if let Some(last_recorded_height) = self.gas_price_db.get_recorded_height()? { - self.da_source.set_last_value(last_recorded_height).await?; - tracing::info!("Set last recorded height to {}", last_recorded_height); - } + let recorded_height = self.gas_price_db.get_recorded_height()?; let poll_duration = self .config .da_poll_interval .map(|x| Duration::from_millis(x.into())); - let da_service = DaSourceService::new(self.da_source, poll_duration); + let latest_l2_height = Arc::new(std::sync::Mutex::new(BlockHeight::new(0))); + let da_service = DaSourceService::new( + self.da_source, + poll_duration, + latest_l2_height.clone(), + recorded_height, + ); let da_service_runner = ServiceRunner::new(da_service); da_service_runner.start_and_await().await?; @@ -209,6 +212,7 @@ where self.algo_updater, da_service_runner, self.gas_price_db, + latest_l2_height, ); Ok(service) } else { @@ -231,6 +235,7 @@ where self.algo_updater, da_service_runner, self.gas_price_db, + latest_l2_height, ); Ok(service) } diff --git a/crates/services/src/service.rs b/crates/services/src/service.rs index 93b1b179ff6..2964841d6f6 100644 --- a/crates/services/src/service.rs +++ b/crates/services/src/service.rs @@ -84,6 +84,7 @@ pub trait RunnableService: Send { } /// The result of a single iteration of the service task +#[derive(Debug)] pub enum TaskNextAction { /// Request the task to be run again Continue, diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index aa8a4631c40..a09586d055f 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -602,6 +602,11 @@ fn produce_block__l1_committed_block_affects_gas_price() { .await .unwrap(); tokio::time::sleep(Duration::from_millis(20)).await; + // Won't accept DA costs until l2_height is > 1 + driver.client.produce_blocks(1, None).await.unwrap(); + // Wait for DaBlockCosts to be accepted + tokio::time::sleep(Duration::from_millis(2)).await; + // Produce new block to update gas price driver.client.produce_blocks(1, None).await.unwrap(); tokio::time::sleep(Duration::from_millis(20)).await; driver.client.estimate_gas_price(0).await.unwrap().gas_price From 6ea6a5c1df33295fc02eabb4c0c2b16822bc2e57 Mon Sep 17 00:00:00 2001 From: rymnc <43716372+rymnc@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:20:58 +0530 Subject: [PATCH 130/131] fix: default da poll interval to be higher than 10ms --- crates/fuel-core/src/service/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-core/src/service/config.rs b/crates/fuel-core/src/service/config.rs index 6bc3ddf1182..ace51098781 100644 --- a/crates/fuel-core/src/service/config.rs +++ b/crates/fuel-core/src/service/config.rs @@ -224,7 +224,7 @@ impl Config { activity_decrease_range_size: 0, da_committer_url, block_activity_threshold: 0, - da_poll_interval: Some(10), + da_poll_interval: Some(1_000), } } From ae40ee4826e4007a5e3d771ecfa8152ce98ae611 Mon Sep 17 00:00:00 2001 From: Aaryamann Challani <43716372+rymnc@users.noreply.github.com> Date: Thu, 9 Jan 2025 23:22:06 +0530 Subject: [PATCH 131/131] feat(gas_price_service_v1): include `max_da_gas_price` to control volatility (#2541) ## Linked Issues/PRs - #2469 ## Description Adds a new config param `max_da_gas_price`, which `new_scaled_da_gas_price` depends upon while mutating it. ## Checklist - [ ] Breaking changes are clearly marked as such in the PR description and changelog - [ ] New behavior is reflected in tests - [ ] [The specification](https://github.com/FuelLabs/fuel-specs/) matches the implemented behavior (link update PR if changes are needed) ### Before requesting review - [ ] I have reviewed the code myself - [ ] I have created follow-up issues caused by this PR and linked them here ### After merging, notify other teams [Add or remove entries as needed] - [ ] [Rust SDK](https://github.com/FuelLabs/fuels-rs/) - [ ] [Sway compiler](https://github.com/FuelLabs/sway/) - [ ] [Platform documentation](https://github.com/FuelLabs/devrel-requests/issues/new?assignees=&labels=new+request&projects=&template=NEW-REQUEST.yml&title=%5BRequest%5D%3A+) (for out-of-organization contributors, the person merging the PR will do this) - [ ] Someone else? --------- Co-authored-by: Brandon Kite Co-authored-by: Mitchell Turner --- bin/fuel-core/src/cli/run.rs | 7 + .../service/adapters/gas_price_adapters.rs | 1 + crates/fuel-core/src/service/config.rs | 2 + crates/fuel-gas-price-algorithm/src/v1.rs | 22 +++- .../fuel-gas-price-algorithm/src/v1/tests.rs | 8 ++ .../v1/tests/update_da_record_data_tests.rs | 122 ++++++++++++++++++ .../v1/tests/update_l2_block_data_tests.rs | 10 ++ .../services/gas_price_service/src/ports.rs | 2 + .../gas_price_service/src/v1/metadata.rs | 3 + .../gas_price_service/src/v1/service.rs | 3 + .../gas_price_service/src/v1/tests.rs | 2 + tests/tests/gas_price.rs | 3 + 12 files changed, 181 insertions(+), 4 deletions(-) diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index 94c1afe0213..93c9fc95b65 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -212,6 +212,11 @@ pub struct Command { #[arg(long = "min-da-gas-price", default_value = "10000000", env)] pub min_da_gas_price: u64, + /// Maximum DA gas price + // DEV: ensure that the max_da_gas_price default is > then the min_da_gas_price default + #[arg(long = "max-da-gas-price", default_value = "10000001", env)] + pub max_da_gas_price: u64, + /// P component of DA gas price calculation /// **NOTE**: This is the **inverse** gain of a typical P controller. /// Increasing this value will reduce gas price fluctuations. @@ -339,6 +344,7 @@ impl Command { min_gas_price, gas_price_threshold_percent, min_da_gas_price, + max_da_gas_price, da_p_component, da_d_component, max_da_gas_price_change_percent, @@ -660,6 +666,7 @@ impl Command { memory_pool_size, da_gas_price_factor: NonZeroU64::new(100).expect("100 is not zero"), min_da_gas_price, + max_da_gas_price, max_da_gas_price_change_percent, da_p_component, da_d_component, diff --git a/crates/fuel-core/src/service/adapters/gas_price_adapters.rs b/crates/fuel-core/src/service/adapters/gas_price_adapters.rs index fe131ddadbf..d5eb0c71cb9 100644 --- a/crates/fuel-core/src/service/adapters/gas_price_adapters.rs +++ b/crates/fuel-core/src/service/adapters/gas_price_adapters.rs @@ -71,6 +71,7 @@ impl From for GasPriceServiceConfig { value.exec_gas_price_threshold_percent, value.da_gas_price_factor, value.min_da_gas_price, + value.max_da_gas_price, value.max_da_gas_price_change_percent, value.da_p_component, value.da_d_component, diff --git a/crates/fuel-core/src/service/config.rs b/crates/fuel-core/src/service/config.rs index ace51098781..d3acbe58d55 100644 --- a/crates/fuel-core/src/service/config.rs +++ b/crates/fuel-core/src/service/config.rs @@ -86,6 +86,7 @@ pub struct Config { pub memory_pool_size: usize, pub da_gas_price_factor: NonZeroU64, pub min_da_gas_price: u64, + pub max_da_gas_price: u64, pub max_da_gas_price_change_percent: u16, pub da_p_component: i64, pub da_d_component: i64, @@ -216,6 +217,7 @@ impl Config { memory_pool_size: 4, da_gas_price_factor: NonZeroU64::new(100).expect("100 is not zero"), min_da_gas_price: 0, + max_da_gas_price: 1, max_da_gas_price_change_percent: 0, da_p_component: 0, da_d_component: 0, diff --git a/crates/fuel-gas-price-algorithm/src/v1.rs b/crates/fuel-gas-price-algorithm/src/v1.rs index 628ef7d0254..bef07066275 100644 --- a/crates/fuel-gas-price-algorithm/src/v1.rs +++ b/crates/fuel-gas-price-algorithm/src/v1.rs @@ -1,6 +1,9 @@ use crate::utils::cumulative_percentage_change; use std::{ - cmp::max, + cmp::{ + max, + min, + }, collections::BTreeMap, num::NonZeroU64, ops::{ @@ -147,6 +150,8 @@ pub struct AlgorithmUpdaterV1 { pub gas_price_factor: NonZeroU64, /// The lowest the algorithm allows the da gas price to go pub min_da_gas_price: u64, + /// The highest the algorithm allows the da gas price to go + pub max_da_gas_price: u64, /// The maximum percentage that the DA portion of the gas price can change in a single block /// Using `u16` because it can go above 100% and possibly over 255% pub max_da_gas_price_change_percent: u16, @@ -514,9 +519,12 @@ impl AlgorithmUpdaterV1 { scaled_da_change, maybe_new_scaled_da_gas_price ); - self.new_scaled_da_gas_price = max( - self.min_scaled_da_gas_price(), - maybe_new_scaled_da_gas_price, + self.new_scaled_da_gas_price = min( + max( + self.min_scaled_da_gas_price(), + maybe_new_scaled_da_gas_price, + ), + self.max_scaled_da_gas_price(), ); } @@ -540,6 +548,12 @@ impl AlgorithmUpdaterV1 { .saturating_mul(self.gas_price_factor.into()) } + fn max_scaled_da_gas_price(&self) -> u64 { + // note: here we make sure that a correct maximum is used; hacky :( + max(self.max_da_gas_price, self.min_da_gas_price) + .saturating_mul(self.gas_price_factor.into()) + } + fn p(&self) -> i128 { let upcast_p = i128::from(self.da_p_component); let checked_p = self.last_profit.checked_div(upcast_p); diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests.rs index c2dcaba36fa..535ff23b3e5 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests.rs @@ -26,6 +26,7 @@ pub struct BlockBytes { pub struct UpdaterBuilder { min_exec_gas_price: u64, min_da_gas_price: u64, + max_da_gas_price: u64, starting_exec_gas_price: u64, starting_da_gas_price: u64, exec_gas_price_change_percent: u16, @@ -53,6 +54,7 @@ impl UpdaterBuilder { Self { min_exec_gas_price: 0, min_da_gas_price: 0, + max_da_gas_price: 1, starting_exec_gas_price: 1, starting_da_gas_price: 1, exec_gas_price_change_percent: 0, @@ -86,6 +88,11 @@ impl UpdaterBuilder { self } + fn with_max_da_gas_price(mut self, max_price: u64) -> Self { + self.max_da_gas_price = max_price; + self + } + fn with_starting_exec_gas_price(mut self, starting_da_gas_price: u64) -> Self { self.starting_exec_gas_price = starting_da_gas_price; self @@ -192,6 +199,7 @@ impl UpdaterBuilder { last_profit: self.last_profit, second_to_last_profit: self.second_to_last_profit, min_da_gas_price: self.min_da_gas_price, + max_da_gas_price: self.max_da_gas_price, gas_price_factor: self .da_gas_price_factor .try_into() diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs index da6812cce2f..5817227de8a 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_da_record_data_tests.rs @@ -302,6 +302,7 @@ fn update_da_record_data__da_block_lowers_da_gas_price() { fn update_da_record_data__da_block_increases_da_gas_price() { // given let da_cost_per_byte = 40; + let max_da_gas_price = u64::MAX; let l2_block_height = 11; let original_known_total_cost = 150; let mut unrecorded_blocks: BTreeMap<_, _> = [(11, 3000)].into_iter().collect(); @@ -320,6 +321,7 @@ fn update_da_record_data__da_block_increases_da_gas_price() { .with_projected_total_cost(projected_total_cost as u128) .with_known_total_cost(original_known_total_cost as u128) .with_unrecorded_blocks(&unrecorded_blocks) + .with_max_da_gas_price(max_da_gas_price) .build(); let new_cost_per_byte = 100; @@ -355,6 +357,126 @@ fn update_da_record_data__da_block_increases_da_gas_price() { assert_ne!(old_da_gas_price, new_da_gas_price); } +#[test] +fn update_da_record_data__da_block_increases_da_gas_price_within_the_min_max_range() { + // given + let min_da_gas_price = 0; + let max_da_gas_price = 5; + let da_cost_per_byte = 40; + let l2_block_height = 11; + let original_known_total_cost = 150; + let mut unrecorded_blocks: BTreeMap<_, _> = [(11, 3000)].into_iter().collect(); + let da_p_component = 2; + let guessed_cost: u64 = unrecorded_blocks + .values() + .map(|bytes| bytes * da_cost_per_byte) + .sum(); + let projected_total_cost = original_known_total_cost + guessed_cost; + + let mut updater = UpdaterBuilder::new() + .with_da_cost_per_byte(da_cost_per_byte as u128) + .with_da_p_component(da_p_component) + .with_last_profit(-10, 0) + .with_l2_block_height(l2_block_height) + .with_projected_total_cost(projected_total_cost as u128) + .with_known_total_cost(original_known_total_cost as u128) + .with_unrecorded_blocks(&unrecorded_blocks) + .with_min_da_gas_price(min_da_gas_price) + .with_max_da_gas_price(max_da_gas_price) + .build(); + + let new_cost_per_byte = 100; + let (recorded_heights, recorded_cost) = unrecorded_blocks.iter().fold( + (vec![], 0), + |(mut range, cost), (height, bytes)| { + range.push(height); + (range, cost + bytes * new_cost_per_byte) + }, + ); + + let min = *recorded_heights.iter().min().unwrap(); + let max = *recorded_heights.iter().max().unwrap(); + let recorded_range = *min..=*max; + let recorded_bytes = 500; + + let old_da_gas_price = updater.new_scaled_da_gas_price; + + // when + updater + .update_da_record_data( + recorded_range, + recorded_bytes, + recorded_cost as u128, + &mut unrecorded_blocks, + ) + .unwrap(); + + // then + let new_da_gas_price = updater.new_scaled_da_gas_price; + // because the profit is -10 and the da_p_component is 2, the new da gas price should be greater than the previous one. + assert_eq!(new_da_gas_price, max_da_gas_price); + assert_ne!(old_da_gas_price, new_da_gas_price); +} + +#[test] +fn update_da_record_data__sets_da_gas_price_to_min_da_gas_price_when_max_lt_min() { + // given + let min_da_gas_price = 1; + let max_da_gas_price = 0; + let da_cost_per_byte = 40; + let l2_block_height = 11; + let original_known_total_cost = 150; + let mut unrecorded_blocks: BTreeMap<_, _> = [(11, 3000)].into_iter().collect(); + let da_p_component = 2; + let guessed_cost: u64 = unrecorded_blocks + .values() + .map(|bytes| bytes * da_cost_per_byte) + .sum(); + let projected_total_cost = original_known_total_cost + guessed_cost; + + let mut updater = UpdaterBuilder::new() + .with_da_cost_per_byte(da_cost_per_byte as u128) + .with_da_p_component(da_p_component) + .with_last_profit(-10, 0) + .with_l2_block_height(l2_block_height) + .with_projected_total_cost(projected_total_cost as u128) + .with_known_total_cost(original_known_total_cost as u128) + .with_unrecorded_blocks(&unrecorded_blocks) + .with_min_da_gas_price(min_da_gas_price) + .with_max_da_gas_price(max_da_gas_price) + .build(); + + let new_cost_per_byte = 100; + let (recorded_heights, recorded_cost) = unrecorded_blocks.iter().fold( + (vec![], 0), + |(mut range, cost), (height, bytes)| { + range.push(height); + (range, cost + bytes * new_cost_per_byte) + }, + ); + + let min = *recorded_heights.iter().min().unwrap(); + let max = *recorded_heights.iter().max().unwrap(); + let recorded_range = *min..=*max; + let recorded_bytes = 500; + + // when + updater + .update_da_record_data( + recorded_range, + recorded_bytes, + recorded_cost as u128, + &mut unrecorded_blocks, + ) + .unwrap(); + + // then + let new_da_gas_price = updater.new_scaled_da_gas_price; + + // because max_da_gas_price = 0 and < min_da_gas_price = 1, the new da gas price should be min_da_gas_price + assert_eq!(new_da_gas_price, min_da_gas_price); +} + #[test] fn update_da_record_data__da_block_will_not_change_da_gas_price() { // given diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs index f4555551579..994318172d9 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/update_l2_block_data_tests.rs @@ -24,6 +24,7 @@ fn negative_profit_updater_builder() -> UpdaterBuilder { let starting_da_gas_price = 100; let starting_cost = u128::MAX; let latest_gas_per_byte = i32::MAX; // DA is very expensive + let max_da_gas_price = u64::MAX; let da_p_component = 100; let da_d_component = 10; let last_profit = i128::MIN; @@ -39,6 +40,7 @@ fn negative_profit_updater_builder() -> UpdaterBuilder { .with_projected_total_cost(starting_cost) .with_da_cost_per_byte(latest_gas_per_byte as u128) .with_last_profit(last_profit, last_last_profit) + .with_max_da_gas_price(max_da_gas_price) } fn positive_profit_updater_builder() -> UpdaterBuilder { @@ -399,6 +401,7 @@ fn update_l2_block_data__price_does_not_decrease_more_than_max_percent() { let last_profit = i128::MAX; // Large, positive profit to decrease da price let last_last_profit = 0; let max_da_change_percent = 5; + let max_da_gas_price = u64::MAX; let large_starting_reward = i128::MAX; let mut updater = UpdaterBuilder::new() .with_starting_exec_gas_price(starting_exec_gas_price) @@ -411,6 +414,7 @@ fn update_l2_block_data__price_does_not_decrease_more_than_max_percent() { .with_da_cost_per_byte(latest_gas_per_byte as u128) .with_last_profit(last_profit, last_last_profit) .with_da_max_change_percent(max_da_change_percent) + .with_max_da_gas_price(max_da_gas_price) .build(); let unrecorded_blocks = &mut empty_unrecorded_blocks(); @@ -446,6 +450,7 @@ fn update_l2_block_data__da_price_does_not_increase_more_than_max_percent() { let last_profit = i128::MIN; // Large, negative profit to increase da price let last_last_profit = 0; let max_da_change_percent = 5; + let max_da_gas_price = u64::MAX; let large_starting_reward = 0; let unrecorded_blocks = &mut empty_unrecorded_blocks(); let mut updater = UpdaterBuilder::new() @@ -459,6 +464,7 @@ fn update_l2_block_data__da_price_does_not_increase_more_than_max_percent() { .with_da_cost_per_byte(latest_gas_per_byte) .with_last_profit(last_profit, last_last_profit) .with_da_max_change_percent(max_da_change_percent) + .with_max_da_gas_price(max_da_gas_price) .build(); // when @@ -487,6 +493,7 @@ fn update_l2_block_data__never_drops_below_minimum_da_gas_price() { let starting_exec_gas_price = 0; let last_da_gas_price = 100; let min_da_gas_price = 100; + let max_da_gas_price = min_da_gas_price + 1; let starting_cost = 0; let latest_gas_per_byte = 0; // DA is free let da_p_component = 100; @@ -507,6 +514,7 @@ fn update_l2_block_data__never_drops_below_minimum_da_gas_price() { .with_da_cost_per_byte(latest_gas_per_byte as u128) .with_last_profit(last_profit, avg_window) .with_min_da_gas_price(min_da_gas_price) + .with_max_da_gas_price(max_da_gas_price) .build(); // when @@ -534,6 +542,7 @@ fn update_l2_block_data__even_profit_maintains_price() { // given let starting_exec_gas_price = 100; let starting_da_gas_price = 100; + let max_da_gas_price = u64::MAX; let starting_cost = 500; let latest_cost_per_byte = 10; let da_gas_price_denominator = 1; @@ -548,6 +557,7 @@ fn update_l2_block_data__even_profit_maintains_price() { .with_known_total_cost(starting_cost as u128) .with_projected_total_cost(starting_cost as u128) .with_da_cost_per_byte(latest_cost_per_byte as u128) + .with_max_da_gas_price(max_da_gas_price) .build(); // when diff --git a/crates/services/gas_price_service/src/ports.rs b/crates/services/gas_price_service/src/ports.rs index 7670b3e0eab..80ec23e20e4 100644 --- a/crates/services/gas_price_service/src/ports.rs +++ b/crates/services/gas_price_service/src/ports.rs @@ -98,6 +98,7 @@ impl GasPriceServiceConfig { l2_block_fullness_threshold_percent: u8, gas_price_factor: NonZeroU64, min_da_gas_price: u64, + max_da_gas_price: u64, max_da_gas_price_change_percent: u16, da_p_component: i64, da_d_component: i64, @@ -114,6 +115,7 @@ impl GasPriceServiceConfig { l2_block_fullness_threshold_percent, gas_price_factor, min_da_gas_price, + max_da_gas_price, max_da_gas_price_change_percent, da_p_component, da_d_component, diff --git a/crates/services/gas_price_service/src/v1/metadata.rs b/crates/services/gas_price_service/src/v1/metadata.rs index 76f886dde39..f182818eee5 100644 --- a/crates/services/gas_price_service/src/v1/metadata.rs +++ b/crates/services/gas_price_service/src/v1/metadata.rs @@ -67,6 +67,7 @@ pub struct V1AlgorithmConfig { // https://github.com/FuelLabs/fuel-core/issues/2481 pub gas_price_factor: NonZeroU64, pub min_da_gas_price: u64, + pub max_da_gas_price: u64, pub max_da_gas_price_change_percent: u16, pub da_p_component: i64, pub da_d_component: i64, @@ -108,6 +109,7 @@ pub fn updater_from_config(value: &V1AlgorithmConfig) -> AlgorithmUpdaterV1 { .l2_block_fullness_threshold_percent .into(), min_da_gas_price: value.min_da_gas_price, + max_da_gas_price: value.max_da_gas_price, max_da_gas_price_change_percent: value.max_da_gas_price_change_percent, da_p_component: value.da_p_component, da_d_component: value.da_d_component, @@ -166,6 +168,7 @@ pub fn v1_algorithm_from_metadata( .l2_block_fullness_threshold_percent .into(), min_da_gas_price: config.min_da_gas_price, + max_da_gas_price: config.max_da_gas_price, max_da_gas_price_change_percent: config.max_da_gas_price_change_percent, da_p_component: config.da_p_component, da_d_component: config.da_d_component, diff --git a/crates/services/gas_price_service/src/v1/service.rs b/crates/services/gas_price_service/src/v1/service.rs index 87bb46445df..619d55642a4 100644 --- a/crates/services/gas_price_service/src/v1/service.rs +++ b/crates/services/gas_price_service/src/v1/service.rs @@ -556,6 +556,7 @@ mod tests { l2_block_fullness_threshold_percent: 20, gas_price_factor: NonZeroU64::new(10).unwrap(), min_da_gas_price: 10, + max_da_gas_price: 11, max_da_gas_price_change_percent: 20, da_p_component: 4, da_d_component: 2, @@ -635,6 +636,7 @@ mod tests { l2_block_fullness_threshold_percent: 20, gas_price_factor: NonZeroU64::new(10).unwrap(), min_da_gas_price: 0, + max_da_gas_price: 1, max_da_gas_price_change_percent: 100, da_p_component: 4, da_d_component: 2, @@ -714,6 +716,7 @@ mod tests { l2_block_fullness_threshold_percent: 20, gas_price_factor: NonZeroU64::new(10).unwrap(), min_da_gas_price: 0, + max_da_gas_price: 1, max_da_gas_price_change_percent: 100, da_p_component: 4, da_d_component: 2, diff --git a/crates/services/gas_price_service/src/v1/tests.rs b/crates/services/gas_price_service/src/v1/tests.rs index fd29fb9e1b1..c8024db81cd 100644 --- a/crates/services/gas_price_service/src/v1/tests.rs +++ b/crates/services/gas_price_service/src/v1/tests.rs @@ -293,6 +293,7 @@ fn zero_threshold_arbitrary_config() -> V1AlgorithmConfig { l2_block_fullness_threshold_percent: 0, gas_price_factor: NonZeroU64::new(100).unwrap(), min_da_gas_price: 0, + max_da_gas_price: 1, max_da_gas_price_change_percent: 0, da_p_component: 0, da_d_component: 0, @@ -327,6 +328,7 @@ fn different_arb_config() -> V1AlgorithmConfig { l2_block_fullness_threshold_percent: 0, gas_price_factor: NonZeroU64::new(100).unwrap(), min_da_gas_price: 0, + max_da_gas_price: 1, max_da_gas_price_change_percent: 0, da_p_component: 0, da_d_component: 0, diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index a09586d055f..5908bb7c270 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -208,6 +208,7 @@ async fn produce_block__raises_gas_price() { node_config.da_d_component = 0; node_config.max_da_gas_price_change_percent = 0; node_config.min_da_gas_price = 0; + node_config.max_da_gas_price = 1; let srv = FuelService::new_node(node_config.clone()).await.unwrap(); let client = FuelClient::from(srv.bound_address); @@ -257,6 +258,7 @@ async fn produce_block__lowers_gas_price() { node_config.da_d_component = 0; node_config.max_da_gas_price_change_percent = 0; node_config.min_da_gas_price = 0; + node_config.max_da_gas_price = 1; let srv = FuelService::new_node(node_config.clone()).await.unwrap(); let client = FuelClient::from(srv.bound_address); @@ -705,6 +707,7 @@ fn node_config_with_da_committer_url(url: &str) -> Config { let starting_gas_price = 10_000_000; node_config.block_producer.coinbase_recipient = Some([5; 32].into()); node_config.min_da_gas_price = starting_gas_price; + node_config.max_da_gas_price = u64::MAX; node_config.max_da_gas_price_change_percent = 15; node_config.block_production = Trigger::Never; node_config.da_committer_url = Some(url.to_string());