From 8218159ac28e2e4cf66feeaf4edb62b7eb28816f Mon Sep 17 00:00:00 2001 From: DCG-Claude Date: Fri, 11 Sep 2026 16:45:14 -0500 Subject: [PATCH 1/8] fix(platform): resolve fee versions by registered number The fee version registry resolved a fee_version_number by array position and as_static aborted on an unknown number. Lookup is now by number on every entry point, zero and unregistered numbers are errors, as_static is fallible, and a compile time assertion keeps the registry numbered contiguously from one. Tests pin that every schedule a platform version references is registered and agrees with the registered generation on every group the fee history serves, and that the number one storage rates stay frozen for replay. Co-Authored-By: Claude Fable 5.1 --- .../src/version/fee/mod.rs | 236 ++++++++++++++++-- 1 file changed, 218 insertions(+), 18 deletions(-) diff --git a/packages/rs-platform-version/src/version/fee/mod.rs b/packages/rs-platform-version/src/version/fee/mod.rs index 29e02b56012..37b515909d8 100644 --- a/packages/rs-platform-version/src/version/fee/mod.rs +++ b/packages/rs-platform-version/src/version/fee/mod.rs @@ -32,8 +32,44 @@ pub mod vote_resolution_fund_fees; pub type FeeVersionNumber = u32; +/// The registry of fee-history generations, keyed by `fee_version_number`. +/// +/// A fee-history generation is the set of values the persisted fee history can be asked for +/// through `KnownCostItem`: the storage, processing, hashing and signature groups. The epoch +/// change hook records the number of the active schedule in platform state, and saved state +/// stores that number, so every number that was ever recorded must stay registered here. +/// +/// Entries are ordered by number, starting at 1 and without gaps. A schedule that changes any +/// value the fee history serves (storage rates in particular, because they price refunds) must +/// be registered under a new number. A schedule that only changes a group the history never +/// serves (for example `FEE_VERSION2`, which changed only `data_contract_registration`) keeps +/// the number of the generation it agrees with. pub const FEE_VERSIONS: &[FeeVersion] = &[FEE_VERSION1]; +const _: () = assert!( + !FEE_VERSIONS.is_empty(), + "the fee version registry must hold at least one generation" +); + +const _: () = assert!( + registry_numbers_are_contiguous_from_one(FEE_VERSIONS), + "fee version numbers must be registered in order, starting at 1 and without gaps" +); + +/// Returns true when the registry entries carry the numbers 1, 2, ... in order. +/// +/// Evaluated at compile time so a mis-numbered entry cannot be built into a node. +const fn registry_numbers_are_contiguous_from_one(registry: &[FeeVersion]) -> bool { + let mut index = 0; + while index < registry.len() { + if registry[index].fee_version_number != index as FeeVersionNumber + 1 { + return false; + } + index += 1; + } + true +} + #[derive(Clone, Debug, Encode, Decode, Default, PartialEq, Eq)] pub struct FeeVersion { pub fee_version_number: FeeVersionNumber, @@ -50,37 +86,46 @@ pub struct FeeVersion { } impl FeeVersion { - pub fn as_static(&self) -> &'static FeeVersion { - FeeVersion::get(self.fee_version_number).expect("expected fee version to exist") + /// Returns the registered fee-history generation this schedule's number names. + /// + /// The registered entry agrees with this schedule on every value the fee history serves, + /// but it is not necessarily the same schedule constant: several schedules may share one + /// number when they differ only in groups the history never reads. Fails when the number + /// is not registered. + pub fn as_static(&self) -> Result<&'static FeeVersion, PlatformVersionError> { + FeeVersion::get(self.fee_version_number) } + + /// Resolves a fee version number to its registered generation. + /// + /// Lookup is by number, never by position in the registry. Zero and any unregistered number + /// are errors. pub fn get<'a>(version: FeeVersionNumber) -> Result<&'a Self, PlatformVersionError> { - if version > 0 { - FEE_VERSIONS.get(version as usize - 1).ok_or_else(|| { - PlatformVersionError::UnknownVersionError(format!("no fee version {version}")) - }) - } else { - Err(PlatformVersionError::UnknownVersionError(format!( - "no fee version {version}" - ))) - } + FeeVersion::get_optional(version).ok_or_else(|| { + PlatformVersionError::UnknownVersionError(format!("no fee version {version}")) + }) } + /// Resolves a fee version number to its registered generation, or `None` when the number + /// is zero or not registered. pub fn get_optional<'a>(version: FeeVersionNumber) -> Option<&'a Self> { - if version > 0 { - FEE_VERSIONS.get(version as usize - 1) - } else { - None - } + FEE_VERSIONS + .iter() + .find(|fee_version| fee_version.fee_version_number == version) } + /// The earliest registered generation. This is what an empty fee history resolves to. pub fn first<'a>() -> &'a Self { FEE_VERSIONS .first() - .expect("expected to have a fee version") + .expect("the compile time assertion above proves the registry is not empty") } + /// The most recently registered generation. pub fn latest<'a>() -> &'a Self { - FEE_VERSIONS.last().expect("expected to have a fee version") + FEE_VERSIONS + .last() + .expect("the compile time assertion above proves the registry is not empty") } } @@ -149,3 +194,158 @@ impl From for FeeVersion { } } } + +#[cfg(test)] +mod tests { + use super::*; + #[cfg(feature = "mock-versions")] + use crate::version::mocks::v2_test::TEST_PLATFORM_V2; + #[cfg(feature = "mock-versions")] + use crate::version::mocks::v3_test::TEST_PLATFORM_V3; + use crate::version::PLATFORM_VERSIONS; + + /// Every schedule a platform version references, plus the mock schedules when they are + /// compiled in. + fn referenced_schedules() -> Vec<(&'static str, &'static FeeVersion)> { + let shipped = PLATFORM_VERSIONS + .iter() + .map(|platform_version| ("shipped platform version", &platform_version.fee_version)); + #[cfg(feature = "mock-versions")] + let mocks = [ + ("TEST_PLATFORM_V2", &TEST_PLATFORM_V2.fee_version), + ("TEST_PLATFORM_V3", &TEST_PLATFORM_V3.fee_version), + ]; + #[cfg(not(feature = "mock-versions"))] + let mocks: [(&'static str, &'static FeeVersion); 0] = []; + shipped.chain(mocks).collect() + } + + #[test] + fn should_resolve_every_registered_number_to_the_entry_with_that_number() { + for registered in FEE_VERSIONS { + let number = registered.fee_version_number; + let resolved = FeeVersion::get(number).expect("registered number resolves"); + assert_eq!( + resolved.fee_version_number, number, + "lookup must return the entry carrying the requested number" + ); + assert_eq!(resolved, registered); + assert_eq!(FeeVersion::get_optional(number), Some(registered)); + } + } + + #[test] + fn should_reject_zero_and_unregistered_numbers() { + let unregistered = [0, FEE_VERSIONS.len() as FeeVersionNumber + 1, u32::MAX]; + for number in unregistered { + let error = FeeVersion::get(number).expect_err("unregistered number is an error"); + assert!( + error.to_string().contains(&number.to_string()), + "error names the unknown number: {error}" + ); + assert!(FeeVersion::get_optional(number).is_none()); + } + } + + #[test] + fn should_keep_registered_numbers_unique_and_contiguous_from_one() { + let numbers: Vec = FEE_VERSIONS + .iter() + .map(|fee_version| fee_version.fee_version_number) + .collect(); + let expected: Vec = + (1..=FEE_VERSIONS.len() as FeeVersionNumber).collect(); + assert_eq!(numbers, expected); + assert!(registry_numbers_are_contiguous_from_one(FEE_VERSIONS)); + + let skipped = [ + FEE_VERSION1, + FeeVersion { + fee_version_number: 3, + ..FEE_VERSION1 + }, + ]; + assert!(!registry_numbers_are_contiguous_from_one(&skipped)); + let duplicated = [FEE_VERSION1, FEE_VERSION1]; + assert!(!registry_numbers_are_contiguous_from_one(&duplicated)); + let starting_at_zero = [FeeVersion { + fee_version_number: 0, + ..FEE_VERSION1 + }]; + assert!(!registry_numbers_are_contiguous_from_one(&starting_at_zero)); + } + + #[test] + fn should_register_the_number_every_platform_version_references() { + for (origin, schedule) in referenced_schedules() { + let registered = FeeVersion::get(schedule.fee_version_number).unwrap_or_else(|error| { + panic!( + "{origin} references fee version number {} which is not registered: {error}", + schedule.fee_version_number + ) + }); + // The registered generation must agree with the schedule on every group the fee + // history can serve. A schedule that changes one of these needs a new number. + assert_eq!( + registered.storage, schedule.storage, + "{origin}: storage group" + ); + assert_eq!( + registered.processing, schedule.processing, + "{origin}: processing group" + ); + assert_eq!( + registered.hashing, schedule.hashing, + "{origin}: hashing group" + ); + assert_eq!( + registered.signature, schedule.signature, + "{origin}: signature group" + ); + } + } + + #[test] + fn should_keep_fee_version_one_storage_rates_frozen_for_replay() { + // Every storage byte written on the network so far was priced with these rates, and + // every refund of those bytes is priced with them again through the fee history. + // Changing them under number 1 would re-price historical refunds during replay, so a + // change in storage rates must be registered under a new number instead. + let storage = &FeeVersion::get(1) + .expect("fee version 1 is registered") + .storage; + assert_eq!(storage.storage_disk_usage_credit_per_byte, 27000); + assert_eq!(storage.storage_processing_credit_per_byte, 400); + assert_eq!(storage.storage_load_credit_per_byte, 20); + assert_eq!(storage.non_storage_load_credit_per_byte, 10); + assert_eq!(storage.storage_seek_cost, 2000); + } + + #[test] + fn should_return_the_registered_entry_from_as_static() { + for (origin, schedule) in referenced_schedules() { + let registered = schedule + .as_static() + .unwrap_or_else(|error| panic!("{origin}: {error}")); + assert_eq!(registered.fee_version_number, schedule.fee_version_number); + assert_eq!( + registered, + FeeVersion::get(schedule.fee_version_number).unwrap() + ); + } + } + + #[test] + fn should_error_from_as_static_for_an_unregistered_number() { + for number in [0, 99] { + let schedule = FeeVersion { + fee_version_number: number, + ..FEE_VERSION1 + }; + let error = schedule + .as_static() + .expect_err("an unregistered number cannot resolve"); + assert!(error.to_string().contains(&number.to_string())); + } + } +} From fb6caeb59e5beb277a3b9ad6a50d40657ae1d246 Mon Sep 17 00:00:00 2001 From: DCG-Claude Date: Fri, 11 Sep 2026 17:09:09 -0500 Subject: [PATCH 2/8] fix(dpp): price storage refunds at the rate active when the bytes were stored A refund is the unpaid remainder of the storage fee originally charged, and that fee was priced with the storage table active at the storage epoch. The refund now resolves the rate through the fee history at the storage epoch instead of the removal epoch. Every shipped schedule carries the same storage table under fee version number 1, so no reachable input changes; the tests pin both the boundary behaviour and the shipped-input equivalence. Also adds tests for the epoch fee-history resolution and for agreement between every platform version's schedule and its registered generation on every known cost item. Co-Authored-By: Claude Fable 5.1 --- packages/rs-dpp/src/fee/default_costs/mod.rs | 131 ++++++++++++++ packages/rs-dpp/src/fee/fee_result/refunds.rs | 171 +++++++++++++++++- 2 files changed, 301 insertions(+), 1 deletion(-) diff --git a/packages/rs-dpp/src/fee/default_costs/mod.rs b/packages/rs-dpp/src/fee/default_costs/mod.rs index 59359361abc..52be461287d 100644 --- a/packages/rs-dpp/src/fee/default_costs/mod.rs +++ b/packages/rs-dpp/src/fee/default_costs/mod.rs @@ -162,3 +162,134 @@ impl EpochCosts for Epoch { cost_item.lookup_cost_on_epoch(self, cached_fee_version) } } + +#[cfg(test)] +mod tests { + use super::*; + use platform_version::version::fee::storage::FeeStorageVersion; + use platform_version::version::fee::v1::FEE_VERSION1; + use platform_version::version::PLATFORM_VERSIONS; + + /// A second generation that is not registered anywhere, so a boundary in the fee history is + /// observable through the storage rate it carries. + static SYNTHETIC_FEE_VERSION_2: FeeVersion = FeeVersion { + fee_version_number: 2, + storage: FeeStorageVersion { + storage_disk_usage_credit_per_byte: 54000, + ..FEE_VERSION1.storage + }, + ..FEE_VERSION1 + }; + + /// Every `KnownCostItem` variant, with a few sizes for the two sized variants. + fn every_known_cost_item() -> Vec { + let mut items = vec![ + KnownCostItem::StorageDiskUsageCreditPerByte, + KnownCostItem::StorageProcessingCreditPerByte, + KnownCostItem::StorageLoadCreditPerByte, + KnownCostItem::NonStorageLoadCreditPerByte, + KnownCostItem::StorageSeekCost, + KnownCostItem::FetchIdentityBalanceProcessingCost, + KnownCostItem::FetchSingleIdentityKeyProcessingCost, + KnownCostItem::VerifySignatureEcdsaSecp256k1, + KnownCostItem::VerifySignatureBLS12_381, + KnownCostItem::VerifySignatureEcdsaHash160, + KnownCostItem::VerifySignatureBip13ScriptHash, + KnownCostItem::VerifySignatureEddsa25519Hash160, + ]; + for size in [0, 1, 64] { + items.push(KnownCostItem::SingleSHA256(size)); + items.push(KnownCostItem::Blake3(size)); + } + items + } + + fn epoch(index: EpochIndex) -> Epoch { + Epoch::new(index).expect("epoch index fits") + } + + #[test] + fn should_use_the_first_fee_version_when_the_history_is_empty() { + // The epoch change hook only records a generation on the first non-genesis epoch change, + // so genesis epoch refunds always resolve through an empty history. That must land on + // the first registered generation, which is number 1. + let empty = CachedEpochIndexFeeVersions::default(); + for index in [0, 1, 500] { + let resolved = epoch(index).active_fee_version(&empty); + assert_eq!(resolved, FeeVersion::first()); + assert_eq!( + resolved, + FeeVersion::get(1).expect("number 1 is registered") + ); + } + } + + #[test] + fn should_use_the_exact_epoch_entry_when_present() { + let history: CachedEpochIndexFeeVersions = BTreeMap::from([ + (0, FeeVersion::get(1).expect("registered")), + (10, &SYNTHETIC_FEE_VERSION_2), + ]); + assert_eq!(epoch(0).active_fee_version(&history).fee_version_number, 1); + assert_eq!(epoch(10).active_fee_version(&history).fee_version_number, 2); + assert_eq!( + epoch(10) + .cost_for_known_cost_item(&history, KnownCostItem::StorageDiskUsageCreditPerByte), + 54000 + ); + } + + #[test] + fn should_use_the_nearest_lower_epoch_entry() { + let history: CachedEpochIndexFeeVersions = BTreeMap::from([ + (0, FeeVersion::get(1).expect("registered")), + (10, &SYNTHETIC_FEE_VERSION_2), + ]); + assert_eq!(epoch(9).active_fee_version(&history).fee_version_number, 1); + assert_eq!(epoch(11).active_fee_version(&history).fee_version_number, 2); + assert_eq!( + epoch(u16::MAX - 300) + .active_fee_version(&history) + .fee_version_number, + 2 + ); + } + + #[test] + fn should_use_the_first_fee_version_before_the_earliest_entry() { + let history: CachedEpochIndexFeeVersions = BTreeMap::from([(10, &SYNTHETIC_FEE_VERSION_2)]); + let resolved = epoch(3).active_fee_version(&history); + assert_eq!(resolved, FeeVersion::first()); + assert_eq!( + epoch(3) + .cost_for_known_cost_item(&history, KnownCostItem::StorageDiskUsageCreditPerByte), + FEE_VERSION1.storage.storage_disk_usage_credit_per_byte + ); + } + + #[test] + fn should_agree_with_the_registered_entry_on_every_known_cost_item_for_every_platform_version() + { + // The fee history stores a number and serves values through KnownCostItem. For the stored + // number to resolve to the intended table, the registered generation must return the same + // value as the schedule the platform version actually carries, for every item. + for platform_version in PLATFORM_VERSIONS { + let schedule = &platform_version.fee_version; + let registered = FeeVersion::get(schedule.fee_version_number).unwrap_or_else(|error| { + panic!( + "protocol version {} references an unregistered fee version: {error}", + platform_version.protocol_version + ) + }); + for item in every_known_cost_item() { + assert_eq!( + item.lookup_cost(schedule), + item.lookup_cost(registered), + "protocol version {} disagrees with registered fee version {} on a cost item", + platform_version.protocol_version, + schedule.fee_version_number + ); + } + } + } +} diff --git a/packages/rs-dpp/src/fee/fee_result/refunds.rs b/packages/rs-dpp/src/fee/fee_result/refunds.rs index 95142a4f82d..48b57c56598 100644 --- a/packages/rs-dpp/src/fee/fee_result/refunds.rs +++ b/packages/rs-dpp/src/fee/fee_result/refunds.rs @@ -36,6 +36,12 @@ pub struct FeeRefunds(pub CreditsPerEpochByIdentifier); impl FeeRefunds { /// Create fee refunds from GroveDB's StorageRemovalPerEpochByIdentifier + /// + /// A refund is the unpaid remainder of the storage fee originally charged for the removed + /// bytes. That fee was priced with the storage table active when the bytes were written, so + /// the rate is resolved at the storage epoch (the key of each removal entry) through the fee + /// history. The current epoch only decides how many era shares of that fee were already paid + /// out to proposers. pub fn from_storage_removal( storage_removal: I, current_epoch_index: EpochIndex, @@ -58,8 +64,11 @@ impl FeeRefunds { // TODO Add in multipliers once they have been made + let storage_rate = Epoch::new(epoch_index)? + .cost_for_known_cost_item(previous_fee_versions, StorageDiskUsageCreditPerByte); + let credits: Credits = (bytes as Credits) - .checked_mul(Epoch::new(current_epoch_index)?.cost_for_known_cost_item(previous_fee_versions, StorageDiskUsageCreditPerByte)) + .checked_mul(storage_rate) .ok_or(ProtocolError::Overflow("storage written bytes cost overflow"))?; let (amount, _) = calculate_storage_fee_refund_amount_and_leftovers( @@ -179,16 +188,73 @@ impl IntoIterator for FeeRefunds { mod tests { use super::*; use once_cell::sync::Lazy; + use platform_version::version::fee::storage::FeeStorageVersion; + use platform_version::version::fee::v1::FEE_VERSION1; use platform_version::version::fee::FeeVersion; static EPOCH_CHANGE_FEE_VERSION_TEST: Lazy = Lazy::new(|| BTreeMap::from([(0, FeeVersion::first())])); + /// Storage rate of the first registered generation, which priced every byte written so far. + const FIRST_GENERATION_RATE: Credits = 27000; + + /// A second storage table that is not registered anywhere. It exists only so a rate boundary + /// in the fee history is observable in these tests. + const SYNTHETIC_RATE: Credits = 54000; + + static SYNTHETIC_FEE_VERSION_2: FeeVersion = FeeVersion { + fee_version_number: 2, + storage: FeeStorageVersion { + storage_disk_usage_credit_per_byte: SYNTHETIC_RATE, + ..FEE_VERSION1.storage + }, + ..FEE_VERSION1 + }; + mod from_storage_removal { use super::*; use nohash_hasher::IntMap; use std::iter::FromIterator; + const EPOCHS_PER_ERA: u16 = 20; + + fn expected_refund( + bytes: u32, + rate: Credits, + storage_epoch: EpochIndex, + current_epoch: EpochIndex, + ) -> Credits { + let (amount, _) = calculate_storage_fee_refund_amount_and_leftovers( + bytes as Credits * rate, + storage_epoch, + current_epoch, + EPOCHS_PER_ERA, + ) + .expect("refund amount"); + amount + } + + fn refunds_for_one_identity( + bytes_per_epoch: IntMap, + current_epoch: EpochIndex, + fee_history: &CachedEpochIndexFeeVersions, + ) -> CreditsPerEpoch { + let identity_id = [7; 32]; + let storage_removal = + BytesPerEpochByIdentifier::from_iter([(identity_id, bytes_per_epoch)]); + + FeeRefunds::from_storage_removal( + storage_removal, + current_epoch, + EPOCHS_PER_ERA, + fee_history, + ) + .expect("should create fee refunds") + .get(&identity_id) + .expect("identity has refunds") + .clone() + } + #[test] fn should_filter_out_refunds_under_the_limit() { let identity_id = [0; 32]; @@ -210,5 +276,108 @@ mod tests { assert!(credits_per_epoch.get(&0).is_none()); assert!(credits_per_epoch.get(&1).is_some()); } + + #[test] + fn should_price_each_removed_epoch_at_the_storage_table_active_when_the_bytes_were_stored() + { + // Rate boundary at epoch 10: bytes written before it were charged at the first + // generation's rate, bytes written from it on at the synthetic rate. + let fee_history: CachedEpochIndexFeeVersions = BTreeMap::from([ + (0, FeeVersion::get(1).expect("registered")), + (10, &SYNTHETIC_FEE_VERSION_2), + ]); + let current_epoch = 15; + + let refunds = refunds_for_one_identity( + IntMap::from_iter([(5, 100), (12, 100)]), + current_epoch, + &fee_history, + ); + + assert_eq!( + refunds.get(&5).copied(), + Some(expected_refund( + 100, + FIRST_GENERATION_RATE, + 5, + current_epoch + )), + "bytes stored before the boundary refund at the rate they were charged" + ); + assert_eq!( + refunds.get(&12).copied(), + Some(expected_refund(100, SYNTHETIC_RATE, 12, current_epoch)), + "bytes stored after the boundary refund at the new rate" + ); + assert_ne!( + refunds.get(&5).copied(), + Some(expected_refund(100, SYNTHETIC_RATE, 5, current_epoch)), + "pre-boundary bytes must not be re-priced at the current epoch's rate" + ); + } + + #[test] + fn should_price_removals_before_the_earliest_history_entry_with_the_first_generation() { + let fee_history: CachedEpochIndexFeeVersions = + BTreeMap::from([(10, &SYNTHETIC_FEE_VERSION_2)]); + let current_epoch = 12; + + let refunds = refunds_for_one_identity( + IntMap::from_iter([(3, 100)]), + current_epoch, + &fee_history, + ); + + assert_eq!( + refunds.get(&3).copied(), + Some(expected_refund( + 100, + FIRST_GENERATION_RATE, + 3, + current_epoch + )) + ); + } + + #[test] + fn should_match_the_current_epoch_rate_whenever_every_generation_shares_one_storage_table() + { + // Every shipped input: an empty history (the fee version number 1 path in Drive) or a + // history whose entries all resolve to number 1. The storage epoch and the current + // epoch then resolve to the same rate, so the result is identical to what pricing at + // the current epoch produced before the rule changed. + let same_table_histories: [CachedEpochIndexFeeVersions; 2] = [ + BTreeMap::default(), + BTreeMap::from([ + (0, FeeVersion::get(1).expect("registered")), + (7, FeeVersion::get(1).expect("registered")), + ]), + ]; + let current_epoch = 9; + + for fee_history in &same_table_histories { + let refunds = refunds_for_one_identity( + IntMap::from_iter([(2, 100), (8, 100)]), + current_epoch, + fee_history, + ); + let current_epoch_rate = Epoch::new(current_epoch) + .expect("epoch") + .cost_for_known_cost_item(fee_history, StorageDiskUsageCreditPerByte); + assert_eq!(current_epoch_rate, FIRST_GENERATION_RATE); + + for storage_epoch in [2, 8] { + assert_eq!( + refunds.get(&storage_epoch).copied(), + Some(expected_refund( + 100, + current_epoch_rate, + storage_epoch, + current_epoch + )) + ); + } + } + } } } From 67edc404f2b6f057fcd65b4f37bc297fda3284bf Mon Sep 17 00:00:00 2001 From: DCG-Claude Date: Fri, 11 Sep 2026 17:28:28 -0500 Subject: [PATCH 3/8] test(drive): cover refund pricing paths in consume_to_fees_v0 Pins the legacy empty-history path for fee version number one, the fee history requirement for any other number, storage-epoch rate resolution across a history boundary, and the calculate_fee dispatcher forwarding the platform schedule and history. Co-Authored-By: Claude Fable 5.1 --- .../rs-drive/src/fees/calculate_fee/mod.rs | 65 +++++++ packages/rs-drive/src/fees/op.rs | 174 ++++++++++++++++++ 2 files changed, 239 insertions(+) diff --git a/packages/rs-drive/src/fees/calculate_fee/mod.rs b/packages/rs-drive/src/fees/calculate_fee/mod.rs index f2a6516a5ed..8c9dee0e40e 100644 --- a/packages/rs-drive/src/fees/calculate_fee/mod.rs +++ b/packages/rs-drive/src/fees/calculate_fee/mod.rs @@ -55,3 +55,68 @@ impl Drive { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::fees::op::LowLevelDriveOperation::CalculatedCostOperation; + use dpp::fee::default_costs::CachedEpochIndexFeeVersions; + use grovedb_costs::storage_cost::removal::StorageRemovedBytes::SectionedStorageRemoval; + use grovedb_costs::storage_cost::StorageCost; + use grovedb_costs::OperationCost; + use intmap::IntMap; + use platform_version::version::fee::FeeVersion; + use platform_version::version::PlatformVersion; + use std::collections::BTreeMap; + + #[test] + fn should_forward_the_platform_fee_schedule_and_the_fee_history_to_the_implementation() { + let platform_version = PlatformVersion::latest(); + let identity = [9; 32]; + let operation = || { + let mut removal = BTreeMap::new(); + removal.insert(identity, IntMap::from_iter([(2u16, 200u32)])); + CalculatedCostOperation(OperationCost { + storage_cost: StorageCost { + added_bytes: 10, + replaced_bytes: 0, + removed_bytes: SectionedStorageRemoval(removal), + }, + ..Default::default() + }) + }; + let epoch = Epoch::new(6).expect("epoch"); + let history: CachedEpochIndexFeeVersions = + BTreeMap::from([(0, FeeVersion::get(1).expect("registered"))]); + + let through_dispatcher = Drive::calculate_fee( + None, + Some(vec![operation()]), + &epoch, + 20, + platform_version, + Some(&history), + ) + .expect("latest platform version dispatches calculate_fee"); + + let expected = Drive::calculate_fee_v0( + None, + Some(vec![operation()]), + &epoch, + 20, + &platform_version.fee_version, + Some(&history), + ) + .expect("direct implementation call"); + + assert_eq!(through_dispatcher, expected); + assert_eq!( + through_dispatcher.storage_fee, + 10 * platform_version + .fee_version + .storage + .storage_disk_usage_credit_per_byte + ); + assert!(through_dispatcher.fee_refunds.get(&identity).is_some()); + } +} diff --git a/packages/rs-drive/src/fees/op.rs b/packages/rs-drive/src/fees/op.rs index e501c02f8b2..269a09afc8a 100644 --- a/packages/rs-drive/src/fees/op.rs +++ b/packages/rs-drive/src/fees/op.rs @@ -2816,4 +2816,178 @@ mod tests { "expected overflow error when summing large components" ); } + + // --------------------------------------------------------------- + // 9. consume_to_fees_v0 — refunds and the fee history requirement + // --------------------------------------------------------------- + + mod consume_to_fees_v0 { + use super::*; + use dpp::fee::epoch::distribution::calculate_storage_fee_refund_amount_and_leftovers; + use intmap::IntMap; + use platform_version::version::fee::v1::FEE_VERSION1; + + const EPOCHS_PER_ERA: u16 = 20; + const CURRENT_EPOCH: u16 = 15; + const IDENTITY: [u8; 32] = [3; 32]; + + /// Storage rate of the first registered generation. + const FIRST_GENERATION_RATE: Credits = 27000; + + /// A second storage table that is not registered anywhere, so a rate boundary in the + /// fee history is observable. Its number is not 1, which makes Drive require the + /// history to price refunds. + const SYNTHETIC_RATE: Credits = 54000; + static SYNTHETIC_FEE_VERSION_2: FeeVersion = FeeVersion { + fee_version_number: 2, + storage: FeeStorageVersion { + storage_disk_usage_credit_per_byte: SYNTHETIC_RATE, + ..FEE_VERSION1.storage + }, + ..FEE_VERSION1 + }; + + /// Fee history with a rate boundary at epoch 10. + fn boundary_history() -> CachedEpochIndexFeeVersions { + BTreeMap::from([ + (0, FeeVersion::get(1).expect("registered")), + (10, &SYNTHETIC_FEE_VERSION_2), + ]) + } + + /// One identity removing 100 bytes stored at epoch 5 and 100 bytes stored at epoch 12, + /// plus 40 system bytes with an unknown storage epoch. + fn removal_operation() -> LowLevelDriveOperation { + let mut removal = BTreeMap::new(); + removal.insert( + IDENTITY, + IntMap::from_iter([(5u16, 100u32), (12u16, 100u32)]), + ); + removal.insert( + Identifier::default(), + IntMap::from_iter([(u16::MAX, 40u32)]), + ); + CalculatedCostOperation(OperationCost { + storage_cost: StorageCost { + added_bytes: 0, + replaced_bytes: 0, + removed_bytes: SectionedStorageRemoval(removal), + }, + ..Default::default() + }) + } + + fn expected_refund(bytes: u32, rate: Credits, storage_epoch: u16) -> Credits { + let (amount, _) = calculate_storage_fee_refund_amount_and_leftovers( + bytes as Credits * rate, + storage_epoch, + CURRENT_EPOCH, + EPOCHS_PER_ERA, + ) + .expect("refund amount"); + amount + } + + fn refunds_of(fee_result: &FeeResult) -> BTreeMap { + fee_result + .fee_refunds + .get(&IDENTITY) + .expect("identity has refunds") + .iter() + .map(|(epoch_index, credits)| (*epoch_index, *credits)) + .collect() + } + + #[test] + fn should_refund_through_the_legacy_empty_history_path_when_the_fee_version_number_is_one() + { + let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + let mut results = LowLevelDriveOperation::consume_to_fees_v0( + vec![removal_operation()], + &epoch, + EPOCHS_PER_ERA, + &FEE_VERSION1, + None, + ) + .expect("number 1 never needs the fee history"); + let fee_result = results.remove(0); + + assert_eq!(fee_result.removed_bytes_from_system, 40); + assert_eq!(fee_result.storage_fee, 0); + assert_eq!( + refunds_of(&fee_result), + BTreeMap::from([ + (5, expected_refund(100, FIRST_GENERATION_RATE, 5)), + (12, expected_refund(100, FIRST_GENERATION_RATE, 12)), + ]) + ); + } + + #[test] + fn should_require_fee_history_when_the_fee_version_number_is_not_one() { + let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + let error = LowLevelDriveOperation::consume_to_fees_v0( + vec![removal_operation()], + &epoch, + EPOCHS_PER_ERA, + &SYNTHETIC_FEE_VERSION_2, + None, + ) + .expect_err("a later generation cannot price refunds without the history"); + assert!( + matches!(error, Error::Drive(DriveError::CorruptedCodeExecution(_))), + "unexpected error {error}" + ); + } + + #[test] + fn should_refund_at_the_storage_epoch_rate_across_a_history_boundary_when_the_fee_version_number_is_not_one( + ) { + let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + let history = boundary_history(); + let mut results = LowLevelDriveOperation::consume_to_fees_v0( + vec![removal_operation()], + &epoch, + EPOCHS_PER_ERA, + &SYNTHETIC_FEE_VERSION_2, + Some(&history), + ) + .expect("history supplied"); + let fee_result = results.remove(0); + + assert_eq!( + refunds_of(&fee_result), + BTreeMap::from([ + (5, expected_refund(100, FIRST_GENERATION_RATE, 5)), + (12, expected_refund(100, SYNTHETIC_RATE, 12)), + ]), + "each epoch refunds at the rate its bytes were charged" + ); + } + + #[test] + fn should_ignore_fee_history_when_the_fee_version_number_is_one() { + // Shipped replay path: every schedule a released protocol version references + // carries number 1, and that branch prices refunds against an empty history. + let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + let history = boundary_history(); + let mut results = LowLevelDriveOperation::consume_to_fees_v0( + vec![removal_operation()], + &epoch, + EPOCHS_PER_ERA, + &FEE_VERSION1, + Some(&history), + ) + .expect("number 1 accepts but does not read the history"); + let fee_result = results.remove(0); + + assert_eq!( + refunds_of(&fee_result), + BTreeMap::from([ + (5, expected_refund(100, FIRST_GENERATION_RATE, 5)), + (12, expected_refund(100, FIRST_GENERATION_RATE, 12)), + ]) + ); + } + } } From 4d53f1c711311260c6c0a00ba12bc67b44c9c485 Mon Sep 17 00:00:00 2001 From: DCG-Claude Date: Fri, 11 Sep 2026 19:49:08 -0500 Subject: [PATCH 4/8] fix(drive-abci): reject saved state that stores an unknown fee version number Restoring PlatformState from the version 1 saving format resolved every stored fee version number with expect, so a state written by a newer build aborted the node at start. The conversion is now TryFrom and returns a CorruptedCachedState error naming the number and the epoch. The struct and its encoding are unchanged; the legacy version 0 mapping to the first generation is unchanged. Tests cover both fixture formats resolving to number 1, every registered number round-tripping through saved state, in-memory versus reloaded fee history agreeing on every known cost item, and the unknown-number rejection. Co-Authored-By: Claude Fable 5.1 --- .../block_processing_end_events/tests.rs | 10 +- .../src/platform_types/platform_state/mod.rs | 236 +++++++++++++++++- .../platform_state_for_saving/v1/mod.rs | 40 +-- .../platform_state_for_saving/v2/mod.rs | 29 ++- 4 files changed, 285 insertions(+), 30 deletions(-) diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_processing_end_events/tests.rs b/packages/rs-drive-abci/src/execution/platform_events/block_processing_end_events/tests.rs index 266a5091af3..c8082856f5a 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_processing_end_events/tests.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_processing_end_events/tests.rs @@ -867,9 +867,13 @@ mod refund_tests { let mut platform_state = platform.state.load().clone().deref().clone(); - platform_state - .previous_fee_versions_mut() - .insert(5, platform_version_with_higher_fees.fee_version.as_static()); + platform_state.previous_fee_versions_mut().insert( + 5, + platform_version_with_higher_fees + .fee_version + .as_static() + .expect("registered fee version"), + ); let (mut fee_results, _) = process_state_transitions( &platform, diff --git a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs index f4467c9cce9..fe2b6c4c119 100644 --- a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs @@ -337,7 +337,7 @@ impl TryFromPlatformVersioned for PlatformState { } PlatformStateForSaving::V1(v1) => { match platform_version.drive_abci.structs.platform_state_structure { - 0 => Ok(PlatformState::from(v1)), + 0 => PlatformState::try_from(v1), version => Err(Error::Execution(ExecutionError::UnknownVersionMismatch { method: "PlatformState::try_from_platform_versioned(PlatformStateForSavingV1)" @@ -367,13 +367,105 @@ mod tests { mod versioned_deserialize { use super::*; + use crate::platform_types::platform_state::platform_state_for_saving::v2::{ + serialize_masternode_entry, serialize_validator_set_entry, + }; use crate::test::fixture::platform_state::{ PLATFORM_STATE_V3_TESTNET, PLATFORM_STATE_V8_DEVNET, }; + use dpp::block::epoch::{Epoch, EpochIndex}; + use dpp::fee::default_costs::{EpochCosts, KnownCostItem}; + use dpp::version::fee::{FeeVersion, FEE_VERSIONS}; use platform_version::version::v3::PLATFORM_V3; use platform_version::version::v9::PLATFORM_V9; + use platform_version::version::LATEST_VERSION; use std::ops::Deref; + /// Every `KnownCostItem` variant, with a few sizes for the two sized variants. + fn every_known_cost_item() -> Vec { + let mut items = vec![ + KnownCostItem::StorageDiskUsageCreditPerByte, + KnownCostItem::StorageProcessingCreditPerByte, + KnownCostItem::StorageLoadCreditPerByte, + KnownCostItem::NonStorageLoadCreditPerByte, + KnownCostItem::StorageSeekCost, + KnownCostItem::FetchIdentityBalanceProcessingCost, + KnownCostItem::FetchSingleIdentityKeyProcessingCost, + KnownCostItem::VerifySignatureEcdsaSecp256k1, + KnownCostItem::VerifySignatureBLS12_381, + KnownCostItem::VerifySignatureEcdsaHash160, + KnownCostItem::VerifySignatureBip13ScriptHash, + KnownCostItem::VerifySignatureEddsa25519Hash160, + ]; + for size in [0, 1, 64] { + items.push(KnownCostItem::SingleSHA256(size)); + items.push(KnownCostItem::Blake3(size)); + } + items + } + + fn latest_state() -> PlatformState { + PlatformState::default_with_protocol_versions( + LATEST_VERSION, + LATEST_VERSION, + &PlatformConfig::default_testnet(), + ) + .expect("default state") + } + + /// Through the standalone snapshot, which always writes structure 0: the + /// structure 1 record the latest version writes per block is only readable + /// together with its entries, which `round_trip_through_entries` covers. + fn round_trip(state: &PlatformState) -> PlatformState { + let bytes = state + .serialize_standalone_to_bytes() + .expect("serialize state"); + PlatformState::versioned_deserialize_trusted(&bytes, PlatformVersion::latest()) + .expect("deserialize state") + } + + /// Through the structure 1 record and its per-member entries, the path + /// a node takes on restart. + fn round_trip_through_entries(state: &PlatformState) -> PlatformState { + let record = PlatformStateForSavingV2::from(state); + let masternode_entries = state + .full_masternode_list() + .iter() + .map(|(pro_tx_hash, masternode)| { + Ok(( + pro_tx_hash.to_byte_array().to_vec(), + serialize_masternode_entry(masternode, PlatformVersion::latest())?, + )) + }) + .collect::, Error>>() + .expect("serialize masternode entries"); + let validator_set_entries = state + .validator_sets() + .iter() + .map(|(quorum_hash, validator_set)| { + Ok(( + quorum_hash.to_byte_array().to_vec(), + serialize_validator_set_entry(validator_set)?, + )) + }) + .collect::, Error>>() + .expect("serialize validator set entries"); + record + .into_platform_state(masternode_entries, validator_set_entries) + .expect("rebuild state from record and entries") + } + + fn assert_every_entry_is_fee_version_one(state: &PlatformState) { + let registered = FeeVersion::get(1).expect("number 1 is registered"); + for (epoch_index, fee_version) in state.previous_fee_versions() { + assert_eq!( + fee_version.fee_version_number, 1, + "epoch {epoch_index} must resolve to fee version number 1" + ); + assert_eq!(*fee_version, registered); + } + } + #[test] fn should_deserialize_state_stored_in_version_0_from_testnet() { let serialized_state = @@ -411,5 +503,147 @@ mod tests { PlatformState::versioned_deserialize_trusted(&serialized_state, &PLATFORM_V9) .expect("failed to deserialize state"); } + + #[test] + fn should_still_load_legacy_v0_states_as_fee_version_one() { + // The pre-1.4 format stored whole fee version structs. Only number 1 existed then, + // so every stored epoch maps to the first registered generation. + let serialized_state = + hex::decode(PLATFORM_STATE_V3_TESTNET.deref()).expect("failed to decode hex"); + let state = + PlatformState::versioned_deserialize_trusted(&serialized_state, &PLATFORM_V3) + .expect("failed to deserialize state"); + + assert_every_entry_is_fee_version_one(&state); + } + + #[test] + fn should_resolve_stored_numbers_in_v1_states_to_registered_fee_versions() { + let serialized_state = + hex::decode(PLATFORM_STATE_V8_DEVNET.deref()).expect("failed to decode hex"); + let state = + PlatformState::versioned_deserialize_trusted(&serialized_state, &PLATFORM_V9) + .expect("failed to deserialize state"); + + assert_every_entry_is_fee_version_one(&state); + } + + #[test] + fn should_round_trip_every_registered_fee_version_number_through_saved_state() { + let mut state = latest_state(); + for registered in FEE_VERSIONS { + let epoch_index = registered.fee_version_number as EpochIndex; + state + .previous_fee_versions_mut() + .insert(epoch_index, registered); + } + + let reloaded = round_trip(&state); + + assert_eq!( + reloaded.previous_fee_versions().len(), + FEE_VERSIONS.len(), + "every registered number survives the round trip" + ); + for (epoch_index, fee_version) in state.previous_fee_versions() { + let reloaded_fee_version = reloaded + .previous_fee_versions() + .get(epoch_index) + .expect("epoch entry survives the round trip"); + assert_eq!( + reloaded_fee_version.fee_version_number, + fee_version.fee_version_number + ); + assert_eq!(*reloaded_fee_version, *fee_version); + assert_eq!( + *reloaded_fee_version, + FeeVersion::get(fee_version.fee_version_number).expect("registered") + ); + } + } + + #[test] + fn should_agree_with_the_in_memory_fee_history_on_every_known_cost_item_after_reload() { + // The epoch change hook stores a reference into the platform version table, not the + // registry entry. After a reload the map holds the registry entry. Both must serve + // the same values for every epoch and every cost item. + let mut state = latest_state(); + state + .previous_fee_versions_mut() + .insert(1, &PlatformVersion::latest().fee_version); + + let reloaded = round_trip(&state); + + assert_eq!( + reloaded.previous_fee_versions().keys().collect::>(), + state.previous_fee_versions().keys().collect::>() + ); + for epoch_index in 0..=3 { + let epoch = Epoch::new(epoch_index).expect("epoch"); + for item in every_known_cost_item() { + assert_eq!( + epoch.cost_for_known_cost_item(state.previous_fee_versions(), item), + epoch.cost_for_known_cost_item(reloaded.previous_fee_versions(), item), + "epoch {epoch_index} disagrees after reload" + ); + } + } + } + + #[test] + fn should_round_trip_every_registered_fee_version_number_through_the_structure_1_record() { + let mut state = latest_state(); + for registered in FEE_VERSIONS { + let epoch_index = registered.fee_version_number as EpochIndex; + state + .previous_fee_versions_mut() + .insert(epoch_index, registered); + } + + let reloaded = round_trip_through_entries(&state); + + assert_eq!( + reloaded.previous_fee_versions(), + state.previous_fee_versions(), + "every registered number survives the structure 1 round trip" + ); + } + + #[test] + fn should_reject_a_structure_1_record_that_stores_an_unknown_fee_version_number() { + let state = latest_state(); + let mut record = PlatformStateForSavingV2::from(&state); + record.previous_fee_versions.insert(3, 99); + + let error = record + .into_platform_state(Vec::new(), Vec::new()) + .expect_err("an unknown fee version number must not load"); + let message = error.to_string(); + assert!(message.contains("99"), "error names the number: {message}"); + assert!( + message.contains("epoch 3"), + "error names the epoch: {message}" + ); + } + + #[test] + fn should_reject_a_saved_state_that_stores_an_unknown_fee_version_number() { + let state = latest_state(); + let mut saving = PlatformStateForSavingV1::try_from(state).expect("saving form"); + saving.previous_fee_versions.insert(3, 99); + let config = config::standard().with_big_endian().with_no_limit(); + let bytes = bincode::encode_to_vec(PlatformStateForSaving::V1(saving), config) + .expect("encode saving form"); + + let error = + PlatformState::versioned_deserialize_trusted(&bytes, PlatformVersion::latest()) + .expect_err("an unknown fee version number must not load"); + let message = error.to_string(); + assert!(message.contains("99"), "error names the number: {message}"); + assert!( + message.contains("epoch 3"), + "error names the epoch: {message}" + ); + } } } diff --git a/packages/rs-drive-abci/src/platform_types/platform_state/platform_state_for_saving/v1/mod.rs b/packages/rs-drive-abci/src/platform_types/platform_state/platform_state_for_saving/v1/mod.rs index b91987dbe1e..0a925e68889 100644 --- a/packages/rs-drive-abci/src/platform_types/platform_state/platform_state_for_saving/v1/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform_state/platform_state_for_saving/v1/mod.rs @@ -1,3 +1,4 @@ +use crate::error::execution::ExecutionError; use crate::error::Error; use crate::platform_types::masternode::Masternode; use crate::platform_types::platform_state::accessors::PlatformStateV0Methods; @@ -120,9 +121,28 @@ impl TryFrom for PlatformStateForSavingV1 { } } -impl From for PlatformState { - fn from(value: PlatformStateForSavingV1) -> Self { - PlatformState { +impl TryFrom for PlatformState { + type Error = Error; + + /// Restores the in-memory state, resolving every stored fee version number through the fee + /// version registry. A number this build does not know is a load error, never a fallback to + /// another generation, because the refund rates it stands for would be wrong. + fn try_from(value: PlatformStateForSavingV1) -> Result { + let previous_fee_versions = value + .previous_fee_versions + .into_iter() + .map(|(epoch_index, fee_version_number)| { + FeeVersion::get(fee_version_number) + .map(|fee_version| (epoch_index, fee_version)) + .map_err(|_| { + Error::Execution(ExecutionError::CorruptedCachedState(format!( + "platform state stores fee version {fee_version_number} for epoch {epoch_index}, which this build does not know" + ))) + }) + }) + .collect::>()?; + + Ok(PlatformState { genesis_block_info: value.genesis_block_info, last_committed_block_info: value.last_committed_block_info, current_protocol_version_in_consensus: value.current_protocol_version_in_consensus, @@ -150,22 +170,12 @@ impl From for PlatformState { .into_iter() .map(|(k, v)| (ProTxHash::from_byte_array(k.to_buffer()), v.into())) .collect(), - previous_fee_versions: value - .previous_fee_versions - .into_iter() - .map(|(epoch_index, fee_version_number)| { - ( - epoch_index, - FeeVersion::get(fee_version_number) - .expect("expected fee version number to exist"), - ) - }) - .collect(), + previous_fee_versions, // a state read back from disk has not been written in full since heavy_fields_dirty: true, // this record carried the collections itself, so no entries back it on disk masternode_changes: EntryChanges::all(), validator_set_changes: EntryChanges::all(), - } + }) } } diff --git a/packages/rs-drive-abci/src/platform_types/platform_state/platform_state_for_saving/v2/mod.rs b/packages/rs-drive-abci/src/platform_types/platform_state/platform_state_for_saving/v2/mod.rs index a5edd43fffc..0f613dfe4f6 100644 --- a/packages/rs-drive-abci/src/platform_types/platform_state/platform_state_for_saving/v2/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform_state/platform_state_for_saving/v2/mod.rs @@ -213,6 +213,23 @@ impl PlatformStateForSavingV2 { ); } + // A stored fee version number this build does not know is a load + // error, never a fallback to another generation: the refund rates it + // stands for would be wrong. + let previous_fee_versions = self + .previous_fee_versions + .into_iter() + .map(|(epoch_index, fee_version_number)| { + FeeVersion::get(fee_version_number) + .map(|fee_version| (epoch_index, fee_version)) + .map_err(|_| { + corrupted(format!( + "platform state stores fee version {fee_version_number} for epoch {epoch_index}, which this build does not know" + )) + }) + }) + .collect::>()?; + Ok(PlatformState { genesis_block_info: self.genesis_block_info, last_committed_block_info: self.last_committed_block_info, @@ -229,17 +246,7 @@ impl PlatformStateForSavingV2 { instant_lock_validating_quorums: self.instant_lock_validating_quorums.into(), full_masternode_list, hpmn_masternode_list, - previous_fee_versions: self - .previous_fee_versions - .into_iter() - .map(|(epoch_index, fee_version_number)| { - ( - epoch_index, - FeeVersion::get(fee_version_number) - .expect("expected fee version number to exist"), - ) - }) - .collect(), + previous_fee_versions, // The record was written under structure 1, so its entries are on // disk and match it: nothing is pending. The flag only drives the // structure 0 store, which starts from a full write after a restart. From b014357da54c54789cd81d7ee53c33d906404f72 Mon Sep 17 00:00:00 2001 From: DCG-Claude Date: Fri, 11 Sep 2026 19:49:13 -0500 Subject: [PATCH 5/8] docs(platform): describe fee version generations and refund pricing Co-Authored-By: Claude Fable 5.1 --- book/src/fees/overview.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/book/src/fees/overview.md b/book/src/fees/overview.md index 6dc8a66b8d5..37403beb064 100644 --- a/book/src/fees/overview.md +++ b/book/src/fees/overview.md @@ -450,6 +450,15 @@ out to proposers. There is a **dust limit**: refunds below 32 bytes worth of storage credits are discarded to prevent micro-refund spam. +A refund is priced at the storage table that was active when the bytes were +written. `FeeRefunds::from_storage_removal` resolves that rate through the fee +history at the storage epoch (the epoch recorded in the element's storage +flags), never at the epoch of the removal. The current epoch only fixes how many +era shares of the original fee were already paid out to proposers. Every +schedule shipped so far carries the same storage table, so this distinction +first becomes observable when a schedule with new storage rates is registered +under a new fee version number. + ## Epoch-Based Fee Distribution Fees do not go directly to the block proposer. Instead, they accumulate in @@ -523,6 +532,27 @@ Fee versions are stored in the `FEE_VERSIONS` array and looked up by number. The `uses_version_fee_multiplier_permille` field allows a global scaling factor (permille = divide by 1000; a value of 1000 means no change). +`fee_version_number` names a fee-history generation: the set of values the +persisted fee history can serve through `KnownCostItem` (storage, processing, +hashing and signature costs). It is what the epoch change hook records in +platform state and what saved state stores. `FEE_VERSIONS` is the registry of +those generations, keyed by that number; `FeeVersion::get` resolves a number by +matching it, never by array position, and a number that is not registered is an +error. A saved state that stores an unknown number therefore fails to load with +a descriptive error instead of aborting the node. + +Several schedules may share one number when they differ only in groups the +history never serves: `FEE_VERSION1` and `FEE_VERSION2` both carry number 1 +because only `data_contract_registration` changed between them. A schedule that +changes storage rates must be registered under a new number, because that +number is what prices refunds of bytes written while it was active. Unit tests +in `rs-platform-version` pin that every schedule a protocol version references +is registered and agrees with the registered generation on every served value. + +An empty fee history resolves to the first registered generation. This is +deliberate: the hook records a generation only on the first non-genesis epoch +change, so genesis epoch costs always take that path. + ## Key Source Files | File | Contents | From b3b2e2e1f3eb3ab53419c6eb7a4eeb5571801801 Mon Sep 17 00:00:00 2001 From: DCG-Claude Date: Fri, 11 Sep 2026 20:55:36 -0500 Subject: [PATCH 6/8] test(platform): exercise fee version lookup and dispatcher refunds with a mock generation Adds a mock fee-history generation under the mock-versions feature whose number lives above the test protocol version shift, so it is never a position in the shipped registry and can never collide with a persisted number. The registry consults it after the shipped entries. Lookup by carried number is factored into a helper and tested against a registry whose numbers do not line up with positions. Drive::calculate_fee is now driven through the dispatcher with a platform version carrying the mock generation: refunds across a storage-rate boundary are priced at the storage-epoch rate, and a missing fee history is rejected. Saved state round-trips the mock number and prices storage identically after reload. Co-Authored-By: Claude Fable 5.1 --- .../src/platform_types/platform_state/mod.rs | 59 +++++++ .../rs-drive/src/fees/calculate_fee/mod.rs | 153 +++++++++++++++--- .../src/version/fee/mod.rs | 97 ++++++++++- .../src/version/mocks/fee_test.rs | 49 ++++++ .../src/version/mocks/mod.rs | 1 + 5 files changed, 335 insertions(+), 24 deletions(-) create mode 100644 packages/rs-platform-version/src/version/mocks/fee_test.rs diff --git a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs index fe2b6c4c119..95858008b39 100644 --- a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs @@ -376,6 +376,9 @@ mod tests { use dpp::block::epoch::{Epoch, EpochIndex}; use dpp::fee::default_costs::{EpochCosts, KnownCostItem}; use dpp::version::fee::{FeeVersion, FEE_VERSIONS}; + use dpp::version::mocks::fee_test::{ + TEST_FEE_VERSION_DOUBLED_STORAGE_RATE, TEST_FEE_VERSION_NUMBER_DOUBLED_STORAGE_RATE, + }; use platform_version::version::v3::PLATFORM_V3; use platform_version::version::v9::PLATFORM_V9; use platform_version::version::LATEST_VERSION; @@ -562,6 +565,62 @@ mod tests { } } + #[test] + fn should_round_trip_a_fee_version_number_that_is_not_a_registry_position() { + // The mock generation's number lives above the test shift, so it is never a position + // in the shipped registry. It must come back from saved state through a lookup by + // carried number, and the reloaded entry must price storage at its own rate. + let mut state = latest_state(); + let mock = TEST_FEE_VERSION_DOUBLED_STORAGE_RATE + .as_static() + .expect("mock generation is registered"); + state + .previous_fee_versions_mut() + .insert(0, FeeVersion::get(1).expect("registered")); + state.previous_fee_versions_mut().insert(10, mock); + + let saving = PlatformStateForSavingV1::try_from(state.clone()).expect("saving form"); + assert_eq!( + saving.previous_fee_versions.get(&10).copied(), + Some(TEST_FEE_VERSION_NUMBER_DOUBLED_STORAGE_RATE), + "the stored number is the carried number, not a position" + ); + + let reloaded = round_trip(&state); + let reloaded_mock = reloaded + .previous_fee_versions() + .get(&10) + .expect("boundary entry survives the round trip"); + assert_eq!( + reloaded_mock.fee_version_number, + TEST_FEE_VERSION_NUMBER_DOUBLED_STORAGE_RATE + ); + assert_eq!(*reloaded_mock, mock); + for epoch_index in [9, 10, 11] { + let epoch = Epoch::new(epoch_index).expect("epoch"); + assert_eq!( + epoch.cost_for_known_cost_item( + reloaded.previous_fee_versions(), + KnownCostItem::StorageDiskUsageCreditPerByte + ), + epoch.cost_for_known_cost_item( + state.previous_fee_versions(), + KnownCostItem::StorageDiskUsageCreditPerByte + ), + "epoch {epoch_index} prices storage the same before and after reload" + ); + } + assert_eq!( + Epoch::new(10).expect("epoch").cost_for_known_cost_item( + reloaded.previous_fee_versions(), + KnownCostItem::StorageDiskUsageCreditPerByte + ), + TEST_FEE_VERSION_DOUBLED_STORAGE_RATE + .storage + .storage_disk_usage_credit_per_byte + ); + } + #[test] fn should_agree_with_the_in_memory_fee_history_on_every_known_cost_item_after_reload() { // The epoch change hook stores a reference into the platform version table, not the diff --git a/packages/rs-drive/src/fees/calculate_fee/mod.rs b/packages/rs-drive/src/fees/calculate_fee/mod.rs index 8c9dee0e40e..f0ab38d6f20 100644 --- a/packages/rs-drive/src/fees/calculate_fee/mod.rs +++ b/packages/rs-drive/src/fees/calculate_fee/mod.rs @@ -61,6 +61,10 @@ mod tests { use super::*; use crate::fees::op::LowLevelDriveOperation::CalculatedCostOperation; use dpp::fee::default_costs::CachedEpochIndexFeeVersions; + use dpp::fee::epoch::distribution::calculate_storage_fee_refund_amount_and_leftovers; + use dpp::fee::Credits; + use dpp::version::mocks::fee_test::TEST_FEE_VERSION_DOUBLED_STORAGE_RATE; + use dpp::version::mocks::v2_test::TEST_PLATFORM_V2; use grovedb_costs::storage_cost::removal::StorageRemovedBytes::SectionedStorageRemoval; use grovedb_costs::storage_cost::StorageCost; use grovedb_costs::OperationCost; @@ -69,31 +73,82 @@ mod tests { use platform_version::version::PlatformVersion; use std::collections::BTreeMap; + const EPOCHS_PER_ERA: u16 = 20; + const CURRENT_EPOCH: u16 = 15; + const IDENTITY: [u8; 32] = [9; 32]; + + /// One identity removing 100 bytes stored at epoch 5 and 100 bytes stored at epoch 12, + /// plus 10 freshly added bytes. + fn removal_operation() -> LowLevelDriveOperation { + let mut removal = BTreeMap::new(); + removal.insert( + IDENTITY, + IntMap::from_iter([(5u16, 100u32), (12u16, 100u32)]), + ); + CalculatedCostOperation(OperationCost { + storage_cost: StorageCost { + added_bytes: 10, + replaced_bytes: 0, + removed_bytes: SectionedStorageRemoval(removal), + }, + ..Default::default() + }) + } + + /// Fee history with a storage-rate boundary at epoch 10. + fn boundary_history() -> CachedEpochIndexFeeVersions { + BTreeMap::from([ + (0, FeeVersion::get(1).expect("registered")), + ( + 10, + TEST_FEE_VERSION_DOUBLED_STORAGE_RATE + .as_static() + .expect("mock generation is registered"), + ), + ]) + } + + /// A mock platform version whose schedule is the mock generation, so the dispatcher must + /// hand the fee history through for refunds to be priced at all. + fn platform_version_with_doubled_storage_rate() -> PlatformVersion { + PlatformVersion { + fee_version: TEST_FEE_VERSION_DOUBLED_STORAGE_RATE, + ..TEST_PLATFORM_V2 + } + } + + fn expected_refund(bytes: u32, rate: Credits, storage_epoch: u16) -> Credits { + let (amount, _) = calculate_storage_fee_refund_amount_and_leftovers( + bytes as Credits * rate, + storage_epoch, + CURRENT_EPOCH, + EPOCHS_PER_ERA, + ) + .expect("refund amount"); + amount + } + + fn refunds_of(fee_result: &FeeResult) -> BTreeMap { + fee_result + .fee_refunds + .get(&IDENTITY) + .expect("identity has refunds") + .iter() + .map(|(epoch_index, credits)| (*epoch_index, *credits)) + .collect() + } + #[test] fn should_forward_the_platform_fee_schedule_and_the_fee_history_to_the_implementation() { let platform_version = PlatformVersion::latest(); - let identity = [9; 32]; - let operation = || { - let mut removal = BTreeMap::new(); - removal.insert(identity, IntMap::from_iter([(2u16, 200u32)])); - CalculatedCostOperation(OperationCost { - storage_cost: StorageCost { - added_bytes: 10, - replaced_bytes: 0, - removed_bytes: SectionedStorageRemoval(removal), - }, - ..Default::default() - }) - }; - let epoch = Epoch::new(6).expect("epoch"); - let history: CachedEpochIndexFeeVersions = - BTreeMap::from([(0, FeeVersion::get(1).expect("registered"))]); + let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + let history = boundary_history(); let through_dispatcher = Drive::calculate_fee( None, - Some(vec![operation()]), + Some(vec![removal_operation()]), &epoch, - 20, + EPOCHS_PER_ERA, platform_version, Some(&history), ) @@ -101,9 +156,9 @@ mod tests { let expected = Drive::calculate_fee_v0( None, - Some(vec![operation()]), + Some(vec![removal_operation()]), &epoch, - 20, + EPOCHS_PER_ERA, &platform_version.fee_version, Some(&history), ) @@ -117,6 +172,62 @@ mod tests { .storage .storage_disk_usage_credit_per_byte ); - assert!(through_dispatcher.fee_refunds.get(&identity).is_some()); + assert!(through_dispatcher.fee_refunds.get(&IDENTITY).is_some()); + } + + #[test] + fn should_price_refunds_across_a_rate_boundary_through_the_dispatcher_for_a_later_generation() { + let platform_version = platform_version_with_doubled_storage_rate(); + let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + let history = boundary_history(); + let first_rate = FeeVersion::get(1) + .expect("registered") + .storage + .storage_disk_usage_credit_per_byte; + let doubled_rate = TEST_FEE_VERSION_DOUBLED_STORAGE_RATE + .storage + .storage_disk_usage_credit_per_byte; + assert_ne!(first_rate, doubled_rate); + + let fee_result = Drive::calculate_fee( + None, + Some(vec![removal_operation()]), + &epoch, + EPOCHS_PER_ERA, + &platform_version, + Some(&history), + ) + .expect("history supplied through the dispatcher"); + + assert_eq!(fee_result.storage_fee, 10 * doubled_rate); + assert_eq!( + refunds_of(&fee_result), + BTreeMap::from([ + (5, expected_refund(100, first_rate, 5)), + (12, expected_refund(100, doubled_rate, 12)), + ]), + "bytes stored on each side of the boundary refund at the rate they were charged" + ); + } + + #[test] + fn should_reject_a_missing_fee_history_through_the_dispatcher_for_a_later_generation() { + let platform_version = platform_version_with_doubled_storage_rate(); + let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + + let error = Drive::calculate_fee( + None, + Some(vec![removal_operation()]), + &epoch, + EPOCHS_PER_ERA, + &platform_version, + None, + ) + .expect_err("a later generation cannot price refunds without the history"); + + assert!( + matches!(error, Error::Drive(DriveError::CorruptedCodeExecution(_))), + "unexpected error {error}" + ); } } diff --git a/packages/rs-platform-version/src/version/fee/mod.rs b/packages/rs-platform-version/src/version/fee/mod.rs index 37b515909d8..4c86624527e 100644 --- a/packages/rs-platform-version/src/version/fee/mod.rs +++ b/packages/rs-platform-version/src/version/fee/mod.rs @@ -16,6 +16,8 @@ use crate::version::fee::v1::FEE_VERSION1; use crate::version::fee::vote_resolution_fund_fees::{ VoteResolutionFundFees, VoteResolutionFundFeesFieldsBeforeVersion4, }; +#[cfg(feature = "mock-versions")] +use crate::version::mocks::fee_test::FEE_TEST_VERSIONS; use bincode::{Decode, Encode}; pub mod data_contract_registration; @@ -70,6 +72,17 @@ const fn registry_numbers_are_contiguous_from_one(registry: &[FeeVersion]) -> bo true } +/// Finds the entry that carries `number` in a registry slice. +/// +/// Only the carried number is compared. The position of an entry in the slice is never used, +/// so a registry whose numbers do not line up with positions still resolves correctly and a +/// number that no entry carries is `None`. +fn find_registered(registry: &[FeeVersion], number: FeeVersionNumber) -> Option<&FeeVersion> { + registry + .iter() + .find(|fee_version| fee_version.fee_version_number == number) +} + #[derive(Clone, Debug, Encode, Decode, Default, PartialEq, Eq)] pub struct FeeVersion { pub fee_version_number: FeeVersionNumber, @@ -108,10 +121,15 @@ impl FeeVersion { /// Resolves a fee version number to its registered generation, or `None` when the number /// is zero or not registered. + /// + /// With the `mock-versions` feature the mock generations are consulted after the shipped + /// registry. Their numbers live above the test shift, so they can never shadow a shipped one. pub fn get_optional<'a>(version: FeeVersionNumber) -> Option<&'a Self> { - FEE_VERSIONS - .iter() - .find(|fee_version| fee_version.fee_version_number == version) + #[cfg(feature = "mock-versions")] + if let Some(mock_fee_version) = find_registered(FEE_TEST_VERSIONS, version) { + return Some(mock_fee_version); + } + find_registered(FEE_VERSIONS, version) } /// The earliest registered generation. This is what an empty fee history resolves to. @@ -199,6 +217,10 @@ impl From for FeeVersion { mod tests { use super::*; #[cfg(feature = "mock-versions")] + use crate::version::mocks::fee_test::{ + TEST_FEE_VERSION_DOUBLED_STORAGE_RATE, TEST_FEE_VERSION_NUMBER_DOUBLED_STORAGE_RATE, + }; + #[cfg(feature = "mock-versions")] use crate::version::mocks::v2_test::TEST_PLATFORM_V2; #[cfg(feature = "mock-versions")] use crate::version::mocks::v3_test::TEST_PLATFORM_V3; @@ -234,6 +256,75 @@ mod tests { } } + #[test] + fn should_find_entries_by_carried_number_regardless_of_position() { + // A registry whose numbers do not line up with positions: number 3 sits at position 0 + // and number 1 at position 1. Position-based lookup would return the wrong entry for 1 + // and nothing for 3; matching on the carried number must find both and reject 2. + let registry = [ + FeeVersion { + fee_version_number: 3, + ..FEE_VERSION1 + }, + FeeVersion { + fee_version_number: 1, + ..FEE_VERSION1 + }, + ]; + + let found_one = find_registered(®istry, 1).expect("number 1 is carried"); + assert!(std::ptr::eq(found_one, ®istry[1])); + let found_three = find_registered(®istry, 3).expect("number 3 is carried"); + assert!(std::ptr::eq(found_three, ®istry[0])); + assert!(find_registered(®istry, 2).is_none()); + assert!(find_registered(®istry, 0).is_none()); + + // The contrast a position-based lookup would have produced. + assert_ne!(registry[1 - 1].fee_version_number, 1); + assert!(registry.get(3 - 1).is_none()); + } + + #[cfg(feature = "mock-versions")] + #[test] + fn should_resolve_a_mock_generation_whose_number_is_not_a_registry_position() { + let number = TEST_FEE_VERSION_NUMBER_DOUBLED_STORAGE_RATE; + // The mock number lives above the test shift, so no position in the shipped registry + // corresponds to it; only a lookup by carried number can find it. + assert!(FEE_VERSIONS.get(number as usize - 1).is_none()); + assert!(!FEE_VERSIONS.contains(&TEST_FEE_VERSION_DOUBLED_STORAGE_RATE)); + + let resolved = FeeVersion::get(number).expect("mock generation is registered"); + assert_eq!(resolved.fee_version_number, number); + assert_eq!(*resolved, TEST_FEE_VERSION_DOUBLED_STORAGE_RATE); + assert_eq!( + FeeVersion::get_optional(number), + Some(&TEST_FEE_VERSION_DOUBLED_STORAGE_RATE) + ); + assert_eq!( + TEST_FEE_VERSION_DOUBLED_STORAGE_RATE + .as_static() + .expect("mock generation resolves"), + resolved + ); + assert_eq!( + resolved.storage.storage_disk_usage_credit_per_byte, + 2 * FeeVersion::get(1) + .expect("registered") + .storage + .storage_disk_usage_credit_per_byte + ); + + // Neighbours of the mock number are not registered, and the mock never becomes the + // first or latest shipped generation. + assert!(FeeVersion::get(number - 1).is_err()); + assert!(FeeVersion::get(number + 1).is_err()); + assert_eq!(FeeVersion::first().fee_version_number, 1); + assert_eq!( + FeeVersion::latest(), + FEE_VERSIONS.last().expect("non-empty") + ); + } + #[test] fn should_reject_zero_and_unregistered_numbers() { let unregistered = [0, FEE_VERSIONS.len() as FeeVersionNumber + 1, u32::MAX]; diff --git a/packages/rs-platform-version/src/version/mocks/fee_test.rs b/packages/rs-platform-version/src/version/mocks/fee_test.rs new file mode 100644 index 00000000000..e8e9c5e0ee1 --- /dev/null +++ b/packages/rs-platform-version/src/version/mocks/fee_test.rs @@ -0,0 +1,49 @@ +use crate::version::fee::storage::FeeStorageVersion; +use crate::version::fee::v1::FEE_VERSION1; +use crate::version::fee::{FeeVersion, FeeVersionNumber}; +use crate::version::mocks::TEST_PROTOCOL_VERSION_SHIFT_BYTES; + +/// Number of the mock fee-history generation. +/// +/// Test identifiers live above the same shift as test protocol versions, so a mock number can +/// never collide with a number a released network persisted, and it is never a position in the +/// shipped registry. +pub const TEST_FEE_VERSION_NUMBER_DOUBLED_STORAGE_RATE: FeeVersionNumber = + (1 << TEST_PROTOCOL_VERSION_SHIFT_BYTES) + 1; + +/// A fee-history generation whose disk usage rate is twice the shipped one. +/// +/// It exists so tests can put a storage-rate boundary into the fee history and observe it +/// through every consumer: the registry lookup, Drive's refund pricing and the saved-state +/// round trip. It is registered only when the `mock-versions` feature is on. +pub const TEST_FEE_VERSION_DOUBLED_STORAGE_RATE: FeeVersion = FeeVersion { + fee_version_number: TEST_FEE_VERSION_NUMBER_DOUBLED_STORAGE_RATE, + storage: FeeStorageVersion { + storage_disk_usage_credit_per_byte: FEE_VERSION1.storage.storage_disk_usage_credit_per_byte + * 2, + ..FEE_VERSION1.storage + }, + ..FEE_VERSION1 +}; + +/// Mock generations consulted by `FeeVersion::get` after the shipped registry. +/// +/// Never part of a release build: `mock-versions` is a development-only feature. +pub const FEE_TEST_VERSIONS: &[FeeVersion] = &[TEST_FEE_VERSION_DOUBLED_STORAGE_RATE]; + +const _: () = assert!( + every_number_is_above_the_test_shift(FEE_TEST_VERSIONS), + "mock fee version numbers must live above the test shift so they cannot collide with shipped numbers" +); + +/// Returns true when every mock entry carries a number above the test shift. +const fn every_number_is_above_the_test_shift(registry: &[FeeVersion]) -> bool { + let mut index = 0; + while index < registry.len() { + if registry[index].fee_version_number >> TEST_PROTOCOL_VERSION_SHIFT_BYTES == 0 { + return false; + } + index += 1; + } + true +} diff --git a/packages/rs-platform-version/src/version/mocks/mod.rs b/packages/rs-platform-version/src/version/mocks/mod.rs index aef47dea3d5..5bae5da8aa9 100644 --- a/packages/rs-platform-version/src/version/mocks/mod.rs +++ b/packages/rs-platform-version/src/version/mocks/mod.rs @@ -1,3 +1,4 @@ +pub mod fee_test; pub mod v2_test; pub mod v3_test; From b59012301024597b64e52fa23024ee782cb96a76 Mon Sep 17 00:00:00 2001 From: DCG-Claude Date: Sat, 12 Sep 2026 00:33:23 -0500 Subject: [PATCH 7/8] fix(drive): move storage-epoch refund pricing behind calculate_fee version 1 Review pointed out that changing FeeRefunds::from_storage_removal in place edits a shipped generation even though no released protocol version can observe the difference. The shipped helper, consume_to_fees_v0 and calculate_fee_v0 are restored byte-identical to the base branch. The storage-epoch rule now lives in from_storage_removal_v1, consume_to_fees_v1 and calculate_fee_v1, reachable only through calculate_fee version 1, which no drive version table selects yet. The unreleased protocol version that registers a schedule under a new fee version number flips the slot. Tests run both generations: the shipped rule is pinned to current-epoch pricing across a boundary, generation 1 prices at the storage epoch, both agree whenever every history entry is number 1, and the dispatcher covers both arms plus an unknown slot. Co-Authored-By: Claude Fable 5.1 --- book/src/fees/overview.md | 19 +- packages/rs-dpp/src/fee/fee_result/refunds.rs | 167 +++++++++++-- .../rs-drive/src/fees/calculate_fee/mod.rs | 164 +++++++++++-- .../rs-drive/src/fees/calculate_fee/v1/mod.rs | 56 +++++ packages/rs-drive/src/fees/op.rs | 227 +++++++++++++----- 5 files changed, 518 insertions(+), 115 deletions(-) create mode 100644 packages/rs-drive/src/fees/calculate_fee/v1/mod.rs diff --git a/book/src/fees/overview.md b/book/src/fees/overview.md index 37403beb064..3b2c4cea2d8 100644 --- a/book/src/fees/overview.md +++ b/book/src/fees/overview.md @@ -450,14 +450,17 @@ out to proposers. There is a **dust limit**: refunds below 32 bytes worth of storage credits are discarded to prevent micro-refund spam. -A refund is priced at the storage table that was active when the bytes were -written. `FeeRefunds::from_storage_removal` resolves that rate through the fee -history at the storage epoch (the epoch recorded in the element's storage -flags), never at the epoch of the removal. The current epoch only fixes how many -era shares of the original fee were already paid out to proposers. Every -schedule shipped so far carries the same storage table, so this distinction -first becomes observable when a schedule with new storage rates is registered -under a new fee version number. +Refund pricing is versioned through `Drive::calculate_fee`. Generation 0, which +every released protocol version selects, prices every removed byte at the storage +rate active at the removal epoch (`FeeRefunds::from_storage_removal`). Generation +1 prices each removed epoch at the storage table that was active when the bytes +were written: `FeeRefunds::from_storage_removal_v1` resolves that rate through +the fee history at the storage epoch (the epoch recorded in the element's storage +flags), and the current epoch only fixes how many era shares of the original fee +were already paid out to proposers. Every schedule shipped so far carries the same +storage table, so the two generations agree on every reachable input; generation +1 is switched on by the unreleased protocol version that registers a schedule +with new storage rates under a new fee version number. ## Epoch-Based Fee Distribution diff --git a/packages/rs-dpp/src/fee/fee_result/refunds.rs b/packages/rs-dpp/src/fee/fee_result/refunds.rs index 48b57c56598..2046b307fce 100644 --- a/packages/rs-dpp/src/fee/fee_result/refunds.rs +++ b/packages/rs-dpp/src/fee/fee_result/refunds.rs @@ -37,12 +37,66 @@ pub struct FeeRefunds(pub CreditsPerEpochByIdentifier); impl FeeRefunds { /// Create fee refunds from GroveDB's StorageRemovalPerEpochByIdentifier /// + /// Shipped generation: every removed epoch is priced at the storage rate active at the + /// current epoch. Selected by `calculate_fee` version 0, which every released protocol + /// version uses. Frozen; see `from_storage_removal_v1` for the corrected rule. + pub fn from_storage_removal( + storage_removal: I, + current_epoch_index: EpochIndex, + epochs_per_era: u16, + previous_fee_versions: &CachedEpochIndexFeeVersions, + ) -> Result + where + I: IntoIterator, + C: IntoIterator, + E: TryInto, + { + let refunds_per_epoch_by_identifier = storage_removal + .into_iter() + .map(|(identifier, bytes_per_epochs)| { + bytes_per_epochs + .into_iter() + .filter(|(_, bytes)| bytes >= &MIN_REFUND_LIMIT_BYTES) + .map(|(encoded_epoch_index, bytes)| { + let epoch_index : u16 = encoded_epoch_index.try_into().map_err(|_| ProtocolError::Overflow("can't fit u64 epoch index from StorageRemovalPerEpochByIdentifier to u16 EpochIndex"))?; + + // TODO Add in multipliers once they have been made + + let credits: Credits = (bytes as Credits) + .checked_mul(Epoch::new(current_epoch_index)?.cost_for_known_cost_item(previous_fee_versions, StorageDiskUsageCreditPerByte)) + .ok_or(ProtocolError::Overflow("storage written bytes cost overflow"))?; + + let (amount, _) = calculate_storage_fee_refund_amount_and_leftovers( + credits, + epoch_index, + current_epoch_index, + epochs_per_era, + )?; + + Ok((epoch_index, amount)) + }) + .collect::>() + .map(|credits_per_epochs| (identifier, credits_per_epochs)) + }) + .collect::>()?; + + Ok(Self(refunds_per_epoch_by_identifier)) + } + + /// Create fee refunds from GroveDB's StorageRemovalPerEpochByIdentifier, pricing each + /// removed epoch at the storage rate that was active when the bytes were stored. + /// /// A refund is the unpaid remainder of the storage fee originally charged for the removed /// bytes. That fee was priced with the storage table active when the bytes were written, so /// the rate is resolved at the storage epoch (the key of each removal entry) through the fee /// history. The current epoch only decides how many era shares of that fee were already paid /// out to proposers. - pub fn from_storage_removal( + /// + /// Generation 1 of the refund pricing rule, selected by `calculate_fee` version 1. No + /// released protocol version selects it yet; the protocol version that registers a schedule + /// under a new fee version number is the boundary at which it takes effect. Until then every + /// registered generation shares one storage table, so both rules produce identical refunds. + pub fn from_storage_removal_v1( storage_removal: I, current_epoch_index: EpochIndex, epochs_per_era: u16, @@ -234,7 +288,17 @@ mod tests { amount } + /// Which generation of the pricing rule a test drives. + #[derive(Clone, Copy)] + enum Generation { + /// `from_storage_removal`: the shipped rule, priced at the current epoch. + Shipped, + /// `from_storage_removal_v1`: priced at the storage epoch. + V1, + } + fn refunds_for_one_identity( + generation: Generation, bytes_per_epoch: IntMap, current_epoch: EpochIndex, fee_history: &CachedEpochIndexFeeVersions, @@ -243,16 +307,25 @@ mod tests { let storage_removal = BytesPerEpochByIdentifier::from_iter([(identity_id, bytes_per_epoch)]); - FeeRefunds::from_storage_removal( - storage_removal, - current_epoch, - EPOCHS_PER_ERA, - fee_history, - ) - .expect("should create fee refunds") - .get(&identity_id) - .expect("identity has refunds") - .clone() + let refunds = match generation { + Generation::Shipped => FeeRefunds::from_storage_removal( + storage_removal, + current_epoch, + EPOCHS_PER_ERA, + fee_history, + ), + Generation::V1 => FeeRefunds::from_storage_removal_v1( + storage_removal, + current_epoch, + EPOCHS_PER_ERA, + fee_history, + ), + }; + refunds + .expect("should create fee refunds") + .get(&identity_id) + .expect("identity has refunds") + .clone() } #[test] @@ -263,18 +336,56 @@ mod tests { let storage_removal = BytesPerEpochByIdentifier::from_iter([(identity_id, bytes_per_epoch)]); - let fee_refunds = FeeRefunds::from_storage_removal( - storage_removal, - 3, - 20, - &EPOCH_CHANGE_FEE_VERSION_TEST, - ) - .expect("should create fee refunds"); + for fee_refunds in [ + FeeRefunds::from_storage_removal( + storage_removal.clone(), + 3, + 20, + &EPOCH_CHANGE_FEE_VERSION_TEST, + ) + .expect("should create fee refunds"), + FeeRefunds::from_storage_removal_v1( + storage_removal.clone(), + 3, + 20, + &EPOCH_CHANGE_FEE_VERSION_TEST, + ) + .expect("should create fee refunds"), + ] { + let credits_per_epoch = fee_refunds.get(&identity_id).expect("should exists"); + + assert!(credits_per_epoch.get(&0).is_none()); + assert!(credits_per_epoch.get(&1).is_some()); + } + } - let credits_per_epoch = fee_refunds.get(&identity_id).expect("should exists"); + #[test] + fn should_keep_pricing_every_removed_epoch_at_the_current_epoch_rate_in_the_shipped_generation( + ) { + // Frozen replay behaviour of `from_storage_removal`: the rate is the one active at + // the removal epoch, whatever epoch the bytes were stored in. Selected by + // `calculate_fee` version 0 on every released protocol version. + let fee_history: CachedEpochIndexFeeVersions = BTreeMap::from([ + (0, FeeVersion::get(1).expect("registered")), + (10, &SYNTHETIC_FEE_VERSION_2), + ]); + let current_epoch = 15; + + let refunds = refunds_for_one_identity( + Generation::Shipped, + IntMap::from_iter([(5, 100), (12, 100)]), + current_epoch, + &fee_history, + ); - assert!(credits_per_epoch.get(&0).is_none()); - assert!(credits_per_epoch.get(&1).is_some()); + assert_eq!( + refunds.get(&5).copied(), + Some(expected_refund(100, SYNTHETIC_RATE, 5, current_epoch)) + ); + assert_eq!( + refunds.get(&12).copied(), + Some(expected_refund(100, SYNTHETIC_RATE, 12, current_epoch)) + ); } #[test] @@ -289,6 +400,7 @@ mod tests { let current_epoch = 15; let refunds = refunds_for_one_identity( + Generation::V1, IntMap::from_iter([(5, 100), (12, 100)]), current_epoch, &fee_history, @@ -323,6 +435,7 @@ mod tests { let current_epoch = 12; let refunds = refunds_for_one_identity( + Generation::V1, IntMap::from_iter([(3, 100)]), current_epoch, &fee_history, @@ -344,8 +457,8 @@ mod tests { { // Every shipped input: an empty history (the fee version number 1 path in Drive) or a // history whose entries all resolve to number 1. The storage epoch and the current - // epoch then resolve to the same rate, so the result is identical to what pricing at - // the current epoch produced before the rule changed. + // epoch then resolve to the same rate, so both generations of the rule produce the + // same refunds. let same_table_histories: [CachedEpochIndexFeeVersions; 2] = [ BTreeMap::default(), BTreeMap::from([ @@ -356,11 +469,19 @@ mod tests { let current_epoch = 9; for fee_history in &same_table_histories { + let shipped = refunds_for_one_identity( + Generation::Shipped, + IntMap::from_iter([(2, 100), (8, 100)]), + current_epoch, + fee_history, + ); let refunds = refunds_for_one_identity( + Generation::V1, IntMap::from_iter([(2, 100), (8, 100)]), current_epoch, fee_history, ); + assert_eq!(shipped, refunds, "both generations agree on shipped inputs"); let current_epoch_rate = Epoch::new(current_epoch) .expect("epoch") .cost_for_known_cost_item(fee_history, StorageDiskUsageCreditPerByte); diff --git a/packages/rs-drive/src/fees/calculate_fee/mod.rs b/packages/rs-drive/src/fees/calculate_fee/mod.rs index f0ab38d6f20..6a07efa61b4 100644 --- a/packages/rs-drive/src/fees/calculate_fee/mod.rs +++ b/packages/rs-drive/src/fees/calculate_fee/mod.rs @@ -9,6 +9,7 @@ use dpp::version::PlatformVersion; use enum_map::EnumMap; mod v0; +mod v1; impl Drive { /// Calculates fees for the given operations. Returns the storage and processing costs. @@ -47,9 +48,17 @@ impl Drive { &platform_version.fee_version, previous_fee_versions, ), + 1 => Self::calculate_fee_v1( + base_operations, + drive_operations, + epoch, + epochs_per_era, + &platform_version.fee_version, + previous_fee_versions, + ), version => Err(Error::Drive(DriveError::UnknownVersionMismatch { method: "Drive::calculate_fee".to_string(), - known_versions: vec![0], + known_versions: vec![0, 1], received: version, })), } @@ -69,6 +78,8 @@ mod tests { use grovedb_costs::storage_cost::StorageCost; use grovedb_costs::OperationCost; use intmap::IntMap; + use platform_version::version::drive_versions::DriveFeesMethodVersions; + use platform_version::version::drive_versions::{DriveMethodVersions, DriveVersion}; use platform_version::version::fee::FeeVersion; use platform_version::version::PlatformVersion; use std::collections::BTreeMap; @@ -109,14 +120,35 @@ mod tests { } /// A mock platform version whose schedule is the mock generation, so the dispatcher must - /// hand the fee history through for refunds to be priced at all. - fn platform_version_with_doubled_storage_rate() -> PlatformVersion { + /// hand the fee history through for refunds to be priced at all, running the requested + /// `calculate_fee` generation. + fn platform_version_with_doubled_storage_rate(calculate_fee: u16) -> PlatformVersion { PlatformVersion { fee_version: TEST_FEE_VERSION_DOUBLED_STORAGE_RATE, + drive: DriveVersion { + methods: DriveMethodVersions { + fees: DriveFeesMethodVersions { calculate_fee }, + ..TEST_PLATFORM_V2.drive.methods + }, + ..TEST_PLATFORM_V2.drive + }, ..TEST_PLATFORM_V2 } } + fn first_rate() -> Credits { + FeeVersion::get(1) + .expect("registered") + .storage + .storage_disk_usage_credit_per_byte + } + + fn doubled_rate() -> Credits { + TEST_FEE_VERSION_DOUBLED_STORAGE_RATE + .storage + .storage_disk_usage_credit_per_byte + } + fn expected_refund(bytes: u32, rate: Credits, storage_epoch: u16) -> Credits { let (amount, _) = calculate_storage_fee_refund_amount_and_leftovers( bytes as Credits * rate, @@ -176,18 +208,13 @@ mod tests { } #[test] - fn should_price_refunds_across_a_rate_boundary_through_the_dispatcher_for_a_later_generation() { - let platform_version = platform_version_with_doubled_storage_rate(); + fn should_price_every_removed_epoch_at_the_current_epoch_rate_through_generation_zero() { + // Shipped rule, selected by every released protocol version: the rate active at the + // removal epoch (15, after the boundary) prices every removed epoch. + let platform_version = platform_version_with_doubled_storage_rate(0); let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); let history = boundary_history(); - let first_rate = FeeVersion::get(1) - .expect("registered") - .storage - .storage_disk_usage_credit_per_byte; - let doubled_rate = TEST_FEE_VERSION_DOUBLED_STORAGE_RATE - .storage - .storage_disk_usage_credit_per_byte; - assert_ne!(first_rate, doubled_rate); + assert_ne!(first_rate(), doubled_rate()); let fee_result = Drive::calculate_fee( None, @@ -199,20 +226,114 @@ mod tests { ) .expect("history supplied through the dispatcher"); - assert_eq!(fee_result.storage_fee, 10 * doubled_rate); + assert_eq!(fee_result.storage_fee, 10 * doubled_rate()); assert_eq!( refunds_of(&fee_result), BTreeMap::from([ - (5, expected_refund(100, first_rate, 5)), - (12, expected_refund(100, doubled_rate, 12)), + (5, expected_refund(100, doubled_rate(), 5)), + (12, expected_refund(100, doubled_rate(), 12)), ]), - "bytes stored on each side of the boundary refund at the rate they were charged" + "generation 0 prices both epochs at the current epoch's rate" + ); + } + + #[test] + fn should_price_refunds_across_a_rate_boundary_through_generation_one() { + let platform_version = platform_version_with_doubled_storage_rate(1); + let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + let history = boundary_history(); + + let fee_result = Drive::calculate_fee( + None, + Some(vec![removal_operation()]), + &epoch, + EPOCHS_PER_ERA, + &platform_version, + Some(&history), + ) + .expect("history supplied through the dispatcher"); + + assert_eq!(fee_result.storage_fee, 10 * doubled_rate()); + assert_eq!( + refunds_of(&fee_result), + BTreeMap::from([ + (5, expected_refund(100, first_rate(), 5)), + (12, expected_refund(100, doubled_rate(), 12)), + ]), + "generation 1 refunds each epoch at the rate its bytes were charged" + ); + } + + #[test] + fn should_agree_across_generations_whenever_every_generation_shares_one_storage_table() { + // Every input reachable on a released protocol version: a history where every entry is + // number 1. Both generations then resolve the same rate at every epoch. + let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + let history: CachedEpochIndexFeeVersions = BTreeMap::from([ + (0, FeeVersion::get(1).expect("registered")), + (10, FeeVersion::get(1).expect("registered")), + ]); + let results: Vec = [0, 1] + .into_iter() + .map(|calculate_fee| { + let platform_version = PlatformVersion { + drive: DriveVersion { + methods: DriveMethodVersions { + fees: DriveFeesMethodVersions { calculate_fee }, + ..TEST_PLATFORM_V2.drive.methods + }, + ..TEST_PLATFORM_V2.drive + }, + ..TEST_PLATFORM_V2 + }; + Drive::calculate_fee( + None, + Some(vec![removal_operation()]), + &epoch, + EPOCHS_PER_ERA, + &platform_version, + Some(&history), + ) + .expect("dispatches") + }) + .collect(); + + assert_eq!(results[0], results[1]); + assert_eq!( + refunds_of(&results[1]), + BTreeMap::from([ + (5, expected_refund(100, first_rate(), 5)), + (12, expected_refund(100, first_rate(), 12)), + ]) ); } #[test] fn should_reject_a_missing_fee_history_through_the_dispatcher_for_a_later_generation() { - let platform_version = platform_version_with_doubled_storage_rate(); + for calculate_fee in [0, 1] { + let platform_version = platform_version_with_doubled_storage_rate(calculate_fee); + let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + + let error = Drive::calculate_fee( + None, + Some(vec![removal_operation()]), + &epoch, + EPOCHS_PER_ERA, + &platform_version, + None, + ) + .expect_err("a later generation cannot price refunds without the history"); + + assert!( + matches!(error, Error::Drive(DriveError::CorruptedCodeExecution(_))), + "calculate_fee {calculate_fee}: unexpected error {error}" + ); + } + } + + #[test] + fn should_reject_an_unknown_calculate_fee_version() { + let platform_version = platform_version_with_doubled_storage_rate(2); let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); let error = Drive::calculate_fee( @@ -223,10 +344,13 @@ mod tests { &platform_version, None, ) - .expect_err("a later generation cannot price refunds without the history"); + .expect_err("version 2 is not known"); assert!( - matches!(error, Error::Drive(DriveError::CorruptedCodeExecution(_))), + matches!( + error, + Error::Drive(DriveError::UnknownVersionMismatch { received: 2, .. }) + ), "unexpected error {error}" ); } diff --git a/packages/rs-drive/src/fees/calculate_fee/v1/mod.rs b/packages/rs-drive/src/fees/calculate_fee/v1/mod.rs new file mode 100644 index 00000000000..294e7f78ccf --- /dev/null +++ b/packages/rs-drive/src/fees/calculate_fee/v1/mod.rs @@ -0,0 +1,56 @@ +use crate::drive::Drive; +use crate::error::fee::FeeError; +use crate::error::Error; +use crate::fees::op::{BaseOp, LowLevelDriveOperation}; +use dpp::block::epoch::Epoch; +use dpp::fee::fee_result::FeeResult; + +use dpp::fee::default_costs::CachedEpochIndexFeeVersions; +use enum_map::EnumMap; +use platform_version::version::fee::FeeVersion; + +impl Drive { + /// Calculates fees for the given operations. Returns the storage and processing costs. + /// + /// Generation 1 differs from generation 0 only in refund pricing: removed bytes are refunded + /// at the storage rate active when they were stored (`consume_to_fees_v1`), not at the rate + /// active when they are removed. No released protocol version selects this generation; the + /// unreleased protocol version that registers a schedule under a new fee version number is + /// where it is switched on. + #[inline(always)] + pub(crate) fn calculate_fee_v1( + base_operations: Option>, + drive_operations: Option>, + epoch: &Epoch, + epochs_per_era: u16, + fee_version: &FeeVersion, + previous_fee_versions: Option<&CachedEpochIndexFeeVersions>, + ) -> Result { + let mut aggregate_fee_result = FeeResult::default(); + if let Some(base_operations) = base_operations { + for (base_op, count) in base_operations.iter() { + match base_op.cost().checked_mul(*count) { + None => return Err(Error::Fee(FeeError::Overflow("overflow error"))), + Some(cost) => match aggregate_fee_result.processing_fee.checked_add(cost) { + None => return Err(Error::Fee(FeeError::Overflow("overflow error"))), + Some(value) => aggregate_fee_result.processing_fee = value, + }, + } + } + } + + if let Some(drive_operations) = drive_operations { + for drive_fee_result in LowLevelDriveOperation::consume_to_fees_v1( + drive_operations, + epoch, + epochs_per_era, + fee_version, + previous_fee_versions, + )? { + aggregate_fee_result.checked_add_assign(drive_fee_result)?; + } + } + + Ok(aggregate_fee_result) + } +} diff --git a/packages/rs-drive/src/fees/op.rs b/packages/rs-drive/src/fees/op.rs index 269a09afc8a..e4194d03cf0 100644 --- a/packages/rs-drive/src/fees/op.rs +++ b/packages/rs-drive/src/fees/op.rs @@ -368,6 +368,81 @@ impl LowLevelDriveOperation { .collect() } + /// Returns a list of the costs of the Drive operations, refunding removed bytes at the + /// storage rate active when they were stored. + /// Should only be used by Calculate fee (generation 1). + /// + /// Identical to `consume_to_fees_v0` except that refunds go through + /// `FeeRefunds::from_storage_removal_v1`. The fee version number 1 arm keeps pricing against + /// an empty history, so on every schedule shipped so far the two generations agree. + pub fn consume_to_fees_v1( + drive_operations: Vec, + epoch: &Epoch, + epochs_per_era: u16, + fee_version: &FeeVersion, + previous_fee_versions: Option<&CachedEpochIndexFeeVersions>, + ) -> Result, Error> { + drive_operations + .into_iter() + .map(|operation| match operation { + PreCalculatedFeeResult(f) => Ok(f), + FunctionOperation(op) => Ok(FeeResult { + processing_fee: op.cost(fee_version), + ..Default::default() + }), + _ => { + let cost = operation.operation_cost()?; + // There is no need for a checked multiply here because added bytes are u64 and + // storage disk usage credit per byte should never be high enough to cause an overflow + let storage_fee = cost.storage_cost.added_bytes as u64 * fee_version.storage.storage_disk_usage_credit_per_byte; + let processing_fee = cost.ephemeral_cost(fee_version)?; + let (fee_refunds, removed_bytes_from_system) = + match cost.storage_cost.removed_bytes { + NoStorageRemoval => (FeeRefunds::default(), 0), + BasicStorageRemoval(amount) => { + // this is not always considered an error + (FeeRefunds::default(), amount) + } + SectionedStorageRemoval(mut removal_per_epoch_by_identifier) => { + + let system_amount = removal_per_epoch_by_identifier + .remove(&Identifier::default()) + .map_or(0, |a| a.values().sum()); + if fee_version.fee_version_number == 1 { + ( + FeeRefunds::from_storage_removal_v1( + removal_per_epoch_by_identifier, + epoch.index, + epochs_per_era, + &BTreeMap::default(), + )?, + system_amount, + ) + } else { + let previous_fee_versions = previous_fee_versions.ok_or(Error::Drive(DriveError::CorruptedCodeExecution("expected previous epoch index fee versions to be able to offer refunds")))?; + ( + FeeRefunds::from_storage_removal_v1( + removal_per_epoch_by_identifier, + epoch.index, + epochs_per_era, + previous_fee_versions, + )?, + system_amount, + ) + } + } + }; + Ok(FeeResult { + storage_fee, + processing_fee, + fee_refunds, + removed_bytes_from_system, + }) + } + }) + .collect() + } + /// Returns the cost of this operation pub fn operation_cost(self) -> Result { match self { @@ -2818,10 +2893,10 @@ mod tests { } // --------------------------------------------------------------- - // 9. consume_to_fees_v0 — refunds and the fee history requirement + // 9. consume_to_fees_v0 / consume_to_fees_v1 — refunds and the fee history requirement // --------------------------------------------------------------- - mod consume_to_fees_v0 { + mod consume_to_fees { use super::*; use dpp::fee::epoch::distribution::calculate_storage_fee_refund_amount_and_leftovers; use intmap::IntMap; @@ -2898,62 +2973,90 @@ mod tests { .collect() } + /// Runs the requested generation of `consume_to_fees`. + fn consume( + generation: u16, + fee_version: &FeeVersion, + previous_fee_versions: Option<&CachedEpochIndexFeeVersions>, + ) -> Result { + let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + let results = match generation { + 0 => LowLevelDriveOperation::consume_to_fees_v0( + vec![removal_operation()], + &epoch, + EPOCHS_PER_ERA, + fee_version, + previous_fee_versions, + ), + 1 => LowLevelDriveOperation::consume_to_fees_v1( + vec![removal_operation()], + &epoch, + EPOCHS_PER_ERA, + fee_version, + previous_fee_versions, + ), + other => panic!("no consume_to_fees generation {other}"), + }?; + Ok(results + .into_iter() + .next() + .expect("one operation, one result")) + } + #[test] fn should_refund_through_the_legacy_empty_history_path_when_the_fee_version_number_is_one() { - let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); - let mut results = LowLevelDriveOperation::consume_to_fees_v0( - vec![removal_operation()], - &epoch, - EPOCHS_PER_ERA, - &FEE_VERSION1, - None, - ) - .expect("number 1 never needs the fee history"); - let fee_result = results.remove(0); + for generation in [0, 1] { + let fee_result = consume(generation, &FEE_VERSION1, None) + .expect("number 1 never needs the fee history"); + + assert_eq!(fee_result.removed_bytes_from_system, 40); + assert_eq!(fee_result.storage_fee, 0); + assert_eq!( + refunds_of(&fee_result), + BTreeMap::from([ + (5, expected_refund(100, FIRST_GENERATION_RATE, 5)), + (12, expected_refund(100, FIRST_GENERATION_RATE, 12)), + ]), + "generation {generation}" + ); + } + } + + #[test] + fn should_require_fee_history_when_the_fee_version_number_is_not_one() { + for generation in [0, 1] { + let error = consume(generation, &SYNTHETIC_FEE_VERSION_2, None) + .expect_err("a later generation cannot price refunds without the history"); + assert!( + matches!(error, Error::Drive(DriveError::CorruptedCodeExecution(_))), + "generation {generation}: unexpected error {error}" + ); + } + } + + #[test] + fn should_price_every_removed_epoch_at_the_current_epoch_rate_in_generation_zero() { + // Shipped rule: the rate active at the removal epoch (15, past the boundary) prices + // every removed epoch, including bytes stored at epoch 5 before the boundary. + let history = boundary_history(); + let fee_result = + consume(0, &SYNTHETIC_FEE_VERSION_2, Some(&history)).expect("history supplied"); - assert_eq!(fee_result.removed_bytes_from_system, 40); - assert_eq!(fee_result.storage_fee, 0); assert_eq!( refunds_of(&fee_result), BTreeMap::from([ - (5, expected_refund(100, FIRST_GENERATION_RATE, 5)), - (12, expected_refund(100, FIRST_GENERATION_RATE, 12)), + (5, expected_refund(100, SYNTHETIC_RATE, 5)), + (12, expected_refund(100, SYNTHETIC_RATE, 12)), ]) ); } #[test] - fn should_require_fee_history_when_the_fee_version_number_is_not_one() { - let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); - let error = LowLevelDriveOperation::consume_to_fees_v0( - vec![removal_operation()], - &epoch, - EPOCHS_PER_ERA, - &SYNTHETIC_FEE_VERSION_2, - None, - ) - .expect_err("a later generation cannot price refunds without the history"); - assert!( - matches!(error, Error::Drive(DriveError::CorruptedCodeExecution(_))), - "unexpected error {error}" - ); - } - - #[test] - fn should_refund_at_the_storage_epoch_rate_across_a_history_boundary_when_the_fee_version_number_is_not_one( - ) { - let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + fn should_refund_at_the_storage_epoch_rate_across_a_history_boundary_in_generation_one() { let history = boundary_history(); - let mut results = LowLevelDriveOperation::consume_to_fees_v0( - vec![removal_operation()], - &epoch, - EPOCHS_PER_ERA, - &SYNTHETIC_FEE_VERSION_2, - Some(&history), - ) - .expect("history supplied"); - let fee_result = results.remove(0); + let fee_result = + consume(1, &SYNTHETIC_FEE_VERSION_2, Some(&history)).expect("history supplied"); assert_eq!( refunds_of(&fee_result), @@ -2968,26 +3071,22 @@ mod tests { #[test] fn should_ignore_fee_history_when_the_fee_version_number_is_one() { // Shipped replay path: every schedule a released protocol version references - // carries number 1, and that branch prices refunds against an empty history. - let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + // carries number 1, and that branch prices refunds against an empty history in + // both generations. let history = boundary_history(); - let mut results = LowLevelDriveOperation::consume_to_fees_v0( - vec![removal_operation()], - &epoch, - EPOCHS_PER_ERA, - &FEE_VERSION1, - Some(&history), - ) - .expect("number 1 accepts but does not read the history"); - let fee_result = results.remove(0); - - assert_eq!( - refunds_of(&fee_result), - BTreeMap::from([ - (5, expected_refund(100, FIRST_GENERATION_RATE, 5)), - (12, expected_refund(100, FIRST_GENERATION_RATE, 12)), - ]) - ); + for generation in [0, 1] { + let fee_result = consume(generation, &FEE_VERSION1, Some(&history)) + .expect("number 1 accepts but does not read the history"); + + assert_eq!( + refunds_of(&fee_result), + BTreeMap::from([ + (5, expected_refund(100, FIRST_GENERATION_RATE, 5)), + (12, expected_refund(100, FIRST_GENERATION_RATE, 12)), + ]), + "generation {generation}" + ); + } } } } From dfa3b637990966f49ba2ef3aab631369cbafd16e Mon Sep 17 00:00:00 2001 From: DCG-Claude Date: Thu, 24 Sep 2026 11:28:42 -0500 Subject: [PATCH 8/8] fix(drive): keep ephemeral storage billing in calculate_fee generation 1 consume_to_fees_v1 was copied before the base added the ephemeral-cost arm to generation 0, so TTL writes reaching generation 1 fell through to the wildcard and were priced as permanent storage, and a sectioned removal on an ephemeral batch was no longer rejected. The arm is ported verbatim; the only difference between the two generations is again the refund helper. Tests at the op and dispatcher layers pin ephemeral pricing and the sectioned-removal rejection in both generations; all four fail when the arm is removed from generation 1. Co-Authored-By: Claude Fable 5.1 --- .../rs-drive/src/fees/calculate_fee/mod.rs | 94 +++++++++++- packages/rs-drive/src/fees/op.rs | 145 +++++++++++++++++- 2 files changed, 232 insertions(+), 7 deletions(-) diff --git a/packages/rs-drive/src/fees/calculate_fee/mod.rs b/packages/rs-drive/src/fees/calculate_fee/mod.rs index 6a07efa61b4..4e910099493 100644 --- a/packages/rs-drive/src/fees/calculate_fee/mod.rs +++ b/packages/rs-drive/src/fees/calculate_fee/mod.rs @@ -71,10 +71,13 @@ mod tests { use crate::fees::op::LowLevelDriveOperation::CalculatedCostOperation; use dpp::fee::default_costs::CachedEpochIndexFeeVersions; use dpp::fee::epoch::distribution::calculate_storage_fee_refund_amount_and_leftovers; + use dpp::fee::fee_result::refunds::FeeRefunds; use dpp::fee::Credits; use dpp::version::mocks::fee_test::TEST_FEE_VERSION_DOUBLED_STORAGE_RATE; use dpp::version::mocks::v2_test::TEST_PLATFORM_V2; - use grovedb_costs::storage_cost::removal::StorageRemovedBytes::SectionedStorageRemoval; + use grovedb_costs::storage_cost::removal::StorageRemovedBytes::{ + NoStorageRemoval, SectionedStorageRemoval, + }; use grovedb_costs::storage_cost::StorageCost; use grovedb_costs::OperationCost; use intmap::IntMap; @@ -331,6 +334,95 @@ mod tests { } } + /// An ephemeral (TTL) batch adding 100 bytes, as TTL batch application and estimation + /// produce it. + fn ephemeral_operation() -> LowLevelDriveOperation { + LowLevelDriveOperation::CalculatedEphemeralCostOperation(OperationCost { + storage_cost: StorageCost { + added_bytes: 100, + replaced_bytes: 0, + removed_bytes: NoStorageRemoval, + }, + ..Default::default() + }) + } + + #[test] + fn should_bill_ephemeral_bytes_to_processing_through_both_generations() { + // A TTL write must never be charged as permanent storage, whichever generation the + // platform version selects. Both generations produce the same fee result: zero storage + // fee and the ephemeral bytes fee inside processing. + let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + let results: Vec = [0, 1] + .into_iter() + .map(|calculate_fee| { + let platform_version = platform_version_with_doubled_storage_rate(calculate_fee); + Drive::calculate_fee( + None, + Some(vec![ephemeral_operation()]), + &epoch, + EPOCHS_PER_ERA, + &platform_version, + None, + ) + .expect("ephemeral operations never need the fee history") + }) + .collect(); + + let ephemeral_rate = TEST_FEE_VERSION_DOUBLED_STORAGE_RATE + .storage + .ttl_ephemeral_disk_usage_credit_per_byte; + for (calculate_fee, fee_result) in results.iter().enumerate() { + assert_eq!(fee_result.storage_fee, 0, "calculate_fee {calculate_fee}"); + assert_eq!( + fee_result.processing_fee, + 100 * ephemeral_rate + + 100 + * TEST_FEE_VERSION_DOUBLED_STORAGE_RATE + .storage + .storage_processing_credit_per_byte, + "calculate_fee {calculate_fee}" + ); + assert_eq!(fee_result.fee_refunds, FeeRefunds::default()); + } + assert_eq!(results[0], results[1]); + } + + #[test] + fn should_reject_a_sectioned_removal_on_an_ephemeral_batch_through_both_generations() { + let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); + let history = boundary_history(); + for calculate_fee in [0, 1] { + let platform_version = platform_version_with_doubled_storage_rate(calculate_fee); + let mut removal = BTreeMap::new(); + removal.insert(IDENTITY, IntMap::from_iter([(5u16, 100u32)])); + let operation = + LowLevelDriveOperation::CalculatedEphemeralCostOperation(OperationCost { + storage_cost: StorageCost { + added_bytes: 0, + replaced_bytes: 0, + removed_bytes: SectionedStorageRemoval(removal), + }, + ..Default::default() + }); + + let error = Drive::calculate_fee( + None, + Some(vec![operation]), + &epoch, + EPOCHS_PER_ERA, + &platform_version, + Some(&history), + ) + .expect_err("TTL subtrees carry no storage flags"); + + assert!( + matches!(error, Error::Drive(DriveError::CorruptedCodeExecution(_))), + "calculate_fee {calculate_fee}: unexpected error {error}" + ); + } + } + #[test] fn should_reject_an_unknown_calculate_fee_version() { let platform_version = platform_version_with_doubled_storage_rate(2); diff --git a/packages/rs-drive/src/fees/op.rs b/packages/rs-drive/src/fees/op.rs index e4194d03cf0..a9b2a8c5609 100644 --- a/packages/rs-drive/src/fees/op.rs +++ b/packages/rs-drive/src/fees/op.rs @@ -373,8 +373,11 @@ impl LowLevelDriveOperation { /// Should only be used by Calculate fee (generation 1). /// /// Identical to `consume_to_fees_v0` except that refunds go through - /// `FeeRefunds::from_storage_removal_v1`. The fee version number 1 arm keeps pricing against - /// an empty history, so on every schedule shipped so far the two generations agree. + /// `FeeRefunds::from_storage_removal_v1`. Ephemeral (TTL) operations keep generation 0's + /// pricing: added bytes bill to processing at the ephemeral rate, never to storage, and a + /// sectioned removal on an ephemeral batch is an error. The fee version number 1 arm keeps + /// pricing against an empty history, so on every schedule shipped so far the two + /// generations agree. pub fn consume_to_fees_v1( drive_operations: Vec, epoch: &Epoch, @@ -390,6 +393,46 @@ impl LowLevelDriveOperation { processing_fee: op.cost(fee_version), ..Default::default() }), + CalculatedEphemeralCostOperation(cost) => { + // TTL'd-subtree bytes: the added bytes bill to + // PROCESSING at the ephemeral rate instead of to + // storage — they provably live at most `ttl` plus a + // bounded drainage lag, so the perpetual-retention + // storage price does not apply. No refunds by + // construction: TTL elements carry no storage flags, + // so their removal can only ever be basic. + let ephemeral_bytes_fee = (cost.storage_cost.added_bytes as u64) + .checked_mul( + fee_version + .storage + .ttl_ephemeral_disk_usage_credit_per_byte, + ) + .ok_or(Error::Fee(FeeError::Overflow( + "overflow pricing ephemeral bytes", + )))?; + let processing_fee = cost + .ephemeral_cost(fee_version)? + .checked_add(ephemeral_bytes_fee) + .ok_or(Error::Fee(FeeError::Overflow( + "overflow adding ephemeral bytes fee", + )))?; + let removed_bytes_from_system = match cost.storage_cost.removed_bytes { + NoStorageRemoval => 0, + BasicStorageRemoval(amount) => amount, + SectionedStorageRemoval(_) => { + return Err(Error::Drive(DriveError::CorruptedCodeExecution( + "TTL'd subtrees carry no storage flags, so an ephemeral \ + batch cannot produce sectioned (refundable) removal", + ))) + } + }; + Ok(FeeResult { + storage_fee: 0, + processing_fee, + fee_refunds: FeeRefunds::default(), + removed_bytes_from_system, + }) + } _ => { let cost = operation.operation_cost()?; // There is no need for a checked multiply here because added bytes are u64 and @@ -2973,23 +3016,51 @@ mod tests { .collect() } - /// Runs the requested generation of `consume_to_fees`. - fn consume( + /// An ephemeral (TTL) batch adding 100 bytes and removing 30 basic bytes. + fn ephemeral_operation() -> LowLevelDriveOperation { + CalculatedEphemeralCostOperation(OperationCost { + storage_cost: StorageCost { + added_bytes: 100, + replaced_bytes: 0, + removed_bytes: BasicStorageRemoval(30), + }, + ..Default::default() + }) + } + + /// An ephemeral (TTL) batch that claims a sectioned removal, which TTL subtrees can + /// never produce. + fn ephemeral_operation_with_sectioned_removal() -> LowLevelDriveOperation { + let mut removal = BTreeMap::new(); + removal.insert(IDENTITY, IntMap::from_iter([(5u16, 100u32)])); + CalculatedEphemeralCostOperation(OperationCost { + storage_cost: StorageCost { + added_bytes: 100, + replaced_bytes: 0, + removed_bytes: SectionedStorageRemoval(removal), + }, + ..Default::default() + }) + } + + /// Runs the requested generation of `consume_to_fees` on one operation. + fn consume_operation( generation: u16, + operation: LowLevelDriveOperation, fee_version: &FeeVersion, previous_fee_versions: Option<&CachedEpochIndexFeeVersions>, ) -> Result { let epoch = Epoch::new(CURRENT_EPOCH).expect("epoch"); let results = match generation { 0 => LowLevelDriveOperation::consume_to_fees_v0( - vec![removal_operation()], + vec![operation], &epoch, EPOCHS_PER_ERA, fee_version, previous_fee_versions, ), 1 => LowLevelDriveOperation::consume_to_fees_v1( - vec![removal_operation()], + vec![operation], &epoch, EPOCHS_PER_ERA, fee_version, @@ -3003,6 +3074,68 @@ mod tests { .expect("one operation, one result")) } + /// Runs the requested generation of `consume_to_fees` on the removal operation. + fn consume( + generation: u16, + fee_version: &FeeVersion, + previous_fee_versions: Option<&CachedEpochIndexFeeVersions>, + ) -> Result { + consume_operation( + generation, + removal_operation(), + fee_version, + previous_fee_versions, + ) + } + + #[test] + fn should_bill_ephemeral_bytes_to_processing_at_the_ephemeral_rate_in_both_generations() { + // TTL subtree bytes never pay the perpetual storage price. Both generations must + // price them identically: no storage fee, processing carries the ephemeral bytes + // fee on top of the operation's ephemeral cost, no refunds, basic removal reported. + let expected_processing = ephemeral_operation() + .operation_cost() + .expect("calculated cost") + .ephemeral_cost(&FEE_VERSION1) + .expect("ephemeral cost") + + 100 + * FEE_VERSION1 + .storage + .ttl_ephemeral_disk_usage_credit_per_byte; + + for generation in [0, 1] { + let fee_result = + consume_operation(generation, ephemeral_operation(), &FEE_VERSION1, None) + .expect("ephemeral operations never need the fee history"); + + assert_eq!(fee_result.storage_fee, 0, "generation {generation}"); + assert_eq!( + fee_result.processing_fee, expected_processing, + "generation {generation}" + ); + assert_eq!(fee_result.fee_refunds, FeeRefunds::default()); + assert_eq!(fee_result.removed_bytes_from_system, 30); + } + } + + #[test] + fn should_reject_a_sectioned_removal_on_an_ephemeral_batch_in_both_generations() { + let history = boundary_history(); + for generation in [0, 1] { + let error = consume_operation( + generation, + ephemeral_operation_with_sectioned_removal(), + &FEE_VERSION1, + Some(&history), + ) + .expect_err("TTL subtrees carry no storage flags"); + assert!( + matches!(error, Error::Drive(DriveError::CorruptedCodeExecution(_))), + "generation {generation}: unexpected error {error}" + ); + } + } + #[test] fn should_refund_through_the_legacy_empty_history_path_when_the_fee_version_number_is_one() {