diff --git a/data-model/src/lengthy_entry.rs b/data-model/src/lengthy_entry.rs index 609b4ba..1941062 100644 --- a/data-model/src/lengthy_entry.rs +++ b/data-model/src/lengthy_entry.rs @@ -36,6 +36,23 @@ where pub fn available(&self) -> u64 { self.available } + + /// Turn this into a regular [`Entry`]. + pub fn into_entry(self) -> Entry { + self.entry + } +} + +impl + AsRef> for LengthyEntry +where + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, +{ + fn as_ref(&self) -> &Entry { + &self.entry + } } /// An [`AuthorisedEntry`] together with information about how much of its payload a given [`Store`] holds. @@ -81,4 +98,23 @@ where pub fn available(&self) -> u64 { self.available } + + /// Turn this into a [`AuthorisedEntry`]. + pub fn into_authorised_entry(self) -> AuthorisedEntry { + self.entry + } +} + +impl + AsRef> + for LengthyAuthorisedEntry +where + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, + AT: AuthorisationToken, +{ + fn as_ref(&self) -> &AuthorisedEntry { + &self.entry + } } diff --git a/data-model/src/store.rs b/data-model/src/store.rs index c9d86df..a690a76 100644 --- a/data-model/src/store.rs +++ b/data-model/src/store.rs @@ -4,7 +4,7 @@ use ufotofu::{local_nb::Producer, nb::BulkProducer}; use crate::{ entry::AuthorisedEntry, - grouping::{AreaOfInterest, Range3d}, + grouping::{Area, AreaOfInterest, Range3d}, parameters::{AuthorisationToken, NamespaceId, PayloadDigest, SubspaceId}, LengthyAuthorisedEntry, LengthyEntry, Path, }; @@ -21,8 +21,6 @@ pub enum EntryIngestionSuccess< > { /// The entry was successfully ingested. Success, - /// The entry was successfully ingested and prefix pruned some entries. - SuccessAndPruned(Vec>), /// The entry was not ingested because a newer entry with same Obsolete { /// The obsolete entry which was not ingested. @@ -41,19 +39,31 @@ pub enum EntryIngestionError< S: SubspaceId, PD: PayloadDigest, AT, + OE, > { /// The entry belonged to another namespace. WrongNamespace(AuthorisedEntry), /// The ingestion would have triggered prefix pruning when that was not desired. PruningPrevented, + /// Something specific to this store implementation went wrong. + OperationsError(OE), } /// A tuple of an [`AuthorisedEntry`] and how a [`Store`] responded to its ingestion. -pub type BulkIngestionResult = ( +pub type BulkIngestionResult< + const MCL: usize, + const MCC: usize, + const MPL: usize, + N, + S, + PD, + AT, + OE, +> = ( AuthorisedEntry, Result< EntryIngestionSuccess, - EntryIngestionError, + EntryIngestionError, >, ); @@ -66,9 +76,10 @@ pub struct BulkIngestionError< S: SubspaceId, PD: PayloadDigest, AT: AuthorisationToken, + OE, IngestionError, > { - pub ingested: Vec>, + pub ingested: Vec>, pub error: IngestionError, } @@ -86,32 +97,46 @@ where } /// Returned when a payload fails to be appended into the [`Store`]. -pub enum PayloadAppendError { +pub enum PayloadAppendError { /// None of the entries in the store reference this payload. NotEntryReference, /// The payload is already held in storage. AlreadyHaveIt, - /// The received payload is larger than was expected. - PayloadTooLarge, + /// The payload source produced more bytes than were expected for this payload. + TooManyBytes, /// The completed payload's digest is not what was expected. DigestMismatch, - /// Try deleting some files!!! - SomethingElseWentWrong, + /// Something specific to this store implementation went wrong. + OperationError(OE), } /// Returned when no entry was found for some criteria. -pub struct NoSuchEntryError(); +pub struct NoSuchEntryError; -/// Orderings for a +/// The order by which entries should be returned for a given query. pub enum QueryOrder { - /// Ordered by subspace, then path, then timestamp + /// Ordered by subspace, then path, then timestamp. Subspace, - /// Ordered by path, them timestamp, then subspace + /// Ordered by path, then by an arbitrary order determined by the implementation. Path, - /// Ordered by timestamp, then subspace, then path + /// Ordered by timestamp, then by an arbitrary order determined by the implementation. Timestamp, - /// Whichever order is most efficient. - Efficient, + /// An arbitrary order chosen by the implementation, hopefully the most efficient one. + Arbitrary, +} + +/// Describes an [`AuthorisedEntry`] which was pruned and the [`AuthorisedEntry`] which triggered the pruning. +pub struct PruneEvent +where + N: NamespaceId, + S: SubspaceId, + PD: PayloadDigest, + AT: AuthorisationToken, +{ + /// The subspace ID and path of the entry which was pruned. + pub pruned: (S, Path), + /// The entry which triggered the pruning. + pub by: AuthorisedEntry, } /// An event which took place within a [`Store`]. @@ -128,18 +153,41 @@ where /// An existing entry received a portion of its corresponding payload. Appended(u64, LengthyAuthorisedEntry), /// An entry was forgotten. - EntryForgotten(u64, AuthorisedEntry), + EntryForgotten(u64, (S, Path)), /// A payload was forgotten. PayloadForgotten(u64, PD), - /// An entry was overwritten. - Overwritten(u64, AuthorisedEntry), /// An entry was pruned via prefix pruning. - Pruned(u64, AuthorisedEntry), + Pruned(u64, PruneEvent), } -/// Returned when the store could not resume a subscription because the provided ID was too outdated or not present. +/// Returned when the store chooses to not resume a subscription. pub struct ResumptionFailedError(pub u64); +/// Describes which entries to ignore during a query. +#[derive(Default)] +pub struct QueryIgnoreParams { + /// Omit entries with locally incomplete corresponding payloads. + pub ignore_incomplete_payloads: bool, + /// Omit entries whose payload is the empty string. + pub ignore_empty_payloads: bool, +} + +impl QueryIgnoreParams { + pub fn ignore_incomplete_payloads(&mut self) { + self.ignore_incomplete_payloads = true; + } + + pub fn ignore_empty_payloads(&mut self) { + self.ignore_empty_payloads = true; + } +} + +/// Returned when a payload could not be forgotten. +pub enum ForgetPayloadError { + NoSuchEntry, + ReferredToByOtherEntries, +} + /// A [`Store`] is a set of [`AuthorisedEntry`] belonging to a single namespace, and a (possibly partial) corresponding set of payloads. pub trait Store where @@ -150,8 +198,7 @@ where { type FlushError; type BulkIngestionError; - type EntryProducer: Producer>; - type EventProducer: Producer>; + type OperationsError; /// The [namespace](https://willowprotocol.org/specs/data-model/index.html#namespace) which all of this store's [`AuthorisedEntry`] belong to. fn namespace_id() -> N; @@ -165,7 +212,7 @@ where ) -> impl Future< Output = Result< EntryIngestionSuccess, - EntryIngestionError, + EntryIngestionError, >, >; @@ -178,8 +225,18 @@ where prevent_pruning: bool, ) -> impl Future< Output = Result< - Vec>, - BulkIngestionError, + Vec>, + BulkIngestionError< + MCL, + MCC, + MPL, + N, + S, + PD, + AT, + Self::BulkIngestionError, + Self::OperationsError, + >, >, >; @@ -188,78 +245,98 @@ where /// Will fail if: /// - The payload digest is not referred to by any of the store's entries. /// - A complete payload with the same digest is already held in storage. - /// - The payload exceeded the expected size + /// - The payload source produced more bytes than were expected for this payload. /// - The final payload's digest did not match the expected digest /// - Something else went wrong, e.g. there was no space for the payload on disk. /// - /// This method **cannot** verify the integrity of partial payload. This means that arbitrary (and possibly malicious) payloads smaller than the expected size will be stored unless partial verification is implemented upstream (e.g. during [the Willow General Sync Protocol's payload transformation](https://willowprotocol.org/specs/sync/index.html#sync_payloads_transform)). + /// This method **cannot** verify the integrity of partial payloads. This means that arbitrary (and possibly malicious) payloads smaller than the expected size will be stored unless partial verification is implemented upstream (e.g. during [the Willow General Sync Protocol's payload transformation](https://willowprotocol.org/specs/sync/index.html#sync_payloads_transform)). fn append_payload( &self, - expected_digest: PD, + expected_digest: &PD, expected_size: u64, - producer: &mut Producer, - ) -> impl Future, PayloadAppendError>> + payload_source: &mut Producer, + ) -> impl Future< + Output = Result< + PayloadAppendSuccess, + PayloadAppendError, + >, + > where Producer: BulkProducer; /// Locally forget an entry with a given [`Path`] and [subspace](https://willowprotocol.org/specs/data-model/index.html#subspace) id, returning the forgotten entry, or an error if no entry with that path and subspace ID are held by this store. /// - /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the entry. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. + /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the entry. If `false`, it *may* persist what was forgotten for an arbitrary amount of time. /// - /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten entry back. + /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten entry back. fn forget_entry( + &self, path: &Path, - subspace_id: S, + subspace_id: &S, traceless: bool, - ) -> impl Future, NoSuchEntryError>>; + ) -> impl Future>; /// Locally forget all [`AuthorisedEntry`] [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all forgotten entries /// + /// If `protected` is `Some`, then all entries [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by that [`Area`] will be prevented from being forgotten, even though they are included by `area`. + /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten entries. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// - /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten entries back. + /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten entries back. fn forget_area( + &self, area: &AreaOfInterest, + protected: Option>, traceless: bool, ) -> impl Future>>; - /// Locally forget all [`AuthorisedEntry`] **not** [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all forgotten entries + /// Locally forget the corresponding payload of the entry with a given path and subspace, or an error if no entry with that path and subspace ID is held by this store or if the entry's payload corresponds to other entries. /// - /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten entries. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. + /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the payload. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// - /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten entries back. - fn forget_everything_but_area( - area: &AreaOfInterest, + /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten payload back. + fn forget_payload( + path: &Path, + subspace_id: S, traceless: bool, - ) -> impl Future>>; + ) -> impl Future>; - /// Locally forget a payload with a given [`PayloadDigest`], or an error if no payload with that digest is held by this store. + /// Locally forget the corresponding payload of the entry with a given path and subspace, or an error if no entry with that path and subspace ID is held by this store. **The payload will be forgotten even if it corresponds to other entries**. /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the payload. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// - /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payload back. - fn forget_payload( - digest: PD, + /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten payload back. + fn force_forget_payload( + path: &Path, + subspace_id: S, traceless: bool, ) -> impl Future>; - /// Locally forget all payloads with corresponding ['AuthorisedEntry'] [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all [`PayloadDigest`] of forgotten payloads. + /// Locally forget all payloads with corresponding ['AuthorisedEntry'] [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all [`PayloadDigest`] of forgotten payloads. Payloads corresponding to entries *outside* of the given `area` param will be be prevented from being forgotten. + /// + /// If `protected` is `Some`, then all payloads corresponding to entries [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by that [`Area`] will be prevented from being forgotten, even though they are included by `area`. /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten payloads. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// - /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payloads back. + /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten payloads back. fn forget_area_payloads( + &self, area: &AreaOfInterest, + protected: Option>, traceless: bool, ) -> impl Future>; - /// Locally forget all payloads with corresponding [`AuthorisedEntry`] **not** [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all [`PayloadDigest`] of forgotten payloads. + /// Locally forget all payloads with corresponding ['AuthorisedEntry'] [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by a given [`AreaOfInterest`], returning all [`PayloadDigest`] of forgotten payloads. **Payloads will be forgotten even if it corresponds to other entries outside the given area**. + /// + /// If `protected` is `Some`, then all payloads corresponding to entries [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by that [`Area`] will be prevented from being forgotten, even though they are included by `area`. /// /// If the `traceless` parameter is `true`, the store will keep no record of ever having had the forgotten payloads. If `false`, it *may* persist what was forgetten for an arbitrary amount of time. /// - /// Forgetting is not the same as deleting! Subsequent joins with other [`Store`]s may bring the forgotten payloads back. - fn forget_everything_but_area_payloads( + /// Forgetting is not the same as [pruning](https://willowprotocol.org/specs/data-model/index.html#prefix_pruning)! Subsequent joins with other [`Store`]s may bring the forgotten payloads back. + fn force_forget_area_payloads( + &self, area: &AreaOfInterest, + protected: Option>, traceless: bool, ) -> impl Future>; @@ -271,39 +348,38 @@ where /// If `ignore_incomplete_payloads` is `true`, will return `None` if the entry's corresponding payload is incomplete, even if there is an entry present. /// If `ignore_empty_payloads` is `true`, will return `None` if the entry's payload length is `0`, even if there is an entry present. fn entry( - path: Path, - subspace_id: S, - ignore_incomplete_payloads: bool, - ignore_empty_payloads: bool, - ) -> Option>; + &self, + path: &Path, + subspace_id: &S, + ignore: Option, + ) -> impl Future>; /// Query which entries are [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by an [`AreaOfInterest`], returning a producer of [`LengthyAuthorisedEntry`]. /// /// If `ignore_incomplete_payloads` is `true`, the producer will not produce entries with incomplete corresponding payloads. /// If `ignore_empty_payloads` is `true`, the producer will not produce entries with a `payload_length` of `0`. fn query_area( - area: AreaOfInterest, - order: QueryOrder, + &self, + area: &AreaOfInterest, + order: &QueryOrder, reverse: bool, - ignore_incomplete_payloads: bool, - ignore_empty_payloads: bool, - ) -> Self::EntryProducer; + ignore: Option, + ) -> impl Producer>; /// Subscribe to events concerning entries [included](https://willowprotocol.org/specs/grouping-entries/index.html#area_include) by an [`AreaOfInterest`], returning a producer of `StoreEvent`s which occurred since the moment of calling this function. /// /// If `ignore_incomplete_payloads` is `true`, the producer will not produce entries with incomplete corresponding payloads. /// If `ignore_empty_payloads` is `true`, the producer will not produce entries with a `payload_length` of `0`. fn subscribe_area( - area: AreaOfInterest, - ignore_incomplete_payloads: bool, - ignore_empty_payloads: bool, - ) -> Self::EventProducer; + &self, + area: &AreaOfInterest, + ignore: Option, + ) -> impl Producer>; /// Attempt to resume a subscription using a *progress ID* obtained from a previous subscription, or return an error if this store implementation is unable to resume the subscription. fn resume_subscription( progress_id: u64, area: AreaOfInterest, - ignore_incomplete_payloads: bool, - ignore_empty_payloads: bool, - ) -> Result; + ignore: Option, + ) -> Result>, ResumptionFailedError>; }