From bcbddbb446ef002dbde1899a5674430eeecd401e Mon Sep 17 00:00:00 2001 From: qima Date: Tue, 29 Oct 2024 18:31:27 +0800 Subject: [PATCH 01/38] fix(node): fairly pick verification candidates --- sn_node/src/node.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sn_node/src/node.rs b/sn_node/src/node.rs index 0d74551751..03156de40e 100644 --- a/sn_node/src/node.rs +++ b/sn_node/src/node.rs @@ -718,10 +718,10 @@ impl Node { // The `rolling_index` is rotating among 0-511, // meanwhile the returned `kbuckets` only holding non-empty buckets. // Hence using the `remainder` calculate to achieve a rolling check. - // A further `divide by 2` is used to allow `upper or lower part` index within - // a bucket, to further reduce the concurrent queries. + // A further `remainder of 2` is used to allow `upper or lower part` + // index within a bucket, to further reduce the concurrent queries. let mut bucket_index = (rolling_index / 2) % kbuckets.len(); - let part_index = rolling_index / 2; + let part_index = rolling_index % 2; for (distance, peers) in kbuckets.iter() { if bucket_index == 0 { From c964a094b6e4f0a585add41da7dbd3baa6a18f64 Mon Sep 17 00:00:00 2001 From: qima Date: Mon, 28 Oct 2024 19:07:33 +0800 Subject: [PATCH 02/38] fix(CI): address failed CI --- .github/workflows/memcheck.yml | 9 ++++++--- .github/workflows/merge.yml | 17 +++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/memcheck.yml b/.github/workflows/memcheck.yml index cbfb52d4cc..d16b417fca 100644 --- a/.github/workflows/memcheck.yml +++ b/.github/workflows/memcheck.yml @@ -5,9 +5,9 @@ on: # on main, we want to know that all commits are passing at a glance, any deviation should help bisecting errors # the merge run checks should show on master and enable this clear test/passing history merge_group: - branches: [ main, alpha*, beta*, rc* ] + branches: [main, alpha*, beta*, rc*] pull_request: - branches: [ "*" ] + branches: ["*"] env: SAFE_DATA_PATH: /home/runner/.local/share/safe @@ -126,6 +126,9 @@ jobs: - name: Assert we've reloaded some chunks run: rg "Existing record found" $RESTART_TEST_NODE_DATA_PATH + - name: Wait at least 1min for replication to happen # it is throttled to once/30s. + run: sleep 60 + - name: Verify data replication using rg shell: bash timeout-minutes: 1 @@ -232,7 +235,7 @@ jobs: # Logging of handling time is on Trace level, # meanwhile the local_network startup tool sets the logging level on Debug. - # + # # - name: Check node swarm_driver handling statistics # shell: bash # # With the latest improvements, swarm_driver will be in high chance diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index d639924585..67e6dad9ac 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -807,19 +807,24 @@ jobs: # Otherwise kad will remove a `dropped out node` directly from RT. # So, the detection of the removal explicity will now have much less chance, # due to the removal of connection_issue tracking. + # + # With the further reduction of replication frequency, + # it now becomes harder to detect a `dropped out node` as a `failed to replicate` node. + # Hence now remove the assertion check and replace with a print out only. run: | + node_count=$(ls "${{ matrix.node_data_path }}" | wc -l) + echo "Node dir count is $node_count" restart_count=$(rg "Node is restarting in" "${{ matrix.node_data_path }}" -c --stats | \ rg "(\d+) matches" | rg "\d+" -o) echo "Restart $restart_count nodes" + if ! rg "PeerRemovedFromRoutingTable" "${{ matrix.node_data_path }}" -c --stats + then + echo "No peer removal count found" + exit 0 + fi peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.node_data_path }}" -c --stats | \ rg "(\d+) matches" | rg "\d+" -o) echo "PeerRemovedFromRoutingTable $peer_removed times" - if [ -z "$peer_removed" ]; then - echo "No peer removal count found" - exit 1 - fi - node_count=$(ls "${{ matrix.node_data_path }}" | wc -l) - echo "Node dir count is $node_count" # Only error out after uploading the logs - name: Don't log raw data From cc15ee51403d48ad3b394e0a18526c9ca7b3d656 Mon Sep 17 00:00:00 2001 From: loziniak Date: Sun, 27 Oct 2024 23:53:38 +0100 Subject: [PATCH 03/38] chore(autonomi): minor clarifications --- autonomi/Cargo.toml | 2 +- autonomi/src/lib.rs | 6 ++++++ evmlib/src/wallet.rs | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/autonomi/Cargo.toml b/autonomi/Cargo.toml index 3da273183e..7f4fb00ac0 100644 --- a/autonomi/Cargo.toml +++ b/autonomi/Cargo.toml @@ -14,7 +14,7 @@ crate-type = ["cdylib", "rlib"] [features] default = ["data", "vault"] -full = ["data", "registers", "vault"] +full = ["data", "registers", "vault", "fs"] data = [] vault = ["data", "registers"] fs = ["tokio/fs", "data"] diff --git a/autonomi/src/lib.rs b/autonomi/src/lib.rs index c73bef1378..d7441a6736 100644 --- a/autonomi/src/lib.rs +++ b/autonomi/src/lib.rs @@ -24,7 +24,13 @@ //! //! # Features //! +//! - `fs`: Up/download files and directories from filesystem +//! - `registers`: Operate on register datatype +//! - `data`: Operate on raw bytes and chunks +//! - `vault`: Operate on Vault datatype +//! - `full`: All of above //! - `local`: Discover local peers using mDNS. Useful for development. +//! - `loud`: Print debug information to stdout // docs.rs generation will enable unstable `doc_cfg` feature #![cfg_attr(docsrs, feature(doc_cfg))] diff --git a/evmlib/src/wallet.rs b/evmlib/src/wallet.rs index b9504f69a1..8b82834456 100644 --- a/evmlib/src/wallet.rs +++ b/evmlib/src/wallet.rs @@ -52,7 +52,7 @@ impl Wallet { Self::new(network, random()) } - /// Creates a new Wallet based on the given private_key. It will fail with Error::PrivateKeyInvalid if private_key is invalid. + /// Creates a new Wallet based on the given Ethereum private key. It will fail with Error::PrivateKeyInvalid if private_key is invalid. pub fn new_from_private_key(network: Network, private_key: &str) -> Result { let wallet = from_private_key(private_key)?; Ok(Self::new(network, wallet)) From 58b3a297f7d0e9329b7d490eb50b6ccd8e520719 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 25 Oct 2024 15:34:14 +0200 Subject: [PATCH 04/38] test(global): take SAFE_PEERS from compile time This is required for WASM --- test_utils/src/lib.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 3151878ade..cb13f35c55 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -15,6 +15,19 @@ use libp2p::Multiaddr; use rand::Rng; use sn_peers_acquisition::parse_peer_addr; +// Get environment variable from runtime or build time, in that order. Returns `None` if not set. +macro_rules! env_from_runtime_or_compiletime { + ($var:literal) => {{ + if let Ok(val) = std::env::var($var) { + Some(val) + } else if let Some(val) = option_env!($var) { + Some(val.to_string()) + } else { + None + } + }}; +} + /// Generate random data of the given length. pub fn gen_random_data(len: usize) -> Bytes { let mut data = vec![0u8; len]; @@ -28,7 +41,7 @@ pub fn gen_random_data(len: usize) -> Bytes { pub fn peers_from_env() -> Result> { let bootstrap_peers = if cfg!(feature = "local") { Ok(vec![]) - } else if let Ok(peers_str) = std::env::var("SAFE_PEERS") { + } else if let Some(peers_str) = env_from_runtime_or_compiletime!("SAFE_PEERS") { peers_str.split(',').map(parse_peer_addr).collect() } else { Ok(vec![]) From fa6926144fe843749ff29febb3534316c352264b Mon Sep 17 00:00:00 2001 From: Lautaro Mazzitelli Date: Tue, 29 Oct 2024 17:33:38 +0100 Subject: [PATCH 05/38] feat(launchpad): more error handling --- node-launchpad/src/node_mgmt.rs | 111 ++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 50 deletions(-) diff --git a/node-launchpad/src/node_mgmt.rs b/node-launchpad/src/node_mgmt.rs index 2c3b6205a9..8c43b202e0 100644 --- a/node-launchpad/src/node_mgmt.rs +++ b/node-launchpad/src/node_mgmt.rs @@ -23,20 +23,18 @@ pub fn stop_nodes(services: Vec, action_sender: UnboundedSender) sn_node_manager::cmd::node::stop(vec![], services, VerbosityLevel::Minimal).await { error!("Error while stopping services {err:?}"); - if let Err(err) = - action_sender.send(Action::StatusActions(StatusActions::ErrorStoppingNodes { + send_action( + action_sender, + Action::StatusActions(StatusActions::ErrorStoppingNodes { raw_error: err.to_string(), - })) - { - error!("Error while sending action: {err:?}"); - } + }), + ); } else { info!("Successfully stopped services"); - } - if let Err(err) = - action_sender.send(Action::StatusActions(StatusActions::StopNodesCompleted)) - { - error!("Error while sending action: {err:?}"); + send_action( + action_sender, + Action::StatusActions(StatusActions::StopNodesCompleted), + ); } }); } @@ -94,12 +92,10 @@ pub fn maintain_n_running_nodes(args: MaintainNodesArgs) { } debug!("Finished maintaining {} nodes", args.count); - if let Err(err) = args - .action_sender - .send(Action::StatusActions(StatusActions::StartNodesCompleted)) - { - error!("Error while sending action: {err:?}"); - } + send_action( + args.action_sender, + Action::StatusActions(StatusActions::StartNodesCompleted), + ); }); } @@ -108,28 +104,32 @@ pub fn reset_nodes(action_sender: UnboundedSender, start_nodes_after_res tokio::task::spawn_local(async move { if let Err(err) = sn_node_manager::cmd::node::reset(true, VerbosityLevel::Minimal).await { error!("Error while resetting services {err:?}"); - if let Err(err) = - action_sender.send(Action::StatusActions(StatusActions::ErrorResettingNodes { + send_action( + action_sender, + Action::StatusActions(StatusActions::ErrorResettingNodes { raw_error: err.to_string(), - })) - { - error!("Error while sending action: {err:?}"); - } + }), + ); } else { info!("Successfully reset services"); - } - if let Err(err) = - action_sender.send(Action::StatusActions(StatusActions::ResetNodesCompleted { - trigger_start_node: start_nodes_after_reset, - })) - { - error!("Error while sending action: {err:?}"); + send_action( + action_sender, + Action::StatusActions(StatusActions::ResetNodesCompleted { + trigger_start_node: start_nodes_after_reset, + }), + ); } }); } // --- Helper functions --- +fn send_action(action_sender: UnboundedSender, action: Action) { + if let Err(err) = action_sender.send(action) { + error!("Error while sending action: {err:?}"); + } +} + /// Load the node registry and handle errors async fn load_node_registry( action_sender: &UnboundedSender, @@ -291,7 +291,7 @@ async fn scale_down_nodes(config: &NodeConfig, count: u16) { config.data_dir_path.clone(), true, None, - Some(EvmNetwork::ArbitrumSepolia), //FIXME: should come from an UI element. + Some(EvmNetwork::ArbitrumSepolia), config.home_network, false, None, @@ -344,16 +344,15 @@ async fn add_nodes( if *current_port > max_port { error!("Reached maximum port number. Unable to find an available port."); - if let Err(err) = - action_sender.send(Action::StatusActions(StatusActions::ErrorScalingUpNodes { + send_action( + action_sender.clone(), + Action::StatusActions(StatusActions::ErrorScalingUpNodes { raw_error: format!( "Reached maximum port number ({}).\nUnable to find an available port.", max_port ), - })) - { - error!("Error while sending action: {err:?}"); - } + }), + ); break; } @@ -366,7 +365,7 @@ async fn add_nodes( config.data_dir_path.clone(), true, None, - Some(EvmNetwork::ArbitrumSepolia), //FIXME: Should come from an UI element + Some(EvmNetwork::ArbitrumSepolia), config.home_network, false, None, @@ -412,16 +411,29 @@ async fn add_nodes( .contains("Failed to add one or more services") && retry_count >= NODE_ADD_MAX_RETRIES { - if let Err(err) = action_sender.send(Action::StatusActions( - StatusActions::ErrorScalingUpNodes { + send_action( + action_sender.clone(), + Action::StatusActions(StatusActions::ErrorScalingUpNodes { raw_error: "When trying to add a node, we failed.\n\ Maybe you ran out of disk space?\n\ Maybe you need to change the port range?" .to_string(), - }, - )) { - error!("Error while sending action: {err:?}"); - } + }), + ); + } else if err + .to_string() + .contains("contains a virus or potentially unwanted software") + && retry_count >= NODE_ADD_MAX_RETRIES + { + send_action( + action_sender.clone(), + Action::StatusActions(StatusActions::ErrorScalingUpNodes { + raw_error: "When trying to add a node, we failed.\n\ + You may be running an old version of safenode service?\n\ + Did you whitelisted safenode and the launchpad?" + .to_string(), + }), + ); } else { error!("Range of ports to be used {:?}", *current_port..max_port); error!("Error while adding node on port {}: {err:?}", current_port); @@ -433,17 +445,16 @@ async fn add_nodes( } } if retry_count >= NODE_ADD_MAX_RETRIES { - if let Err(err) = - action_sender.send(Action::StatusActions(StatusActions::ErrorScalingUpNodes { + send_action( + action_sender.clone(), + Action::StatusActions(StatusActions::ErrorScalingUpNodes { raw_error: format!( "When trying run a node, we reached the maximum amount of retries ({}).\n\ Could this be a firewall blocking nodes starting?\n\ Or ports on your router already in use?", NODE_ADD_MAX_RETRIES ), - })) - { - error!("Error while sending action: {err:?}"); - } + }), + ); } } From ea1d36d71b45738547c016b7d3f49ac7cb0c83a6 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Tue, 29 Oct 2024 21:05:05 +0100 Subject: [PATCH 06/38] feat(autonomi): add method to get custom evm net --- autonomi/WASM_docs.md | 13 +++++++++++-- autonomi/src/client/wasm.rs | 24 ++++++++++++++++++++++++ evmlib/src/utils.rs | 5 +++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/autonomi/WASM_docs.md b/autonomi/WASM_docs.md index 6cf080113f..ee62681aba 100644 --- a/autonomi/WASM_docs.md +++ b/autonomi/WASM_docs.md @@ -1,6 +1,6 @@ -# JavaScript Autonomi API Documentation +# Autonomi JS API -Note that this is a first version and will be subject to change. +Note: the JS API is experimental and will be subject to change. The entry point for connecting to the network is {@link Client.connect}. @@ -12,6 +12,8 @@ For addresses (chunk, data, archives, etc) we're using hex-encoded strings conta ## Example +Note: `getEvmNetwork` will use hardcoded EVM network values that should be set during compilation of this library. + ```javascript import init, { Client, Wallet, getEvmNetwork } from 'autonomi'; @@ -28,3 +30,10 @@ console.log("Data stored at:", result); let fetchedData = await client.get(result); console.log("Data retrieved:", fetchedData); ``` + +## Funded wallet from custom local network + +```js +const evmNetwork = getEvmNetworkCustom("http://localhost:4343", "", ""); +const wallet = getFundedWalletWithCustomNetwork(evmNetwork, "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"); +``` diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index a102626ea3..2e619fe813 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -407,6 +407,22 @@ pub fn evm_network() -> Result { Ok(js_value) } +/// Create an `EvmNetwork` with custom values. +/// +/// # Example +/// +/// ```js +/// const [quotes, quotePayments, free_chunks] = await client.getQuotes(data); +/// const evmNetwork = getEvmNetworkCustom("http://localhost:4343", "", ""); +/// const payForQuotesCalldata = getPayForQuotesCalldata(evmNetwork, quotePayments); +/// ``` +#[wasm_bindgen(js_name = getEvmNetworkCustom)] +pub fn evm_network_custom(rpc_url: String, payment_token_address: String, data_payments_address: String) -> Result { + let evm_network = evmlib::utils::get_evm_network(&rpc_url, &payment_token_address, &data_payments_address); + let js_value = serde_wasm_bindgen::to_value(&evm_network)?; + Ok(js_value) +} + #[wasm_bindgen(js_name = Wallet)] pub struct JsWallet(evmlib::wallet::Wallet); @@ -431,6 +447,14 @@ pub fn funded_wallet() -> JsWallet { JsWallet(wallet) } +/// Get a funded wallet with a custom network. +#[wasm_bindgen(js_name = getFundedWalletWithCustomNetwork)] +pub fn funded_wallet_with_custom_network(network: JsValue, private_key: String) -> Result { + let network: evmlib::Network = serde_wasm_bindgen::from_value(network)?; + let wallet = evmlib::wallet::Wallet::new_from_private_key(network, &private_key)?; + Ok(JsWallet(wallet)) +} + /// Enable tracing logging in the console. /// /// A level could be passed like `trace` or `warn`. Or set for a specific module/crate diff --git a/evmlib/src/utils.rs b/evmlib/src/utils.rs index fbd838843f..143af255d5 100644 --- a/evmlib/src/utils.rs +++ b/evmlib/src/utils.rs @@ -57,6 +57,11 @@ pub fn get_evm_testnet_csv_path() -> Result { Ok(file) } +/// Create a custom `Network` from the given values +pub fn get_evm_network(rpc_url: &str, payment_token_address: &str, data_payments_address: &str) -> Network { + Network::Custom(CustomNetwork::new(rpc_url, payment_token_address, data_payments_address)) +} + /// Get the `Network` from environment variables /// Returns an error if we cannot obtain the network from any means. pub fn get_evm_network_from_env() -> Result { From d6a733853ff876bb3dcc1df0c90bac2b54a03908 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Tue, 29 Oct 2024 21:06:54 +0100 Subject: [PATCH 07/38] style(autonomi): cargo fmt --- autonomi/src/client/wasm.rs | 16 ++++++++++++---- evmlib/src/utils.rs | 12 ++++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index 2e619fe813..f67c16babf 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -408,7 +408,7 @@ pub fn evm_network() -> Result { } /// Create an `EvmNetwork` with custom values. -/// +/// /// # Example /// /// ```js @@ -417,8 +417,13 @@ pub fn evm_network() -> Result { /// const payForQuotesCalldata = getPayForQuotesCalldata(evmNetwork, quotePayments); /// ``` #[wasm_bindgen(js_name = getEvmNetworkCustom)] -pub fn evm_network_custom(rpc_url: String, payment_token_address: String, data_payments_address: String) -> Result { - let evm_network = evmlib::utils::get_evm_network(&rpc_url, &payment_token_address, &data_payments_address); +pub fn evm_network_custom( + rpc_url: String, + payment_token_address: String, + data_payments_address: String, +) -> Result { + let evm_network = + evmlib::utils::get_evm_network(&rpc_url, &payment_token_address, &data_payments_address); let js_value = serde_wasm_bindgen::to_value(&evm_network)?; Ok(js_value) } @@ -449,7 +454,10 @@ pub fn funded_wallet() -> JsWallet { /// Get a funded wallet with a custom network. #[wasm_bindgen(js_name = getFundedWalletWithCustomNetwork)] -pub fn funded_wallet_with_custom_network(network: JsValue, private_key: String) -> Result { +pub fn funded_wallet_with_custom_network( + network: JsValue, + private_key: String, +) -> Result { let network: evmlib::Network = serde_wasm_bindgen::from_value(network)?; let wallet = evmlib::wallet::Wallet::new_from_private_key(network, &private_key)?; Ok(JsWallet(wallet)) diff --git a/evmlib/src/utils.rs b/evmlib/src/utils.rs index 143af255d5..ea82471f6f 100644 --- a/evmlib/src/utils.rs +++ b/evmlib/src/utils.rs @@ -58,8 +58,16 @@ pub fn get_evm_testnet_csv_path() -> Result { } /// Create a custom `Network` from the given values -pub fn get_evm_network(rpc_url: &str, payment_token_address: &str, data_payments_address: &str) -> Network { - Network::Custom(CustomNetwork::new(rpc_url, payment_token_address, data_payments_address)) +pub fn get_evm_network( + rpc_url: &str, + payment_token_address: &str, + data_payments_address: &str, +) -> Network { + Network::Custom(CustomNetwork::new( + rpc_url, + payment_token_address, + data_payments_address, + )) } /// Get the `Network` from environment variables From 737a9140338644ba73c5f1dec604845eccc7a814 Mon Sep 17 00:00:00 2001 From: grumbach Date: Wed, 30 Oct 2024 15:21:07 +0900 Subject: [PATCH 08/38] fix: allow split records during crdt record puts verifications --- autonomi/src/client/registers.rs | 2 +- autonomi/src/client/vault.rs | 2 +- sn_networking/src/driver.rs | 2 ++ sn_networking/src/lib.rs | 5 +++++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/autonomi/src/client/registers.rs b/autonomi/src/client/registers.rs index 52f8944e1e..40f79ead0b 100644 --- a/autonomi/src/client/registers.rs +++ b/autonomi/src/client/registers.rs @@ -209,7 +209,7 @@ impl Client { put_quorum: Quorum::All, retry_strategy: None, use_put_record_to: None, - verification: Some((VerificationKind::Network, get_cfg)), + verification: Some((VerificationKind::Crdt, get_cfg)), }; // Store the updated register on the network diff --git a/autonomi/src/client/vault.rs b/autonomi/src/client/vault.rs index 55103b0578..ea48f4c15f 100644 --- a/autonomi/src/client/vault.rs +++ b/autonomi/src/client/vault.rs @@ -257,7 +257,7 @@ impl Client { retry_strategy: Some(RetryStrategy::Balanced), use_put_record_to: None, verification: Some(( - VerificationKind::Network, + VerificationKind::Crdt, GetRecordCfg { get_quorum: Quorum::Majority, retry_strategy: None, diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index e70cc6c68d..b76c14f82c 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -231,6 +231,8 @@ pub struct PutRecordCfg { pub enum VerificationKind { /// Uses the default KAD GET to perform verification. Network, + /// Uses the default KAD GET to perform verification, but don't error out on split records + Crdt, /// Uses the hash based verification for chunks. ChunkProof { expected_proof: ChunkProof, diff --git a/sn_networking/src/lib.rs b/sn_networking/src/lib.rs index 01e5d6c9f6..4b15e66a08 100644 --- a/sn_networking/src/lib.rs +++ b/sn_networking/src/lib.rs @@ -1017,6 +1017,11 @@ impl Network { NetworkAddress::from_record_key(&record_key), )); } + Err(NetworkError::GetRecordError(GetRecordError::SplitRecord { .. })) + if matches!(verification_kind, VerificationKind::Crdt) => + { + warn!("Record {pretty_key:?} is split, which is okay since we're dealing with CRDTs"); + } Err(e) => { debug!( "Failed to verify record {pretty_key:?} to be stored with error: {e:?}" From 6c2c3e893a16f0864a8b0035276df0eae559c261 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 29 Oct 2024 15:57:12 +0530 Subject: [PATCH 09/38] feat(autonomi): download chunks in parallel --- autonomi/src/client/data.rs | 16 ++++++++++++++++ autonomi/src/client/utils.rs | 31 ++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/autonomi/src/client/data.rs b/autonomi/src/client/data.rs index 0a6be8598a..86f500bc57 100644 --- a/autonomi/src/client/data.rs +++ b/autonomi/src/client/data.rs @@ -40,6 +40,22 @@ pub static CHUNK_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { batch_size }); +/// Number of chunks to download in parallel. +/// Can be overridden by the `CHUNK_DOWNLOAD_BATCH_SIZE` environment variable. +pub static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { + let batch_size = std::env::var("CHUNK_DOWNLOAD_BATCH_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or( + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + * 8, + ); + info!("Chunk download batch size: {}", batch_size); + batch_size +}); + /// Raw Data Address (points to a DataMap) pub type DataAddr = XorName; /// Raw Chunk Address (points to a [`Chunk`]) diff --git a/autonomi/src/client/utils.rs b/autonomi/src/client/utils.rs index 95d70b6e4d..fc3679e01b 100644 --- a/autonomi/src/client/utils.rs +++ b/autonomi/src/client/utils.rs @@ -24,7 +24,7 @@ use std::{collections::HashMap, future::Future, num::NonZero}; use xor_name::XorName; use super::{ - data::{CostError, GetError, PayError, PutError}, + data::{CostError, GetError, PayError, PutError, CHUNK_DOWNLOAD_BATCH_SIZE}, Client, }; use crate::self_encryption::DataMapLevel; @@ -35,16 +35,25 @@ impl Client { pub(crate) async fn fetch_from_data_map(&self, data_map: &DataMap) -> Result { let mut encrypted_chunks = vec![]; - for info in data_map.infos() { - let chunk = self - .chunk_get(info.dst_hash) - .await - .inspect_err(|err| error!("Error fetching chunk {:?}: {err:?}", info.dst_hash))?; - let chunk = EncryptedChunk { - index: info.index, - content: chunk.value, - }; - encrypted_chunks.push(chunk); + let mut stream = futures::stream::iter(data_map.infos().into_iter()) + .map(|info| { + let dst_hash = info.dst_hash; + async move { + self.chunk_get(dst_hash) + .await + .inspect_err(move |err| { + error!("Error fetching chunk {:?}: {err:?}", dst_hash) + }) + .map(|chunk| EncryptedChunk { + index: info.index, + content: chunk.value, + }) + } + }) + .buffered(*CHUNK_DOWNLOAD_BATCH_SIZE); + + while let Some(encrypted_chunk_result) = stream.next().await { + encrypted_chunks.push(encrypted_chunk_result?); } let data = decrypt_full_set(data_map, &encrypted_chunks).map_err(|e| { From b671669fba2fada0a51d00169b57d4a8f2b65fa8 Mon Sep 17 00:00:00 2001 From: grumbach Date: Wed, 30 Oct 2024 15:59:59 +0900 Subject: [PATCH 10/38] chore: use universal batching fn --- autonomi/src/client/utils.rs | 43 +++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/autonomi/src/client/utils.rs b/autonomi/src/client/utils.rs index fc3679e01b..53c7f88747 100644 --- a/autonomi/src/client/utils.rs +++ b/autonomi/src/client/utils.rs @@ -33,29 +33,32 @@ use crate::utils::payment_proof_from_quotes_and_payments; impl Client { /// Fetch and decrypt all chunks in the data map. pub(crate) async fn fetch_from_data_map(&self, data_map: &DataMap) -> Result { - let mut encrypted_chunks = vec![]; - - let mut stream = futures::stream::iter(data_map.infos().into_iter()) - .map(|info| { - let dst_hash = info.dst_hash; - async move { - self.chunk_get(dst_hash) - .await - .inspect_err(move |err| { - error!("Error fetching chunk {:?}: {err:?}", dst_hash) - }) - .map(|chunk| EncryptedChunk { - index: info.index, - content: chunk.value, - }) + let mut download_tasks = vec![]; + for info in data_map.infos() { + download_tasks.push(async move { + match self + .chunk_get(info.dst_hash) + .await + .inspect_err(|err| error!("Error fetching chunk {:?}: {err:?}", info.dst_hash)) + { + Ok(chunk) => Ok(EncryptedChunk { + index: info.index, + content: chunk.value, + }), + Err(err) => { + error!("Error fetching chunk {:?}: {err:?}", info.dst_hash); + Err(err) + } } - }) - .buffered(*CHUNK_DOWNLOAD_BATCH_SIZE); - - while let Some(encrypted_chunk_result) = stream.next().await { - encrypted_chunks.push(encrypted_chunk_result?); + }); } + let encrypted_chunks = + process_tasks_with_max_concurrency(download_tasks, *CHUNK_DOWNLOAD_BATCH_SIZE) + .await + .into_iter() + .collect::, GetError>>()?; + let data = decrypt_full_set(data_map, &encrypted_chunks).map_err(|e| { error!("Error decrypting encrypted_chunks: {e:?}"); GetError::Decryption(crate::self_encryption::Error::SelfEncryption(e)) From 54c07630aaa40dcf0cf4e1c3a8fa2657654600af Mon Sep 17 00:00:00 2001 From: Lautaro Mazzitelli Date: Wed, 30 Oct 2024 11:42:14 +0100 Subject: [PATCH 11/38] fix(launchpad): starting status when not running --- node-launchpad/src/components/status.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/node-launchpad/src/components/status.rs b/node-launchpad/src/components/status.rs index e2ee7e09f4..041a992de8 100644 --- a/node-launchpad/src/components/status.rs +++ b/node-launchpad/src/components/status.rs @@ -191,7 +191,9 @@ impl Status<'_> { // Starting is not part of ServiceStatus so we do it manually if let Some(LockRegistryState::StartingNodes) = self.lock_registry { NodeItem::update_spinner_state(&mut item.spinner_state); - item.status = NodeStatus::Starting; + if item.status != NodeStatus::Running { + item.status = NodeStatus::Starting; + } } // Update peers count From 62a88a47c71ebdcc97af5c693c60dc5ac552f04a Mon Sep 17 00:00:00 2001 From: Ermine Jose Date: Mon, 28 Oct 2024 23:43:14 +0530 Subject: [PATCH 12/38] test: add vault CLI testcases --- .github/workflows/merge.yml | 134 +++++++++++++++++++++++++++++++++++- 1 file changed, 133 insertions(+), 1 deletion(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index db89c867be..007218429a 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -16,6 +16,16 @@ env: GENESIS_SK: 5ec88891c1098a0fede5b98b07f8abc931d7247b7aa310d21ab430cc957f9f02 jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Display Python version + run: python --version + cargo-udeps: if: "!startsWith(github.event.head_commit.message, 'chore(release):')" name: Unused dependency check @@ -142,7 +152,6 @@ jobs: # of proptesting # we do many more runs on the nightly run PROPTEST_CASES: 50 - e2e: if: "!startsWith(github.event.head_commit.message, 'chore(release):')" name: E2E tests @@ -340,6 +349,129 @@ jobs: SN_LOG: "v" timeout-minutes: 2 + - name: Estimate cost to create a vault + run: | + echo "test-file" > upload-test.txt + ./target/release/autonomi --log-output-dest=data-dir file upload ./upload-test.txt + ./target/release/autonomi --log-output-dest=data-dir register create sample_new_register 1234 + ./target/release/autonomi --log-output-dest=data-dir vault cost + ./target/release/autonomi --log-output-dest=data-dir file list 2>&1 | tee file_list.txt + ./target/release/autonomi --log-output-dest=data-dir register list 2>&1 | tee register_list.txt + env: + SN_LOG: "v" + timeout-minutes: 2 + + - name: create a vault with existing user data as above + run: | + ./target/release/autonomi --log-output-dest=data-dir vault create + ./target/release/autonomi --log-output-dest=data-dir vault load 2>&1 | tee vault_data.txt + env: + SN_LOG: "v" + timeout-minutes: 2 + + - name: vault create validation + run: | + NUM_OF_PUBLIC_FILES=`cat file_list.txt | grep "public" | grep -o '[0-9]\+'` + NUM_OF_PRIVATE_FILES=`cat file_list.txt | grep "private" | grep -o '[0-9]\+'` + NUM_OF_REGISTERS=`cat register_list.txt | grep "register" | grep -o '[0-9]\+'` + + NUM_OF_PUBLIC_FILES_IN_VAULT=`cat vault_data.txt | grep "public" | grep -o '[0-9]\+'` + NUM_OF_PRIVATE_FILES_IN_VAULT=`cat vault_data.txt| grep "private" | grep -o '[0-9]\+'` + NUM_OF_REGISTERS_IN_VAULT=`cat vault_data.txt| grep "register" | grep -o '[0-9]\+'` + + rm -rf file_list.txt register_list.txt vault_data.txt + + + python3 -c 'import sys; print("Equal" if sys.argv[1] == sys.argv[2] else "Not Equal")' $NUM_OF_PUBLIC_FILES $NUM_OF_PUBLIC_FILES_IN_VAULT + + python3 -c 'import sys; print("Equal" if sys.argv[1] == sys.argv[2] else "Not Equal")' $NUM_OF_PRIVATE_FILES $NUM_OF_PRIVATE_FILES_IN_VAULT + + python3 -c 'import sys; print("Equal" if sys.argv[1] == sys.argv[2] else "Not Equal")' $NUM_OF_REGISTERS $NUM_OF_REGISTERS_IN_VAULT + + echo "vault created successfully!" + env: + SN_LOG: "v" + timeout-minutes: 4 + + - name: add more files and registers + run: | + echo "testing-file2" > sync-test.txt + ./target/release/autonomi --log-output-dest=data-dir file upload ./sync-test.txt + ./target/release/autonomi --log-output-dest=data-dir register create sample_new_register_2 12345 + env: + SN_LOG: "v" + timeout-minutes: 2 + + - name: sync the vault + run: ./target/release/autonomi --log-output-dest=data-dir vault sync + env: + SN_LOG: "v" + timeout-minutes: 2 + + - name: vault sync validation + run: | + NUM_OF_PUBLIC_FILES="" + NUM_OF_PRIVATE_FILES="" + NUM_OF_REGISTERS="" + NUM_OF_PUBLIC_FILES_IN_VAULT="" + NUM_OF_PRIVATE_FILES_IN_VAULT="" + NUM_OF_REGISTERS_IN_VAULT="" + + ./target/release/autonomi --log-output-dest=data-dir file list 2>&1 | tee file_list.txt + ./target/release/autonomi --log-output-dest=data-dir register list 2>&1 | tee register_list.txt + + NUM_OF_PUBLIC_FILES=`cat file_list.txt | grep "public" | grep -o '[0-9]\+'` + NUM_OF_PRIVATE_FILES=`cat file_list.txt | grep "private" | grep -o '[0-9]\+'` + NUM_OF_REGISTERS=`cat register_list.txt | grep "register" | grep -o '[0-9]\+'` + + ./target/release/autonomi --log-output-dest=data-dir vault load 2>&1 | tee vault_data.txt + + NUM_OF_PUBLIC_FILES_IN_VAULT=`cat vault_data.txt | grep "public" | grep -o '[0-9]\+'` + NUM_OF_PRIVATE_FILES_IN_VAULT=`cat vault_data.txt| grep "private" | grep -o '[0-9]\+'` + NUM_OF_REGISTERS_IN_VAULT=`cat vault_data.txt| grep "register" | grep -o '[0-9]\+'` + + rm -rf file_list.txt register_list.txt vault_data.txt + + python3 -c 'import sys; print("Equal" if sys.argv[1] == sys.argv[2] else "Not Equal")' $NUM_OF_PUBLIC_FILES $NUM_OF_PUBLIC_FILES_IN_VAULT + + python3 -c 'import sys; print("Equal" if sys.argv[1] == sys.argv[2] else "Not Equal")' $NUM_OF_PRIVATE_FILES $NUM_OF_PRIVATE_FILES_IN_VAULT + + python3 -c 'import sys; print("Equal" if sys.argv[1] == sys.argv[2] else "Not Equal")' $NUM_OF_REGISTERS $NUM_OF_REGISTERS_IN_VAULT + + echo "vault synced successfully!" + env: + SN_LOG: "v" + timeout-minutes: 2 + + - name: load an existing vault from the network + run: ./target/release/autonomi --log-output-dest=data-dir vault load + env: + SN_LOG: "v" + timeout-minutes: 2 + + - name: Time profiling for Different files + run: | + # 1 MB + python3 -c "with open('random_1MB.bin', 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))" + # 10 MB + python3 -c "with open('random_10MB.bin', 'wb') as f: f.write(bytearray([0xff] * 10 * 1024 * 1024))" + # 100 MB + python3 -c "with open('random_100MB.bin', 'wb') as f: f.write(bytearray([0xff] * 100 * 1024 * 1024))" + # 1 GB + python3 -c "with open('random_1GB.bin', 'wb') as f: f.write(bytearray([0xff] * 1000 * 1024 * 1024))" + + ./target/release/autonomi --log-output-dest=data-dir file list + time ./target/release/autonomi --log-output-dest=data-dir file upload random_1MB.bin + time ./target/release/autonomi --log-output-dest=data-dir file upload random_10MB.bin + time ./target/release/autonomi --log-output-dest=data-dir file upload random_100MB.bin + time ./target/release/autonomi --log-output-dest=data-dir file upload random_1GB.bin + + rm -rf random*.bin + + env: + SN_LOG: "v" + timeout-minutes: 10 + - name: Stop the local network and upload logs if: always() uses: maidsafe/sn-local-testnet-action@main From 1cfef6adddf2db461ce99d78e34326bbfe98779c Mon Sep 17 00:00:00 2001 From: Lautaro Mazzitelli Date: Wed, 30 Oct 2024 15:12:12 +0100 Subject: [PATCH 13/38] fix(launchpad): help section changed after beta --- node-launchpad/src/components/help.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/node-launchpad/src/components/help.rs b/node-launchpad/src/components/help.rs index 9270616d27..c091c18ba5 100644 --- a/node-launchpad/src/components/help.rs +++ b/node-launchpad/src/components/help.rs @@ -96,17 +96,17 @@ impl Component for Help { let quickstart_guide_link = Hyperlink::new( Span::styled( - "docs.autonomi.com/getstarted", + "autonomi.com/getstarted", Style::default().fg(VIVID_SKY_BLUE).underlined(), ), - "https://docs.autonomi.com/getstarted", + "https://autonomi.com/getstarted", ); - let beta_rewards_link = Hyperlink::new( + let terms_and_conditions_link = Hyperlink::new( Span::styled( - "autonomi.com/beta", + "autonomi.com/terms", Style::default().fg(VIVID_SKY_BLUE).underlined(), ), - "https://autonomi.com/beta", + "https://autonomi.com/terms", ); let get_direct_support_link = Hyperlink::new( Span::styled( @@ -134,7 +134,7 @@ impl Component for Help { // Render hyperlinks in the new area f.render_widget( Span::styled( - "See the quick start guides:", + "Read the quick start guides:", Style::default().fg(GHOST_WHITE), ), left_column[0], @@ -147,20 +147,17 @@ impl Component for Help { f.render_widget_ref(get_direct_support_link, left_column[3]); f.render_widget( Span::styled( - "To join the Beta Rewards Program:", + "Download the latest launchpad:", Style::default().fg(GHOST_WHITE), ), right_column[0], ); - f.render_widget_ref(beta_rewards_link, right_column[1]); + f.render_widget_ref(download_latest_link, right_column[1]); f.render_widget( - Span::styled( - "Download the latest launchpad:", - Style::default().fg(GHOST_WHITE), - ), + Span::styled("Terms & Conditions:", Style::default().fg(GHOST_WHITE)), right_column[2], ); - f.render_widget_ref(download_latest_link, right_column[3]); + f.render_widget_ref(terms_and_conditions_link, right_column[3]); f.render_widget(block, layout[1]); From 873fe5b591146a7b898c217b1b2240db47813c12 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 30 Oct 2024 15:38:08 +0100 Subject: [PATCH 14/38] feat(autonomi): support prepaid put operations --- autonomi-cli/src/commands/vault.rs | 4 +- autonomi/src/client/archive.rs | 2 +- autonomi/src/client/archive_private.rs | 6 +- autonomi/src/client/data.rs | 17 ++-- autonomi/src/client/data_private.rs | 13 +-- autonomi/src/client/external_signer.rs | 72 ++------------ autonomi/src/client/fs.rs | 4 +- autonomi/src/client/fs_private.rs | 6 +- autonomi/src/client/mod.rs | 1 + autonomi/src/client/payment.rs | 49 ++++++++++ autonomi/src/client/utils.rs | 20 ++-- autonomi/src/client/vault.rs | 80 +++++++++------- autonomi/src/client/vault/user_data.rs | 25 ++++- autonomi/src/client/wasm.rs | 6 +- autonomi/src/lib.rs | 2 +- autonomi/src/utils.rs | 20 +++- autonomi/tests/external_signer.rs | 128 +++++++++++++++++++++---- autonomi/tests/put.rs | 2 +- autonomi/tests/wasm.rs | 2 +- sn_node/tests/data_with_churn.rs | 2 +- sn_node/tests/verify_data_location.rs | 2 +- sn_protocol/src/storage/scratchpad.rs | 10 +- 22 files changed, 310 insertions(+), 163 deletions(-) create mode 100644 autonomi/src/client/payment.rs diff --git a/autonomi-cli/src/commands/vault.rs b/autonomi-cli/src/commands/vault.rs index e7ce3f95c8..b5446f8962 100644 --- a/autonomi-cli/src/commands/vault.rs +++ b/autonomi-cli/src/commands/vault.rs @@ -40,7 +40,7 @@ pub async fn create(peers: Vec) -> Result<()> { println!("Pushing to network vault..."); let total_cost = client - .put_user_data_to_vault(&vault_sk, &wallet, local_user_data) + .put_user_data_to_vault(&vault_sk, wallet.into(), local_user_data) .await?; if total_cost.is_zero() { @@ -82,7 +82,7 @@ pub async fn sync(peers: Vec, force: bool) -> Result<()> { let private_file_archives_len = local_user_data.private_file_archives.len(); let registers_len = local_user_data.registers.len(); client - .put_user_data_to_vault(&vault_sk, &wallet, local_user_data) + .put_user_data_to_vault(&vault_sk, wallet.into(), local_user_data) .await?; println!("✅ Successfully synced vault"); diff --git a/autonomi/src/client/archive.rs b/autonomi/src/client/archive.rs index 04ad120b19..9d5f1de78a 100644 --- a/autonomi/src/client/archive.rs +++ b/autonomi/src/client/archive.rs @@ -168,7 +168,7 @@ impl Client { let bytes = archive .into_bytes() .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; - self.data_put(bytes, wallet).await + self.data_put(bytes, wallet.into()).await } /// Get the cost to upload an archive diff --git a/autonomi/src/client/archive_private.rs b/autonomi/src/client/archive_private.rs index a7ba854380..7354634140 100644 --- a/autonomi/src/client/archive_private.rs +++ b/autonomi/src/client/archive_private.rs @@ -19,9 +19,9 @@ use super::{ data_private::PrivateDataAccess, Client, }; +use crate::client::payment::PaymentOption; use bytes::Bytes; use serde::{Deserialize, Serialize}; -use sn_evm::EvmWallet; /// The address of a private archive /// Contains the [`PrivateDataAccess`] leading to the [`PrivateArchive`] data @@ -130,11 +130,11 @@ impl Client { pub async fn private_archive_put( &self, archive: PrivateArchive, - wallet: &EvmWallet, + payment_option: PaymentOption, ) -> Result { let bytes = archive .into_bytes() .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; - self.private_data_put(bytes, wallet).await + self.private_data_put(bytes, payment_option).await } } diff --git a/autonomi/src/client/data.rs b/autonomi/src/client/data.rs index 86f500bc57..ec7ebf6d70 100644 --- a/autonomi/src/client/data.rs +++ b/autonomi/src/client/data.rs @@ -13,11 +13,12 @@ use std::collections::HashSet; use std::sync::LazyLock; use xor_name::XorName; +use crate::client::payment::PaymentOption; use crate::client::utils::process_tasks_with_max_concurrency; use crate::client::{ClientEvent, UploadSummary}; use crate::{self_encryption::encrypt, Client}; +use sn_evm::EvmWalletError; use sn_evm::{Amount, AttoTokens}; -use sn_evm::{EvmWallet, EvmWalletError}; use sn_networking::{GetRecordCfg, NetworkError}; use sn_protocol::{ storage::{try_deserialize_record, Chunk, ChunkAddress, RecordHeader, RecordKind}, @@ -136,7 +137,11 @@ impl Client { /// Upload a piece of data to the network. /// Returns the Data Address at which the data was stored. /// This data is publicly accessible. - pub async fn data_put(&self, data: Bytes, wallet: &EvmWallet) -> Result { + pub async fn data_put( + &self, + data: Bytes, + payment_option: PaymentOption, + ) -> Result { let now = sn_networking::target_arch::Instant::now(); let (data_map_chunk, chunks) = encrypt(data)?; let data_map_addr = data_map_chunk.address(); @@ -152,8 +157,8 @@ impl Client { // Pay for all chunks + data map chunk info!("Paying for {} addresses", xor_names.len()); - let (payment_proofs, _free_chunks) = self - .pay(xor_names.into_iter(), wallet) + let receipt = self + .pay_for_content_addrs(xor_names.into_iter(), payment_option) .await .inspect_err(|err| error!("Error paying for data: {err:?}"))?; @@ -163,7 +168,7 @@ impl Client { for chunk in chunks.into_iter().chain(std::iter::once(data_map_chunk)) { let self_clone = self.clone(); let address = *chunk.address(); - if let Some(proof) = payment_proofs.get(chunk.name()) { + if let Some(proof) = receipt.get(chunk.name()) { let proof_clone = proof.clone(); upload_tasks.push(async move { self_clone @@ -191,7 +196,7 @@ impl Client { // Reporting if let Some(channel) = self.client_event_sender.as_ref() { - let tokens_spent = payment_proofs + let tokens_spent = receipt .values() .map(|proof| proof.quote.cost.as_atto()) .sum::(); diff --git a/autonomi/src/client/data_private.rs b/autonomi/src/client/data_private.rs index d2ecaf0a2b..29925b915b 100644 --- a/autonomi/src/client/data_private.rs +++ b/autonomi/src/client/data_private.rs @@ -10,11 +10,12 @@ use std::hash::{DefaultHasher, Hash, Hasher}; use bytes::Bytes; use serde::{Deserialize, Serialize}; -use sn_evm::{Amount, EvmWallet}; +use sn_evm::Amount; use sn_protocol::storage::Chunk; use super::data::CHUNK_UPLOAD_BATCH_SIZE; use super::data::{GetError, PutError}; +use crate::client::payment::PaymentOption; use crate::client::utils::process_tasks_with_max_concurrency; use crate::client::{ClientEvent, UploadSummary}; use crate::{self_encryption::encrypt, Client}; @@ -64,7 +65,7 @@ impl Client { pub async fn private_data_put( &self, data: Bytes, - wallet: &EvmWallet, + payment_option: PaymentOption, ) -> Result { let now = sn_networking::target_arch::Instant::now(); let (data_map_chunk, chunks) = encrypt(data)?; @@ -73,8 +74,8 @@ impl Client { // Pay for all chunks let xor_names: Vec<_> = chunks.iter().map(|chunk| *chunk.name()).collect(); info!("Paying for {} addresses", xor_names.len()); - let (payment_proofs, _free_chunks) = self - .pay(xor_names.into_iter(), wallet) + let receipt = self + .pay_for_content_addrs(xor_names.into_iter(), payment_option) .await .inspect_err(|err| error!("Error paying for data: {err:?}"))?; @@ -84,7 +85,7 @@ impl Client { for chunk in chunks { let self_clone = self.clone(); let address = *chunk.address(); - if let Some(proof) = payment_proofs.get(chunk.name()) { + if let Some(proof) = receipt.get(chunk.name()) { let proof_clone = proof.clone(); upload_tasks.push(async move { self_clone @@ -112,7 +113,7 @@ impl Client { // Reporting if let Some(channel) = self.client_event_sender.as_ref() { - let tokens_spent = payment_proofs + let tokens_spent = receipt .values() .map(|proof| proof.quote.cost.as_atto()) .sum::(); diff --git a/autonomi/src/client/external_signer.rs b/autonomi/src/client/external_signer.rs index b17002bd9c..c4f473d378 100644 --- a/autonomi/src/client/external_signer.rs +++ b/autonomi/src/client/external_signer.rs @@ -1,36 +1,23 @@ -use crate::client::data::{DataAddr, PutError}; +use crate::client::data::PutError; use crate::client::utils::extract_quote_payments; use crate::self_encryption::encrypt; use crate::Client; use bytes::Bytes; -use sn_evm::{PaymentQuote, ProofOfPayment, QuotePayment}; +use sn_evm::{PaymentQuote, QuotePayment}; use sn_protocol::storage::Chunk; use std::collections::HashMap; use xor_name::XorName; +use crate::utils::cost_map_to_quotes; #[allow(unused_imports)] pub use sn_evm::external_signer::*; impl Client { - /// Upload a piece of data to the network. This data will be self-encrypted. - /// Payment will not be done automatically as opposed to the regular `data_put`, so the proof of payment has to be provided. - /// Returns the Data Address at which the data was stored. - pub async fn data_put_with_proof_of_payment( - &self, - data: Bytes, - proof: HashMap, - ) -> Result { - let (data_map_chunk, chunks, _) = encrypt_data(data)?; - self.upload_data_map(&proof, &data_map_chunk).await?; - self.upload_chunks(&chunks, &proof).await?; - Ok(*data_map_chunk.address().xorname()) - } - /// Get quotes for data. /// Returns a cost map, data payments to be executed and a list of free (already paid for) chunks. - pub async fn get_quotes_for_data( + pub async fn get_quotes_for_content_addresses( &self, - data: Bytes, + content_addrs: impl Iterator, ) -> Result< ( HashMap, @@ -39,59 +26,18 @@ impl Client { ), PutError, > { - // Encrypt the data as chunks - let (_data_map_chunk, _chunks, xor_names) = encrypt_data(data)?; - - let cost_map: HashMap = self - .get_store_quotes(xor_names.into_iter()) - .await? - .into_iter() - .map(|(name, (_, _, q))| (name, q)) - .collect(); - + let cost_map = self.get_store_quotes(content_addrs).await?; let (quote_payments, free_chunks) = extract_quote_payments(&cost_map); - Ok((cost_map, quote_payments, free_chunks)) - } + let quotes = cost_map_to_quotes(cost_map); - async fn upload_data_map( - &self, - payment_proofs: &HashMap, - data_map_chunk: &Chunk, - ) -> Result<(), PutError> { - let map_xor_name = data_map_chunk.name(); - - if let Some(proof) = payment_proofs.get(map_xor_name) { - debug!("Uploading data map chunk: {map_xor_name:?}"); - self.chunk_upload_with_payment(data_map_chunk.clone(), proof.clone()) - .await - .inspect_err(|err| error!("Error uploading data map chunk: {err:?}")) - } else { - Ok(()) - } - } - - async fn upload_chunks( - &self, - chunks: &[Chunk], - payment_proofs: &HashMap, - ) -> Result<(), PutError> { - debug!("Uploading {} chunks", chunks.len()); - for chunk in chunks { - if let Some(proof) = payment_proofs.get(chunk.name()) { - let address = *chunk.address(); - self.chunk_upload_with_payment(chunk.clone(), proof.clone()) - .await - .inspect_err(|err| error!("Error uploading chunk {address:?} :{err:?}"))?; - } - } - Ok(()) + Ok((quotes, quote_payments, free_chunks)) } } /// Encrypts data as chunks. /// /// Returns the data map chunk, file chunks and a list of all content addresses including the data map. -fn encrypt_data(data: Bytes) -> Result<(Chunk, Vec, Vec), PutError> { +pub fn encrypt_data(data: Bytes) -> Result<(Chunk, Vec, Vec), PutError> { let now = sn_networking::target_arch::Instant::now(); let result = encrypt(data)?; diff --git a/autonomi/src/client/fs.rs b/autonomi/src/client/fs.rs index c1505224bc..40a43b9fba 100644 --- a/autonomi/src/client/fs.rs +++ b/autonomi/src/client/fs.rs @@ -154,7 +154,7 @@ impl Client { // upload archive let archive_serialized = archive.into_bytes()?; - let arch_addr = self.data_put(archive_serialized, wallet).await?; + let arch_addr = self.data_put(archive_serialized, wallet.into()).await?; info!("Complete archive upload completed in {:?}", start.elapsed()); #[cfg(feature = "loud")] @@ -175,7 +175,7 @@ impl Client { let data = tokio::fs::read(path).await?; let data = Bytes::from(data); - let addr = self.data_put(data, wallet).await?; + let addr = self.data_put(data, wallet.into()).await?; Ok(addr) } diff --git a/autonomi/src/client/fs_private.rs b/autonomi/src/client/fs_private.rs index 08d453ae37..d158916373 100644 --- a/autonomi/src/client/fs_private.rs +++ b/autonomi/src/client/fs_private.rs @@ -102,7 +102,9 @@ impl Client { // upload archive let archive_serialized = archive.into_bytes()?; - let arch_addr = self.private_data_put(archive_serialized, wallet).await?; + let arch_addr = self + .private_data_put(archive_serialized, wallet.into()) + .await?; info!( "Complete private archive upload completed in {:?}", @@ -126,7 +128,7 @@ impl Client { let data = tokio::fs::read(path).await?; let data = Bytes::from(data); - let addr = self.private_data_put(data, wallet).await?; + let addr = self.private_data_put(data, wallet.into()).await?; Ok(addr) } } diff --git a/autonomi/src/client/mod.rs b/autonomi/src/client/mod.rs index 0938dcbf9d..c4a2919347 100644 --- a/autonomi/src/client/mod.rs +++ b/autonomi/src/client/mod.rs @@ -7,6 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. pub mod address; +pub mod payment; #[cfg(feature = "data")] pub mod archive; diff --git a/autonomi/src/client/payment.rs b/autonomi/src/client/payment.rs new file mode 100644 index 0000000000..fbff226294 --- /dev/null +++ b/autonomi/src/client/payment.rs @@ -0,0 +1,49 @@ +use crate::client::data::PayError; +use crate::Client; +use sn_evm::{EvmWallet, ProofOfPayment}; +use std::collections::HashMap; +use xor_name::XorName; + +/// Contains the proof of payment for XOR addresses. +pub type Receipt = HashMap; + +/// Payment options for data payments. +#[derive(Clone)] +pub enum PaymentOption { + Wallet(EvmWallet), + Receipt(Receipt), +} + +impl From for PaymentOption { + fn from(value: EvmWallet) -> Self { + PaymentOption::Wallet(value) + } +} + +impl From<&EvmWallet> for PaymentOption { + fn from(value: &EvmWallet) -> Self { + PaymentOption::Wallet(value.clone()) + } +} + +impl From for PaymentOption { + fn from(value: Receipt) -> Self { + PaymentOption::Receipt(value) + } +} + +impl Client { + pub(crate) async fn pay_for_content_addrs( + &self, + content_addrs: impl Iterator, + payment_option: PaymentOption, + ) -> Result { + match payment_option { + PaymentOption::Wallet(wallet) => { + let (receipt, _) = self.pay(content_addrs, &wallet).await?; + Ok(receipt) + } + PaymentOption::Receipt(receipt) => Ok(receipt), + } + } +} diff --git a/autonomi/src/client/utils.rs b/autonomi/src/client/utils.rs index 53c7f88747..bc17f9e58f 100644 --- a/autonomi/src/client/utils.rs +++ b/autonomi/src/client/utils.rs @@ -6,12 +6,14 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. +use crate::client::payment::Receipt; +use crate::utils::receipt_from_cost_map_and_payments; use bytes::Bytes; use futures::stream::{FuturesUnordered, StreamExt}; use libp2p::kad::{Quorum, Record}; use rand::{thread_rng, Rng}; use self_encryption::{decrypt_full_set, DataMap, EncryptedChunk}; -use sn_evm::{EvmWallet, PaymentQuote, ProofOfPayment, QuotePayment}; +use sn_evm::{EvmWallet, ProofOfPayment, QuotePayment}; use sn_networking::{ GetRecordCfg, Network, NetworkError, PayeeQuote, PutRecordCfg, VerificationKind, }; @@ -28,7 +30,6 @@ use super::{ Client, }; use crate::self_encryption::DataMapLevel; -use crate::utils::payment_proof_from_quotes_and_payments; impl Client { /// Fetch and decrypt all chunks in the data map. @@ -160,13 +161,8 @@ impl Client { &self, content_addrs: impl Iterator, wallet: &EvmWallet, - ) -> Result<(HashMap, Vec), PayError> { - let cost_map = self - .get_store_quotes(content_addrs) - .await? - .into_iter() - .map(|(name, (_, _, q))| (name, q)) - .collect(); + ) -> Result<(Receipt, Vec), PayError> { + let cost_map = self.get_store_quotes(content_addrs).await?; let (quote_payments, skipped_chunks) = extract_quote_payments(&cost_map); @@ -187,7 +183,7 @@ impl Client { drop(lock_guard); debug!("Unlocked wallet"); - let proofs = payment_proof_from_quotes_and_payments(&cost_map, &payments); + let proofs = receipt_from_cost_map_and_payments(cost_map, &payments); trace!( "Chunk payments of {} chunks completed. {} chunks were free / already paid for", @@ -254,12 +250,12 @@ async fn fetch_store_quote( /// Form to be executed payments and already executed payments from a cost map. pub(crate) fn extract_quote_payments( - cost_map: &HashMap, + cost_map: &HashMap, ) -> (Vec, Vec) { let mut to_be_paid = vec![]; let mut already_paid = vec![]; - for (chunk_address, quote) in cost_map.iter() { + for (chunk_address, (_, _, quote)) in cost_map.iter() { if quote.cost.is_zero() { already_paid.push(*chunk_address); } else { diff --git a/autonomi/src/client/vault.rs b/autonomi/src/client/vault.rs index 55103b0578..209d174842 100644 --- a/autonomi/src/client/vault.rs +++ b/autonomi/src/client/vault.rs @@ -11,12 +11,13 @@ pub mod user_data; pub use key::{derive_vault_key, VaultSecretKey}; pub use user_data::UserData; -use xor_name::XorName; +use super::data::CostError; use crate::client::data::PutError; +use crate::client::payment::PaymentOption; use crate::client::Client; use libp2p::kad::{Quorum, Record}; -use sn_evm::{Amount, AttoTokens, EvmWallet}; +use sn_evm::{Amount, AttoTokens}; use sn_networking::{GetRecordCfg, GetRecordError, NetworkError, PutRecordCfg, VerificationKind}; use sn_protocol::storage::{ try_serialize_record, RecordKind, RetryStrategy, Scratchpad, ScratchpadAddress, @@ -27,8 +28,6 @@ use std::collections::HashSet; use std::hash::{DefaultHasher, Hash, Hasher}; use tracing::info; -use super::data::CostError; - #[derive(Debug, thiserror::Error)] pub enum VaultError { #[error("Could not generate Vault secret key from entropy: {0:?}")] @@ -171,35 +170,15 @@ impl Client { pub async fn write_bytes_to_vault( &self, data: Bytes, - wallet: &EvmWallet, + payment_option: PaymentOption, secret_key: &VaultSecretKey, content_type: VaultContentType, ) -> Result { let mut total_cost = AttoTokens::zero(); - let client_pk = secret_key.public_key(); - - let pad_res = self.get_vault_from_network(secret_key).await; - let mut is_new = true; - - let mut scratch = if let Ok(existing_data) = pad_res { - info!("Scratchpad already exists, returning existing data"); - - info!( - "scratch already exists, is version {:?}", - existing_data.count() - ); - is_new = false; - - if existing_data.owner() != &client_pk { - return Err(PutError::VaultBadOwner); - } - - existing_data - } else { - trace!("new scratchpad creation"); - Scratchpad::new(client_pk, content_type) - }; + let (mut scratch, is_new) = self + .get_or_create_scratchpad(secret_key, content_type) + .await?; let _ = scratch.update_and_sign(data, secret_key); debug_assert!(scratch.is_valid(), "Must be valid after being signed. This is a bug, please report it by opening an issue on our github"); @@ -210,21 +189,18 @@ impl Client { info!("Writing to vault at {scratch_address:?}",); let record = if is_new { - let scratch_xor = [&scratch_address] - .iter() - .filter_map(|f| f.as_xorname()) - .collect::>(); - let (payment_proofs, _) = self - .pay(scratch_xor.iter().cloned(), wallet) + let receipt = self + .pay_for_content_addrs(scratch.to_xor_name_vec().into_iter(), payment_option) .await .inspect_err(|err| { error!("Failed to pay for new vault at addr: {scratch_address:?} : {err}"); })?; - let proof = match payment_proofs.values().next() { + let proof = match receipt.values().next() { Some(proof) => proof, None => return Err(PutError::PaymentUnexpectedlyInvalid(scratch_address)), }; + total_cost = proof.quote.cost; Record { @@ -280,4 +256,38 @@ impl Client { Ok(total_cost) } + + /// Returns an existing scratchpad or creates a new one if it does not exist. + pub async fn get_or_create_scratchpad( + &self, + secret_key: &VaultSecretKey, + content_type: VaultContentType, + ) -> Result<(Scratchpad, bool), PutError> { + let client_pk = secret_key.public_key(); + + let pad_res = self.get_vault_from_network(secret_key).await; + let mut is_new = true; + + let scratch = if let Ok(existing_data) = pad_res { + info!("Scratchpad already exists, returning existing data"); + + info!( + "scratch already exists, is version {:?}", + existing_data.count() + ); + + is_new = false; + + if existing_data.owner() != &client_pk { + return Err(PutError::VaultBadOwner); + } + + existing_data + } else { + trace!("new scratchpad creation"); + Scratchpad::new(client_pk, content_type) + }; + + Ok((scratch, is_new)) + } } diff --git a/autonomi/src/client/vault/user_data.rs b/autonomi/src/client/vault/user_data.rs index 1f91b547bb..a0f217bda8 100644 --- a/autonomi/src/client/vault/user_data.rs +++ b/autonomi/src/client/vault/user_data.rs @@ -12,13 +12,13 @@ use crate::client::archive::ArchiveAddr; use crate::client::archive_private::PrivateArchiveAccess; use crate::client::data::GetError; use crate::client::data::PutError; +use crate::client::payment::PaymentOption; use crate::client::registers::RegisterAddress; use crate::client::vault::VaultError; use crate::client::vault::{app_name_to_vault_content_type, VaultContentType, VaultSecretKey}; use crate::client::Client; use serde::{Deserialize, Serialize}; use sn_evm::AttoTokens; -use sn_evm::EvmWallet; use sn_protocol::Bytes; use std::sync::LazyLock; @@ -75,11 +75,30 @@ impl UserData { self.file_archives.insert(archive, name) } + /// Add a private archive. Returning `Option::Some` with the old name if the archive was already in the set. + pub fn add_private_file_archive(&mut self, archive: PrivateArchiveAccess) -> Option { + self.private_file_archives.insert(archive, "".into()) + } + + /// Add a private archive with a name. Returning `Option::Some` with the old name if the archive was already in the set. + pub fn add_private_file_archive_with_name( + &mut self, + archive: PrivateArchiveAccess, + name: String, + ) -> Option { + self.private_file_archives.insert(archive, name) + } + /// Remove an archive. Returning `Option::Some` with the old name if the archive was already in the set. pub fn remove_file_archive(&mut self, archive: ArchiveAddr) -> Option { self.file_archives.remove(&archive) } + /// Remove a private archive. Returning `Option::Some` with the old name if the archive was already in the set. + pub fn remove_private_file_archive(&mut self, archive: PrivateArchiveAccess) -> Option { + self.private_file_archives.remove(&archive) + } + /// To bytes pub fn to_bytes(&self) -> Result { let bytes = rmp_serde::to_vec(&self)?; @@ -121,7 +140,7 @@ impl Client { pub async fn put_user_data_to_vault( &self, secret_key: &VaultSecretKey, - wallet: &EvmWallet, + payment_option: PaymentOption, user_data: UserData, ) -> Result { let bytes = user_data @@ -130,7 +149,7 @@ impl Client { let total_cost = self .write_bytes_to_vault( bytes, - wallet, + payment_option, secret_key, *USER_DATA_VAULT_CONTENT_IDENTIFIER, ) diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index f67c16babf..1dbec2fd15 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -81,7 +81,7 @@ impl JsClient { #[wasm_bindgen(js_name = dataPut)] pub async fn data_put(&self, data: Vec, wallet: &JsWallet) -> Result { let data = crate::Bytes::from(data); - let xorname = self.0.data_put(data, &wallet.0).await?; + let xorname = self.0.data_put(data, (&wallet.0).into()).await?; Ok(addr_to_str(xorname)) } @@ -284,7 +284,7 @@ mod vault { #[cfg(feature = "external-signer")] mod external_signer { use super::*; - use crate::payment_proof_from_quotes_and_payments; + use crate::receipt_from_quotes_and_payments; use sn_evm::external_signer::{approve_to_spend_tokens_calldata, pay_for_quotes_calldata}; use sn_evm::EvmNetwork; use sn_evm::ProofOfPayment; @@ -377,7 +377,7 @@ mod external_signer { ) -> Result { let quotes: HashMap = serde_wasm_bindgen::from_value(quotes)?; let payments: BTreeMap = serde_wasm_bindgen::from_value(payments)?; - let proof = payment_proof_from_quotes_and_payments("es, &payments); + let proof = receipt_from_quotes_and_payments("es, &payments); let js_value = serde_wasm_bindgen::to_value(&proof)?; Ok(js_value) } diff --git a/autonomi/src/lib.rs b/autonomi/src/lib.rs index d7441a6736..2f29d04926 100644 --- a/autonomi/src/lib.rs +++ b/autonomi/src/lib.rs @@ -48,7 +48,7 @@ pub use sn_evm::EvmNetwork; pub use sn_evm::EvmWallet as Wallet; pub use sn_evm::RewardsAddress; #[cfg(feature = "external-signer")] -pub use utils::payment_proof_from_quotes_and_payments; +pub use utils::receipt_from_quotes_and_payments; #[doc(no_inline)] // Place this under 'Re-exports' in the docs. pub use bytes::Bytes; diff --git a/autonomi/src/utils.rs b/autonomi/src/utils.rs index fc9ceb7718..b664581901 100644 --- a/autonomi/src/utils.rs +++ b/autonomi/src/utils.rs @@ -1,11 +1,27 @@ +use crate::client::payment::Receipt; use sn_evm::{PaymentQuote, ProofOfPayment, QuoteHash, TxHash}; +use sn_networking::PayeeQuote; use std::collections::{BTreeMap, HashMap}; use xor_name::XorName; -pub fn payment_proof_from_quotes_and_payments( +pub fn cost_map_to_quotes( + cost_map: HashMap, +) -> HashMap { + cost_map.into_iter().map(|(k, (_, _, v))| (k, v)).collect() +} + +pub fn receipt_from_cost_map_and_payments( + cost_map: HashMap, + payments: &BTreeMap, +) -> Receipt { + let quotes = cost_map_to_quotes(cost_map); + receipt_from_quotes_and_payments("es, payments) +} + +pub fn receipt_from_quotes_and_payments( quotes: &HashMap, payments: &BTreeMap, -) -> HashMap { +) -> Receipt { quotes .iter() .filter_map(|(xor_name, quote)| { diff --git a/autonomi/tests/external_signer.rs b/autonomi/tests/external_signer.rs index d97107cb39..53dce804db 100644 --- a/autonomi/tests/external_signer.rs +++ b/autonomi/tests/external_signer.rs @@ -2,7 +2,14 @@ use alloy::network::TransactionBuilder; use alloy::providers::Provider; -use autonomi::Client; +use autonomi::client::archive::Metadata; +use autonomi::client::archive_private::PrivateArchive; +use autonomi::client::external_signer::encrypt_data; +use autonomi::client::payment::Receipt; +use autonomi::client::vault::user_data::USER_DATA_VAULT_CONTENT_IDENTIFIER; +use autonomi::client::vault::VaultSecretKey; +use autonomi::{receipt_from_quotes_and_payments, Client, Wallet}; +use bytes::Bytes; use sn_evm::{QuoteHash, TxHash}; use sn_logging::LogBuilder; use std::collections::BTreeMap; @@ -10,18 +17,21 @@ use std::time::Duration; use test_utils::evm::get_funded_wallet; use test_utils::{gen_random_data, peers_from_env}; use tokio::time::sleep; +use xor_name::XorName; -// Example of how put would be done using external signers. -#[tokio::test] -async fn external_signer_put() -> eyre::Result<()> { - let _log_appender_guard = - LogBuilder::init_single_threaded_tokio_test("external_signer_put", false); - - let client = Client::connect(&peers_from_env()?).await?; - let wallet = get_funded_wallet(); - let data = gen_random_data(1024 * 1024 * 10); +async fn pay_for_data(client: &Client, wallet: &Wallet, data: Bytes) -> eyre::Result { + let (_data_map_chunk, _chunks, xor_names) = encrypt_data(data)?; + pay_for_content_addresses(client, wallet, xor_names.into_iter()).await +} - let (quotes, quote_payments, _free_chunks) = client.get_quotes_for_data(data.clone()).await?; +async fn pay_for_content_addresses( + client: &Client, + wallet: &Wallet, + content_addrs: impl Iterator, +) -> eyre::Result { + let (quotes, quote_payments, _free_chunks) = client + .get_quotes_for_content_addresses(content_addrs) + .await?; // Form quotes payment transaction data let pay_for_quotes_calldata = autonomi::client::external_signer::pay_for_quotes_calldata( @@ -76,16 +86,100 @@ async fn external_signer_put() -> eyre::Result<()> { } // Payment proofs - let proofs = autonomi::payment_proof_from_quotes_and_payments("es, &payments); + Ok(receipt_from_quotes_and_payments("es, &payments)) +} - let addr = client - .data_put_with_proof_of_payment(data.clone(), proofs) +// Example of how put would be done using external signers. +#[tokio::test] +async fn external_signer_put() -> eyre::Result<()> { + let _log_appender_guard = + LogBuilder::init_single_threaded_tokio_test("external_signer_put", false); + + let client = Client::connect(&peers_from_env()?).await?; + let wallet = get_funded_wallet(); + let data = gen_random_data(1024 * 1024 * 10); + + let receipt = pay_for_data(&client, &wallet, data.clone()).await?; + + sleep(Duration::from_secs(5)).await; + + let private_data_access = client + .private_data_put(data.clone(), receipt.into()) .await?; - sleep(Duration::from_secs(10)).await; + let mut private_archive = PrivateArchive::new(); + private_archive.add_file("test-file".into(), private_data_access, Metadata::default()); + + let archive_serialized = private_archive.into_bytes()?; - let data_fetched = client.data_get(addr).await?; - assert_eq!(data, data_fetched, "data fetched should match data put"); + let receipt = pay_for_data(&client, &wallet, archive_serialized.clone()).await?; + + sleep(Duration::from_secs(5)).await; + + let private_archive_access = client + .private_data_put(archive_serialized, receipt.into()) + .await?; + + let vault_key = VaultSecretKey::random(); + + let mut user_data = client + .get_user_data_from_vault(&vault_key) + .await + .unwrap_or_default(); + + user_data.add_private_file_archive_with_name( + private_archive_access.clone(), + "test-archive".to_string(), + ); + + let (scratch, is_new) = client + .get_or_create_scratchpad(&vault_key, *USER_DATA_VAULT_CONTENT_IDENTIFIER) + .await?; + + assert!(is_new, "Scratchpad is not new"); + + let scratch_addresses = if is_new { + scratch.to_xor_name_vec() + } else { + vec![] + }; + + let receipt = + pay_for_content_addresses(&client, &wallet, scratch_addresses.into_iter()).await?; + + sleep(Duration::from_secs(5)).await; + + let _ = client + .put_user_data_to_vault(&vault_key, receipt.into(), user_data) + .await?; + + let fetched_user_data = client.get_user_data_from_vault(&vault_key).await?; + + let fetched_private_archive_access = fetched_user_data + .private_file_archives + .keys() + .next() + .expect("No private archive present in the UserData") + .clone(); + + let fetched_private_archive = client + .private_archive_get(fetched_private_archive_access) + .await?; + + let (_, (fetched_private_file_access, _)) = fetched_private_archive + .map() + .iter() + .next() + .expect("No file present in private archive"); + + let fetched_private_file = client + .private_data_get(fetched_private_file_access.clone()) + .await?; + + assert_eq!( + fetched_private_file, data, + "Fetched private data is not identical to the uploaded data" + ); Ok(()) } diff --git a/autonomi/tests/put.rs b/autonomi/tests/put.rs index dbced37d00..27bd18fafb 100644 --- a/autonomi/tests/put.rs +++ b/autonomi/tests/put.rs @@ -23,7 +23,7 @@ async fn put() -> Result<()> { let wallet = get_funded_wallet(); let data = gen_random_data(1024 * 1024 * 10); - let addr = client.data_put(data.clone(), &wallet).await?; + let addr = client.data_put(data.clone(), wallet.into()).await?; sleep(Duration::from_secs(10)).await; diff --git a/autonomi/tests/wasm.rs b/autonomi/tests/wasm.rs index 8f27576f06..70dd347ffa 100644 --- a/autonomi/tests/wasm.rs +++ b/autonomi/tests/wasm.rs @@ -25,7 +25,7 @@ async fn put() -> Result<(), Box> { let wallet = get_funded_wallet(); let data = gen_random_data(1024 * 1024 * 10); - let addr = client.data_put(data.clone(), &wallet).await?; + let addr = client.data_put(data.clone(), wallet.into()).await?; sleep(Duration::from_secs(10)).await; diff --git a/sn_node/tests/data_with_churn.rs b/sn_node/tests/data_with_churn.rs index 347c74dc44..c23248a6ba 100644 --- a/sn_node/tests/data_with_churn.rs +++ b/sn_node/tests/data_with_churn.rs @@ -338,7 +338,7 @@ fn store_chunks_task( let mut retries = 1; loop { match client - .data_put(random_data.clone(), &wallet) + .data_put(random_data.clone(), (&wallet).into()) .await .inspect_err(|err| { println!("Error to put chunk: {err:?}"); diff --git a/sn_node/tests/verify_data_location.rs b/sn_node/tests/verify_data_location.rs index 8649d07909..c043f795ce 100644 --- a/sn_node/tests/verify_data_location.rs +++ b/sn_node/tests/verify_data_location.rs @@ -332,7 +332,7 @@ async fn store_chunks( let random_bytes = Bytes::from(random_bytes); - client.data_put(random_bytes, wallet).await?; + client.data_put(random_bytes, wallet.into()).await?; uploaded_chunks_count += 1; diff --git a/sn_protocol/src/storage/scratchpad.rs b/sn_protocol/src/storage/scratchpad.rs index 94b5c633a5..1022941de2 100644 --- a/sn_protocol/src/storage/scratchpad.rs +++ b/sn_protocol/src/storage/scratchpad.rs @@ -126,11 +126,19 @@ impl Scratchpad { &self.address } - /// Returns the NetworkAddress + /// Returns the NetworkAddress. pub fn network_address(&self) -> NetworkAddress { NetworkAddress::ScratchpadAddress(self.address) } + /// Returns a VEC with the XOR name. + pub fn to_xor_name_vec(&self) -> Vec { + [self.network_address()] + .iter() + .filter_map(|f| f.as_xorname()) + .collect::>() + } + /// Returns the name. pub fn name(&self) -> XorName { self.address.xorname() From 96b94d615f7a0f4c4afbdb00a23b4431dec788bf Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 30 Oct 2024 17:10:14 +0100 Subject: [PATCH 15/38] feat!: changed wasm/js api endpoints --- autonomi/src/client/wasm.rs | 169 +++++++++++++++++++++++++++++++----- 1 file changed, 148 insertions(+), 21 deletions(-) diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index 1dbec2fd15..16d391d5ae 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -1,8 +1,8 @@ +use super::address::{addr_to_str, str_to_addr}; +use crate::client::data_private::PrivateDataAccess; use libp2p::Multiaddr; use wasm_bindgen::prelude::*; -use super::address::{addr_to_str, str_to_addr}; - #[cfg(feature = "vault")] use super::vault::UserData; @@ -86,6 +86,22 @@ impl JsClient { Ok(addr_to_str(xorname)) } + /// Upload private data to the network. + /// + /// Returns the `PrivateDataAccess` chunk of the data. + #[wasm_bindgen(js_name = privateDataPut)] + pub async fn private_data_put( + &self, + data: Vec, + wallet: &JsWallet, + ) -> Result { + let data = crate::Bytes::from(data); + let private_data_access = self.0.private_data_put(data, (&wallet.0).into()).await?; + let js_value = serde_wasm_bindgen::to_value(&private_data_access)?; + + Ok(js_value) + } + /// Fetch the data from the network. #[wasm_bindgen(js_name = dataGet)] pub async fn data_get(&self, addr: String) -> Result, JsError> { @@ -95,6 +111,16 @@ impl JsClient { Ok(data.to_vec()) } + /// Fetch the data from the network. + #[wasm_bindgen(js_name = privateDataGet)] + pub async fn private_data_get(&self, private_data_access: JsValue) -> Result, JsError> { + let private_data_access: PrivateDataAccess = + serde_wasm_bindgen::from_value(private_data_access)?; + let data = self.0.private_data_get(private_data_access).await?; + + Ok(data.to_vec()) + } + /// Get the cost of uploading data to the network. #[wasm_bindgen(js_name = dataCost)] pub async fn data_cost(&self, data: Vec) -> Result { @@ -176,6 +202,79 @@ mod archive { } } +mod archive_private { + use super::*; + use crate::client::archive_private::{PrivateArchive, PrivateArchiveAccess}; + use crate::client::data_private::PrivateDataAccess; + use std::path::PathBuf; + use wasm_bindgen::JsValue; + + /// Structure mapping paths to data addresses. + #[wasm_bindgen(js_name = PrivateArchive)] + pub struct JsPrivateArchive(PrivateArchive); + + #[wasm_bindgen(js_class = PrivateArchive)] + impl JsPrivateArchive { + /// Create a new private archive. + #[wasm_bindgen(constructor)] + pub fn new() -> Self { + Self(PrivateArchive::new()) + } + + /// Add a new file to the private archive. + #[wasm_bindgen(js_name = addNewFile)] + pub fn add_new_file(&mut self, path: String, data_map: JsValue) -> Result<(), JsError> { + let path = PathBuf::from(path); + let data_map: PrivateDataAccess = serde_wasm_bindgen::from_value(data_map)?; + self.0.add_new_file(path, data_map); + + Ok(()) + } + + #[wasm_bindgen] + pub fn map(&self) -> Result { + let files = serde_wasm_bindgen::to_value(self.0.map())?; + Ok(files) + } + } + + #[wasm_bindgen(js_class = Client)] + impl JsClient { + /// Fetch a private archive from the network. + #[wasm_bindgen(js_name = privateArchiveGet)] + pub async fn private_archive_get( + &self, + private_archive_access: JsValue, + ) -> Result { + let private_archive_access: PrivateArchiveAccess = + serde_wasm_bindgen::from_value(private_archive_access)?; + let archive = self.0.private_archive_get(private_archive_access).await?; + let archive = JsPrivateArchive(archive); + + Ok(archive) + } + + /// Upload a private archive to the network. + /// + /// Returns the `PrivateArchiveAccess` chunk of the archive. + #[wasm_bindgen(js_name = privateArchivePut)] + pub async fn private_archive_put( + &self, + archive: &JsPrivateArchive, + wallet: &JsWallet, + ) -> Result { + let private_archive_access = self + .0 + .private_archive_put(archive.0.clone(), (&wallet.0).into()) + .await?; + + let js_value = serde_wasm_bindgen::to_value(&private_archive_access)?; + + Ok(js_value) + } + } +} + #[cfg(feature = "vault")] mod vault { use super::*; @@ -273,7 +372,7 @@ mod vault { secret_key: &SecretKeyJs, ) -> Result<(), JsError> { self.0 - .put_user_data_to_vault(&secret_key.0, &wallet.0, user_data.0.clone()) + .put_user_data_to_vault(&secret_key.0, (&wallet.0).into(), user_data.0.clone()) .await?; Ok(()) @@ -284,10 +383,12 @@ mod vault { #[cfg(feature = "external-signer")] mod external_signer { use super::*; + use crate::client::address::str_to_addr; + use crate::client::external_signer::encrypt_data; + use crate::client::payment::Receipt; use crate::receipt_from_quotes_and_payments; use sn_evm::external_signer::{approve_to_spend_tokens_calldata, pay_for_quotes_calldata}; use sn_evm::EvmNetwork; - use sn_evm::ProofOfPayment; use sn_evm::QuotePayment; use sn_evm::{Amount, PaymentQuote}; use sn_evm::{EvmAddress, QuoteHash, TxHash}; @@ -298,38 +399,64 @@ mod external_signer { #[wasm_bindgen(js_class = Client)] impl JsClient { - /// Get quotes for given data. + /// Encrypt data. /// /// # Example /// /// ```js - /// const [quotes, quotePayments, free_chunks] = await client.getQuotes(data); + /// const [dataMapChunk, dataChunks, [chunkAddresses]] = await client.encryptData(data); /// `` - #[wasm_bindgen(js_name = getQuotes)] - pub async fn get_quotes_for_data(&self, data: Vec) -> Result { + #[wasm_bindgen(js_name = encryptData)] + pub async fn encrypt_data(&self, data: Vec) -> Result { let data = crate::Bytes::from(data); - let result = self.0.get_quotes_for_data(data).await?; + let result = encrypt_data(data)?; + let js_value = serde_wasm_bindgen::to_value(&result)?; + Ok(js_value) + } + + /// Get quotes for given chunk addresses. + /// + /// # Example + /// + /// ```js + /// const [quotes, quotePayments, free_chunks] = await client.getQuotes(chunkAddresses); + /// `` + #[wasm_bindgen(js_name = getQuotes)] + pub async fn get_quotes(&self, chunk_addresses: Vec) -> Result { + let mut xor_addresses: Vec = vec![]; + + for chunk_address_str in &chunk_addresses { + let xor_address = str_to_addr(chunk_address_str)?; + xor_addresses.push(xor_address); + } + + let result = self + .0 + .get_quotes_for_content_addresses(xor_addresses.into_iter()) + .await?; + let js_value = serde_wasm_bindgen::to_value(&result)?; + Ok(js_value) } - /// Upload data with a proof of payment. + /// Upload data with a receipt. /// /// # Example /// /// ```js - /// const proof = getPaymentProofFromQuotesAndPayments(quotes, payments); - /// const addr = await client.dataPutWithProof(data, proof); + /// const receipt = getReceiptFromQuotesAndPayments(quotes, payments); + /// const addr = await client.dataPutWithReceipt(data, receipt); /// ``` - #[wasm_bindgen(js_name = dataPutWithProof)] - pub async fn data_put_with_proof_of_payment( + #[wasm_bindgen(js_name = dataPutWithReceipt)] + pub async fn data_put_with_receipt( &self, data: Vec, - proof: JsValue, + receipt: JsValue, ) -> Result { let data = crate::Bytes::from(data); - let proof: HashMap = serde_wasm_bindgen::from_value(proof)?; - let xorname = self.0.data_put_with_proof_of_payment(data, proof).await?; + let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?; + let xorname = self.0.data_put(data, receipt.into()).await?; Ok(addr_to_str(xorname)) } } @@ -370,15 +497,15 @@ mod external_signer { } /// Generate payment proof. - #[wasm_bindgen(js_name = getPaymentProofFromQuotesAndPayments)] - pub fn get_payment_proof_from_quotes_and_payments( + #[wasm_bindgen(js_name = getReceiptFromQuotesAndPayments)] + pub fn get_receipt_from_quotes_and_payments( quotes: JsValue, payments: JsValue, ) -> Result { let quotes: HashMap = serde_wasm_bindgen::from_value(quotes)?; let payments: BTreeMap = serde_wasm_bindgen::from_value(payments)?; - let proof = receipt_from_quotes_and_payments("es, &payments); - let js_value = serde_wasm_bindgen::to_value(&proof)?; + let receipt = receipt_from_quotes_and_payments("es, &payments); + let js_value = serde_wasm_bindgen::to_value(&receipt)?; Ok(js_value) } } From d62b252e5f8d88f2c56c684d2f2707454c70addd Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 30 Oct 2024 17:47:44 +0100 Subject: [PATCH 16/38] fix(test): fs test --- autonomi/tests/fs.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/autonomi/tests/fs.rs b/autonomi/tests/fs.rs index b952852bc2..93fa7e3964 100644 --- a/autonomi/tests/fs.rs +++ b/autonomi/tests/fs.rs @@ -93,7 +93,12 @@ async fn file_into_vault() -> Result<()> { let archive = client.archive_get(addr).await?; let set_version = 0; client - .write_bytes_to_vault(archive.into_bytes()?, &wallet, &client_sk, set_version) + .write_bytes_to_vault( + archive.into_bytes()?, + wallet.into(), + &client_sk, + set_version, + ) .await?; // now assert over the stored account packet From a1eb0bfa821a788ac1e6e974a5fd84ef82f4bdd9 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 30 Oct 2024 17:54:56 +0100 Subject: [PATCH 17/38] feat(autonomi): add wasm/js functions: `privateDataPutWithReceipt` & `privateArchivePutWithReceipt` --- autonomi/src/client/wasm.rs | 47 ++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index 16d391d5ae..e8aae2161d 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -1,11 +1,11 @@ use super::address::{addr_to_str, str_to_addr}; +#[cfg(feature = "vault")] +use super::vault::UserData; use crate::client::data_private::PrivateDataAccess; +use crate::client::payment::Receipt; use libp2p::Multiaddr; use wasm_bindgen::prelude::*; -#[cfg(feature = "vault")] -use super::vault::UserData; - /// The `Client` object allows interaction with the network to store and retrieve data. /// /// To connect to the network, see {@link Client.connect}. @@ -102,6 +102,24 @@ impl JsClient { Ok(js_value) } + /// Upload private data to the network. + /// Uses a `Receipt` as payment. + /// + /// Returns the `PrivateDataAccess` chunk of the data. + #[wasm_bindgen(js_name = privateDataPutWithReceipt)] + pub async fn private_data_put_with_receipt( + &self, + data: Vec, + receipt: JsValue, + ) -> Result { + let data = crate::Bytes::from(data); + let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?; + let private_data_access = self.0.private_data_put(data, receipt.into()).await?; + let js_value = serde_wasm_bindgen::to_value(&private_data_access)?; + + Ok(js_value) + } + /// Fetch the data from the network. #[wasm_bindgen(js_name = dataGet)] pub async fn data_get(&self, addr: String) -> Result, JsError> { @@ -206,6 +224,7 @@ mod archive_private { use super::*; use crate::client::archive_private::{PrivateArchive, PrivateArchiveAccess}; use crate::client::data_private::PrivateDataAccess; + use crate::client::payment::Receipt; use std::path::PathBuf; use wasm_bindgen::JsValue; @@ -272,6 +291,28 @@ mod archive_private { Ok(js_value) } + + /// Upload a private archive to the network. + /// Uses a `Receipt` as payment. + /// + /// Returns the `PrivateArchiveAccess` chunk of the archive. + #[wasm_bindgen(js_name = privateArchivePutWithReceipt)] + pub async fn private_archive_put_with_receipt( + &self, + archive: &JsPrivateArchive, + receipt: JsValue, + ) -> Result { + let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?; + + let private_archive_access = self + .0 + .private_archive_put(archive.0.clone(), receipt.into()) + .await?; + + let js_value = serde_wasm_bindgen::to_value(&private_archive_access)?; + + Ok(js_value) + } } } From 685c881391990456291fbe437fe4f32f43756fc6 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 30 Oct 2024 18:00:33 +0100 Subject: [PATCH 18/38] refactor: change `encryptData` function to be synchronous and move out of `Client` --- autonomi/src/client/wasm.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index e8aae2161d..4400252f33 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -440,21 +440,6 @@ mod external_signer { #[wasm_bindgen(js_class = Client)] impl JsClient { - /// Encrypt data. - /// - /// # Example - /// - /// ```js - /// const [dataMapChunk, dataChunks, [chunkAddresses]] = await client.encryptData(data); - /// `` - #[wasm_bindgen(js_name = encryptData)] - pub async fn encrypt_data(&self, data: Vec) -> Result { - let data = crate::Bytes::from(data); - let result = encrypt_data(data)?; - let js_value = serde_wasm_bindgen::to_value(&result)?; - Ok(js_value) - } - /// Get quotes for given chunk addresses. /// /// # Example @@ -502,6 +487,21 @@ mod external_signer { } } + /// Encrypt data. + /// + /// # Example + /// + /// ```js + /// const [dataMapChunk, dataChunks, [chunkAddresses]] = client.encryptData(data); + /// `` + #[wasm_bindgen(js_name = encryptData)] + pub fn encrypt(data: Vec) -> Result { + let data = crate::Bytes::from(data); + let result = encrypt_data(data)?; + let js_value = serde_wasm_bindgen::to_value(&result)?; + Ok(js_value) + } + /// Get the calldata for paying for quotes. /// /// # Example From 561656a77980e0c91efc27c2efd4e622cc3925bd Mon Sep 17 00:00:00 2001 From: Ermine Jose Date: Wed, 30 Oct 2024 16:32:46 +0530 Subject: [PATCH 19/38] test: fix conditional checks --- .github/workflows/merge.yml | 52 ++++++++++++------------------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 007218429a..1a81143b41 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -350,6 +350,7 @@ jobs: timeout-minutes: 2 - name: Estimate cost to create a vault + if: matrix.os != 'windows-latest' run: | echo "test-file" > upload-test.txt ./target/release/autonomi --log-output-dest=data-dir file upload ./upload-test.txt @@ -362,6 +363,7 @@ jobs: timeout-minutes: 2 - name: create a vault with existing user data as above + if: matrix.os != 'windows-latest' run: | ./target/release/autonomi --log-output-dest=data-dir vault create ./target/release/autonomi --log-output-dest=data-dir vault load 2>&1 | tee vault_data.txt @@ -369,40 +371,21 @@ jobs: SN_LOG: "v" timeout-minutes: 2 - - name: vault create validation - run: | - NUM_OF_PUBLIC_FILES=`cat file_list.txt | grep "public" | grep -o '[0-9]\+'` - NUM_OF_PRIVATE_FILES=`cat file_list.txt | grep "private" | grep -o '[0-9]\+'` - NUM_OF_REGISTERS=`cat register_list.txt | grep "register" | grep -o '[0-9]\+'` - - NUM_OF_PUBLIC_FILES_IN_VAULT=`cat vault_data.txt | grep "public" | grep -o '[0-9]\+'` - NUM_OF_PRIVATE_FILES_IN_VAULT=`cat vault_data.txt| grep "private" | grep -o '[0-9]\+'` - NUM_OF_REGISTERS_IN_VAULT=`cat vault_data.txt| grep "register" | grep -o '[0-9]\+'` - - rm -rf file_list.txt register_list.txt vault_data.txt - - - python3 -c 'import sys; print("Equal" if sys.argv[1] == sys.argv[2] else "Not Equal")' $NUM_OF_PUBLIC_FILES $NUM_OF_PUBLIC_FILES_IN_VAULT - - python3 -c 'import sys; print("Equal" if sys.argv[1] == sys.argv[2] else "Not Equal")' $NUM_OF_PRIVATE_FILES $NUM_OF_PRIVATE_FILES_IN_VAULT - - python3 -c 'import sys; print("Equal" if sys.argv[1] == sys.argv[2] else "Not Equal")' $NUM_OF_REGISTERS $NUM_OF_REGISTERS_IN_VAULT - - echo "vault created successfully!" - env: - SN_LOG: "v" - timeout-minutes: 4 - - - name: add more files and registers + - name: add more files + if: matrix.os != 'windows-latest' run: | - echo "testing-file2" > sync-test.txt - ./target/release/autonomi --log-output-dest=data-dir file upload ./sync-test.txt - ./target/release/autonomi --log-output-dest=data-dir register create sample_new_register_2 12345 + for i in {1..100}; do + dd if=/dev/urandom of=random_file_$i.bin bs=1M count=1 status=none + ./target/release/autonomi --log-output-dest=data-dir file upload random_file_$i.bin --public + ./target/release/autonomi --log-output-dest=data-dir file upload random_file_$i.bin + ./target/release/autonomi --log-output-dest=data-dir register create $i random_file_$i.bin + done env: SN_LOG: "v" timeout-minutes: 2 - name: sync the vault + if: matrix.os != 'windows-latest' run: ./target/release/autonomi --log-output-dest=data-dir vault sync env: SN_LOG: "v" @@ -432,18 +415,16 @@ jobs: rm -rf file_list.txt register_list.txt vault_data.txt - python3 -c 'import sys; print("Equal" if sys.argv[1] == sys.argv[2] else "Not Equal")' $NUM_OF_PUBLIC_FILES $NUM_OF_PUBLIC_FILES_IN_VAULT - - python3 -c 'import sys; print("Equal" if sys.argv[1] == sys.argv[2] else "Not Equal")' $NUM_OF_PRIVATE_FILES $NUM_OF_PRIVATE_FILES_IN_VAULT - - python3 -c 'import sys; print("Equal" if sys.argv[1] == sys.argv[2] else "Not Equal")' $NUM_OF_REGISTERS $NUM_OF_REGISTERS_IN_VAULT - + python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: {sys.argv[1]} and {sys.argv[2]} are Not Equal"' $NUM_OF_PUBLIC_FILES $NUM_OF_PUBLIC_FILES_IN_VAULT + python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: {sys.argv[1]} and {sys.argv[2]} are Not Equal"' $NUM_OF_PRIVATE_FILES $NUM_OF_PRIVATE_FILES_IN_VAULT + python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: {sys.argv[1]} and {sys.argv[2]} are Not Equal"' $NUM_OF_REGISTERS $NUM_OF_REGISTERS_IN_VAULT echo "vault synced successfully!" env: SN_LOG: "v" timeout-minutes: 2 - name: load an existing vault from the network + if: matrix.os != 'windows-latest' run: ./target/release/autonomi --log-output-dest=data-dir vault load env: SN_LOG: "v" @@ -465,8 +446,9 @@ jobs: time ./target/release/autonomi --log-output-dest=data-dir file upload random_10MB.bin time ./target/release/autonomi --log-output-dest=data-dir file upload random_100MB.bin time ./target/release/autonomi --log-output-dest=data-dir file upload random_1GB.bin - + ./target/release/autonomi --log-output-dest=data-dir vault sync rm -rf random*.bin + rm -rf ${{ matrix.safe_path }}/autonomi env: SN_LOG: "v" From 5fb5a0dc460c73be1c8e133270216b189afe1f7a Mon Sep 17 00:00:00 2001 From: Ermine Jose Date: Thu, 31 Oct 2024 16:59:47 +0530 Subject: [PATCH 20/38] test: increase timeout --- .github/workflows/merge.yml | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 1a81143b41..47ddd0dc5b 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -152,6 +152,7 @@ jobs: # of proptesting # we do many more runs on the nightly run PROPTEST_CASES: 50 + e2e: if: "!startsWith(github.event.head_commit.message, 'chore(release):')" name: E2E tests @@ -364,9 +365,7 @@ jobs: - name: create a vault with existing user data as above if: matrix.os != 'windows-latest' - run: | - ./target/release/autonomi --log-output-dest=data-dir vault create - ./target/release/autonomi --log-output-dest=data-dir vault load 2>&1 | tee vault_data.txt + run: ./target/release/autonomi --log-output-dest=data-dir vault create env: SN_LOG: "v" timeout-minutes: 2 @@ -382,7 +381,7 @@ jobs: done env: SN_LOG: "v" - timeout-minutes: 2 + timeout-minutes: 25 - name: sync the vault if: matrix.os != 'windows-latest' @@ -392,6 +391,7 @@ jobs: timeout-minutes: 2 - name: vault sync validation + if: matrix.os != 'windows-latest' run: | NUM_OF_PUBLIC_FILES="" NUM_OF_PRIVATE_FILES="" @@ -400,28 +400,33 @@ jobs: NUM_OF_PRIVATE_FILES_IN_VAULT="" NUM_OF_REGISTERS_IN_VAULT="" - ./target/release/autonomi --log-output-dest=data-dir file list 2>&1 | tee file_list.txt - ./target/release/autonomi --log-output-dest=data-dir register list 2>&1 | tee register_list.txt - + ./target/release/autonomi --log-output-dest=data-dir file list 2>&1 > file_list.txt + + # ./target/release/autonomi --log-output-dest=data-dir register list | grep archives > register_list.txt + NUM_OF_PUBLIC_FILES=`cat file_list.txt | grep "public" | grep -o '[0-9]\+'` NUM_OF_PRIVATE_FILES=`cat file_list.txt | grep "private" | grep -o '[0-9]\+'` NUM_OF_REGISTERS=`cat register_list.txt | grep "register" | grep -o '[0-9]\+'` - ./target/release/autonomi --log-output-dest=data-dir vault load 2>&1 | tee vault_data.txt + ./target/release/autonomi --log-output-dest=data-dir vault load 2>&1 > vault_data.txt NUM_OF_PUBLIC_FILES_IN_VAULT=`cat vault_data.txt | grep "public" | grep -o '[0-9]\+'` NUM_OF_PRIVATE_FILES_IN_VAULT=`cat vault_data.txt| grep "private" | grep -o '[0-9]\+'` - NUM_OF_REGISTERS_IN_VAULT=`cat vault_data.txt| grep "register" | grep -o '[0-9]\+'` + # NUM_OF_REGISTERS_IN_VAULT=`cat vault_data.txt | grep "register" | grep -o '[0-9]\+'` + echo "Total Num of local public files is $NUM_OF_PUBLIC_FILES and in vault is $NUM_OF_PUBLIC_FILES_IN_VAULT" + echo "Total Num of local private files is $NUM_OF_PRIVATE_FILES and in vault is $NUM_OF_PRIVATE_FILES_IN_VAULT" + # echo "Total Num of local registers is $NUM_OF_REGISTERS and in vault is $NUM_OF_REGISTERS_IN_VAULT" + rm -rf file_list.txt register_list.txt vault_data.txt - python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: {sys.argv[1]} and {sys.argv[2]} are Not Equal"' $NUM_OF_PUBLIC_FILES $NUM_OF_PUBLIC_FILES_IN_VAULT - python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: {sys.argv[1]} and {sys.argv[2]} are Not Equal"' $NUM_OF_PRIVATE_FILES $NUM_OF_PRIVATE_FILES_IN_VAULT - python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: {sys.argv[1]} and {sys.argv[2]} are Not Equal"' $NUM_OF_REGISTERS $NUM_OF_REGISTERS_IN_VAULT + python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local public Files: {sys.argv[1]} and vault public files: {sys.argv[2]} are Not Equal"' $NUM_OF_PUBLIC_FILES $NUM_OF_PUBLIC_FILES_IN_VAULT + python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local private Files: {sys.argv[1]} and vault private files: {sys.argv[2]} are Not Equal"' $NUM_OF_PRIVATE_FILES $NUM_OF_PRIVATE_FILES_IN_VAULT + # python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local registers: {sys.argv[1]} and vault registers: {sys.argv[2]} are Not Equal"' $NUM_OF_REGISTERS $NUM_OF_REGISTERS_IN_VAULT echo "vault synced successfully!" env: SN_LOG: "v" - timeout-minutes: 2 + timeout-minutes: 15 - name: load an existing vault from the network if: matrix.os != 'windows-latest' @@ -431,6 +436,7 @@ jobs: timeout-minutes: 2 - name: Time profiling for Different files + if: matrix.os != 'windows-latest' run: | # 1 MB python3 -c "with open('random_1MB.bin', 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))" @@ -448,11 +454,10 @@ jobs: time ./target/release/autonomi --log-output-dest=data-dir file upload random_1GB.bin ./target/release/autonomi --log-output-dest=data-dir vault sync rm -rf random*.bin - rm -rf ${{ matrix.safe_path }}/autonomi - + rm -rf ${{ matrix.safe_path }}/autonomi env: SN_LOG: "v" - timeout-minutes: 10 + timeout-minutes: 15 - name: Stop the local network and upload logs if: always() From 899afd910491c98a25f5152e56135a34f01b9069 Mon Sep 17 00:00:00 2001 From: Lautaro Mazzitelli Date: Mon, 4 Nov 2024 10:02:34 +0100 Subject: [PATCH 21/38] chore(launchpad): update ratatui and throbbber versions --- Cargo.lock | 36 ++++++++++++++++--------- node-launchpad/Cargo.toml | 4 +-- node-launchpad/src/components/status.rs | 16 +++-------- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dfcaa5e8c7..3bec6540c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1793,7 +1793,7 @@ dependencies = [ "strsim", "terminal_size", "unicase", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -1925,7 +1925,7 @@ dependencies = [ "encode_unicode 0.3.6", "lazy_static", "libc", - "unicode-width", + "unicode-width 0.1.14", "windows-sys 0.52.0", ] @@ -4436,9 +4436,15 @@ dependencies = [ "number_prefix", "portable-atomic", "tokio", - "unicode-width", + "unicode-width 0.1.14", ] +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + [[package]] name = "inferno" version = "0.11.21" @@ -6568,7 +6574,7 @@ dependencies = [ "is-terminal", "lazy_static", "term", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -7091,24 +7097,24 @@ dependencies = [ [[package]] name = "ratatui" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdef7f9be5c0122f890d58bdf4d964349ba6a6161f705907526d891efabba57d" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" dependencies = [ "bitflags 2.6.0", "cassowary", "compact_str", "crossterm 0.28.1", + "indoc", "instability", "itertools 0.13.0", "lru", "paste", "serde", "strum", - "strum_macros", "unicode-segmentation", "unicode-truncate", - "unicode-width", + "unicode-width 0.2.0", ] [[package]] @@ -8943,9 +8949,9 @@ dependencies = [ [[package]] name = "throbber-widgets-tui" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad9e055cadd9da8b4a67662b962e3e67e96af491ae9cec7e88aaff92e7c3666" +checksum = "1d36b5738d666a2b4c91b7c24998a8588db724b3107258343ebf8824bf55b06d" dependencies = [ "rand 0.8.5", "ratatui", @@ -9498,7 +9504,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3e785f863a3af4c800a2a669d0b64c879b538738e352607e2624d03f868dc01" dependencies = [ "crossterm 0.27.0", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -9600,7 +9606,7 @@ checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" dependencies = [ "itertools 0.13.0", "unicode-segmentation", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -9609,6 +9615,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unicode-xid" version = "0.2.6" diff --git a/node-launchpad/Cargo.toml b/node-launchpad/Cargo.toml index b9ee73af76..d6d77fffcc 100644 --- a/node-launchpad/Cargo.toml +++ b/node-launchpad/Cargo.toml @@ -44,7 +44,7 @@ libc = "0.2.148" log = "0.4.20" pretty_assertions = "1.4.0" prometheus-parse = "0.2.5" -ratatui = { version = "0.28.1", features = ["serde", "macros", "unstable-widget-ref"] } +ratatui = { version = "0.29.0", features = ["serde", "macros", "unstable-widget-ref"] } reqwest = { version = "0.12.2", default-features = false, features = [ "rustls-tls-manual-roots", ] } @@ -69,7 +69,7 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter", "serde"] } tui-input = "0.8.0" which = "6.0.1" faccess = "0.2.4" -throbber-widgets-tui = "0.7.0" +throbber-widgets-tui = "0.8.0" regex = "1.11.0" [build-dependencies] diff --git a/node-launchpad/src/components/status.rs b/node-launchpad/src/components/status.rs index 3c82045f7b..7c476ca9aa 100644 --- a/node-launchpad/src/components/status.rs +++ b/node-launchpad/src/components/status.rs @@ -180,7 +180,7 @@ impl Status<'_> { // Update status based on current node status item.status = match node_item.status { ServiceStatus::Running => { - NodeItem::update_spinner_state(&mut item.spinner_state); + item.spinner_state.calc_next(); NodeStatus::Running } ServiceStatus::Stopped => NodeStatus::Stopped, @@ -190,7 +190,7 @@ impl Status<'_> { // Starting is not part of ServiceStatus so we do it manually if let Some(LockRegistryState::StartingNodes) = self.lock_registry { - NodeItem::update_spinner_state(&mut item.spinner_state); + item.spinner_state.calc_next(); item.status = NodeStatus::Starting; } @@ -848,7 +848,7 @@ impl Component for Status<'_> { let table = Table::new(items, node_widths) .header(header_row) .column_spacing(1) - .highlight_style(Style::default().bg(INDIGO)) + .row_highlight_style(Style::default().bg(INDIGO)) .highlight_spacing(HighlightSpacing::Always); f.render_widget(table, inner_area); @@ -1057,16 +1057,6 @@ pub struct NodeItem<'a> { } impl NodeItem<'_> { - fn update_spinner_state(state: &mut ThrobberState) { - // Call calc_next on the spinner state - // https://github.com/arkbig/throbber-widgets-tui/issues/19 - if state.index() == i8::MAX { - *state = ThrobberState::default(); - } else { - state.calc_next(); - } - } - fn render_as_row(&mut self, index: usize, area: Rect, f: &mut Frame<'_>) -> Row { let mut row_style = Style::default().fg(GHOST_WHITE); let mut spinner_state = self.spinner_state.clone(); From df00ff40d7babda40886766fe116e099deaaf2b2 Mon Sep 17 00:00:00 2001 From: Lautaro Mazzitelli Date: Mon, 4 Nov 2024 12:14:10 +0100 Subject: [PATCH 22/38] feat(launchpad): ctrl v pasting on rewards address --- Cargo.lock | 291 ++++++++++++++++++ node-launchpad/Cargo.toml | 1 + .../src/components/popup/rewards_address.rs | 19 +- 3 files changed, 310 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index dfcaa5e8c7..e575f15506 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -726,6 +726,24 @@ version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +[[package]] +name = "arboard" +version = "3.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df099ccb16cd014ff054ac1bf392c67feeef57164b05c42f037cd40f5d4357f4" +dependencies = [ + "clipboard-win", + "core-graphics", + "image", + "log", + "objc2", + "objc2-app-kit", + "objc2-foundation", + "parking_lot", + "windows-sys 0.48.0", + "x11rb", +] + [[package]] name = "arc-swap" version = "1.7.1" @@ -1409,6 +1427,15 @@ dependencies = [ "generic-array 0.14.7", ] +[[package]] +name = "block2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c132eebf10f5cad5289222520a4a058514204aed6d791f1cf4fe8088b82d15f" +dependencies = [ + "objc2", +] + [[package]] name = "blst" version = "0.3.13" @@ -1545,6 +1572,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.7.2" @@ -1814,6 +1847,15 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "clipboard-win" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15efe7a882b08f34e38556b14f2fb3daa98769d06c7f0c1b076dfd0d983bc892" +dependencies = [ + "error-code", +] + [[package]] name = "cloudabi" version = "0.0.3" @@ -2009,6 +2051,30 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core-graphics" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-graphics-types", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "libc", +] + [[package]] name = "core2" version = "0.4.0" @@ -2786,6 +2852,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "error-code" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d9305ccc6942a704f4335694ecd3de2ea531b114ac2d51f5f843750787a92f" + [[package]] name = "event-listener" version = "5.3.1" @@ -2883,6 +2955,15 @@ dependencies = [ "bytes", ] +[[package]] +name = "fdeflate" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07c6f4c64c1d33a3111c4466f7365ebdcc37c5bd1ea0d62aae2e3d722aacbedb" +dependencies = [ + "simd-adler32", +] + [[package]] name = "ff" version = "0.12.1" @@ -3014,6 +3095,33 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -3225,6 +3333,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "gethostname" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0176e0459c2e4a1fe232f984bca6890e681076abb9934f6cea7c326f3fc47818" +dependencies = [ + "libc", + "windows-targets 0.48.5", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -4377,6 +4495,19 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "image" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc144d44a31d753b02ce64093d532f55ff8dc4ebf2ffb8a63c0dda691385acae" +dependencies = [ + "bytemuck", + "byteorder-lite", + "num-traits", + "png", + "tiff", +] + [[package]] name = "impl-codec" version = "0.6.0" @@ -4566,6 +4697,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" + [[package]] name = "js-sys" version = "0.3.70" @@ -5469,6 +5606,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -5749,6 +5887,7 @@ dependencies = [ name = "node-launchpad" version = "0.4.2" dependencies = [ + "arboard", "atty", "better-panic", "chrono", @@ -5940,6 +6079,105 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "objc-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb91bdd390c7ce1a8607f35f3ca7151b65afc0ff5ff3b34fa350f7d7c7e4310" + +[[package]] +name = "objc2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a785d4eeff09c14c487497c162e92766fbb3e4059a71840cecc03d9a50b804" +dependencies = [ + "objc-sys", + "objc2-encode", +] + +[[package]] +name = "objc2-app-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4e89ad9e3d7d297152b17d39ed92cd50ca8063a89a9fa569046d41568891eff" +dependencies = [ + "bitflags 2.6.0", + "block2", + "libc", + "objc2", + "objc2-core-data", + "objc2-core-image", + "objc2-foundation", + "objc2-quartz-core", +] + +[[package]] +name = "objc2-core-data" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617fbf49e071c178c0b24c080767db52958f716d9eabdf0890523aeae54773ef" +dependencies = [ + "bitflags 2.6.0", + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-image" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55260963a527c99f1819c4f8e3b47fe04f9650694ef348ffd2227e8196d34c80" +dependencies = [ + "block2", + "objc2", + "objc2-foundation", + "objc2-metal", +] + +[[package]] +name = "objc2-encode" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7891e71393cd1f227313c9379a26a584ff3d7e6e7159e988851f0934c993f0f8" + +[[package]] +name = "objc2-foundation" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8" +dependencies = [ + "bitflags 2.6.0", + "block2", + "libc", + "objc2", +] + +[[package]] +name = "objc2-metal" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0cba1276f6023976a406a14ffa85e1fdd19df6b0f737b063b95f6c8c7aadd6" +dependencies = [ + "bitflags 2.6.0", + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e42bee7bff906b14b167da2bac5efe6b6a07e6f7c0a21a7308d40c960242dc7a" +dependencies = [ + "bitflags 2.6.0", + "block2", + "objc2", + "objc2-foundation", + "objc2-metal", +] + [[package]] name = "object" version = "0.32.2" @@ -6423,6 +6661,19 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "png" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f9d46a34a05a6a57566bc2bfae066ef07585a6e3fa30fbbdff5936380623f0" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.0", +] + [[package]] name = "polling" version = "3.7.3" @@ -8130,6 +8381,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "slab" version = "0.4.9" @@ -8951,6 +9208,17 @@ dependencies = [ "ratatui", ] +[[package]] +name = "tiff" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "time" version = "0.3.36" @@ -9978,6 +10246,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" + [[package]] name = "which" version = "4.4.2" @@ -10289,6 +10563,23 @@ dependencies = [ "tap", ] +[[package]] +name = "x11rb" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d91ffca73ee7f68ce055750bf9f6eca0780b8c85eff9bc046a3b0da41755e12" +dependencies = [ + "gethostname", + "rustix", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec107c4503ea0b4a98ef47356329af139c0a4f7750e621cf2973cd3385ebcb3d" + [[package]] name = "x25519-dalek" version = "2.0.1" diff --git a/node-launchpad/Cargo.toml b/node-launchpad/Cargo.toml index b9ee73af76..aee33330ea 100644 --- a/node-launchpad/Cargo.toml +++ b/node-launchpad/Cargo.toml @@ -71,6 +71,7 @@ which = "6.0.1" faccess = "0.2.4" throbber-widgets-tui = "0.7.0" regex = "1.11.0" +arboard = "3.4.1" [build-dependencies] vergen = { version = "8.2.6", features = ["build", "git", "gitoxide", "cargo"] } diff --git a/node-launchpad/src/components/popup/rewards_address.rs b/node-launchpad/src/components/popup/rewards_address.rs index 8ec3741034..54c8a34995 100644 --- a/node-launchpad/src/components/popup/rewards_address.rs +++ b/node-launchpad/src/components/popup/rewards_address.rs @@ -14,8 +14,9 @@ use crate::{ style::{clear_area, EUCALYPTUS, GHOST_WHITE, INDIGO, LIGHT_PERIWINKLE, RED, VIVID_SKY_BLUE}, widgets::hyperlink::Hyperlink, }; +use arboard::Clipboard; use color_eyre::Result; -use crossterm::event::{Event, KeyCode, KeyEvent}; +use crossterm::event::{Event, KeyCode, KeyEvent, KeyModifiers}; use ratatui::{prelude::*, widgets::*}; use regex::Regex; use tui_input::{backend::crossterm::EventHandler, Input}; @@ -113,6 +114,22 @@ impl RewardsAddress { self.validate(); vec![] } + KeyCode::Char('v') => { + if key.modifiers.contains(KeyModifiers::CONTROL) { + let mut clipboard = match Clipboard::new() { + Ok(clipboard) => clipboard, + Err(e) => { + error!("Error reading Clipboard : {:?}", e); + return vec![]; + } + }; + if let Ok(content) = clipboard.get_text() { + self.rewards_address_input_field = + self.rewards_address_input_field.clone().with_value(content); + } + } + vec![] + } _ => { if self.rewards_address_input_field.value().chars().count() < INPUT_SIZE_REWARDS_ADDRESS as usize From 21d40446c54066e10024bde38f5d2cf46caa3c71 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 4 Nov 2024 17:34:32 +0100 Subject: [PATCH 23/38] feat: added WASM functions for private data upload to vault using receipts & added metamask test --- autonomi/examples/metamask/index.html | 4 +- autonomi/examples/metamask/index.js | 188 ++++++++++++++++++------- autonomi/src/client/external_signer.rs | 13 +- autonomi/src/client/wasm.rs | 183 +++++++++++++++++++++++- autonomi/tests/external_signer.rs | 12 +- 5 files changed, 332 insertions(+), 68 deletions(-) diff --git a/autonomi/examples/metamask/index.html b/autonomi/examples/metamask/index.html index 50844bd7f9..128273acbc 100644 --- a/autonomi/examples/metamask/index.html +++ b/autonomi/examples/metamask/index.html @@ -7,12 +7,12 @@