From 7e1edee9268b223d7fe8e3255e11b594b0bc37d1 Mon Sep 17 00:00:00 2001 From: stneng Date: Fri, 24 Feb 2023 13:29:19 +0000 Subject: [PATCH 1/6] storage macro: local file system --- src/extensions/storage_macro.rs | 11 ++++++ src/extensions/storage_macro/fs.rs | 58 ++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 src/extensions/storage_macro/fs.rs diff --git a/src/extensions/storage_macro.rs b/src/extensions/storage_macro.rs index 6eb374b..7cef53c 100644 --- a/src/extensions/storage_macro.rs +++ b/src/extensions/storage_macro.rs @@ -1,5 +1,6 @@ mod append; mod chunk; +mod fs; mod redis; type Error = Box; @@ -39,6 +40,10 @@ impl crate::application::CoLink { self._create_entry_redis(&string_before, &string_after, payload) .await } + "fs" => { + self._create_entry_fs(&string_before, &string_after, payload) + .await + } _ => Err(format!( "invalid storage macro, found {} in key name {}", macro_type, key_name @@ -52,6 +57,7 @@ impl crate::application::CoLink { match macro_type.as_str() { "chunk" => self._read_entry_chunk(&string_before).await, "redis" => self._read_entry_redis(&string_before, &string_after).await, + "fs" => self._read_entry_fs(&string_before, &string_after).await, _ => Err(format!( "invalid storage macro, found {} in key name {}", macro_type, key_name @@ -72,6 +78,10 @@ impl crate::application::CoLink { self._update_entry_redis(&string_before, &string_after, payload) .await } + "fs" => { + self._update_entry_fs(&string_before, &string_after, payload) + .await + } "append" => self._update_entry_append(&string_before, payload).await, _ => Err(format!( "invalid storage macro, found {} in key name {}", @@ -89,6 +99,7 @@ impl crate::application::CoLink { self._delete_entry_redis(&string_before, &string_after) .await } + "fs" => self._delete_entry_fs(&string_before, &string_after).await, _ => Err(format!( "invalid storage macro, found {} in key name {}", macro_type, key_name diff --git a/src/extensions/storage_macro/fs.rs b/src/extensions/storage_macro/fs.rs new file mode 100644 index 0000000..c255b4a --- /dev/null +++ b/src/extensions/storage_macro/fs.rs @@ -0,0 +1,58 @@ +use async_recursion::async_recursion; +use tokio::io::AsyncWriteExt; + +type Error = Box; + +impl crate::application::CoLink { + #[async_recursion] + pub(crate) async fn _create_entry_fs( + &self, + _string_before: &str, + path: &str, + payload: &[u8], + ) -> Result { + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .await?; + file.write_all(payload).await?; + Ok("ok".to_string()) + } + + #[async_recursion] + pub(crate) async fn _read_entry_fs( + &self, + _string_before: &str, + path: &str, + ) -> Result, Error> { + let data = tokio::fs::read(path).await?; + Ok(data) + } + + #[async_recursion] + pub(crate) async fn _update_entry_fs( + &self, + _string_before: &str, + path: &str, + payload: &[u8], + ) -> Result { + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .create(true) + .open(path) + .await?; + file.write_all(payload).await?; + Ok("ok".to_string()) + } + + #[async_recursion] + pub(crate) async fn _delete_entry_fs( + &self, + _string_before: &str, + path: &str, + ) -> Result { + tokio::fs::remove_file(path).await?; + Ok("ok".to_string()) + } +} From 91abb279281c0cf8dd2fb7c8278d818afdbde4ce Mon Sep 17 00:00:00 2001 From: stneng Date: Fri, 24 Feb 2023 13:38:58 +0000 Subject: [PATCH 2/6] add test --- src/extensions/storage_macro/fs.rs | 6 +----- tests/test_storage_macro.rs | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/extensions/storage_macro/fs.rs b/src/extensions/storage_macro/fs.rs index c255b4a..14aa533 100644 --- a/src/extensions/storage_macro/fs.rs +++ b/src/extensions/storage_macro/fs.rs @@ -37,11 +37,7 @@ impl crate::application::CoLink { path: &str, payload: &[u8], ) -> Result { - let mut file = tokio::fs::OpenOptions::new() - .write(true) - .create(true) - .open(path) - .await?; + let mut file = tokio::fs::File::create(path).await?; file.write_all(payload).await?; Ok("ok".to_string()) } diff --git a/tests/test_storage_macro.rs b/tests/test_storage_macro.rs index 703dd96..430d1b1 100644 --- a/tests/test_storage_macro.rs +++ b/tests/test_storage_macro.rs @@ -27,6 +27,16 @@ async fn test_storage_macro_redis() -> Result<(), Box Result<(), Box> { + let (_ir, _is, cl) = set_up_test_env_single_user().await?; + + let key_name = "storage_macro_test_fs:$fs:/tmp/colink-sm-fs-test"; + test_crud(&cl, key_name).await?; + + Ok(()) +} + async fn test_crud( cl: &CoLink, key_name: &str, @@ -114,6 +124,17 @@ async fn test_storage_macro_chunk_append( Ok(()) } +#[tokio::test] +async fn test_storage_macro_chunk_fs( +) -> Result<(), Box> { + let (_ir, _is, cl) = set_up_test_env_single_user().await?; + + let key_name = "test_storage_macro_fs_append:$fs:/tmp/colink-sm-fs-append-test"; + test_append(&cl, key_name, 5e6 as usize).await?; + + Ok(()) +} + #[ignore] #[tokio::test] async fn test_storage_macro_redis_chunk( From 5ec976b7cd87887ba97958b0736fd61eef6c8116 Mon Sep 17 00:00:00 2001 From: stneng Date: Fri, 24 Feb 2023 13:47:47 +0000 Subject: [PATCH 3/6] storage macro: fs append --- src/extensions/storage_macro/append.rs | 5 +++++ src/extensions/storage_macro/fs.rs | 22 ++++++++++++++++++++++ tests/test_storage_macro.rs | 2 +- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/extensions/storage_macro/append.rs b/src/extensions/storage_macro/append.rs index fed85a1..237f114 100644 --- a/src/extensions/storage_macro/append.rs +++ b/src/extensions/storage_macro/append.rs @@ -20,6 +20,11 @@ impl crate::application::CoLink { "chunk" => { return self._append_entry_chunk(&string_before, payload).await; } + "fs" => { + return self + ._append_entry_fs(&string_before, &string_after, payload) + .await; + } _ => {} } } diff --git a/src/extensions/storage_macro/fs.rs b/src/extensions/storage_macro/fs.rs index 14aa533..4284e9c 100644 --- a/src/extensions/storage_macro/fs.rs +++ b/src/extensions/storage_macro/fs.rs @@ -42,6 +42,28 @@ impl crate::application::CoLink { Ok("ok".to_string()) } + #[async_recursion] + pub(crate) async fn _append_entry_fs( + &self, + _string_before: &str, + path: &str, + payload: &[u8], + ) -> Result { + let lock_token = self.lock(path).await?; + // use a closure to prevent locking forever caused by errors + let res = async { + let mut file = tokio::fs::OpenOptions::new() + .append(true) + .open(path) + .await?; + file.write_all(payload).await?; + Ok::("ok".to_string()) + } + .await; + self.unlock(lock_token).await?; + res + } + #[async_recursion] pub(crate) async fn _delete_entry_fs( &self, diff --git a/tests/test_storage_macro.rs b/tests/test_storage_macro.rs index 430d1b1..58358e5 100644 --- a/tests/test_storage_macro.rs +++ b/tests/test_storage_macro.rs @@ -125,7 +125,7 @@ async fn test_storage_macro_chunk_append( } #[tokio::test] -async fn test_storage_macro_chunk_fs( +async fn test_storage_macro_fs_append( ) -> Result<(), Box> { let (_ir, _is, cl) = set_up_test_env_single_user().await?; From 8c75437f3fde017708955b4a1e06caab269d2c2e Mon Sep 17 00:00:00 2001 From: stneng Date: Fri, 24 Feb 2023 13:51:25 +0000 Subject: [PATCH 4/6] bump version to 0.3.5 --- Cargo.toml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f465249..8c503cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "colink" -version = "0.3.4" +version = "0.3.5" edition = "2021" description = "CoLink Rust SDK" license = "MIT" diff --git a/README.md b/README.md index 06cd2ad..82acf23 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ CoLink SDK helps both application and protocol developers access the functionali Add this to your Cargo.toml: ```toml [dependencies] -colink = "0.3.4" +colink = "0.3.5" ``` ## Getting Started From f8429928efeab3d2f567131836a22c45f8586d23 Mon Sep 17 00:00:00 2001 From: stneng Date: Tue, 28 Feb 2023 04:01:55 +0000 Subject: [PATCH 5/6] change to mounting --- src/extensions/storage_macro/fs.rs | 46 +++++++++++++++++++++++------- tests/test_storage_macro.rs | 21 ++++++++++++-- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/src/extensions/storage_macro/fs.rs b/src/extensions/storage_macro/fs.rs index 4284e9c..1fa16cd 100644 --- a/src/extensions/storage_macro/fs.rs +++ b/src/extensions/storage_macro/fs.rs @@ -1,16 +1,36 @@ use async_recursion::async_recursion; +use std::path::PathBuf; use tokio::io::AsyncWriteExt; type Error = Box; impl crate::application::CoLink { + async fn _sm_fs_get_path( + &self, + path_key_name: &str, + path_suffix: &str, + ) -> Result { + let path_key = format!("{}:path", path_key_name); + let mut path = String::from_utf8(self.read_entry(&path_key).await?)?; + if !path_suffix.is_empty() { + let path_suffix = path_suffix.replace(":", "/"); + path += "/"; + path += &path_suffix; + } + println!("path: {}", path); + let path = PathBuf::from(path); + tokio::fs::create_dir_all(path.parent().unwrap()).await?; + Ok(path) + } + #[async_recursion] pub(crate) async fn _create_entry_fs( &self, - _string_before: &str, - path: &str, + path_key_name: &str, + path_suffix: &str, payload: &[u8], ) -> Result { + let path = self._sm_fs_get_path(path_key_name, path_suffix).await?; let mut file = tokio::fs::OpenOptions::new() .write(true) .create_new(true) @@ -23,9 +43,10 @@ impl crate::application::CoLink { #[async_recursion] pub(crate) async fn _read_entry_fs( &self, - _string_before: &str, - path: &str, + path_key_name: &str, + path_suffix: &str, ) -> Result, Error> { + let path = self._sm_fs_get_path(path_key_name, path_suffix).await?; let data = tokio::fs::read(path).await?; Ok(data) } @@ -33,10 +54,11 @@ impl crate::application::CoLink { #[async_recursion] pub(crate) async fn _update_entry_fs( &self, - _string_before: &str, - path: &str, + path_key_name: &str, + path_suffix: &str, payload: &[u8], ) -> Result { + let path = self._sm_fs_get_path(path_key_name, path_suffix).await?; let mut file = tokio::fs::File::create(path).await?; file.write_all(payload).await?; Ok("ok".to_string()) @@ -45,11 +67,12 @@ impl crate::application::CoLink { #[async_recursion] pub(crate) async fn _append_entry_fs( &self, - _string_before: &str, - path: &str, + path_key_name: &str, + path_suffix: &str, payload: &[u8], ) -> Result { - let lock_token = self.lock(path).await?; + let path = self._sm_fs_get_path(path_key_name, path_suffix).await?; + let lock_token = self.lock(&path.to_string_lossy()).await?; // use a closure to prevent locking forever caused by errors let res = async { let mut file = tokio::fs::OpenOptions::new() @@ -67,9 +90,10 @@ impl crate::application::CoLink { #[async_recursion] pub(crate) async fn _delete_entry_fs( &self, - _string_before: &str, - path: &str, + path_key_name: &str, + path_suffix: &str, ) -> Result { + let path = self._sm_fs_get_path(path_key_name, path_suffix).await?; tokio::fs::remove_file(path).await?; Ok("ok".to_string()) } diff --git a/tests/test_storage_macro.rs b/tests/test_storage_macro.rs index 58358e5..a4a0e9a 100644 --- a/tests/test_storage_macro.rs +++ b/tests/test_storage_macro.rs @@ -31,7 +31,19 @@ async fn test_storage_macro_redis() -> Result<(), Box Result<(), Box> { let (_ir, _is, cl) = set_up_test_env_single_user().await?; - let key_name = "storage_macro_test_fs:$fs:/tmp/colink-sm-fs-test"; + cl.create_entry("test_storage_macro_fs:path", b"/tmp/colink-sm-fs-test/test") + .await?; + let key_name = "test_storage_macro_fs:$fs"; + test_crud(&cl, key_name).await?; + + cl.create_entry( + "test_storage_macro_fs_dir:path", + b"/tmp/colink-sm-fs-test/test-dir", + ) + .await?; + let key_name = "test_storage_macro_fs_dir:$fs:test-file"; + test_crud(&cl, key_name).await?; + let key_name = "test_storage_macro_fs_dir:$fs:test-dir:test-file"; test_crud(&cl, key_name).await?; Ok(()) @@ -129,7 +141,12 @@ async fn test_storage_macro_fs_append( ) -> Result<(), Box> { let (_ir, _is, cl) = set_up_test_env_single_user().await?; - let key_name = "test_storage_macro_fs_append:$fs:/tmp/colink-sm-fs-append-test"; + cl.create_entry( + "test_storage_macro_fs_append:path", + b"/tmp/colink-sm-fs-test/append-test", + ) + .await?; + let key_name = "test_storage_macro_fs_append:$fs"; test_append(&cl, key_name, 5e6 as usize).await?; Ok(()) From 404dbb96e0d4993df387c499e0ff493e3f98d065 Mon Sep 17 00:00:00 2001 From: stneng Date: Tue, 28 Feb 2023 04:02:28 +0000 Subject: [PATCH 6/6] clippy --- src/extensions/storage_macro/fs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extensions/storage_macro/fs.rs b/src/extensions/storage_macro/fs.rs index 1fa16cd..c7223b2 100644 --- a/src/extensions/storage_macro/fs.rs +++ b/src/extensions/storage_macro/fs.rs @@ -13,7 +13,7 @@ impl crate::application::CoLink { let path_key = format!("{}:path", path_key_name); let mut path = String::from_utf8(self.read_entry(&path_key).await?)?; if !path_suffix.is_empty() { - let path_suffix = path_suffix.replace(":", "/"); + let path_suffix = path_suffix.replace(':', "/"); path += "/"; path += &path_suffix; }