diff --git a/Cargo.lock b/Cargo.lock index d78ffc38..f9fa7272 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -166,7 +166,7 @@ dependencies = [ "parse-size", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.59", ] [[package]] @@ -207,7 +207,7 @@ dependencies = [ "futures-util", "mio", "num_cpus", - "socket2 0.4.7", + "socket2 0.4.10", "tokio", "tracing", ] @@ -289,7 +289,7 @@ dependencies = [ "serde_urlencoded", "smallvec", "socket2 0.5.5", - "time 0.3.14", + "time", "url", ] @@ -319,7 +319,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.59", ] [[package]] @@ -351,6 +351,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "ahash" version = "0.8.6" @@ -394,6 +405,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -409,6 +426,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anyhow" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" + [[package]] name = "arc-swap" version = "1.6.0" @@ -437,13 +460,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.57" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.59", ] [[package]] @@ -484,6 +507,7 @@ dependencies = [ "dotenv", "futures", "instant-acme", + "opendal", "opentelemetry", "opentelemetry-jaeger", "percent-encoding", @@ -560,6 +584,18 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backon" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0" +dependencies = [ + "fastrand", + "futures-core", + "pin-project", + "tokio", +] + [[package]] name = "base64" version = "0.13.0" @@ -572,6 +608,12 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bincode" version = "1.3.3" @@ -611,6 +653,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "brotli" version = "3.3.4" @@ -657,9 +708,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.2.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "bytestring" @@ -676,6 +727,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.0.83" @@ -716,18 +776,17 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.22" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", "serde", - "time 0.1.44", "wasm-bindgen", - "winapi", + "windows-targets 0.52.0", ] [[package]] @@ -763,6 +822,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.0.26" @@ -846,6 +915,32 @@ dependencies = [ "winapi", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.11", + "once_cell", + "tiny-keccak", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -859,7 +954,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05" dependencies = [ "percent-encoding", - "time 0.3.14", + "time", "version_check", ] @@ -1066,7 +1161,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.38", + "syn 2.0.59", ] [[package]] @@ -1077,7 +1172,18 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", "quote", - "syn 2.0.38", + "syn 2.0.59", +] + +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", ] [[package]] @@ -1114,12 +1220,14 @@ checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", + "subtle", ] [[package]] @@ -1183,6 +1291,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + [[package]] name = "doc-comment" version = "0.3.3" @@ -1329,6 +1446,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c5f6c2c942da57e2aaaa84b8a521489486f14e75e7fa91dab70aba913975f98" +[[package]] +name = "flagset" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb3aa5e95cf9aabc17f060cfa0ced7b83f042390760ca53bf09df9968acaa1" + [[package]] name = "flate2" version = "1.0.24" @@ -1401,9 +1524,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.24" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -1411,9 +1534,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.24" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" @@ -1428,38 +1551,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.24" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.24" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.59", ] [[package]] name = "futures-sink" -version = "0.3.24" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.24" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.24" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -1523,8 +1646,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -1610,6 +1735,24 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "html2md" version = "0.2.14" @@ -1646,9 +1789,9 @@ checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" [[package]] name = "http" -version = "0.2.8" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -1701,7 +1844,7 @@ dependencies = [ "httpdate", "itoa 1.0.3", "pin-project-lite", - "socket2 0.4.7", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -1787,6 +1930,16 @@ dependencies = [ "serde", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "instant" version = "0.1.12" @@ -1826,6 +1979,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ea37f355c05dde75b84bba2d767906ad522e97cd9e2eef2be7a4ab7fb442c06" +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + [[package]] name = "is-terminal" version = "0.4.10" @@ -1905,6 +2064,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonwebtoken" +version = "9.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f" +dependencies = [ + "base64 0.21.0", + "js-sys", + "pem", + "ring 0.17.6", + "serde", + "serde_json", + "simple_asn1", +] + [[package]] name = "jwalk" version = "0.8.1" @@ -1939,6 +2113,9 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "lazycell" @@ -1958,6 +2135,12 @@ version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "linux-raw-sys" version = "0.0.46" @@ -2103,6 +2286,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "measure_time" version = "0.8.2" @@ -2279,6 +2472,34 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -2289,6 +2510,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -2296,6 +2528,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -2338,6 +2571,36 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "opendal" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c17c077f23fa2d2c25d9d22af98baa43b8bbe2ef0de80cf66339aa70401467" +dependencies = [ + "anyhow", + "async-trait", + "backon", + "base64 0.21.0", + "bytes", + "chrono", + "flagset", + "futures", + "getrandom 0.2.11", + "http", + "log", + "md-5", + "once_cell", + "percent-encoding", + "quick-xml", + "reqsign", + "reqwest", + "serde", + "serde_json", + "sha2", + "tokio", + "uuid", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -2422,6 +2685,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.1", +] + [[package]] name = "os_str_bytes" version = "6.3.0" @@ -2530,6 +2803,16 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498a099351efa4becc6a19c72aa9270598e8fd274ca47052e37455241c88b696" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "pem" version = "3.0.2" @@ -2540,6 +2823,15 @@ dependencies = [ "serde", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -2670,6 +2962,44 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "pkcs5", + "rand_core 0.6.4", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.25" @@ -2786,9 +3116,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e" dependencies = [ "unicode-ident", ] @@ -2802,11 +3132,21 @@ dependencies = [ "rustyline", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" -version = "1.0.33" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -2843,7 +3183,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha 0.3.1", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -2863,7 +3203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -2877,9 +3217,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom 0.2.11", ] @@ -2930,7 +3270,7 @@ checksum = "48406db8ac1f3cbc7dcdb56ec355343817958a356ff430259bb07baf7607e1e1" dependencies = [ "pem", "ring 0.17.6", - "time 0.3.14", + "time", "yasna", ] @@ -3007,6 +3347,80 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +[[package]] +name = "reqsign" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43e319d9de9ff4d941abf4ac718897118b0fe04577ea3f8e0f5788971784eef5" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.21.0", + "chrono", + "form_urlencoded", + "getrandom 0.2.11", + "hex", + "hmac", + "home", + "http", + "jsonwebtoken", + "log", + "once_cell", + "percent-encoding", + "quick-xml", + "rand 0.8.5", + "reqwest", + "rsa", + "rust-ini", + "serde", + "serde_json", + "sha1", + "sha2", +] + +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64 0.21.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls 0.21.9", + "rustls-native-certs", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-rustls 0.24.1", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "winreg", +] + [[package]] name = "ring" version = "0.16.20" @@ -3053,6 +3467,37 @@ dependencies = [ "rio_api", ] +[[package]] +name = "rsa" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rust-ini" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0698206bcb8882bf2a9ecb4c1e7785db57ff052297085a6efd4fe42302068a" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + [[package]] name = "rust-stemmers" version = "1.2.0" @@ -3123,6 +3568,7 @@ version = "0.21.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ + "log", "ring 0.17.6", "rustls-webpki", "sct", @@ -3207,6 +3653,15 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -3248,6 +3703,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2", +] + [[package]] name = "sct" version = "0.7.0" @@ -3324,7 +3790,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.59", ] [[package]] @@ -3393,7 +3859,7 @@ dependencies = [ "serde", "serde_json", "serde_with_macros", - "time 0.3.14", + "time", ] [[package]] @@ -3405,7 +3871,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.59", ] [[package]] @@ -3429,6 +3895,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -3453,12 +3930,34 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "simple-server-timing-header" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e78919e05c9b8e123d435a4ad104b488ad1585631830e413830985c214086e" +[[package]] +name = "simple_asn1" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror", + "time", +] + [[package]] name = "siphasher" version = "0.3.10" @@ -3507,9 +4006,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -3537,6 +4036,16 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -3592,6 +4101,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "1.0.109" @@ -3605,15 +4120,42 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.38" +version = "2.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tantivy" version = "0.21.1" @@ -3661,7 +4203,7 @@ dependencies = [ "tantivy-tokenizer-api", "tempfile", "thiserror", - "time 0.3.14", + "time", "uuid", "winapi", ] @@ -3701,7 +4243,7 @@ dependencies = [ "byteorder", "ownedbytes", "serde", - "time 0.3.14", + "time", ] [[package]] @@ -3826,7 +4368,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.59", ] [[package]] @@ -3860,17 +4402,6 @@ dependencies = [ "threadpool", ] -[[package]] -name = "time" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - [[package]] name = "time" version = "0.3.14" @@ -3890,6 +4421,15 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -3917,20 +4457,19 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.26.0" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", "bytes", "libc", - "memchr", "mio", "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.4.7", - "windows-sys 0.45.0", + "socket2 0.4.10", + "windows-sys 0.48.0", ] [[package]] @@ -4298,12 +4837,6 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -4331,10 +4864,22 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.59", "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.90" @@ -4353,7 +4898,7 @@ checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.59", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4364,11 +4909,24 @@ version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +[[package]] +name = "wasm-streams" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" -version = "0.3.59" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", @@ -4471,15 +5029,6 @@ dependencies = [ "windows_x86_64_msvc 0.36.1", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -4498,21 +5047,6 @@ dependencies = [ "windows-targets 0.52.0", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.48.5" @@ -4543,12 +5077,6 @@ dependencies = [ "windows_x86_64_msvc 0.52.0", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -4573,12 +5101,6 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -4603,12 +5125,6 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -4633,12 +5149,6 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -4663,12 +5173,6 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -4681,12 +5185,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -4711,12 +5209,6 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -4738,6 +5230,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "xml5ever" version = "0.17.0" @@ -4755,7 +5257,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.14", + "time", ] [[package]] @@ -4775,7 +5277,7 @@ checksum = "a25f293fe55f0a48e7010d65552bb63704f6ceb55a1a385da10d41d8f78e4a3d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.59", ] [[package]] diff --git a/README.md b/README.md index 394bd2cb..de5d3bc9 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,8 @@ _Status: alpha. [Breaking changes](CHANGELOG.md) are expected until 1.0._ - 📖 **Pagination, sorting and filtering** queries using [Atomic Collections](https://docs.atomicdata.dev/schema/collections.html). - 🔐 **Authorization** (read / write permissions) and Hierarchical structures powered by [Atomic Hierarchy](https://docs.atomicdata.dev/hierarchy.html) - 📲 **Invite and sharing system** with [Atomic Invites](https://docs.atomicdata.dev/invitations.html) +- 📂 **File management**: Upload, download and preview attachments with support for using S3 as file storage backend. +- 🖥️ **Desktop app**: Easy desktop installation, with status bar icon, powered by [tauri](https://github.com/tauri-apps/tauri/). - 🌐 **Embedded server** with support for HTTP / HTTPS / HTTP2.0 (TLS) and Built-in LetsEncrypt handshake. - 📚 **Libraries**: [Javascript / Typescript](https://www.npmjs.com/package/@tomic/lib), [React](https://www.npmjs.com/package/@tomic/react), [Svelte](https://www.npmjs.com/package/@tomic/svelte), [Rust](https://crates.io/crates/atomic-lib) @@ -50,6 +52,7 @@ https://user-images.githubusercontent.com/2183313/139728539-d69b899f-6f9b-44cb-a Check out the [documentation] for installation instructions, API docs, and more. +### Configuring S3 for File Storage ## Contribute Issues and PRs are welcome! diff --git a/docs/src/atomicserver/installation.md b/docs/src/atomicserver/installation.md index aafbf7de..4c4d4140 100644 --- a/docs/src/atomicserver/installation.md +++ b/docs/src/atomicserver/installation.md @@ -106,6 +106,21 @@ ATOMIC_HTTPS=false ATOMIC_SERVER_URL=https://example.com ``` +### Configuring S3 for File Storage + +You can configure atomic-server to use S3 (and compatible services) for file storage via environment variables or command line arguments when starting atomic-server. + +Credentials can either be found in the standard location for AWS credentials on your OS (e.g. `~/.aws/credentials` on UNIX systems) or by using the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. + +Available configuration values: + +- bucket: `--s3-bucket="my-bucket-name"` or env var `ATOMIC_S3_BUCKET` (required) +- region: `--s3-region="us-east-2"` or env var `ATOMIC_S3_REGION` +- endpoint: `--s3-endpoint="https://s3.us-east-2.amazonaws.com"` or env var `ATOMIC_S3_ENDPOINT` +0 path: `--s3-path="atomic_uploads"` or env var `ATOMIC_S3_PATH` + +For example, the above configuration would uploads files to `s3://my-bucket-name/atomic_uploads/` in the `us-east-2` region. + ## Using `systemd` to run Atomic-Server as a service In Linux operating systems, you can use `systemd` to manage running processes. diff --git a/lib/src/db.rs b/lib/src/db.rs index 4e66951d..5caa9501 100644 --- a/lib/src/db.rs +++ b/lib/src/db.rs @@ -496,6 +496,7 @@ impl Storelike for Db { let dynamic_span = tracing::span!(tracing::Level::TRACE, "get_resource_extended (dynamic)").entered(); + println!("get_resource_extended: {}", &removed_query_params); let mut resource = self.get_resource(&removed_query_params)?; let _explanation = crate::hierarchy::check_read(self, &resource, for_agent)?; diff --git a/lib/src/storelike.rs b/lib/src/storelike.rs index 095b1758..19736fd7 100644 --- a/lib/src/storelike.rs +++ b/lib/src/storelike.rs @@ -209,6 +209,7 @@ pub trait Storelike: Sized { ) -> AtomicResult { if let Some(self_url) = self.get_self_url() { if subject.starts_with(&self_url) { + println!("Custom backtrace: {}", std::backtrace::Backtrace::force_capture()); return Err(AtomicError::not_found(format!( "Failed to retrieve locally: '{}'", subject diff --git a/server/Cargo.toml b/server/Cargo.toml index ddd63d79..6f36107d 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -33,6 +33,7 @@ dialoguer = "0.11" directories = ">= 2, < 5" dotenv = "0.15" futures = "0.3" +opendal = "0.45.1" percent-encoding = "2.2.0" regex = "1" rio_api = "0.8" diff --git a/server/src/appstate.rs b/server/src/appstate.rs index 945c833a..3511e213 100644 --- a/server/src/appstate.rs +++ b/server/src/appstate.rs @@ -1,6 +1,7 @@ //! App state, which is accessible from handlers use crate::{ - commit_monitor::CommitMonitor, config::Config, errors::AtomicServerResult, search::SearchState, + commit_monitor::CommitMonitor, config::Config, errors::AtomicServerResult, files::FileStore, + search::SearchState, }; use atomic_lib::{ agents::{generate_public_key, Agent}, @@ -23,6 +24,10 @@ pub struct AppState { /// The Actix Address of the CommitMonitor, which should receive updates when a commit is applied pub commit_monitor: actix::Addr, pub search_state: SearchState, + /// stores config values and the active FileStore type, e.g. FS or S3 + pub file_store: FileStore, + /// stores config values for FS filestore regardless of active file store as fallback + pub fs_file_store: FileStore, } /// Creates the AppState (the server's context available in Handlers). @@ -93,11 +98,18 @@ pub fn init(config: Config) -> AtomicServerResult { crate::search::add_all_resources(&search_state, &store)? } + tracing::info!("Initializing file stores"); + // Initialize file stores + let fs_file_store = FileStore::init_fs_from_config(&config); + let file_store = FileStore::init_from_config(&config, fs_file_store.clone()); + Ok(AppState { store, config, commit_monitor, search_state, + file_store, + fs_file_store, }) } diff --git a/server/src/bin.rs b/server/src/bin.rs index 30cc513b..ecf09ff4 100644 --- a/server/src/bin.rs +++ b/server/src/bin.rs @@ -8,6 +8,7 @@ mod commit_monitor; pub mod config; mod content_types; mod errors; +mod files; mod handlers; mod helpers; #[cfg(feature = "https")] diff --git a/server/src/config.rs b/server/src/config.rs index 6a0f60c9..4cc9458b 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -74,6 +74,22 @@ pub struct Opts { #[clap(long, env = "ATOMIC_DATA_DIR")] pub data_dir: Option, + /// bucket name from s3-compatible storage service + #[clap(long, env = "ATOMIC_S3_BUCKET")] + pub s3_bucket: Option, + + /// region for s3-compatible storage service, defaults to "us-east-1" + #[clap(long, env = "ATOMIC_S3_REGION")] + pub s3_region: Option, + + /// endpoint for s3-compatible storage service, defaults to "https://s3.amazonaws.com" + #[clap(long, env = "ATOMIC_S3_ENDPOINT")] + pub s3_endpoint: Option, + + /// path where s3 uploads will be stored + #[clap(long, env = "ATOMIC_S3_PATH")] + pub s3_path: Option, + /// CAUTION: Skip authentication checks, making all data publicly readable. Improves performance. #[clap(long, env = "ATOMIC_PUBLIC_MODE")] pub public_mode: bool, diff --git a/server/src/errors.rs b/server/src/errors.rs index 6cf56521..2f183a50 100644 --- a/server/src/errors.rs +++ b/server/src/errors.rs @@ -179,3 +179,13 @@ impl From for AtomicServerError { } } } + +impl From for AtomicServerError { + fn from(error: opendal::Error) -> Self { + AtomicServerError { + message: error.to_string(), + error_type: AppErrorType::Other, + error_resource: None, + } + } +} diff --git a/server/src/files.rs b/server/src/files.rs new file mode 100644 index 00000000..141bfb45 --- /dev/null +++ b/server/src/files.rs @@ -0,0 +1,175 @@ +use std::{fmt, fs, io::Write, path::PathBuf, time::Duration}; + +use actix_multipart::Field; +use futures::StreamExt; +use opendal::{services::S3, Operator}; + +use crate::{appstate::AppState, config::Config, errors::AtomicServerResult}; + +#[derive(Clone, Debug, PartialEq)] +pub enum FileStore { + S3(S3Config), + FS(FSConfig), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct S3Config { + pub bucket: String, + pub path: String, + pub endpoint: Option, + pub region: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct FSConfig { + pub path: PathBuf, +} + +impl FileStore { + const S3_PREFIX: &'static str = "s3:"; + const FS_PREFIX: &'static str = "fs:"; + + pub fn init_fs_from_config(config: &Config) -> FileStore { + FileStore::FS(FSConfig { + path: config.uploads_path.clone(), + }) + } + + pub fn init_from_config(config: &Config, fs_file_store: FileStore) -> FileStore { + let opts = &config.opts; + if let Some(bucket) = &opts.s3_bucket { + let config = S3Config { + bucket: bucket.clone(), + endpoint: opts.s3_endpoint.clone(), + region: opts.s3_region.clone(), + path: opts.s3_path.clone().unwrap_or("uploads".to_string()), + }; + FileStore::S3(config) + } else { + fs_file_store + } + } + + pub fn get_subject_file_store<'a>(appstate: &'a AppState, subject: &str) -> &'a FileStore { + if subject.contains(Self::S3_PREFIX) { + &appstate.file_store + } else { + &appstate.fs_file_store + } + } + + pub fn get_fs_file_path(&self, file_id: &str) -> AtomicServerResult { + tracing::info!("fs_file_path: {}", file_id); + if let FileStore::FS(config) = self { + let fs_file_id = file_id.strip_prefix(Self::FS_PREFIX).unwrap_or(file_id); + let mut file_path = config.path.clone(); + file_path.push(fs_file_id); + Ok(file_path) + } else { + Err("Wrong FileStore passed to get_fs_file_path".into()) + } + } + + pub fn prefix(&self) -> &str { + match self { + Self::S3(_) => Self::S3_PREFIX, + Self::FS(_) => Self::FS_PREFIX, + } + } + + pub fn encoded(&self) -> String { + urlencoding::encode(self.prefix()).into_owned() + } + + pub async fn upload_file(&self, file_id: &str, field: Field) -> AtomicServerResult { + match self { + FileStore::S3(_) => s3_upload(self, file_id, field).await, + FileStore::FS(config) => fs_upload(self, config, file_id, field).await, + } + } +} + +impl fmt::Display for FileStore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.prefix()) + } +} + +async fn fs_upload( + file_store: &FileStore, + config: &FSConfig, + file_id: &str, + mut field: Field, +) -> AtomicServerResult { + std::fs::create_dir_all(config.path.clone())?; + + let mut file = fs::File::create(file_store.get_fs_file_path(file_id)?)?; + + let byte_count: i64 = file + .metadata()? + .len() + .try_into() + .map_err(|_e| "Too large")?; + + // Field in turn is stream of *Bytes* object + while let Some(chunk) = field.next().await { + let data = chunk.map_err(|e| format!("Error while reading multipart data. {}", e))?; + // TODO: Update a SHA256 hash here for checksum + file.write_all(&data)?; + } + + Ok(byte_count) +} + +async fn s3_upload( + file_store: &FileStore, + file_id: &str, + mut field: Field, +) -> AtomicServerResult { + let mut builder = S3::default(); + + if let FileStore::S3(config) = file_store { + builder.bucket(&config.bucket); + builder.root(&config.path); + config.region.as_ref().map(|r| builder.region(r)); + config.endpoint.as_ref().map(|e| builder.endpoint(e)); + } else { + return Err("Uploading to S3 but no S3 config provided".into()); + } + + let op: Operator = Operator::new(builder)?.finish(); + let mut w = op.writer(file_id).await?; + let mut len = 0; + while let Some(chunk) = field.next().await { + let data = chunk.map_err(|e| format!("Error while reading multipart data. {}", e))?; + len += data.len(); + w.write(data).await?; + } + + let byte_length: i64 = len.try_into().map_err(|_e| "Too large")?; + w.close().await?; + Ok(byte_length) +} + +pub async fn get_s3_signed_url( + file_store: &FileStore, + duration: Duration, + file_id: &str, +) -> AtomicServerResult { + let mut builder = S3::default(); + + if let FileStore::S3(config) = file_store { + builder.bucket(&config.bucket); + builder.root(&config.path); + config.region.as_ref().map(|r| builder.region(r)); + config.endpoint.as_ref().map(|e| builder.endpoint(e)); + } else { + return Err("Downloading from S3 but no S3 config provided".into()); + } + + let op: Operator = Operator::new(builder)?.finish(); + + let uri = op.presign_read(file_id, duration).await?.uri().to_string(); + + Ok(uri) +} diff --git a/server/src/handlers/download.rs b/server/src/handlers/download.rs index 810a1e23..e2d67bef 100644 --- a/server/src/handlers/download.rs +++ b/server/src/handlers/download.rs @@ -1,8 +1,15 @@ +use std::time::Duration; + use actix_files::NamedFile; -use actix_web::{web, HttpRequest, HttpResponse}; -use atomic_lib::{urls, Resource, Storelike}; +use actix_web::{web, HttpRequest, HttpResponse, Responder}; +use atomic_lib::{urls, Storelike}; -use crate::{appstate::AppState, errors::AtomicServerResult, helpers::get_client_agent}; +use crate::{ + appstate::AppState, + errors::AtomicServerResult, + files::{self, FileStore}, + helpers::get_client_agent, +}; /// Downloads the File of the Resource that matches the same URL minus the `/download` path. #[tracing::instrument(skip(appstate, req))] @@ -26,20 +33,39 @@ pub async fn handle_download( let for_agent = get_client_agent(headers, &appstate, subject.clone())?; tracing::info!("handle_download: {}", subject); - let resource = store.get_resource_extended(&subject, false, &for_agent)?; - download_file_handler_partial(&resource, &req, &appstate) + let file_store = FileStore::get_subject_file_store(&appstate, &subject); + let encoded = subject.replace(file_store.prefix(), &file_store.encoded()); + let resource = store.get_resource_extended(&encoded, false, &for_agent)?; + let file_id = resource + .get(urls::INTERNAL_ID) + .map_err(|e| format!("Internal ID of file could not be resolved. {}", e))? + .to_string(); + + if let FileStore::S3(_) = file_store { + signed_url_redirect_handler(file_id.as_str(), &req, &appstate).await + } else { + download_file_handler_partial(file_id.as_str(), &req, &appstate) + } } pub fn download_file_handler_partial( - resource: &Resource, + file_id: &str, req: &HttpRequest, appstate: &AppState, ) -> AtomicServerResult { - let file_name = resource - .get(urls::INTERNAL_ID) - .map_err(|e| format!("Internal ID of file could not be resolved. {}", e))?; - let mut file_path = appstate.config.uploads_path.clone(); - file_path.push(file_name.to_string()); + let file_path = appstate.fs_file_store.get_fs_file_path(file_id)?; let file = NamedFile::open(file_path)?; Ok(file.into_response(req)) } + +async fn signed_url_redirect_handler( + file_id: &str, + req: &HttpRequest, + appstate: &AppState, +) -> AtomicServerResult { + let signed_url = + files::get_s3_signed_url(&appstate.file_store, Duration::from_secs(3600), file_id).await?; + Ok(web::Redirect::to(signed_url) + .respond_to(req) + .map_into_boxed_body()) +} diff --git a/server/src/handlers/get_resource.rs b/server/src/handlers/get_resource.rs index 1ef8f199..1d1582e2 100644 --- a/server/src/handlers/get_resource.rs +++ b/server/src/handlers/get_resource.rs @@ -2,6 +2,7 @@ use crate::{ appstate::AppState, content_types::get_accept, content_types::ContentType, + files::{self, FileStore}, errors::AtomicServerResult, helpers::{get_client_agent, try_extension}, }; @@ -43,6 +44,7 @@ pub async fn handle_get_resource( format!("?{}", req.query_string()) }; let subject = format!("{}/{}{}", server_url, subj_end_string, querystring); + tracing::info!("subject at get_resource: {}", subject); subject } } else { @@ -67,7 +69,9 @@ pub async fn handle_get_resource( "no-store, no-cache, must-revalidate, private", )); - let resource = store.get_resource_extended(&subject, false, &for_agent)?; + let file_store = FileStore::get_subject_file_store(&appstate, &subject); + let encoded = subject.replace(file_store.prefix(), &file_store.encoded()); + let resource = store.get_resource_extended(&encoded, false, &for_agent)?; timer.add("get_resource"); let response_body = match content_type { diff --git a/server/src/handlers/single_page_app.rs b/server/src/handlers/single_page_app.rs index 6140a2c3..7495f1e0 100644 --- a/server/src/handlers/single_page_app.rs +++ b/server/src/handlers/single_page_app.rs @@ -1,7 +1,9 @@ use std::fmt::Display; use std::fmt::Formatter; -use crate::{appstate::AppState, errors::AtomicServerResult}; +use crate::{appstate::AppState, + files::{self, FileStore}, + errors::AtomicServerResult}; use actix_web::HttpResponse; /// Returns the atomic-data-browser single page application @@ -12,10 +14,14 @@ pub async fn single_page( ) -> AtomicServerResult { let template = include_str!("../../assets_tmp/index.html"); let subject = format!("{}/{}", appstate.store.get_server_url(), path); + + let file_store = FileStore::get_subject_file_store(&appstate, &subject); + let encoded = subject.replace(file_store.prefix(), &file_store.encoded()); + println!("encoded from single_page: {}", &encoded); let meta_tags: MetaTags = if let Ok(resource) = appstate .store - .get_resource_extended(&subject, true, &ForAgent::Public) + .get_resource_extended(&encoded, true, &ForAgent::Public) { resource.into() } else { diff --git a/server/src/handlers/upload.rs b/server/src/handlers/upload.rs index e9191562..7b0df221 100644 --- a/server/src/handlers/upload.rs +++ b/server/src/handlers/upload.rs @@ -1,11 +1,12 @@ -use std::{ffi::OsStr, io::Write, path::Path}; +use std::{ffi::OsStr, path::Path}; use actix_multipart::Multipart; use actix_web::{web, HttpResponse}; use atomic_lib::{ commit::CommitResponse, hierarchy::check_write, urls, utils::now, Resource, Storelike, Value, }; -use futures::{StreamExt, TryStreamExt}; + +use futures::TryStreamExt; use serde::Deserialize; use crate::{appstate::AppState, errors::AtomicServerResult, helpers::get_client_agent}; @@ -15,7 +16,7 @@ pub struct UploadQuery { parent: String, } -/// Allows the user to upload files tot the `/upload` endpoint. +/// Allows the user to upload files to the `/upload` endpoint. /// A parent Query parameter is required for checking rights and for placing the file in a Hierarchy. /// Creates new File resources for every submitted file. /// Submission is done using multipart/form-data. @@ -44,36 +45,21 @@ pub async fn upload_handler( let mut created_resources: Vec = Vec::new(); let mut commit_responses: Vec = Vec::new(); - while let Ok(Some(mut field)) = body.try_next().await { + while let Ok(Some(field)) = body.try_next().await { let content_type = field.content_disposition().clone(); let filename = content_type.get_filename().ok_or("Filename is missing")?; - std::fs::create_dir_all(&appstate.config.uploads_path)?; - + let file_store = &appstate.file_store; let file_id = format!( - "{}-{}", + "{}{}-{}", + file_store.prefix(), now(), sanitize_filename::sanitize(filename) // Spacebars lead to very annoying bugs in browsers .replace(' ', "-") ); - let mut file_path = appstate.config.uploads_path.clone(); - file_path.push(&file_id); - let mut file = std::fs::File::create(file_path)?; - - // Field in turn is stream of *Bytes* object - while let Some(chunk) = field.next().await { - let data = chunk.map_err(|e| format!("Error while reading multipart data. {}", e))?; - // TODO: Update a SHA256 hash here for checksum - file.write_all(&data)?; - } - - let byte_count: i64 = file - .metadata()? - .len() - .try_into() - .map_err(|_e| "Too large")?; + let byte_count = file_store.upload_file(&file_id, field).await?; let subject_path = format!("files/{}", urlencoding::encode(&file_id)); let new_subject = format!("{}/{}", store.get_server_url(), subject_path); diff --git a/server/src/lib.rs b/server/src/lib.rs index 9110712d..add17788 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -9,6 +9,7 @@ mod commit_monitor; pub mod config; mod content_types; mod errors; +mod files; mod handlers; mod helpers; #[cfg(feature = "https")] diff --git a/server/src/serve.rs b/server/src/serve.rs index 253be727..5dfdea92 100644 --- a/server/src/serve.rs +++ b/server/src/serve.rs @@ -98,7 +98,7 @@ pub async fn serve(config: crate::config::Config) -> AtomicServerResult<()> { tracing::info!("Binding HTTP server to endpoint {}", endpoint); println!("{}", message); server - .bind(&format!("{}:{}", config.opts.ip, config.opts.port)) + .bind(&endpoint) .map_err(|e| format!("Cannot bind to endpoint {}: {}", &endpoint, e))? .shutdown_timeout(TIMEOUT) .run() diff --git a/server/src/tests.rs b/server/src/tests.rs index 8c472a27..97c105b6 100644 --- a/server/src/tests.rs +++ b/server/src/tests.rs @@ -2,7 +2,7 @@ //! Most of the more rigorous testing is done in the end-to-end tests: //! https://github.com/atomicdata-dev/atomic-data-browser/tree/main/data-browser/tests -use crate::{appstate::AppState, config::Opts}; +use crate::{appstate::AppState, config::Opts, files::FileStore}; use super::*; use actix_web::{ @@ -30,10 +30,9 @@ fn build_request_authenticated(path: &str, appstate: &AppState) -> TestRequest { prereq.insert_header(("Accept", "application/ad+json")) } -#[actix_rt::test] -async fn server_tests() { - let unique_string = atomic_lib::utils::random_string(10); +fn build_test_config() -> crate::config::Config { use clap::Parser; + let unique_string = atomic_lib::utils::random_string(10); let opts = Opts::parse_from([ "atomic-server", "--initialize", @@ -46,8 +45,16 @@ async fn server_tests() { let mut config = config::build_config(opts) .map_err(|e| format!("Initialization failed: {}", e)) .expect("failed init config"); + // This prevents folder access issues when running concurrent tests config.search_index_path = format!("./.temp/{}/search_index", unique_string).into(); + config +} + +#[actix_rt::test] +async fn server_tests() { + let unique_string = atomic_lib::utils::random_string(10); + let config = build_test_config(); let appstate = crate::appstate::init(config.clone()).expect("failed init appstate"); let data = Data::new(appstate.clone()); @@ -163,3 +170,61 @@ fn get_body(resp: ServiceResponse) -> String { let bytes = boxbody.try_into_bytes().unwrap(); String::from_utf8(bytes.as_ref().into()).unwrap() } + +#[actix_rt::test] +async fn file_store_tests() { + let mut config = build_test_config(); + + let fs_store = FileStore::init_fs_from_config(&config); + if let FileStore::FS(fs_config) = &fs_store { + assert!(fs_config.path.to_str().unwrap().contains("uploads")); + } else { + panic!("fs FileStore not initiated"); + } + let store = FileStore::init_from_config(&config, fs_store.clone()); + assert_eq!(fs_store, store); + assert_eq!("fs:", fs_store.prefix()); + assert_eq!("fs%3A", fs_store.encoded()); + + assert!(fs_store + .get_fs_file_path("my-great-file") + .unwrap() + .to_str() + .unwrap() + .contains("uploads/my-great-file")); + assert!(fs_store + .get_fs_file_path("fs:my-great-file") + .unwrap() + .to_str() + .unwrap() + .contains("uploads/my-great-file")); + + // FileStore::S3 init + config.opts.s3_bucket = Some("test-bucket".to_string()); + config.opts.s3_path = Some("uploads".to_string()); + let appstate = crate::appstate::init(config.clone()).expect("failed init appstate"); + + let s3_store = FileStore::init_from_config(&config, fs_store.clone()); + if let FileStore::S3(s3_config) = &s3_store { + assert_eq!(s3_config.bucket, "test-bucket"); + assert_eq!(s3_config.path, "uploads"); + } else { + panic!("s3 FileStore not initiated"); + } + + assert_eq!("s3:", s3_store.prefix()); + assert_eq!("s3%3A", s3_store.encoded()); + + assert_eq!( + &fs_store, + FileStore::get_subject_file_store(&appstate, "my-great-file") + ); + assert_eq!( + &fs_store, + FileStore::get_subject_file_store(&appstate, "fs:my-great-file") + ); + assert_eq!( + &s3_store, + FileStore::get_subject_file_store(&appstate, "s3:my-great-file") + ); +}