From 93d278cef768e5400f81654496b4c307ddfe23ca Mon Sep 17 00:00:00 2001 From: mrq1911 Date: Wed, 13 Dec 2023 12:00:14 +0100 Subject: [PATCH 01/16] subsquid indexer & processor fix (#150) --- rpc_configs/substrate.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/rpc_configs/substrate.yml b/rpc_configs/substrate.yml index c5d5b5c..30a9be6 100644 --- a/rpc_configs/substrate.yml +++ b/rpc_configs/substrate.yml @@ -214,4 +214,5 @@ aliases: - [ chain_getBlockHash, chain_getHead ] - [ state_getKeysPaged, state_getKeysPagedAt ] - [ state_getStorage, state_getStorageAt ] + - [ state_getRuntimeVersion, chain_getRuntimeVersion ] - [ childstate_getKeysPaged, childstate_getKeysPagedAt ] From 939fcc75bc9506e6e93dfa86c461dbf2227ffa37 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Thu, 21 Mar 2024 19:08:19 +0100 Subject: [PATCH 02/16] update deps (#155) --- .github/workflows/test.yml | 7 +- Cargo.lock | 870 ++++++++++++++++++++----------------- rust-toolchain.toml | 2 +- 3 files changed, 481 insertions(+), 398 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2ae7ade..a53e899 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,13 +21,16 @@ jobs: steps: - name: Install toolchain uses: dtolnay/rust-toolchain@stable + with: + toolchain: nightly-2023-09-29 + components: rustfmt, clippy - uses: actions/checkout@v3 with: submodules: recursive - name: Check format - run: cargo fmt --all -- --check + run: cargo +nightly-2023-09-29 fmt --all -- --check - name: Check clippy - run: cargo clippy --all-targets --all-features -- -D warnings + run: cargo +nightly-2023-09-29 clippy --all-targets --all-features -- -D warnings - name: Build run: cargo build --verbose - name: Run tests diff --git a/Cargo.lock b/Cargo.lock index d942438..b9e0d53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,12 +19,12 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if 1.0.0", - "getrandom 0.2.11", + "getrandom 0.2.12", "once_cell", "version_check", "zerocopy", @@ -32,9 +32,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -77,9 +77,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" dependencies = [ "anstyle", "anstyle-parse", @@ -91,43 +91,43 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" [[package]] name = "arrayvec" @@ -137,9 +137,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "async-compression" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5" +checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" dependencies = [ "brotli", "flate2", @@ -162,11 +162,11 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.1.2" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea8b3453dd7cc96711834b75400d671b73e3656975fa68d9f277163b7f7e316" +checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" dependencies = [ - "event-listener 4.0.0", + "event-listener 4.0.3", "event-listener-strategy", "pin-project-lite", ] @@ -190,18 +190,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] name = "async-trait" -version = "0.1.74" +version = "0.1.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "461abc97219de0eaaf81fe3ef974a540158f3d079c2ab200f891f1a2ef201e85" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -278,9 +278,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "beef" @@ -299,9 +299,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "blake2" @@ -353,9 +353,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -374,9 +374,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.8.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", "serde", @@ -384,9 +384,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" [[package]] name = "byte-tools" @@ -402,9 +402,9 @@ checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" [[package]] name = "bytemuck" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" +checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" [[package]] name = "byteorder" @@ -439,9 +439,9 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12024c4645c97566567129c204f65d5815a8c9aecf30fcbe682b2fe034996d36" +checksum = "694c8807f2ae16faecc43dc17d74b3eb042482789fd0eb64b39a2e04e087053f" dependencies = [ "serde", ] @@ -467,9 +467,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.83" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" dependencies = [ "jobserver", "libc", @@ -489,23 +489,23 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "wasm-bindgen", - "windows-targets", + "windows-targets 0.52.4", ] [[package]] name = "ciborium" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" dependencies = [ "ciborium-io", "ciborium-ll", @@ -514,15 +514,15 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" [[package]] name = "ciborium-ll" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", "half", @@ -530,9 +530,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.8" +version = "4.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" +checksum = "949626d00e063efc93b6dca932419ceb5432f99769911c0b995f7e884c778813" dependencies = [ "clap_builder", "clap_derive", @@ -540,9 +540,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.8" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" dependencies = [ "anstream", "anstyle", @@ -552,21 +552,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "90239a040c80f5e14809ca132ddc4176ab33d5e17e49691793296e3fcb34d72f" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "colorchoice" @@ -576,9 +576,9 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "concurrent-queue" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ "crossbeam-utils", ] @@ -622,9 +622,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -632,9 +632,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpp_demangle" @@ -647,18 +647,18 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if 1.0.0", ] @@ -703,46 +703,43 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" dependencies = [ - "cfg-if 1.0.0", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if 1.0.0", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if 1.0.0", "crossbeam-utils", - "memoffset", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "crypto-common" @@ -761,7 +758,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if 1.0.0", - "hashbrown 0.14.2", + "hashbrown 0.14.3", "lock_api", "once_cell", "parking_lot_core 0.9.9", @@ -807,9 +804,9 @@ dependencies = [ [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "encoding_rs" @@ -822,22 +819,22 @@ dependencies = [ [[package]] name = "enumflags2" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5998b4f30320c9d93aed72f63af821bfdac50465b75428fce77b48ec482c3939" +checksum = "3278c9d5fb675e0a51dabcf4c0d355f692b064171535ba72361be1528a9d8e8d" dependencies = [ "enumflags2_derive", ] [[package]] name = "enumflags2_derive" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f95e2801cd355d4a1a3e3953ce6ee5ae9603a5c833455343a8bfe3f44d418246" +checksum = "5c785274071b1b420972453b306eeca06acf4633829db4223b58a2a8c5953bc4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -848,12 +845,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -873,9 +870,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "4.0.0" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" dependencies = [ "concurrent-queue", "parking", @@ -888,7 +885,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" dependencies = [ - "event-listener 4.0.0", + "event-listener 4.0.3", "pin-project-lite", ] @@ -934,9 +931,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -959,9 +956,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -974,9 +971,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -984,15 +981,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -1002,38 +999,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" dependencies = [ "gloo-timers", "send_wrapper", @@ -1041,9 +1038,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -1089,9 +1086,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1100,9 +1097,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "glob" @@ -1112,15 +1109,15 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "globset" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" +checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" dependencies = [ "aho-corasick", "bstr", - "fnv", "log", - "regex", + "regex-automata 0.4.6", + "regex-syntax 0.8.2", ] [[package]] @@ -1180,16 +1177,16 @@ dependencies = [ "no-std-compat", "nonzero_ext", "parking_lot 0.12.1", - "quanta", + "quanta 0.11.1", "rand 0.8.5", "smallvec", ] [[package]] name = "h2" -version = "0.3.22" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "4fbd2820c5e49886948654ab546d0688ff24530286bdcf8fca3cefb16d4618eb" dependencies = [ "bytes 1.5.0", "fnv", @@ -1197,7 +1194,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.1.0", + "indexmap 2.2.5", "slab", "tokio", "tokio-util 0.7.10", @@ -1206,9 +1203,13 @@ dependencies = [ [[package]] name = "half" -version = "1.8.2" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" +dependencies = [ + "cfg-if 1.0.0", + "crunchy", +] [[package]] name = "hashbrown" @@ -1218,17 +1219,17 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "hdrhistogram" -version = "7.5.3" +version = "7.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5b38e5c02b7c7be48c8dc5217c4f1634af2ea221caae2e024bffc7a7651c691" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" dependencies = [ - "base64 0.13.1", + "base64 0.21.7", "byteorder", "flate2", "nom", @@ -1241,17 +1242,23 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes 1.5.0", "fnv", @@ -1260,9 +1267,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes 1.5.0", "http", @@ -1295,9 +1302,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes 1.5.0", "futures-channel", @@ -1310,7 +1317,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2", "tokio", "tower-service", "tracing", @@ -1347,9 +1354,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1370,9 +1377,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1390,22 +1397,22 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", - "hashbrown 0.14.2", + "hashbrown 0.14.3", ] [[package]] name = "inferno" -version = "0.11.18" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abfb2e51b23c338595ae0b6bdaaa7a4a8b860b8d788a4331cb07b50fe5dea71b" +checksum = "321f0f839cd44a4686e9504b0a62b4d69a50b62072144c71c68f5873c167b8d9" dependencies = [ "ahash", - "indexmap 2.1.0", + "indexmap 2.2.5", "is-terminal", "itoa", "log", @@ -1458,13 +1465,13 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ "hermit-abi", - "rustix", - "windows-sys", + "libc", + "windows-sys 0.52.0", ] [[package]] @@ -1487,24 +1494,24 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.65" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -1630,7 +1637,7 @@ name = "jsonrpsee-core" version = "0.20.0" dependencies = [ "anyhow", - "async-lock 3.1.2", + "async-lock 3.3.0", "async-trait", "beef", "futures-timer", @@ -1671,7 +1678,7 @@ dependencies = [ name = "jsonrpsee-proc-macros" version = "0.20.0" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro-crate", "proc-macro2", "quote", @@ -1756,15 +1763,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.150" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "linux-raw-sys" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -1778,15 +1785,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "mach2" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" dependencies = [ "libc", ] @@ -1808,28 +1815,19 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memmap2" -version = "0.8.0" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a5a03cefb0d953ec0be133036f14e109412fa594edc2f77227249db66cc3ed" +checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" dependencies = [ "libc", ] -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] - [[package]] name = "mime" version = "0.3.17" @@ -1854,9 +1852,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -1882,13 +1880,13 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.9" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1917,9 +1915,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.1" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8017ec3548ffe7d4cef7ac0e12b044c01164a74c0f3119420faeaf13490ad8b" +checksum = "b1911e88d5831f748a4097a43862d129e3c6fca831eecac9b8db6d01d93c9de2" dependencies = [ "async-lock 2.8.0", "async-trait", @@ -1929,7 +1927,7 @@ dependencies = [ "futures-util", "once_cell", "parking_lot 0.12.1", - "quanta", + "quanta 0.12.2", "rustc_version", "skeptic", "smallvec", @@ -2005,9 +2003,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", ] @@ -2024,18 +2022,18 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oorandom" @@ -2051,9 +2049,9 @@ checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" [[package]] name = "opaque-debug" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl-probe" @@ -2069,7 +2067,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.1.0", + "indexmap 2.2.5", "js-sys", "once_cell", "pin-project-lite", @@ -2085,7 +2083,7 @@ checksum = "3e09667367cb509f10d7cf5960a83f9c4d96e93715f750b164b4b98d46c3cbf4" dependencies = [ "futures-core", "http", - "indexmap 2.1.0", + "indexmap 2.2.5", "itertools 0.11.0", "once_cell", "opentelemetry", @@ -2138,9 +2136,9 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "968ba3f2ca03e90e5187f5e4f46c791ef7f2c163ae87789c8ce5f5ca3b7b7de5" +checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" dependencies = [ "async-trait", "crossbeam-channel", @@ -2149,7 +2147,7 @@ dependencies = [ "futures-util", "once_cell", "opentelemetry", - "ordered-float 4.1.1", + "ordered-float 4.2.0", "percent-encoding", "rand 0.8.5", "thiserror", @@ -2168,9 +2166,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.1.1" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "536900a8093134cf9ccf00a27deb3532421099e958d9dd431135d0c7543ca1e8" +checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" dependencies = [ "num-traits", ] @@ -2250,7 +2248,7 @@ dependencies = [ "libc", "redox_syscall 0.4.1", "smallvec", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -2261,28 +2259,28 @@ checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -2299,9 +2297,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plotters" @@ -2361,27 +2359,28 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro-crate" -version = "2.0.0" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8" +checksum = "b00f26d3400549137f92511a46ac1cd8ce37cb5598a96d382381458b992a5d24" dependencies = [ + "toml_datetime", "toml_edit", ] [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5a410fc7882af66deb8d01d01737353cf3ad6204c408177ba494291a626312" +checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" dependencies = [ "bytes 1.5.0", "prost-derive", @@ -2389,33 +2388,33 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "065717a5dfaca4a83d2fe57db3487b311365200000551d7a364e715dbf4346bc" +checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] name = "prost-types" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8339f32236f590281e2f6368276441394fcd1b2133b549cc895d0ae80f2f9a52" +checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" dependencies = [ "prost", ] [[package]] name = "pulldown-cmark" -version = "0.9.3" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" +checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "memchr", "unicase", ] @@ -2430,7 +2429,22 @@ dependencies = [ "libc", "mach2", "once_cell", - "raw-cpuid", + "raw-cpuid 10.7.0", + "wasi 0.11.0+wasi-snapshot-preview1", + "web-sys", + "winapi 0.3.9", +] + +[[package]] +name = "quanta" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca0b7bac0b97248c40bb77288fc52029cf1459c0461ea1b05ee32ccf011de2c" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid 11.0.1", "wasi 0.11.0+wasi-snapshot-preview1", "web-sys", "winapi 0.3.9", @@ -2447,9 +2461,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -2513,7 +2527,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.12", ] [[package]] @@ -2534,11 +2548,20 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "raw-cpuid" +version = "11.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "rayon" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd" dependencies = [ "either", "rayon-core", @@ -2546,9 +2569,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -2574,13 +2597,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.3", + "regex-automata 0.4.6", "regex-syntax 0.8.2", ] @@ -2595,9 +2618,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", @@ -2618,11 +2641,11 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "bytes 1.5.0", "encoding_rs", "futures-core", @@ -2641,6 +2664,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tower-service", @@ -2662,16 +2686,17 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.5" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", - "getrandom 0.2.11", + "cfg-if 1.0.0", + "getrandom 0.2.12", "libc", "spin", "untrusted", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -2714,22 +2739,22 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.24" +version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ad981d6c340a49cdc40a1028d9c6084ec7e9fa33fcb839cab656a267071e234" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.21.9" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", "ring", @@ -2755,7 +2780,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", ] [[package]] @@ -2776,9 +2801,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "same-file" @@ -2791,11 +2816,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -2839,9 +2864,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" dependencies = [ "serde", ] @@ -2854,29 +2879,29 @@ checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" [[package]] name = "serde" -version = "1.0.192" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.192" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ "itoa", "ryu", @@ -2897,11 +2922,11 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.27" +version = "0.9.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" +checksum = "a0623d197252096520c6f2a5e1171ee436e5af99a5d7caa2891e55e61950e6d9" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.5", "itoa", "ryu", "serde", @@ -2930,7 +2955,7 @@ dependencies = [ "cfg-if 1.0.0", "cpufeatures", "digest 0.9.0", - "opaque-debug 0.3.0", + "opaque-debug 0.3.1", ] [[package]] @@ -2977,28 +3002,18 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.4.10" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "winapi 0.3.9", -] - -[[package]] -name = "socket2" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" -dependencies = [ - "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -3037,9 +3052,9 @@ checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" [[package]] name = "subtle" @@ -3089,9 +3104,9 @@ dependencies = [ [[package]] name = "symbolic-common" -version = "12.7.0" +version = "12.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39eac77836da383d35edbd9ff4585b4fc1109929ff641232f2e9a1aefdfc9e91" +checksum = "1cccfffbc6bb3bb2d3a26cd2077f4d055f6808d266f9d4d158797a4c60510dfe" dependencies = [ "debugid", "memmap2", @@ -3101,9 +3116,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.7.0" +version = "12.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee1608a1d13061fb0e307a316de29f6c6e737b05459fe6bbf5dd8d7837c4fb7" +checksum = "76a99812da4020a67e76c4eb41f08c87364c14170495ff780f30dd519c221a68" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -3123,9 +3138,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032" dependencies = [ "proc-macro2", "quote", @@ -3167,42 +3182,41 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tempfile" -version = "3.8.1" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if 1.0.0", "fastrand", - "redox_syscall 0.4.1", "rustix", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if 1.0.0", "once_cell", @@ -3257,22 +3271,22 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.34.0" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes 1.5.0", "libc", - "mio 0.8.9", + "mio 0.8.11", "num_cpus", "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.5", + "socket2", "tokio-macros", "tracing", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3293,7 +3307,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -3308,9 +3322,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" dependencies = [ "futures-core", "pin-project-lite", @@ -3348,17 +3362,17 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" [[package]] name = "toml_edit" -version = "0.20.7" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" +checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.5", "toml_datetime", "winnow", ] @@ -3372,7 +3386,7 @@ dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.21.5", + "base64 0.21.7", "bytes 1.5.0", "h2", "http", @@ -3418,8 +3432,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ "async-compression", - "base64 0.21.5", - "bitflags 2.4.1", + "base64 0.21.7", + "bitflags 2.5.0", "bytes 1.5.0", "futures-core", "futures-util", @@ -3473,7 +3487,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -3530,15 +3544,15 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee8098afad3fb0c54a9007aab6804558410503ad676d4633f9c2559a00ac0f" +checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" @@ -3557,9 +3571,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -3569,18 +3583,18 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] [[package]] name = "unsafe-libyaml" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" @@ -3590,9 +3604,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -3613,11 +3627,11 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.5.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.12", ] [[package]] @@ -3634,9 +3648,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -3665,9 +3679,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -3675,24 +3689,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.38" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3702,9 +3716,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3712,28 +3726,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "web-sys" -version = "0.3.65" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -3741,9 +3755,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.2" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "winapi" @@ -3790,11 +3804,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets", + "windows-targets 0.52.4", ] [[package]] @@ -3803,7 +3817,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", ] [[package]] @@ -3812,13 +3835,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] @@ -3827,47 +3865,89 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + [[package]] name = "winnow" -version = "0.5.19" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ "memchr", ] @@ -3879,7 +3959,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ "cfg-if 1.0.0", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3894,22 +3974,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.26" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.26" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 3fb07d1..ba3f7e4 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2023-08-16" +channel = "1.74" components = ["rustfmt", "clippy"] From f6f076d991ac68cf14e5696aaea9ed7b3da5d4e3 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Fri, 22 Mar 2024 10:07:25 +0100 Subject: [PATCH 03/16] bypass cache for unfinalized blocks (#154) --- src/middlewares/methods/block_tag.rs | 55 +++++- src/middlewares/methods/inject_params.rs | 232 ++++++++++++++++++++++- 2 files changed, 270 insertions(+), 17 deletions(-) diff --git a/src/middlewares/methods/block_tag.rs b/src/middlewares/methods/block_tag.rs index 531e497..b71b1ba 100644 --- a/src/middlewares/methods/block_tag.rs +++ b/src/middlewares/methods/block_tag.rs @@ -60,6 +60,8 @@ impl BlockTagMiddleware { } } "latest" => { + // bypass cache for latest block to avoid caching forks + context.insert(BypassCache(true)); let (_, number) = self.api.get_head().read().await; Some(format!("0x{:x}", number).into()) } @@ -68,7 +70,23 @@ impl BlockTagMiddleware { context.insert(BypassCache(true)); None } - _ => None, + number => { + // bypass cache for block number to avoid caching forks unless it's a finalized block + let mut bypass_cache = true; + if let Some((_, finalized_number)) = self.api.current_finalized_head() { + if let Some(hex_number) = number.strip_prefix("0x") { + if let Ok(number) = u64::from_str_radix(hex_number, 16) { + if number <= finalized_number { + bypass_cache = false; + } + } + } + } + if bypass_cache { + context.insert(BypassCache(true)); + } + None + } } } else { None @@ -136,6 +154,10 @@ mod tests { } } + fn bypass_cache(context: &TypeRegistry) -> bool { + context.get::().map_or(false, |x| x.0) + } + async fn create_client() -> (ExecutionContext, EthApi) { let mut builder = TestServerBuilder::new(); @@ -169,7 +191,7 @@ mod tests { #[tokio::test] async fn skip_replacement_if_no_tag() { - let params = vec![json!("0x1234"), json!("0x5678")]; + let params = vec![json!("0x1234"), json!("0x4321")]; let (middleware, mut context) = create_block_tag_middleware(vec![ MethodParam { name: "key".to_string(), @@ -195,8 +217,11 @@ mod tests { .call( CallRequest::new("state_getStorage", params.clone()), Default::default(), - Box::new(move |req: CallRequest, _| { + Box::new(move |req: CallRequest, context| { async move { + // cache bypassed, cannot determine finalized block + assert!(bypass_cache(&context)); + // no replacement assert_eq!(req.params, params); Ok(json!("0x1111")) } @@ -236,7 +261,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(10)).await; let sub = context.subscribe_rx.recv().await.unwrap(); if sub.params.as_array().unwrap().contains(&json!("newFinalizedHeads")) { - sub.run_sink_tasks(vec![SinkTask::Send(json!({ "number": "0x5430", "hash": "0x00" }))]) + sub.run_sink_tasks(vec![SinkTask::Send(json!({ "number": "0x4321", "hash": "0x01" }))]) .await } @@ -252,8 +277,11 @@ mod tests { .call( CallRequest::new("state_getStorage", vec![json!("0x1234"), json!("latest")]), Default::default(), - Box::new(move |req: CallRequest, _| { + Box::new(move |req: CallRequest, context| { async move { + // cache bypassed for latest + assert!(bypass_cache(&context)); + // latest block replaced with block number assert_eq!(req.params, vec![json!("0x1234"), json!("0x4321")]); Ok(json!("0x1111")) } @@ -270,8 +298,11 @@ mod tests { .call( CallRequest::new("state_getStorage", vec![json!("0x1234"), json!("finalized")],), Default::default(), - Box::new(move |req: CallRequest, _| { + Box::new(move |req: CallRequest, context| { async move { + // cache bypassed, block tag not replaced + assert!(bypass_cache(&context)); + // block tag not replaced assert_eq!(req.params, vec![json!("0x1234"), json!("finalized")]); Ok(json!("0x1111")) } @@ -291,9 +322,12 @@ mod tests { .call( CallRequest::new("state_getStorage", vec![json!("0x1234"), json!("finalized")],), Default::default(), - Box::new(move |req: CallRequest, _| { + Box::new(move |req: CallRequest, context| { async move { - assert_eq!(req.params, vec![json!("0x1234"), json!("0x5430")]); + // cache not bypassed, finalized replaced with block number + assert!(!bypass_cache(&context)); + // block tag replaced with block number + assert_eq!(req.params, vec![json!("0x1234"), json!("0x4321")]); Ok(json!("0x1111")) } .boxed() @@ -309,8 +343,11 @@ mod tests { .call( CallRequest::new("state_getStorage", vec![json!("0x1234"), json!("latest")]), Default::default(), - Box::new(move |req: CallRequest, _| { + Box::new(move |req: CallRequest, context| { async move { + // cache bypassed for latest + assert!(bypass_cache(&context)); + // latest block replaced with block number assert_eq!(req.params, vec![json!("0x1234"), json!("0x5432")]); Ok(json!("0x1111")) } diff --git a/src/middlewares/methods/inject_params.rs b/src/middlewares/methods/inject_params.rs index 6992fdc..b07ecad 100644 --- a/src/middlewares/methods/inject_params.rs +++ b/src/middlewares/methods/inject_params.rs @@ -6,7 +6,9 @@ use std::sync::Arc; use crate::{ config::MethodParam, extensions::api::{SubstrateApi, ValueHandle}, - middlewares::{CallRequest, CallResult, Middleware, MiddlewareBuilder, NextFn, RpcMethod, TRACER}, + middlewares::{ + methods::cache::BypassCache, CallRequest, CallResult, Middleware, MiddlewareBuilder, NextFn, RpcMethod, TRACER, + }, utils::errors, utils::{TypeRegistry, TypeRegistryRef}, }; @@ -18,6 +20,7 @@ pub enum InjectType { pub struct InjectParamsMiddleware { head: ValueHandle<(JsonValue, u64)>, + finalized: ValueHandle<(JsonValue, u64)>, inject: InjectType, params: Vec, } @@ -60,6 +63,7 @@ impl InjectParamsMiddleware { pub fn new(api: Arc, inject: InjectType, params: Vec) -> Self { Self { head: api.get_head(), + finalized: api.get_finalized_head(), inject, params, } @@ -99,14 +103,28 @@ impl Middleware for InjectParamsMiddleware { async fn call( &self, mut request: CallRequest, - context: TypeRegistry, + mut context: TypeRegistry, next: NextFn, ) -> CallResult { + let handle_request = |request: CallRequest| async { + for (idx, param) in self.params.iter().enumerate() { + if param.ty == "BlockNumber" { + if let Some(number) = request.params.get(idx).and_then(|x| x.as_u64()) { + let (_, finalized) = self.finalized.read().await; + if number > finalized { + context.insert(BypassCache(true)); + } + } + } + } + next(request, context).await + }; + let idx = self.get_index(); match request.params.len() { len if len == idx + 1 => { // full params with current block - return next(request, context).await; + return handle_request(request).await; } len if len <= idx => { async move { @@ -130,14 +148,14 @@ impl Middleware for InjectParamsMiddleware { } request.params.push(to_inject); - next(request, context).await + handle_request(request).await } .with_context(TRACER.context("inject_params")) .await } _ => { // unexpected number of params - next(request, context).await + handle_request(request).await } } } @@ -160,9 +178,14 @@ mod tests { api: Arc, _server: ServerHandle, head_rx: mpsc::Receiver, - _finalized_head_rx: mpsc::Receiver, + finalized_head_rx: mpsc::Receiver, block_hash_rx: mpsc::Receiver, head_sink: Option, + finalized_head_sink: Option, + } + + fn bypass_cache(context: &TypeRegistry) -> bool { + context.get::().map_or(false, |x| x.0) } async fn create_client() -> ExecutionContext { @@ -171,7 +194,7 @@ mod tests { let head_rx = builder.register_subscription("chain_subscribeNewHeads", "chain_newHead", "chain_unsubscribeNewHeads"); - let _finalized_head_rx = builder.register_subscription( + let finalized_head_rx = builder.register_subscription( "chain_subscribeFinalizedHeads", "chain_finalizedHead", "chain_unsubscribeFinalizedHeads", @@ -188,9 +211,10 @@ mod tests { api: Arc::new(api), _server, head_rx, - _finalized_head_rx, + finalized_head_rx, block_hash_rx, head_sink: None, + finalized_head_sink: None, } } @@ -208,7 +232,15 @@ mod tests { req.respond(json!("0xabcd")); } + let finalized_sub = context.finalized_head_rx.recv().await.unwrap(); + finalized_sub.send(json!({ "number": "0x4321" })).await; + { + let req = context.block_hash_rx.recv().await.unwrap(); + req.respond(json!("0xabcd")); + } + context.head_sink = Some(head_sub.sink); + context.finalized_head_sink = Some(finalized_sub.sink); ( InjectParamsMiddleware::new(context.api.clone(), inject_type, params), @@ -428,6 +460,18 @@ mod tests { let req = context.block_hash_rx.recv().await.unwrap(); req.respond(json!("0xbcde")); } + + // finalized updated + context + .finalized_head_sink + .unwrap() + .send(SubscriptionMessage::from_json(&json!({ "number": "0x5432" })).unwrap()) + .await + .unwrap(); + { + let req = context.block_hash_rx.recv().await.unwrap(); + req.respond(json!("0xbcde")); + } tokio::time::sleep(std::time::Duration::from_millis(1)).await; let result2 = middleware @@ -446,4 +490,176 @@ mod tests { .unwrap(); assert_eq!(result2, json!("0x1111")); } + + #[tokio::test] + async fn skip_cache_if_block_number_not_finalized() { + let (middleware, mut context) = create_inject_middleware( + InjectType::BlockNumberAt(1), + vec![ + MethodParam { + name: "key".to_string(), + ty: "StorageKey".to_string(), + optional: false, + inject: false, + }, + MethodParam { + name: "at".to_string(), + ty: "BlockNumber".to_string(), + optional: true, + inject: true, + }, + ], + ) + .await; + + // head is finalized, cache should not be skipped + { + let result = middleware + .call( + CallRequest::new("state_getStorage", vec![json!("0x1234")]), + Default::default(), + Box::new(move |req: CallRequest, context| { + async move { + // cache not bypassed + assert!(!bypass_cache(&context)); + // block number is not finalized + assert_eq!(req.params, vec![json!("0x1234"), json!(0x4321)]); + Ok(json!("0x1111")) + } + .boxed() + }), + ) + .await + .unwrap(); + assert_eq!(result, json!("0x1111")); + } + + // block head is updated but not finalized, cache should be skipped + { + // head updated but not finalized + context + .head_sink + .unwrap() + .send(SubscriptionMessage::from_json(&json!({ "number": "0x5432" })).unwrap()) + .await + .unwrap(); + { + let req = context.block_hash_rx.recv().await.unwrap(); + req.respond(json!("0xbcde")); + } + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + + let result = middleware + .call( + CallRequest::new("state_getStorage", vec![json!("0x1234")]), + Default::default(), + Box::new(move |req: CallRequest, context| { + async move { + // cache bypassed + assert!(bypass_cache(&context)); + // block number is injected + assert_eq!(req.params, vec![json!("0x1234"), json!(0x5432)]); + Ok(json!("0x1111")) + } + .boxed() + }), + ) + .await + .unwrap(); + assert_eq!(result, json!("0x1111")); + } + + // request with head block number should skip cache + { + let result = middleware + .call( + CallRequest::new("state_getStorage", vec![json!("0x1234"), json!(0x5432)]), + Default::default(), + Box::new(move |req: CallRequest, context| { + async move { + // cache bypassed + assert!(bypass_cache(&context)); + // params not changed + assert_eq!(req.params, vec![json!("0x1234"), json!(0x5432)]); + Ok(json!("0x1111")) + } + .boxed() + }), + ) + .await + .unwrap(); + assert_eq!(result, json!("0x1111")); + } + + // request with finalized block number should not skip cache + { + let result = middleware + .call( + CallRequest::new("state_getStorage", vec![json!("0x1234"), json!(0x4321)]), + Default::default(), + Box::new(move |req: CallRequest, context| { + async move { + // cache not bypassed + assert!(!bypass_cache(&context)); + // params not changed + assert_eq!(req.params, vec![json!("0x1234"), json!(0x4321)]); + Ok(json!("0x1111")) + } + .boxed() + }), + ) + .await + .unwrap(); + assert_eq!(result, json!("0x1111")); + } + + // request with wrong params count will be handled + { + // block is finalized, cache should not be skipped + let result = middleware + .call( + CallRequest::new( + "state_getStorage", + vec![json!("0x1234"), json!(0x4321), json!("0xabcd")], + ), + Default::default(), + Box::new(move |req: CallRequest, context| { + async move { + // cache not bypassed + assert!(!bypass_cache(&context)); + // params not changed + assert_eq!(req.params, vec![json!("0x1234"), json!(0x4321), json!("0xabcd")]); + Ok(json!("0x1111")) + } + .boxed() + }), + ) + .await + .unwrap(); + assert_eq!(result, json!("0x1111")); + + // block is not finalized, cache should be skipped + let result = middleware + .call( + CallRequest::new( + "state_getStorage", + vec![json!("0x1234"), json!(0x5432), json!("0xabcd")], + ), + Default::default(), + Box::new(move |req: CallRequest, context| { + async move { + // cache bypassed + assert!(bypass_cache(&context)); + // params not changed + assert_eq!(req.params, vec![json!("0x1234"), json!(0x5432), json!("0xabcd")]); + Ok(json!("0x1111")) + } + .boxed() + }), + ) + .await + .unwrap(); + assert_eq!(result, json!("0x1111")); + } + } } From cdbdd9bb981db4b2ec68e71eff681d3acc4040fb Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Mon, 8 Apr 2024 09:43:58 +0200 Subject: [PATCH 04/16] endpoint health (#152) --- benches/bench/main.rs | 1 + config.yml | 8 + eth_config.yml | 12 + src/extensions/client/endpoint.rs | 169 ++++++++++ src/extensions/client/health.rs | 159 +++++++++ src/extensions/client/mock.rs | 10 + src/extensions/client/mod.rs | 517 +++++++++++++++++++----------- src/extensions/client/tests.rs | 74 +++++ src/server.rs | 1 + src/tests/merge_subscription.rs | 1 + src/tests/upstream.rs | 1 + 11 files changed, 763 insertions(+), 190 deletions(-) create mode 100644 src/extensions/client/endpoint.rs create mode 100644 src/extensions/client/health.rs diff --git a/benches/bench/main.rs b/benches/bench/main.rs index ea66c99..2f457c4 100644 --- a/benches/bench/main.rs +++ b/benches/bench/main.rs @@ -217,6 +217,7 @@ fn config() -> Config { format!("ws://{}", SERVER_TWO_ENDPOINT), ], shuffle_endpoints: false, + health_check: None, }), server: Some(ServerConfig { listen_address: SUBWAY_SERVER_ADDR.to_string(), diff --git a/config.yml b/config.yml index c284bf2..890f8ae 100644 --- a/config.yml +++ b/config.yml @@ -3,6 +3,14 @@ extensions: endpoints: - wss://acala-rpc.dwellir.com - wss://acala-rpc-0.aca-api.network + health_check: + interval_sec: 10 # check interval, default is 10s + healthy_response_time_ms: 500 # max response time to be considered healthy, default is 500ms + health_method: system_health + response: # response contains { isSyncing: false } + !contains + - - isSyncing + - !eq false event_bus: substrate_api: stale_timeout_seconds: 180 # rotate endpoint if no new blocks for 3 minutes diff --git a/eth_config.yml b/eth_config.yml index 3363d77..85d6955 100644 --- a/eth_config.yml +++ b/eth_config.yml @@ -2,6 +2,18 @@ extensions: client: endpoints: - wss://eth-rpc-karura-testnet.aca-staging.network + health_check: + interval_sec: 10 # check interval, default is 10s + healthy_response_time_ms: 500 # max response time to be considered healthy, default is 500ms + health_method: net_health # eth-rpc-adapter bodhijs + response: # response contains { isHealthy: true, isRPCOK: true } + !contains + - - isHealthy + - !eq true + - - isRPCOK + - !eq true +# health_method: eth_syncing # eth node +# response: !eq false event_bus: eth_api: stale_timeout_seconds: 180 # rotate endpoint if no new blocks for 3 minutes diff --git a/src/extensions/client/endpoint.rs b/src/extensions/client/endpoint.rs new file mode 100644 index 0000000..8ea351f --- /dev/null +++ b/src/extensions/client/endpoint.rs @@ -0,0 +1,169 @@ +use super::health::{Event, Health}; +use crate::{ + extensions::client::{get_backoff_time, HealthCheckConfig}, + utils::errors, +}; +use jsonrpsee::{ + async_client::Client, + core::client::{ClientT, Subscription, SubscriptionClientT}, + ws_client::WsClientBuilder, +}; +use std::{ + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, + time::Duration, +}; + +pub struct Endpoint { + url: String, + health: Arc, + client_rx: tokio::sync::watch::Receiver>>, + on_client_ready: Arc, + background_tasks: Vec>, +} + +impl Drop for Endpoint { + fn drop(&mut self) { + self.background_tasks.drain(..).for_each(|handle| handle.abort()); + } +} + +impl Endpoint { + pub fn new( + url: String, + request_timeout: Option, + connection_timeout: Option, + health_config: HealthCheckConfig, + ) -> Self { + let (client_tx, client_rx) = tokio::sync::watch::channel(None); + let on_client_ready = Arc::new(tokio::sync::Notify::new()); + let health = Arc::new(Health::new(url.clone(), health_config)); + + let url_ = url.clone(); + let health_ = health.clone(); + let on_client_ready_ = on_client_ready.clone(); + + // This task will try to connect to the endpoint and keep the connection alive + let connection_task = tokio::spawn(async move { + let connect_backoff_counter = Arc::new(AtomicU32::new(0)); + + loop { + tracing::info!("Connecting endpoint: {url_}"); + + let client = WsClientBuilder::default() + .request_timeout(request_timeout.unwrap_or(Duration::from_secs(30))) + .connection_timeout(connection_timeout.unwrap_or(Duration::from_secs(30))) + .max_buffer_capacity_per_subscription(2048) + .max_concurrent_requests(2048) + .max_response_size(20 * 1024 * 1024) + .build(&url_); + + match client.await { + Ok(client) => { + let client = Arc::new(client); + health_.update(Event::ConnectionSuccessful); + _ = client_tx.send(Some(client.clone())); + on_client_ready_.notify_waiters(); + tracing::info!("Endpoint connected: {url_}"); + connect_backoff_counter.store(0, Ordering::Relaxed); + client.on_disconnect().await; + } + Err(err) => { + health_.on_error(&err); + _ = client_tx.send(None); + tracing::warn!("Unable to connect to endpoint: {url_} error: {err}"); + tokio::time::sleep(get_backoff_time(&connect_backoff_counter)).await; + } + } + // Wait a second before trying to reconnect + tokio::time::sleep(Duration::from_secs(1)).await; + } + }); + + // This task will check the health of the endpoint and update the health score + let health_checker = Health::monitor(health.clone(), client_rx.clone(), on_client_ready.clone()); + + Self { + url, + health, + client_rx, + on_client_ready, + background_tasks: vec![connection_task, health_checker], + } + } + + pub fn url(&self) -> &str { + &self.url + } + + pub fn health(&self) -> &Health { + self.health.as_ref() + } + + pub async fn connected(&self) { + if self.client_rx.borrow().is_some() { + return; + } + self.on_client_ready.notified().await; + } + + pub async fn request( + &self, + method: &str, + params: Vec, + timeout: Duration, + ) -> Result { + let client = self + .client_rx + .borrow() + .clone() + .ok_or(errors::failed("client not connected"))?; + + match tokio::time::timeout(timeout, client.request(method, params.clone())).await { + Ok(Ok(response)) => Ok(response), + Ok(Err(err)) => { + self.health.on_error(&err); + Err(err) + } + Err(_) => { + tracing::error!("request timed out method: {method} params: {params:?}"); + self.health.on_error(&jsonrpsee::core::Error::RequestTimeout); + Err(jsonrpsee::core::Error::RequestTimeout) + } + } + } + + pub async fn subscribe( + &self, + subscribe_method: &str, + params: Vec, + unsubscribe_method: &str, + timeout: Duration, + ) -> Result, jsonrpsee::core::Error> { + let client = self + .client_rx + .borrow() + .clone() + .ok_or(errors::failed("client not connected"))?; + + match tokio::time::timeout( + timeout, + client.subscribe(subscribe_method, params.clone(), unsubscribe_method), + ) + .await + { + Ok(Ok(response)) => Ok(response), + Ok(Err(err)) => { + self.health.on_error(&err); + Err(err) + } + Err(_) => { + tracing::error!("subscribe timed out subscribe: {subscribe_method} params: {params:?}"); + self.health.on_error(&jsonrpsee::core::Error::RequestTimeout); + Err(jsonrpsee::core::Error::RequestTimeout) + } + } + } +} diff --git a/src/extensions/client/health.rs b/src/extensions/client/health.rs new file mode 100644 index 0000000..acdef15 --- /dev/null +++ b/src/extensions/client/health.rs @@ -0,0 +1,159 @@ +use crate::extensions::client::HealthCheckConfig; +use jsonrpsee::{async_client::Client, core::client::ClientT}; +use std::{ + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, + time::Duration, +}; + +#[derive(Debug)] +pub enum Event { + ResponseOk, + SlowResponse, + RequestTimeout, + ConnectionSuccessful, + ConnectionFailed, + StaleChain, +} + +impl Event { + pub fn update_score(&self, current: u32) -> u32 { + u32::min( + match self { + Event::ResponseOk => current.saturating_add(2), + Event::SlowResponse => current.saturating_sub(5), + Event::RequestTimeout | Event::ConnectionFailed | Event::StaleChain => 0, + Event::ConnectionSuccessful => MAX_SCORE / 5 * 4, // 80% of max score + }, + MAX_SCORE, + ) + } +} + +#[derive(Debug, Default)] +pub struct Health { + url: String, + config: HealthCheckConfig, + score: AtomicU32, + unhealthy: tokio::sync::Notify, +} + +const MAX_SCORE: u32 = 100; +const THRESHOLD: u32 = MAX_SCORE / 2; + +impl Health { + pub fn new(url: String, config: HealthCheckConfig) -> Self { + Self { + url, + config, + score: AtomicU32::new(0), + unhealthy: tokio::sync::Notify::new(), + } + } + + pub fn score(&self) -> u32 { + self.score.load(Ordering::Relaxed) + } + + pub fn update(&self, event: Event) { + let current_score = self.score.load(Ordering::Relaxed); + let new_score = event.update_score(current_score); + if new_score == current_score { + return; + } + self.score.store(new_score, Ordering::Relaxed); + log::trace!( + "Endpoint {:?} score updated from: {current_score} to: {new_score}", + self.url + ); + + // Notify waiters if the score has dropped below the threshold + if current_score >= THRESHOLD && new_score < THRESHOLD { + log::warn!("Endpoint {:?} became unhealthy", self.url); + self.unhealthy.notify_waiters(); + } + } + + pub fn on_error(&self, err: &jsonrpsee::core::Error) { + log::warn!("Endpoint {:?} responded with error: {err:?}", self.url); + match err { + jsonrpsee::core::Error::RequestTimeout => { + self.update(Event::RequestTimeout); + } + jsonrpsee::core::Error::Transport(_) + | jsonrpsee::core::Error::RestartNeeded(_) + | jsonrpsee::core::Error::MaxSlotsExceeded => { + self.update(Event::ConnectionFailed); + } + _ => {} + }; + } + + pub async fn unhealthy(&self) { + self.unhealthy.notified().await; + } +} + +impl Health { + pub fn monitor( + health: Arc, + client_rx_: tokio::sync::watch::Receiver>>, + on_client_ready: Arc, + ) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + // no health method + if health.config.health_method.is_none() { + return; + } + + // Wait for the client to be ready before starting the health check + on_client_ready.notified().await; + + let method_name = health.config.health_method.as_ref().expect("checked above"); + let health_response = health.config.response.clone(); + let interval = Duration::from_secs(health.config.interval_sec); + let healthy_response_time = Duration::from_millis(health.config.healthy_response_time_ms); + + let client = match client_rx_.borrow().clone() { + Some(client) => client, + None => return, + }; + + loop { + // Wait for the next interval + tokio::time::sleep(interval).await; + + let request_start = std::time::Instant::now(); + match client + .request::>(method_name, vec![]) + .await + { + Ok(response) => { + let duration = request_start.elapsed(); + + // Check response + if let Some(ref health_response) = health_response { + if !health_response.validate(&response) { + health.update(Event::StaleChain); + continue; + } + } + + // Check response time + if duration > healthy_response_time { + health.update(Event::SlowResponse); + continue; + } + + health.update(Event::ResponseOk); + } + Err(err) => { + health.on_error(&err); + } + } + } + }) + } +} diff --git a/src/extensions/client/mock.rs b/src/extensions/client/mock.rs index fbe0864..d999093 100644 --- a/src/extensions/client/mock.rs +++ b/src/extensions/client/mock.rs @@ -153,6 +153,16 @@ pub async fn dummy_server() -> ( (addr, handle, rx, sub_rx) } +pub async fn dummy_server_extend(extend: Box) -> (SocketAddr, ServerHandle) { + let mut builder = TestServerBuilder::new(); + + extend(&mut builder); + + let (addr, handle) = builder.build().await; + + (addr, handle) +} + pub enum SinkTask { Sleep(u64), Send(JsonValue), diff --git a/src/extensions/client/mod.rs b/src/extensions/client/mod.rs index a05441d..e016467 100644 --- a/src/extensions/client/mod.rs +++ b/src/extensions/client/mod.rs @@ -1,24 +1,14 @@ use std::{ - sync::{ - atomic::{AtomicU32, AtomicUsize}, - Arc, - }, + sync::{atomic::AtomicU32, Arc}, time::Duration, }; use anyhow::anyhow; use async_trait::async_trait; -use futures::TryFutureExt; -use jsonrpsee::{ - core::{ - client::{ClientT, Subscription, SubscriptionClientT}, - Error, JsonValue, - }, - ws_client::{WsClient, WsClientBuilder}, -}; +use jsonrpsee::core::{client::Subscription, Error, JsonValue}; use opentelemetry::trace::FutureExt; use rand::{seq::SliceRandom, thread_rng}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::sync::Notify; use super::ExtensionRegistry; @@ -28,6 +18,10 @@ use crate::{ utils::{self, errors}, }; +mod endpoint; +mod health; +use endpoint::Endpoint; + #[cfg(test)] pub mod mock; #[cfg(test)] @@ -53,12 +47,72 @@ pub struct ClientConfig { pub endpoints: Vec, #[serde(default = "bool_true")] pub shuffle_endpoints: bool, + pub health_check: Option, } pub fn bool_true() -> bool { true } +#[derive(Deserialize, Debug, Clone)] +pub struct HealthCheckConfig { + #[serde(default = "interval_sec")] + pub interval_sec: u64, + #[serde(default = "healthy_response_time_ms")] + pub healthy_response_time_ms: u64, + pub health_method: Option, + pub response: Option, +} + +impl Default for HealthCheckConfig { + fn default() -> Self { + Self { + interval_sec: interval_sec(), + healthy_response_time_ms: healthy_response_time_ms(), + health_method: None, + response: None, + } + } +} + +pub fn interval_sec() -> u64 { + 10 +} + +pub fn healthy_response_time_ms() -> u64 { + 500 +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum HealthResponse { + Eq(JsonValue), + NotEq(JsonValue), + Contains(Vec<(String, Box)>), +} + +impl HealthResponse { + pub fn validate(&self, response: &JsonValue) -> bool { + match self { + HealthResponse::Eq(value) => value.eq(response), + HealthResponse::NotEq(value) => !value.eq(response), + HealthResponse::Contains(items) => { + for (key, expected) in items { + if let Some(response) = response.get(key) { + if !expected.validate(response) { + return false; + } + } else { + // key missing + return false; + } + } + true + } + } + } +} + #[derive(Debug)] enum Message { Request { @@ -82,12 +136,13 @@ impl Extension for Client { type Config = ClientConfig; async fn from_config(config: &Self::Config, _registry: &ExtensionRegistry) -> Result { + let health_check = config.health_check.clone(); if config.shuffle_endpoints { let mut endpoints = config.endpoints.clone(); endpoints.shuffle(&mut thread_rng()); - Ok(Self::new(endpoints, None, None, None)?) + Ok(Self::new(endpoints, None, None, None, health_check)?) } else { - Ok(Self::new(config.endpoints.clone(), None, None, None)?) + Ok(Self::new(config.endpoints.clone(), None, None, None, health_check)?) } } } @@ -98,14 +153,32 @@ impl Client { request_timeout: Option, connection_timeout: Option, retries: Option, + health_config: Option, ) -> Result { + let health_config = health_config.unwrap_or_default(); let endpoints: Vec<_> = endpoints.into_iter().map(|e| e.as_ref().to_string()).collect(); if endpoints.is_empty() { return Err(anyhow!("No endpoints provided")); } - tracing::debug!("New client with endpoints: {:?}", endpoints); + if let Some(0) = retries { + return Err(anyhow!("Retries need to be at least 1")); + } + + tracing::debug!("New client with endpoints: {endpoints:?}"); + + let endpoints = endpoints + .into_iter() + .map(|e| { + Arc::new(Endpoint::new( + e, + request_timeout, + connection_timeout, + health_config.clone(), + )) + }) + .collect::>(); let (message_tx, mut message_rx) = tokio::sync::mpsc::channel::(100); @@ -115,57 +188,39 @@ impl Client { let rotation_notify_bg = rotation_notify.clone(); let background_task = tokio::spawn(async move { - let connect_backoff_counter = Arc::new(AtomicU32::new(0)); let request_backoff_counter = Arc::new(AtomicU32::new(0)); - let current_endpoint = AtomicUsize::new(0); - - let connect_backoff_counter2 = connect_backoff_counter.clone(); - let build_ws = || async { - let build = || { - let current_endpoint = current_endpoint.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let url = &endpoints[current_endpoint % endpoints.len()]; - - tracing::info!("Connecting to endpoint: {}", url); - - // TODO: make those configurable - WsClientBuilder::default() - .request_timeout(request_timeout.unwrap_or(Duration::from_secs(30))) - .connection_timeout(connection_timeout.unwrap_or(Duration::from_secs(30))) - .max_buffer_capacity_per_subscription(2048) - .max_concurrent_requests(2048) - .max_response_size(20 * 1024 * 1024) - .build(url) - .map_err(|e| (e, url.to_string())) - }; + // Select next endpoint with the highest health score, excluding the current one if provided + let healthiest_endpoint = |exclude: Option>| async { + if endpoints.len() == 1 { + let selected_endpoint = endpoints[0].clone(); + // Ensure it's connected + selected_endpoint.connected().await; + return selected_endpoint; + } - loop { - match build().await { - Ok(ws) => { - let ws = Arc::new(ws); - tracing::info!("Endpoint connected"); - connect_backoff_counter2.store(0, std::sync::atomic::Ordering::Relaxed); - break ws; - } - Err((e, url)) => { - tracing::warn!("Unable to connect to endpoint: '{url}' error: {e}"); - tokio::time::sleep(get_backoff_time(&connect_backoff_counter2)).await; - } - } + let mut endpoints = endpoints.clone(); + // Remove the current endpoint from the list + if let Some(exclude) = exclude { + endpoints.retain(|e| e.url() != exclude.url()); } + // Sort by health score + endpoints.sort_by_key(|endpoint| std::cmp::Reverse(endpoint.health().score())); + // Pick the first one + let selected_endpoint = endpoints[0].clone(); + // Ensure it's connected + selected_endpoint.connected().await; + selected_endpoint }; - let mut ws = build_ws().await; + let mut selected_endpoint = healthiest_endpoint(None).await; - let handle_message = |message: Message, ws: Arc| { + let handle_message = |message: Message, endpoint: Arc, rotation_notify: Arc| { let tx = message_tx_bg.clone(); let request_backoff_counter = request_backoff_counter.clone(); // total timeout for a request - let task_timeout = request_timeout - .unwrap_or(Duration::from_secs(30)) - // buffer 5 seconds for the request to be processed - .saturating_add(Duration::from_secs(5)); + let task_timeout = request_timeout.unwrap_or(Duration::from_secs(30)); tokio::spawn(async move { match message { @@ -182,71 +237,57 @@ impl Client { return; } - if let Ok(result) = - tokio::time::timeout(task_timeout, ws.request(&method, params.clone())).await - { - match result { - result @ Ok(_) => { - request_backoff_counter.store(0, std::sync::atomic::Ordering::Relaxed); - // make sure it's still connected - if response.is_closed() { - return; - } - let _ = response.send(result); + match endpoint.request(&method, params.clone(), task_timeout).await { + result @ Ok(_) => { + request_backoff_counter.store(0, std::sync::atomic::Ordering::Relaxed); + // make sure it's still connected + if response.is_closed() { + return; } - Err(err) => { - tracing::debug!("Request failed: {:?}", err); - match err { - Error::RequestTimeout - | Error::Transport(_) - | Error::RestartNeeded(_) - | Error::MaxSlotsExceeded => { - tokio::time::sleep(get_backoff_time(&request_backoff_counter)).await; - - // make sure it's still connected - if response.is_closed() { - return; - } - - // make sure we still have retries left - if retries == 0 { - let _ = response.send(Err(Error::RequestTimeout)); - return; - } - - if matches!(err, Error::RequestTimeout) { - tx.send(Message::RotateEndpoint) - .await - .expect("Failed to send rotate message"); - } - - tx.send(Message::Request { - method, - params, - response, - retries, - }) - .await - .expect("Failed to send request message"); + let _ = response.send(result); + } + Err(err) => { + tracing::debug!("Request failed: {err:?}"); + match err { + Error::RequestTimeout + | Error::Transport(_) + | Error::RestartNeeded(_) + | Error::MaxSlotsExceeded => { + // Make sure endpoint is rotated + rotation_notify.notified().await; + + tokio::time::sleep(get_backoff_time(&request_backoff_counter)).await; + + // make sure it's still connected + if response.is_closed() { + return; } - err => { - // make sure it's still connected - if response.is_closed() { - return; - } - // not something we can handle, send it back to the caller - let _ = response.send(Err(err)); + + // make sure we still have retries left + if retries == 0 { + let _ = response.send(Err(Error::RequestTimeout)); + return; + } + + tx.send(Message::Request { + method, + params, + response, + retries, + }) + .await + .expect("Failed to send request message"); + } + err => { + // make sure it's still connected + if response.is_closed() { + return; } + // not something we can handle, send it back to the caller + let _ = response.send(Err(err)); } } } - } else { - tracing::error!("request timed out method: {} params: {:?}", method, params); - // make sure it's still connected - if response.is_closed() { - return; - } - let _ = response.send(Err(Error::RequestTimeout)); } } Message::Subscribe { @@ -258,75 +299,61 @@ impl Client { } => { retries = retries.saturating_sub(1); - if let Ok(result) = tokio::time::timeout( - task_timeout, - ws.subscribe(&subscribe, params.clone(), &unsubscribe), - ) - .await + match endpoint + .subscribe(&subscribe, params.clone(), &unsubscribe, task_timeout) + .await { - match result { - result @ Ok(_) => { - request_backoff_counter.store(0, std::sync::atomic::Ordering::Relaxed); - // make sure it's still connected - if response.is_closed() { - return; - } - let _ = response.send(result); + result @ Ok(_) => { + request_backoff_counter.store(0, std::sync::atomic::Ordering::Relaxed); + // make sure it's still connected + if response.is_closed() { + return; } - Err(err) => { - tracing::debug!("Subscribe failed: {:?}", err); - match err { - Error::RequestTimeout - | Error::Transport(_) - | Error::RestartNeeded(_) - | Error::MaxSlotsExceeded => { - tokio::time::sleep(get_backoff_time(&request_backoff_counter)).await; - - // make sure it's still connected - if response.is_closed() { - return; - } - - // make sure we still have retries left - if retries == 0 { - let _ = response.send(Err(Error::RequestTimeout)); - return; - } - - if matches!(err, Error::RequestTimeout) { - tx.send(Message::RotateEndpoint) - .await - .expect("Failed to send rotate message"); - } - - tx.send(Message::Subscribe { - subscribe, - params, - unsubscribe, - response, - retries, - }) - .await - .expect("Failed to send subscribe message") + let _ = response.send(result); + } + Err(err) => { + tracing::debug!("Subscribe failed: {err:?}"); + match err { + Error::RequestTimeout + | Error::Transport(_) + | Error::RestartNeeded(_) + | Error::MaxSlotsExceeded => { + // Make sure endpoint is rotated + rotation_notify.notified().await; + + tokio::time::sleep(get_backoff_time(&request_backoff_counter)).await; + + // make sure it's still connected + if response.is_closed() { + return; + } + + // make sure we still have retries left + if retries == 0 { + let _ = response.send(Err(Error::RequestTimeout)); + return; } - err => { - // make sure it's still connected - if response.is_closed() { - return; - } - // not something we can handle, send it back to the caller - let _ = response.send(Err(err)); + + tx.send(Message::Subscribe { + subscribe, + params, + unsubscribe, + response, + retries, + }) + .await + .expect("Failed to send subscribe message") + } + err => { + // make sure it's still connected + if response.is_closed() { + return; } + // not something we can handle, send it back to the caller + let _ = response.send(Err(err)); } } } - } else { - tracing::error!("subscribe timed out subscribe: {} params: {:?}", subscribe, params); - // make sure it's still connected - if response.is_closed() { - return; - } - let _ = response.send(Err(Error::RequestTimeout)); } } Message::RotateEndpoint => { @@ -338,20 +365,25 @@ impl Client { loop { tokio::select! { - _ = ws.on_disconnect() => { - tracing::info!("Endpoint disconnected"); - tokio::time::sleep(get_backoff_time(&connect_backoff_counter)).await; - ws = build_ws().await; + _ = selected_endpoint.health().unhealthy() => { + // Current selected endpoint is unhealthy, try to rotate to another one. + // In case of all endpoints are unhealthy, we don't want to keep rotating but stick with the healthiest one. + let new_selected_endpoint = healthiest_endpoint(None).await; + if new_selected_endpoint.url() != selected_endpoint.url() { + tracing::warn!("Switch to endpoint: {new_url}", new_url=new_selected_endpoint.url()); + selected_endpoint = new_selected_endpoint; + rotation_notify_bg.notify_waiters(); + } } message = message_rx.recv() => { tracing::trace!("Received message {message:?}"); match message { Some(Message::RotateEndpoint) => { + tracing::info!("Rotating endpoint ..."); + selected_endpoint = healthiest_endpoint(Some(selected_endpoint.clone())).await; rotation_notify_bg.notify_waiters(); - tracing::info!("Rotate endpoint"); - ws = build_ws().await; } - Some(message) => handle_message(message, ws.clone()), + Some(message) => handle_message(message, selected_endpoint.clone(), rotation_notify_bg.clone()), None => { tracing::debug!("Client dropped"); break; @@ -362,10 +394,6 @@ impl Client { } }); - if let Some(0) = retries { - return Err(anyhow!("Retries need to be at least 1")); - } - Ok(Self { sender: message_tx, rotation_notify, @@ -375,7 +403,7 @@ impl Client { } pub fn with_endpoints(endpoints: impl IntoIterator>) -> Result { - Self::new(endpoints, None, None, None) + Self::new(endpoints, None, None, None, None) } pub async fn request(&self, method: &str, params: Vec) -> CallResult { @@ -435,7 +463,7 @@ impl Client { } } -fn get_backoff_time(counter: &Arc) -> Duration { +pub fn get_backoff_time(counter: &Arc) -> Duration { let min_time = 100u64; let step = 100u64; let max_count = 10u32; @@ -465,3 +493,112 @@ fn test_get_backoff_time() { vec![100, 200, 500, 1000, 1700, 2600, 3700, 5000, 6500, 8200, 10100, 10100] ); } + +#[test] +fn health_response_serialize_deserialize_works() { + let response = HealthResponse::Contains(vec![( + "isSyncing".to_string(), + Box::new(HealthResponse::Eq(false.into())), + )]); + + let expected = serde_yaml::from_str::( + r" + !contains + - - isSyncing + - !eq false + ", + ) + .unwrap(); + + assert_eq!(response, expected); +} + +#[test] +fn health_response_validation_works() { + use serde_json::json; + + let expected = serde_yaml::from_str::( + r" + !eq true + ", + ) + .unwrap(); + assert!(expected.validate(&json!(true))); + assert!(!expected.validate(&json!(false))); + + let expected = serde_yaml::from_str::( + r" + !contains + - - isSyncing + - !eq false + ", + ) + .unwrap(); + let cases = [ + (json!({ "isSyncing": false }), true), + (json!({ "isSyncing": true }), false), + (json!({ "isSyncing": false, "peers": 2 }), true), + (json!({ "isSyncing": true, "peers": 2 }), false), + (json!({}), false), + (json!(true), false), + ]; + for (input, output) in cases { + assert_eq!(expected.validate(&input), output); + } + + // multiple items + let expected = serde_yaml::from_str::( + r" + !contains + - - isSyncing + - !eq false + - - peers + - !eq 3 + ", + ) + .unwrap(); + let cases = [ + (json!({ "isSyncing": false, "peers": 3 }), true), + (json!({ "isSyncing": false, "peers": 2 }), false), + (json!({ "isSyncing": true, "peers": 3 }), false), + ]; + for (input, output) in cases { + assert_eq!(expected.validate(&input), output); + } + + // works with strings + let expected = serde_yaml::from_str::( + r" + !contains + - - foo + - !eq bar + ", + ) + .unwrap(); + assert!(expected.validate(&json!({ "foo": "bar" }))); + assert!(!expected.validate(&json!({ "foo": "bar bar" }))); + + // multiple nested items + let expected = serde_yaml::from_str::( + r" + !contains + - - foo + - !contains + - - one + - !eq subway + - - two + - !not_eq subway + ", + ) + .unwrap(); + let cases = [ + (json!({ "foo": { "one": "subway", "two": "not_subway" } }), true), + (json!({ "foo": { "one": "subway", "two": "subway" } }), false), + (json!({ "foo": { "subway": "one" } }), false), + (json!({ "bar" : { "foo": { "subway": "one", "two": "subway" } }}), false), + (json!({ "foo": "subway" }), false), + ]; + for (input, output) in cases { + assert_eq!(expected.validate(&input), output); + } +} diff --git a/src/extensions/client/tests.rs b/src/extensions/client/tests.rs index c8c9c7b..c76a75d 100644 --- a/src/extensions/client/tests.rs +++ b/src/extensions/client/tests.rs @@ -152,6 +152,7 @@ async fn retry_requests_successful() { Some(Duration::from_millis(100)), None, Some(2), + None, ) .unwrap(); @@ -189,6 +190,7 @@ async fn retry_requests_out_of_retries() { Some(Duration::from_millis(100)), None, Some(2), + None, ) .unwrap(); @@ -216,3 +218,75 @@ async fn retry_requests_out_of_retries() { handle1.stop().unwrap(); handle2.stop().unwrap(); } + +#[tokio::test] +async fn health_check_works() { + let (addr1, handle1) = dummy_server_extend(Box::new(|builder| { + let mut system_health = builder.register_method("system_health"); + tokio::spawn(async move { + loop { + tokio::select! { + Some(req) = system_health.recv() => { + req.respond(json!({ "isSyncing": true, "peers": 1, "shouldHavePeers": true })); + } + } + } + }); + })) + .await; + + let (addr2, handle2) = dummy_server_extend(Box::new(|builder| { + let mut system_health = builder.register_method("system_health"); + tokio::spawn(async move { + loop { + tokio::select! { + Some(req) = system_health.recv() => { + req.respond(json!({ "isSyncing": false, "peers": 1, "shouldHavePeers": true })); + } + } + } + }); + })) + .await; + + let client = Client::new( + [format!("ws://{addr1}"), format!("ws://{addr2}")], + None, + None, + None, + Some(HealthCheckConfig { + interval_sec: 1, + healthy_response_time_ms: 250, + health_method: Some("system_health".into()), + response: Some(HealthResponse::Contains(vec![( + "isSyncing".to_string(), + Box::new(HealthResponse::Eq(false.into())), + )])), + }), + ) + .unwrap(); + + // first endpoint is stale + let res = client.request("system_health", vec![]).await; + assert_eq!( + res.unwrap(), + json!({ "isSyncing": true, "peers": 1, "shouldHavePeers": true }) + ); + + // wait for the health check to run + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(1_050)).await; + }) + .await + .unwrap(); + + // second endpoint is healthy + let res = client.request("system_health", vec![]).await; + assert_eq!( + res.unwrap(), + json!({ "isSyncing": false, "peers": 1, "shouldHavePeers": true }) + ); + + handle1.stop().unwrap(); + handle2.stop().unwrap(); +} diff --git a/src/server.rs b/src/server.rs index 0994de3..824a977 100644 --- a/src/server.rs +++ b/src/server.rs @@ -239,6 +239,7 @@ mod tests { client: Some(ClientConfig { endpoints: vec![endpoint], shuffle_endpoints: false, + health_check: None, }), server: Some(ServerConfig { listen_address: "127.0.0.1".to_string(), diff --git a/src/tests/merge_subscription.rs b/src/tests/merge_subscription.rs index a41bb4b..37a7c53 100644 --- a/src/tests/merge_subscription.rs +++ b/src/tests/merge_subscription.rs @@ -49,6 +49,7 @@ async fn merge_subscription_works() { client: Some(ClientConfig { endpoints: vec![format!("ws://{addr}")], shuffle_endpoints: false, + health_check: None, }), server: Some(ServerConfig { listen_address: "0.0.0.0".to_string(), diff --git a/src/tests/upstream.rs b/src/tests/upstream.rs index 84c38f5..9c730da 100644 --- a/src/tests/upstream.rs +++ b/src/tests/upstream.rs @@ -31,6 +31,7 @@ async fn upstream_error_propagate() { client: Some(ClientConfig { endpoints: vec![format!("ws://{addr}")], shuffle_endpoints: false, + health_check: None, }), server: Some(ServerConfig { listen_address: "0.0.0.0".to_string(), From ef1c52421ebedc3594415814e9525f21f14bd922 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Wed, 10 Apr 2024 18:20:11 +0200 Subject: [PATCH 05/16] Await healthy endpoint (#158) --- src/extensions/client/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/extensions/client/mod.rs b/src/extensions/client/mod.rs index e016467..0e20ffb 100644 --- a/src/extensions/client/mod.rs +++ b/src/extensions/client/mod.rs @@ -5,6 +5,7 @@ use std::{ use anyhow::anyhow; use async_trait::async_trait; +use futures::FutureExt as Boxed; use jsonrpsee::core::{client::Subscription, Error, JsonValue}; use opentelemetry::trace::FutureExt; use rand::{seq::SliceRandom, thread_rng}; @@ -204,13 +205,12 @@ impl Client { if let Some(exclude) = exclude { endpoints.retain(|e| e.url() != exclude.url()); } + // wait for at least one endpoint to connect + futures::future::select_all(endpoints.iter().map(|x| x.connected().boxed())).await; // Sort by health score endpoints.sort_by_key(|endpoint| std::cmp::Reverse(endpoint.health().score())); // Pick the first one - let selected_endpoint = endpoints[0].clone(); - // Ensure it's connected - selected_endpoint.connected().await; - selected_endpoint + endpoints[0].clone() }; let mut selected_endpoint = healthiest_endpoint(None).await; From 6ea24ab9ceb3d60492bae6e13f969f98c1270415 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Wed, 10 Apr 2024 18:20:30 +0200 Subject: [PATCH 06/16] use only tracing (#157) --- Cargo.lock | 1 - Cargo.toml | 5 ++--- src/config/mod.rs | 4 ++-- src/extensions/client/health.rs | 6 +++--- src/extensions/telemetry/mod.rs | 2 +- src/middlewares/methods/block_tag.rs | 2 +- 6 files changed, 9 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b9e0d53..3c9bf09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3083,7 +3083,6 @@ dependencies = [ "jsonrpc-pubsub", "jsonrpc-ws-server", "jsonrpsee", - "log", "moka", "opentelemetry", "opentelemetry-datadog", diff --git a/Cargo.toml b/Cargo.toml index 46ddcaa..12f717f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,6 @@ enumflags2 = "0.7.7" futures = "0.3.25" http = "0.2" hyper = "0.14" -log = "0.4.17" moka = { version = "0.12", features = ["future"] } opentelemetry = { version = "0.21.0" } opentelemetry-datadog = { version = "0.9.0", features = ["reqwest-client"] } @@ -39,9 +38,9 @@ serde_yaml = "0.9.17" tokio = { version = "1.24.2", features = ["full"] } tower = { version = "0.4.13", features = ["full"] } tower-http = { version = "0.4", features = ["full"] } -tracing = "0.1.37" +tracing = "0.1.40" tracing-serde = "0.1.3" -tracing-subscriber = { version = "0.3.16", features = ["env-filter", "json"] } +tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json"] } jsonrpsee = { path = "./vendor/jsonrpsee/jsonrpsee", features = ["full"] } governor = { path = "./vendor/governor/governor" } diff --git a/src/config/mod.rs b/src/config/mod.rs index c11095c..9c4d9ad 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -156,7 +156,7 @@ pub fn read_config() -> Result { let mut config: Config = config.into(); if let Ok(endpoints) = std::env::var("ENDPOINTS") { - log::debug!("Override endpoints with env.ENDPOINTS"); + tracing::debug!("Override endpoints with env.ENDPOINTS"); let endpoints = endpoints .split(',') .map(|x| x.trim().to_string()) @@ -171,7 +171,7 @@ pub fn read_config() -> Result { } if let Ok(env_port) = std::env::var("PORT") { - log::debug!("Override port with env.PORT"); + tracing::debug!("Override port with env.PORT"); let port = env_port.parse::(); if let Ok(port) = port { config diff --git a/src/extensions/client/health.rs b/src/extensions/client/health.rs index acdef15..832395b 100644 --- a/src/extensions/client/health.rs +++ b/src/extensions/client/health.rs @@ -64,20 +64,20 @@ impl Health { return; } self.score.store(new_score, Ordering::Relaxed); - log::trace!( + tracing::trace!( "Endpoint {:?} score updated from: {current_score} to: {new_score}", self.url ); // Notify waiters if the score has dropped below the threshold if current_score >= THRESHOLD && new_score < THRESHOLD { - log::warn!("Endpoint {:?} became unhealthy", self.url); + tracing::warn!("Endpoint {:?} became unhealthy", self.url); self.unhealthy.notify_waiters(); } } pub fn on_error(&self, err: &jsonrpsee::core::Error) { - log::warn!("Endpoint {:?} responded with error: {err:?}", self.url); + tracing::warn!("Endpoint {:?} responded with error: {err:?}", self.url); match err { jsonrpsee::core::Error::RequestTimeout => { self.update(Event::RequestTimeout); diff --git a/src/extensions/telemetry/mod.rs b/src/extensions/telemetry/mod.rs index 4426deb..c046f3c 100644 --- a/src/extensions/telemetry/mod.rs +++ b/src/extensions/telemetry/mod.rs @@ -51,7 +51,7 @@ impl Telemetry { pub fn setup_telemetry(options: &TelemetryConfig) -> Result, TraceError> { global::set_error_handler(|e| { - log::warn!("OpenTelemetry error: {}", e); + tracing::warn!("OpenTelemetry error: {}", e); }) .expect("failed to set OpenTelemetry error handler"); diff --git a/src/middlewares/methods/block_tag.rs b/src/middlewares/methods/block_tag.rs index b71b1ba..ed8e657 100644 --- a/src/middlewares/methods/block_tag.rs +++ b/src/middlewares/methods/block_tag.rs @@ -94,7 +94,7 @@ impl BlockTagMiddleware { }; if let Some(value) = maybe_value { - log::trace!( + tracing::trace!( "Replacing params {:?} updated with {:?}", request.params, (self.index, &value), From 354f9171b76e711cb44bbb6a2de22c90f7f84e7a Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Thu, 11 Apr 2024 11:39:51 +0200 Subject: [PATCH 07/16] Filter call errors (#159) --- src/extensions/client/health.rs | 17 +++++++++-------- src/server.rs | 14 +++++++++++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/extensions/client/health.rs b/src/extensions/client/health.rs index 832395b..7dd5612 100644 --- a/src/extensions/client/health.rs +++ b/src/extensions/client/health.rs @@ -14,7 +14,7 @@ pub enum Event { SlowResponse, RequestTimeout, ConnectionSuccessful, - ConnectionFailed, + ServerError, StaleChain, } @@ -24,7 +24,7 @@ impl Event { match self { Event::ResponseOk => current.saturating_add(2), Event::SlowResponse => current.saturating_sub(5), - Event::RequestTimeout | Event::ConnectionFailed | Event::StaleChain => 0, + Event::RequestTimeout | Event::ServerError | Event::StaleChain => 0, Event::ConnectionSuccessful => MAX_SCORE / 5 * 4, // 80% of max score }, MAX_SCORE, @@ -77,17 +77,18 @@ impl Health { } pub fn on_error(&self, err: &jsonrpsee::core::Error) { - tracing::warn!("Endpoint {:?} responded with error: {err:?}", self.url); match err { + jsonrpsee::core::Error::Call(_) => { + // NOT SERVER ERROR + } jsonrpsee::core::Error::RequestTimeout => { + tracing::warn!("Endpoint {:?} request timeout", self.url); self.update(Event::RequestTimeout); } - jsonrpsee::core::Error::Transport(_) - | jsonrpsee::core::Error::RestartNeeded(_) - | jsonrpsee::core::Error::MaxSlotsExceeded => { - self.update(Event::ConnectionFailed); + _ => { + tracing::warn!("Endpoint {:?} responded with error: {err:?}", self.url); + self.update(Event::ServerError); } - _ => {} }; } diff --git a/src/server.rs b/src/server.rs index 824a977..b43bc6a 100644 --- a/src/server.rs +++ b/src/server.rs @@ -4,6 +4,7 @@ use futures::FutureExt; use jsonrpsee::{ core::JsonValue, server::{RpcModule, ServerHandle}, + types::error::INTERNAL_ERROR_CODE, types::ErrorObjectOwned, }; use opentelemetry::trace::FutureExt as _; @@ -96,16 +97,23 @@ pub async fn build(config: Config) -> anyhow::Result { let result = result_rx .await - .map_err(|_| errors::map_error(jsonrpsee::core::Error::RequestTimeout))?; + .map_err(|_| errors::map_error(jsonrpsee::core::Error::RequestTimeout)); match result.as_ref() { - Ok(_) => tracer.span_ok(), + Ok(Ok(_)) => tracer.span_ok(), + Ok(Err(err)) => { + if err.code() == INTERNAL_ERROR_CODE { + tracer.span_error(err) + } else { + tracer.span_ok() + } + } Err(err) => { tracer.span_error(err); } }; - result + result? } .with_context(tracer.context(method_name)) })?; From 50b826d4101c694213f67ebefb533467424bd2b2 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Tue, 16 Apr 2024 03:45:32 +0200 Subject: [PATCH 08/16] validate middleware (#156) * validate middleware * validate extension for config * rename --- README.md | 25 ++++++++++- src/extensions/client/mod.rs | 7 +++ src/extensions/mod.rs | 2 + src/extensions/validator/mod.rs | 67 +++++++++++++++++++++++++++++ src/middlewares/factory.rs | 1 + src/middlewares/methods/mod.rs | 1 + src/middlewares/methods/validate.rs | 52 ++++++++++++++++++++++ src/middlewares/mod.rs | 3 +- 8 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 src/extensions/validator/mod.rs create mode 100644 src/middlewares/methods/validate.rs diff --git a/README.md b/README.md index 3de9e1e..bbeaffc 100644 --- a/README.md +++ b/README.md @@ -75,4 +75,27 @@ It's also possible to run individual benchmarks by: ``` cargo bench --bench bench ws_round_trip -``` \ No newline at end of file +``` + +## Validate Middleware + +This middleware will intercept all method request/responses and compare the result directly with healthy endpoint responses. +This is useful for debugging to make sure the returned values are as expected. +You can enable validate middleware on your config file. +```yml +middlewares: + methods: + - validate +``` +NOTE: Keep in mind that if you place `validate` middleware before `inject_params` you may get false positive errors because the request will not be the same. + +Ignored methods can be defined in extension config: +```yml +extensions: + validator: + ignore_methods: + - system_health + - system_name + - system_version + - author_pendingExtrinsics +``` diff --git a/src/extensions/client/mod.rs b/src/extensions/client/mod.rs index 0e20ffb..9b7a5e6 100644 --- a/src/extensions/client/mod.rs +++ b/src/extensions/client/mod.rs @@ -31,6 +31,7 @@ mod tests; const TRACER: utils::telemetry::Tracer = utils::telemetry::Tracer::new("client"); pub struct Client { + endpoints: Vec>, sender: tokio::sync::mpsc::Sender, rotation_notify: Arc, retries: u32, @@ -187,6 +188,7 @@ impl Client { let rotation_notify = Arc::new(Notify::new()); let rotation_notify_bg = rotation_notify.clone(); + let endpoints_ = endpoints.clone(); let background_task = tokio::spawn(async move { let request_backoff_counter = Arc::new(AtomicU32::new(0)); @@ -395,6 +397,7 @@ impl Client { }); Ok(Self { + endpoints: endpoints_, sender: message_tx, rotation_notify, retries: retries.unwrap_or(3), @@ -406,6 +409,10 @@ impl Client { Self::new(endpoints, None, None, None, None) } + pub fn endpoints(&self) -> &Vec> { + self.endpoints.as_ref() + } + pub async fn request(&self, method: &str, params: Vec) -> CallResult { async move { let (tx, rx) = tokio::sync::oneshot::channel(); diff --git a/src/extensions/mod.rs b/src/extensions/mod.rs index 2cf1880..4959199 100644 --- a/src/extensions/mod.rs +++ b/src/extensions/mod.rs @@ -16,6 +16,7 @@ pub mod merge_subscription; pub mod rate_limit; pub mod server; pub mod telemetry; +pub mod validator; #[async_trait] pub trait Extension: Sized { @@ -138,4 +139,5 @@ define_all_extensions! { server: server::SubwayServerBuilder, event_bus: event_bus::EventBus, rate_limit: rate_limit::RateLimitBuilder, + validator: validator::Validator, } diff --git a/src/extensions/validator/mod.rs b/src/extensions/validator/mod.rs new file mode 100644 index 0000000..00ee5e8 --- /dev/null +++ b/src/extensions/validator/mod.rs @@ -0,0 +1,67 @@ +use crate::extensions::client::Client; +use crate::middlewares::{CallRequest, CallResult}; +use crate::utils::errors; +use async_trait::async_trait; +use serde::Deserialize; +use std::sync::Arc; + +use super::{Extension, ExtensionRegistry}; + +#[derive(Default)] +pub struct Validator { + pub config: ValidateConfig, +} + +#[derive(Deserialize, Default, Debug, Clone)] +pub struct ValidateConfig { + pub ignore_methods: Vec, +} + +#[async_trait] +impl Extension for Validator { + type Config = ValidateConfig; + + async fn from_config(config: &Self::Config, _registry: &ExtensionRegistry) -> Result { + Ok(Self::new(config.clone())) + } +} + +impl Validator { + pub fn new(config: ValidateConfig) -> Self { + Self { config } + } + + pub fn ignore(&self, method: &String) -> bool { + self.config.ignore_methods.contains(method) + } + + pub fn validate(&self, client: Arc, request: CallRequest, response: CallResult) { + tokio::spawn(async move { + let healthy_endpoints = client.endpoints().iter().filter(|x| x.health().score() > 0); + futures::future::join_all(healthy_endpoints.map(|endpoint| async { + let expected = endpoint + .request( + &request.method, + request.params.clone(), + std::time::Duration::from_secs(30), + ) + .await + .map_err(errors::map_error); + + if response != expected { + let request = serde_json::to_string_pretty(&request).unwrap_or_default(); + let actual = match &response { + Ok(value) => serde_json::to_string_pretty(&value).unwrap_or_default(), + Err(e) => e.to_string() + }; + let expected = match &expected { + Ok(value) => serde_json::to_string_pretty(&value).unwrap_or_default(), + Err(e) => e.to_string() + }; + let endpoint_url = endpoint.url(); + tracing::error!("Response mismatch for request:\n{request}\nSubway response:\n{actual}\nEndpoint {endpoint_url} response:\n{expected}"); + } + })).await; + }); + } +} diff --git a/src/middlewares/factory.rs b/src/middlewares/factory.rs index d92da85..2ac12bc 100644 --- a/src/middlewares/factory.rs +++ b/src/middlewares/factory.rs @@ -30,6 +30,7 @@ pub async fn create_method_middleware( "block_tag" => block_tag::BlockTagMiddleware::build(method, extensions).await, "inject_params" => inject_params::InjectParamsMiddleware::build(method, extensions).await, "delay" => delay::DelayMiddleware::build(method, extensions).await, + "validate" => validate::ValidateMiddleware::build(method, extensions).await, #[cfg(test)] "crazy" => testing::CrazyMiddleware::build(method, extensions).await, _ => panic!("Unknown method middleware: {}", name), diff --git a/src/middlewares/methods/mod.rs b/src/middlewares/methods/mod.rs index 20be5e1..420c290 100644 --- a/src/middlewares/methods/mod.rs +++ b/src/middlewares/methods/mod.rs @@ -4,6 +4,7 @@ pub mod delay; pub mod inject_params; pub mod response; pub mod upstream; +pub mod validate; #[cfg(test)] pub mod testing; diff --git a/src/middlewares/methods/validate.rs b/src/middlewares/methods/validate.rs new file mode 100644 index 0000000..24e1971 --- /dev/null +++ b/src/middlewares/methods/validate.rs @@ -0,0 +1,52 @@ +use async_trait::async_trait; +use std::sync::Arc; + +use crate::{ + extensions::{client::Client, validator::Validator}, + middlewares::{CallRequest, CallResult, Middleware, MiddlewareBuilder, NextFn, RpcMethod}, + utils::{TypeRegistry, TypeRegistryRef}, +}; + +pub struct ValidateMiddleware { + validator: Arc, + client: Arc, +} + +impl ValidateMiddleware { + pub fn new(validator: Arc, client: Arc) -> Self { + Self { validator, client } + } +} + +#[async_trait] +impl MiddlewareBuilder for ValidateMiddleware { + async fn build( + _method: &RpcMethod, + extensions: &TypeRegistryRef, + ) -> Option>> { + let validate = extensions.read().await.get::().unwrap_or_default(); + + let client = extensions + .read() + .await + .get::() + .expect("Client extension not found"); + Some(Box::new(ValidateMiddleware::new(validate, client))) + } +} + +#[async_trait] +impl Middleware for ValidateMiddleware { + async fn call( + &self, + request: CallRequest, + context: TypeRegistry, + next: NextFn, + ) -> CallResult { + let result = next(request.clone(), context).await; + if !self.validator.ignore(&request.method) { + self.validator.validate(self.client.clone(), request, result.clone()); + } + result + } +} diff --git a/src/middlewares/mod.rs b/src/middlewares/mod.rs index 03871b9..27656b8 100644 --- a/src/middlewares/mod.rs +++ b/src/middlewares/mod.rs @@ -6,6 +6,7 @@ use jsonrpsee::{ PendingSubscriptionSink, }; use opentelemetry::trace::FutureExt as _; +use serde::Serialize; use std::{ fmt::{Debug, Formatter}, sync::Arc, @@ -20,7 +21,7 @@ pub mod factory; pub mod methods; pub mod subscriptions; -#[derive(Debug)] +#[derive(Clone, Debug, Serialize)] /// Represents a RPC request made to a middleware function. pub struct CallRequest { pub method: String, From 7b5747ee7ddb2abf8b6ed00914c7fe24c10ebb87 Mon Sep 17 00:00:00 2001 From: Jiyuan Zheng Date: Mon, 22 Apr 2024 14:30:36 +0800 Subject: [PATCH 09/16] Improve env handling via preprocessing templated config files (#162) --- Cargo.lock | 5 ++- Cargo.toml | 1 + README.md | 9 +++-- src/config/mod.rs | 99 +++++++++++++++++++++++++++-------------------- src/main.rs | 7 +--- 5 files changed, 69 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c9bf09..57d940f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2597,9 +2597,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.3" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", @@ -3090,6 +3090,7 @@ dependencies = [ "opentelemetry_sdk", "pprof", "rand 0.8.5", + "regex", "serde", "serde_json", "serde_yaml", diff --git a/Cargo.toml b/Cargo.toml index 12f717f..ca0e77f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ opentelemetry-jaeger = { version = "0.20.0", features = ["rt-tokio"] } opentelemetry_sdk = { version = "0.21.1", features = ["rt-tokio", "trace"] } rand = "0.8.5" +regex = "1.10.4" serde = "1.0.152" serde_json = "1.0.92" serde_yaml = "0.9.17" diff --git a/README.md b/README.md index bbeaffc..aefb7cf 100644 --- a/README.md +++ b/README.md @@ -20,12 +20,12 @@ Run with `RUSTFLAGS="--cfg tokio_unstable"` to enable [tokio-console](https://gi - `RUST_LOG` - Log level. Default: `info`. -- `PORT` - - Override port configuration in config file. - `LOG_FORMAT` - Log format. Default: `full`. - Options: `full`, `pretty`, `json`, `compact` +In addition, you can refer env variables in `config.yml` by using `${SOME_ENV}` + ## Features Subway is build with middleware pattern. @@ -62,7 +62,7 @@ Subway is build with middleware pattern. - TODO: Limit batch size, request size and response size. - TODO: Metrics - Getting insights of the RPC calls and server performance. - + ## Benchmarks To run all benchmarks: @@ -82,14 +82,17 @@ cargo bench --bench bench ws_round_trip This middleware will intercept all method request/responses and compare the result directly with healthy endpoint responses. This is useful for debugging to make sure the returned values are as expected. You can enable validate middleware on your config file. + ```yml middlewares: methods: - validate ``` + NOTE: Keep in mind that if you place `validate` middleware before `inject_params` you may get false positive errors because the request will not be the same. Ignored methods can be defined in extension config: + ```yml extensions: validator: diff --git a/src/config/mod.rs b/src/config/mod.rs index 9c4d9ad..41fd6fa 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1,3 +1,6 @@ +use anyhow::{bail, Context}; +use regex::{Captures, Regex}; +use std::env; use std::fs; use clap::Parser; @@ -147,43 +150,18 @@ impl From for Config { } // read config file specified in command line -pub fn read_config() -> Result { +pub fn read_config() -> Result { let cmd = Command::parse(); - let config = fs::File::open(cmd.config).map_err(|e| format!("Unable to open config file: {e}"))?; - let config: ParseConfig = - serde_yaml::from_reader(&config).map_err(|e| format!("Unable to parse config file: {e}"))?; - let mut config: Config = config.into(); - - if let Ok(endpoints) = std::env::var("ENDPOINTS") { - tracing::debug!("Override endpoints with env.ENDPOINTS"); - let endpoints = endpoints - .split(',') - .map(|x| x.trim().to_string()) - .collect::>(); - - config - .extensions - .client - .as_mut() - .expect("Client extension not configured") - .endpoints = endpoints; - } + let templated_config_str = + fs::read_to_string(&cmd.config).with_context(|| format!("Unable to read config file: {}", cmd.config))?; - if let Ok(env_port) = std::env::var("PORT") { - tracing::debug!("Override port with env.PORT"); - let port = env_port.parse::(); - if let Ok(port) = port { - config - .extensions - .server - .as_mut() - .expect("Server extension not configured") - .port = port; - } else { - return Err(format!("Invalid port: {}", env_port)); - } - } + let config_str = render_template(&templated_config_str) + .with_context(|| format!("Unable to preprocess config file: {}", cmd.config))?; + + let config: ParseConfig = + serde_yaml::from_str(&config_str).with_context(|| format!("Unable to parse config file: {}", cmd.config))?; + let config: Config = config.into(); // TODO: shouldn't need to do this here. Creating a server should validates everything validate_config(&config)?; @@ -191,19 +169,42 @@ pub fn read_config() -> Result { Ok(config) } -fn validate_config(config: &Config) -> Result<(), String> { +fn render_template(templated_config_str: &str) -> Result { + // match pattern: ${SOME_VAR} + let re = Regex::new(r"\$\{([^\}]+)\}").unwrap(); + + let mut config_str = String::with_capacity(templated_config_str.len()); + let mut last_match = 0; + // replace pattern: with env variables + let replacement = |caps: &Captures| -> Result { env::var(&caps[1]) }; + + // replace every matches with early return + // when encountering error + for caps in re.captures_iter(templated_config_str) { + let m = caps.get(0).expect("Matched pattern should have at least one capture"); + config_str.push_str(&templated_config_str[last_match..m.start()]); + config_str.push_str( + &replacement(&caps).with_context(|| format!("Unable to replace environment variable {}", &caps[1]))?, + ); + last_match = m.end(); + } + config_str.push_str(&templated_config_str[last_match..]); + Ok(config_str) +} + +fn validate_config(config: &Config) -> Result<(), anyhow::Error> { // TODO: validate logic should be in each individual extensions // validate endpoints for endpoint in &config.extensions.client.as_ref().unwrap().endpoints { if endpoint.parse::().is_err() { - return Err(format!("Invalid endpoint {}", endpoint)); + bail!("Invalid endpoint {}", endpoint); } } // ensure each method has only one param with inject=true for method in &config.rpcs.methods { if method.params.iter().filter(|x| x.inject).count() > 1 { - return Err(format!("Method {} has more than one inject param", method.method)); + bail!("Method {} has more than one inject param", method.method); } } @@ -214,13 +215,29 @@ fn validate_config(config: &Config) -> Result<(), String> { if param.optional { has_optional = true; } else if has_optional { - return Err(format!( - "Method {} has required param after optional param", - method.method - )); + bail!("Method {} has required param after optional param", method.method); } } } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn render_template_basically_works() { + env::set_var("KEY", "value"); + env::set_var("ANOTHER_KEY", "another_value"); + let templated_config_str = "${KEY} ${ANOTHER_KEY}"; + let config_str = render_template(templated_config_str).unwrap(); + assert_eq!(config_str, "value another_value"); + + env::remove_var("KEY"); + let config_str = render_template(templated_config_str); + assert!(config_str.is_err()); + env::remove_var("ANOTHER_KEY"); + } +} diff --git a/src/main.rs b/src/main.rs index aab4d5b..26da8b4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,12 +1,7 @@ #[tokio::main] async fn main() -> anyhow::Result<()> { // read config from file - let config = match subway::config::read_config() { - Ok(config) => config, - Err(e) => { - return Err(anyhow::anyhow!(e)); - } - }; + let config = subway::config::read_config()?; subway::logger::enable_logger(); tracing::trace!("{:#?}", config); From a4f5998115b4463daba97f99870a4a82076b8a03 Mon Sep 17 00:00:00 2001 From: Jiyuan Zheng Date: Tue, 23 Apr 2024 13:07:49 +0800 Subject: [PATCH 10/16] Feat(config): support ${var:-word} and ${var:+word} syntax (#164) --- README.md | 10 ++- config.yml => configs/config.yml | 0 configs/config_with_env.yml | 56 ++++++++++++++ eth_config.yml => configs/eth_config.yml | 0 src/cli.rs | 12 +++ src/config/mod.rs | 97 ++++++++++++++++++------ src/lib.rs | 1 + src/main.rs | 3 +- 8 files changed, 153 insertions(+), 26 deletions(-) rename config.yml => configs/config.yml (100%) create mode 100644 configs/config_with_env.yml rename eth_config.yml => configs/eth_config.yml (100%) create mode 100644 src/cli.rs diff --git a/README.md b/README.md index aefb7cf..01c6794 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ This is a generalized JSON RPC proxy server with features specifically designed Pull vendors: `git submodule update --init --recursive` -Quick start: `cargo run -- --config config.yml` +Quick start: `cargo run -- --config configs/config.yml` -This will run a proxy server with [config.yml](config.yml) as the configuration file. +This will run a proxy server with [config.yml](configs/config.yml) as the configuration file. Run with `RUSTFLAGS="--cfg tokio_unstable"` to enable [tokio-console](https://github.com/tokio-rs/console) @@ -24,7 +24,11 @@ Run with `RUSTFLAGS="--cfg tokio_unstable"` to enable [tokio-console](https://gi - Log format. Default: `full`. - Options: `full`, `pretty`, `json`, `compact` -In addition, you can refer env variables in `config.yml` by using `${SOME_ENV}` +In addition, you can refer env variables in `config.yml` by using following syntax: + +- `${variable}` +- `${variable:-word}` indicates that if variable is set then the result will be that value. If variable is not set then word will be the result. +- `${variable:+word}` indicates that if variable is set then word will be the result, otherwise the result is the empty string. ## Features diff --git a/config.yml b/configs/config.yml similarity index 100% rename from config.yml rename to configs/config.yml diff --git a/configs/config_with_env.yml b/configs/config_with_env.yml new file mode 100644 index 0000000..ff70c97 --- /dev/null +++ b/configs/config_with_env.yml @@ -0,0 +1,56 @@ +extensions: + client: + endpoints: + - wss://acala-rpc.dwellir.com + - wss://acala-rpc-0.aca-api.network + health_check: + interval_sec: 10 # check interval, default is 10s + healthy_response_time_ms: 500 # max response time to be considered healthy, default is 500ms + health_method: system_health + response: # response contains { isSyncing: false } + !contains + - - isSyncing + - !eq false + event_bus: + substrate_api: + stale_timeout_seconds: 180 # rotate endpoint if no new blocks for 3 minutes + telemetry: + provider: none + cache: + default_ttl_seconds: 60 + default_size: 500 + merge_subscription: + keep_alive_seconds: 60 + server: + port: ${SUBWAY_PORT:-9944} + listen_address: '0.0.0.0' + max_connections: ${SUBWAY_MAX_CONNECTIONS:-2000} + http_methods: + - path: /health + method: system_health + - path: /liveness + method: chain_getBlockHash + cors: all + rate_limit: # these are for demo purpose only, please adjust to your needs + connection: # 20 RPC requests per second per connection + burst: 20 + period_secs: 1 + ip: # 500 RPC requests per 10 seconds per ip + burst: 500 + period_secs: 10 + # use X-Forwarded-For header to get real ip, if available (e.g. behind a load balancer). + # WARNING: Use with caution, as this xff header can be forged. + use_xff: true # default is false + +middlewares: + methods: + - delay + - response + - inject_params + - cache + - upstream + subscriptions: + - merge_subscription + - upstream + +rpcs: substrate diff --git a/eth_config.yml b/configs/eth_config.yml similarity index 100% rename from eth_config.yml rename to configs/eth_config.yml diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..4e32bbb --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,12 @@ +use clap::Parser; +use std::path::PathBuf; +#[derive(Parser, Debug)] +#[command(version, about)] +pub struct Command { + /// The config file to use + #[arg(short, long, default_value = ".configs/config.yml")] + pub config: PathBuf, +} +pub fn parse_args() -> Command { + Command::parse() +} diff --git a/src/config/mod.rs b/src/config/mod.rs index 41fd6fa..7366721 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -2,8 +2,8 @@ use anyhow::{bail, Context}; use regex::{Captures, Regex}; use std::env; use std::fs; +use std::path; -use clap::Parser; use serde::Deserialize; use crate::extensions::ExtensionsConfig; @@ -14,14 +14,6 @@ mod rpc; const SUBSTRATE_CONFIG: &str = include_str!("../../rpc_configs/substrate.yml"); const ETHEREUM_CONFIG: &str = include_str!("../../rpc_configs/ethereum.yml"); -#[derive(Parser, Debug)] -#[command(version, about)] -struct Command { - /// The config file to use - #[arg(short, long, default_value = "./config.yml")] - config: String, -} - #[derive(Deserialize, Debug)] pub struct RpcDefinitionsWithBase { #[serde(default)] @@ -150,17 +142,16 @@ impl From for Config { } // read config file specified in command line -pub fn read_config() -> Result { - let cmd = Command::parse(); - +pub fn read_config(path: impl AsRef) -> Result { + let path = path.as_ref(); let templated_config_str = - fs::read_to_string(&cmd.config).with_context(|| format!("Unable to read config file: {}", cmd.config))?; + fs::read_to_string(path).with_context(|| format!("Unable to read config file: {}", path.display()))?; let config_str = render_template(&templated_config_str) - .with_context(|| format!("Unable to preprocess config file: {}", cmd.config))?; + .with_context(|| format!("Unable to preprocess config file: {}", path.display()))?; - let config: ParseConfig = - serde_yaml::from_str(&config_str).with_context(|| format!("Unable to parse config file: {}", cmd.config))?; + let config: ParseConfig = serde_yaml::from_str(&config_str) + .with_context(|| format!("Unable to parse config file: {}", path.display()))?; let config: Config = config.into(); // TODO: shouldn't need to do this here. Creating a server should validates everything @@ -170,18 +161,37 @@ pub fn read_config() -> Result { } fn render_template(templated_config_str: &str) -> Result { - // match pattern: ${SOME_VAR} - let re = Regex::new(r"\$\{([^\}]+)\}").unwrap(); + // match pattern with 1 group: {variable_name} + // match pattern with 3 groups: {variable:-word} or {variable:+word} + // note: incompete syntax like {variable:-} will be matched since group1 is ungreedy match + // but typically it will be rejected due to there is not corresponding env vars + let re = Regex::new(r"\$\{([^}]+?)(?:(:-|:\+)([^}]+))?\}").unwrap(); let mut config_str = String::with_capacity(templated_config_str.len()); let mut last_match = 0; // replace pattern: with env variables - let replacement = |caps: &Captures| -> Result { env::var(&caps[1]) }; + let replacement = |caps: &Captures| -> Result { + match (caps.get(2), caps.get(3)) { + (Some(sign), Some(value_default)) => { + if sign.as_str() == ":-" { + env::var(&caps[1]).or(Ok(value_default.as_str().to_string())) + } else if sign.as_str() == ":+" { + Ok(env::var(&caps[1]).map_or("".to_string(), |_| value_default.as_str().to_string())) + } else { + Err(env::VarError::NotPresent) + } + } + (None, None) => env::var(&caps[1]), + _ => Err(env::VarError::NotPresent), + } + }; // replace every matches with early return // when encountering error for caps in re.captures_iter(templated_config_str) { - let m = caps.get(0).expect("Matched pattern should have at least one capture"); + let m = caps + .get(0) + .expect("i==0 means implicit unnamed group that includes the entire match, which is infalliable"); config_str.push_str(&templated_config_str[last_match..m.start()]); config_str.push_str( &replacement(&caps).with_context(|| format!("Unable to replace environment variable {}", &caps[1]))?, @@ -231,13 +241,56 @@ mod tests { fn render_template_basically_works() { env::set_var("KEY", "value"); env::set_var("ANOTHER_KEY", "another_value"); - let templated_config_str = "${KEY} ${ANOTHER_KEY}"; + let templated_config_str = "${KEY} some random_$tring {inside ${ANOTHER_KEY}"; let config_str = render_template(templated_config_str).unwrap(); - assert_eq!(config_str, "value another_value"); + assert_eq!(config_str, "value some random_$tring {inside another_value"); env::remove_var("KEY"); let config_str = render_template(templated_config_str); assert!(config_str.is_err()); env::remove_var("ANOTHER_KEY"); } + + #[test] + fn render_template_supports_minus_word_syntax() { + // ${variable:-word} indicates that if variable is set then the result will be that value. If variable is not set then word will be the result. + env::set_var("absent_key", "value_set"); + let templated_config_str = "${absent_key:-value_default}"; + let config_str = render_template(templated_config_str).unwrap(); + assert_eq!(config_str, "value_set"); + // remove the env + env::remove_var("absent_key"); + let config_str = render_template(templated_config_str).unwrap(); + assert_eq!(config_str, "value_default") + } + + #[test] + fn render_template_supports_plus_word_syntax() { + // ${variable:+word} indicates that if variable is set then word will be the result, otherwise the result is the empty string. + env::set_var("present_key", "any_value"); + let templated_config_str = "${present_key:+value_default}"; + let config_str = render_template(templated_config_str).unwrap(); + assert_eq!(config_str, "value_default"); + // remove the env + env::remove_var("present_key"); + let config_str = render_template(templated_config_str).unwrap(); + assert_eq!(config_str, "") + } + + #[test] + fn render_template_gets_error_when_syntax_is_incomplete() { + let templated_config_str = "${variable:-}"; + let config_str = render_template(templated_config_str); + assert!(config_str.is_err()); + let template_config_str = "${variable:+}"; + let config_str = render_template(template_config_str); + assert!(config_str.is_err()); + } + + #[test] + fn read_config_with_render_template_works() { + // It's enough to check the replacement works + // if config itself has proper data validation + let _config = read_config("configs/config_with_env.yml").unwrap(); + } } diff --git a/src/lib.rs b/src/lib.rs index 7e244e3..55bdcc0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +pub mod cli; pub mod config; pub mod extensions; pub mod logger; diff --git a/src/main.rs b/src/main.rs index 26da8b4..046ee2d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,8 @@ #[tokio::main] async fn main() -> anyhow::Result<()> { // read config from file - let config = subway::config::read_config()?; + let cli = subway::cli::parse_args(); + let config = subway::config::read_config(&cli.config)?; subway::logger::enable_logger(); tracing::trace!("{:#?}", config); From fe67ac723321d112fdb546f33e0cc52ebc68e6bd Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Thu, 25 Apr 2024 04:47:59 +0200 Subject: [PATCH 11/16] fix cli default value (#169) --- src/cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.rs b/src/cli.rs index 4e32bbb..b935284 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; #[command(version, about)] pub struct Command { /// The config file to use - #[arg(short, long, default_value = ".configs/config.yml")] + #[arg(short, long, default_value = "configs/config.yml")] pub config: PathBuf, } pub fn parse_args() -> Command { From 7cb7c73ab08d53b8c3b181e52acb5d227a5b23a5 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Thu, 25 Apr 2024 09:11:17 +0200 Subject: [PATCH 12/16] improve reconnect wait time (#168) --- src/extensions/client/endpoint.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/extensions/client/endpoint.rs b/src/extensions/client/endpoint.rs index 8ea351f..3a212d3 100644 --- a/src/extensions/client/endpoint.rs +++ b/src/extensions/client/endpoint.rs @@ -74,11 +74,10 @@ impl Endpoint { health_.on_error(&err); _ = client_tx.send(None); tracing::warn!("Unable to connect to endpoint: {url_} error: {err}"); - tokio::time::sleep(get_backoff_time(&connect_backoff_counter)).await; } } - // Wait a second before trying to reconnect - tokio::time::sleep(Duration::from_secs(1)).await; + // Wait a bit before trying to reconnect + tokio::time::sleep(get_backoff_time(&connect_backoff_counter)).await; } }); From 8bbe6634844ca2fdc557d8ff6b831309fff271d5 Mon Sep 17 00:00:00 2001 From: Jan Koscisz Date: Fri, 26 Apr 2024 13:41:33 +0200 Subject: [PATCH 13/16] fmt --- src/extensions/client/mod.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/extensions/client/mod.rs b/src/extensions/client/mod.rs index fe27658..9a91953 100644 --- a/src/extensions/client/mod.rs +++ b/src/extensions/client/mod.rs @@ -5,15 +5,11 @@ use std::{ use anyhow::anyhow; use async_trait::async_trait; -use futures::TryFutureExt; -use jsonrpsee::{ - core::{ - client::{ClientT, Error, Subscription, SubscriptionClientT}, - JsonValue, - }, - ws_client::{WsClient, WsClientBuilder}, -}; use futures::FutureExt as _; +use jsonrpsee::core::{ + client::{Error, Subscription}, + JsonValue, +}; use opentelemetry::trace::FutureExt; use rand::{seq::SliceRandom, thread_rng}; use serde::{Deserialize, Serialize}; From f525d6702d79fbc3e82bce382ae7128d7e31bd5d Mon Sep 17 00:00:00 2001 From: Jan Koscisz Date: Fri, 26 Apr 2024 13:47:38 +0200 Subject: [PATCH 14/16] fix toolchain --- .github/workflows/test.yml | 6 +++--- configs/config.yml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2094291..28a6f36 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,15 +22,15 @@ jobs: - name: Install toolchain uses: dtolnay/rust-toolchain@stable with: - toolchain: nightly-2023-09-29 + toolchain: nightly-2024-02-09 components: rustfmt, clippy - uses: actions/checkout@v3 with: submodules: recursive - name: Check format - run: cargo +nightly-2023-09-29 fmt --all -- --check + run: cargo +nightly-2024-02-09 fmt --all -- --check - name: Check clippy - run: cargo +nightly-2023-09-29 clippy --all-targets --all-features -- -D warnings + run: cargo +nightly-2024-02-09 clippy --all-targets --all-features -- -D warnings - name: Build run: cargo build --verbose - name: Run tests diff --git a/configs/config.yml b/configs/config.yml index 01d211f..75b9404 100644 --- a/configs/config.yml +++ b/configs/config.yml @@ -2,6 +2,7 @@ extensions: client: endpoints: - ws://localhost:9944 + - wss://ws.test.azero.dev event_bus: substrate_api: stale_timeout_seconds: 180 # rotate endpoint if no new blocks for 3 minutes From f9d79790022e14968d2ea59bbdb212992afe479e Mon Sep 17 00:00:00 2001 From: Jan Koscisz Date: Fri, 26 Apr 2024 13:51:18 +0200 Subject: [PATCH 15/16] config path has changed --- Dockerfile | 2 +- configs/config.yml | 1 - src/middlewares/methods/block_tag.rs | 5 +---- src/middlewares/methods/inject_params.rs | 4 +--- src/middlewares/subscriptions/merge_subscription.rs | 4 +--- 5 files changed, 4 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7198be0..e5e28e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN useradd -m -u 1000 -U -s /bin/sh -d /app docker WORKDIR /app COPY --from=builder /app/target/release/subway /usr/local/bin -COPY ./config.yml /app/config.yml +COPY ./configs/config.yml /app/config.yml USER docker diff --git a/configs/config.yml b/configs/config.yml index 75b9404..01d211f 100644 --- a/configs/config.yml +++ b/configs/config.yml @@ -2,7 +2,6 @@ extensions: client: endpoints: - ws://localhost:9944 - - wss://ws.test.azero.dev event_bus: substrate_api: stale_timeout_seconds: 180 # rotate endpoint if no new blocks for 3 minutes diff --git a/src/middlewares/methods/block_tag.rs b/src/middlewares/methods/block_tag.rs index ed8e657..b644e59 100644 --- a/src/middlewares/methods/block_tag.rs +++ b/src/middlewares/methods/block_tag.rs @@ -22,10 +22,7 @@ impl MiddlewareBuilder for BlockTagMiddlewar method: &RpcMethod, extensions: &TypeRegistryRef, ) -> Option>> { - let Some(index) = method.params.iter().position(|p| p.ty == "BlockTag" && p.inject) else { - return None; - }; - + let index = method.params.iter().position(|p| p.ty == "BlockTag" && p.inject)?; let eth_api = extensions .read() .await diff --git a/src/middlewares/methods/inject_params.rs b/src/middlewares/methods/inject_params.rs index b07ecad..141ea98 100644 --- a/src/middlewares/methods/inject_params.rs +++ b/src/middlewares/methods/inject_params.rs @@ -45,9 +45,7 @@ impl MiddlewareBuilder for InjectParamsMiddl method: &RpcMethod, extensions: &TypeRegistryRef, ) -> Option>> { - let Some(inject_type) = inject_type(&method.params) else { - return None; - }; + let inject_type = inject_type(&method.params)?; let api = extensions .read() diff --git a/src/middlewares/subscriptions/merge_subscription.rs b/src/middlewares/subscriptions/merge_subscription.rs index 5f67c46..02da0be 100644 --- a/src/middlewares/subscriptions/merge_subscription.rs +++ b/src/middlewares/subscriptions/merge_subscription.rs @@ -176,9 +176,7 @@ impl MiddlewareBuilder method: &RpcSubscription, extensions: &TypeRegistryRef, ) -> Option>> { - let Some(merge_strategy) = method.merge_strategy else { - return None; - }; + let merge_strategy = method.merge_strategy?; let ext = extensions.read().await; let client = ext.get::().expect("Client extension not found"); From 5ed1da64beae33dc1715d8981bed0f581207c9c2 Mon Sep 17 00:00:00 2001 From: Jan Koscisz Date: Fri, 26 Apr 2024 17:10:11 +0200 Subject: [PATCH 16/16] use slim version of builder image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e5e28e8..69611b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.74-buster as builder +FROM rust:1.74-slim-buster as builder WORKDIR /app ADD . .