diff --git a/code/Cargo.lock b/code/Cargo.lock index 9827dcbb4..5cb4680bc 100644 --- a/code/Cargo.lock +++ b/code/Cargo.lock @@ -23,6 +23,16 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "advisory-lock" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6caee7d48f930f9ad3fc9546f8cbf843365da0c5b0ca4eee1d1ac3dd12d8f93" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "aead" version = "0.5.2" @@ -88,6 +98,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "0.6.15" @@ -483,6 +499,43 @@ dependencies = [ "serde", ] +[[package]] +name = "camino" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cbor4ii" version = "0.3.3" @@ -507,6 +560,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chacha20" version = "0.9.1" @@ -544,6 +603,33 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -698,6 +784,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "crossbeam-channel" version = "0.5.13" @@ -707,6 +829,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -1402,6 +1543,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1837,12 +1988,32 @@ version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +[[package]] +name = "is-terminal" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi 0.4.0", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -1919,9 +2090,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.159" +version = "0.2.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" [[package]] name = "libp2p" @@ -2435,6 +2606,15 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lz4_flex" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" +dependencies = [ + "twox-hash", +] + [[package]] name = "malachite-actors" version = "0.1.0" @@ -2505,7 +2685,7 @@ dependencies = [ "clap", "color-eyre", "directories", - "itertools", + "itertools 0.13.0", "malachite-common", "malachite-config", "malachite-metrics", @@ -2716,7 +2896,7 @@ dependencies = [ "bytesize", "derive-where", "eyre", - "itertools", + "itertools 0.13.0", "libp2p-identity", "malachite-actors", "malachite-blocksync", @@ -2834,6 +3014,22 @@ dependencies = [ "thiserror 2.0.3", ] +[[package]] +name = "malachite-wal" +version = "0.1.0" +dependencies = [ + "advisory-lock", + "bytes", + "cfg-if", + "crc32fast", + "criterion", + "lz4_flex", + "nix 0.29.0", + "rand", + "tempfile", + "testdir", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -3041,6 +3237,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nohash-hasher" version = "0.2.0" @@ -3057,6 +3265,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -3136,6 +3353,12 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "oorandom" +version = "11.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -3290,6 +3513,34 @@ dependencies = [ "spki", ] +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "polling" version = "3.7.3" @@ -3422,7 +3673,7 @@ checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" dependencies = [ "bytes", "heck", - "itertools", + "itertools 0.13.0", "log", "multimap", "once_cell", @@ -3442,7 +3693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", - "itertools", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.87", @@ -3589,6 +3840,26 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rcgen" version = "0.11.3" @@ -3734,7 +4005,7 @@ dependencies = [ "log", "netlink-packet-route", "netlink-proto", - "nix", + "nix 0.24.3", "thiserror 1.0.68", "tokio", ] @@ -3771,9 +4042,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.37" +version = "0.38.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" dependencies = [ "bitflags 2.6.0", "errno", @@ -3846,6 +4117,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -3877,6 +4157,9 @@ name = "semver" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +dependencies = [ + "serde", +] [[package]] name = "serde" @@ -4293,6 +4576,20 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "sysinfo" +version = "0.26.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c18a6156d1f27a9592ee18c1a846ca8dd5c258b7179fc193ae87c74ebb666f5" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "winapi", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -4316,9 +4613,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", @@ -4327,6 +4624,20 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "testdir" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee79e927b64d193f5abb60d20a0eb56be0ee5a242fdeb8ce3bf054177006de52" +dependencies = [ + "anyhow", + "backtrace", + "cargo_metadata", + "once_cell", + "sysinfo", + "whoami", +] + [[package]] name = "thiserror" version = "1.0.68" @@ -4428,6 +4739,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -4651,6 +4972,16 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + [[package]] name = "typenum" version = "1.17.0" @@ -4759,6 +5090,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4774,6 +5115,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.95" @@ -4849,6 +5196,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "whoami" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" +dependencies = [ + "redox_syscall", + "wasite", + "web-sys", +] + [[package]] name = "widestring" version = "1.1.0" @@ -4871,6 +5229,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/code/Cargo.toml b/code/Cargo.toml index 8d66533ea..16b9c348a 100644 --- a/code/Cargo.toml +++ b/code/Cargo.toml @@ -18,6 +18,7 @@ members = [ "crates/proto", "crates/round", "crates/vote", + "crates/wal", # Ed25519 "crates/signing-ed25519", @@ -72,6 +73,7 @@ malachite-proto = { version = "0.1.0", path = "crates/proto" } malachite-round = { version = "0.1.0", path = "crates/round" } malachite-vote = { version = "0.1.0", path = "crates/vote" } malachite-signing-ed25519 = { version = "0.1.0", path = "crates/signing-ed25519" } +malachite-wal = { version = "0.1.0", path = "crates/wal" } # Test malachite-test = { version = "0.1.0", path = "crates/test" } @@ -88,20 +90,22 @@ malachite-starknet-p2p-types = { version = "0.1.0", path = "crates/starknet/p2p- starknet-core = "0.11.1" starknet-crypto = "0.7.3" +advisory-lock = "0.3.0" async-recursion = "1.1" async-trait = "0.1.83" axum = "0.7" base64 = "0.22.0" -bon = "2.3.0" -bytesize = "1.3" bytes = { version = "1", default-features = false } +bytesize = "1.3" clap = "4.5" color-eyre = "0.6" config = { version = "0.14", features = ["toml"], default-features = false } +crc32fast = "1.4.0" +criterion = "0.5.1" dashmap = "6.1.0" derive-where = "1.2.7" -displaydoc = { version = "0.2", default-features = false } directories = "5.0.1" +displaydoc = { version = "0.2", default-features = false } ed25519-consensus = "2.1.0" either = "1" eyre = "0.6" @@ -117,6 +121,7 @@ libp2p = { version = "0.54.1", features = ["macros", "identify", "to libp2p-identity = "0.2.10" libp2p-broadcast = { version = "0.1.0", package = "libp2p-scatter" } multiaddr = "0.18.2" +nix = { version = "0.29.0", features = ["signal"] } num-bigint = "0.4.4" num-traits = "0.2.17" pretty_assertions = "1.4" @@ -135,7 +140,9 @@ serde_with = "3.9" sha3 = "0.10" signature = "2.2.0" tempfile = "3.13.0" +testdir = "0.9.1" thiserror = { version = "2.0", default-features = false } +tikv-jemallocator = "0.6.0" time = "0.3" tokio = "1.41.1" tokio-stream = "0.1" diff --git a/code/crates/wal/Cargo.lock b/code/crates/wal/Cargo.lock new file mode 100644 index 000000000..71efe4684 --- /dev/null +++ b/code/crates/wal/Cargo.lock @@ -0,0 +1,989 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "advisory-lock" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6caee7d48f930f9ad3fc9546f8cbf843365da0c5b0ca4eee1d1ac3dd12d8f93" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anyhow" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "camino" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.5.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crc32fast" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + +[[package]] +name = "is-terminal" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "540654e97a3f4470a492cd30ff187bc95d89557a903a2bbf112e2fae98104ef2" + +[[package]] +name = "js-sys" +version = "0.3.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "malachite-wal" +version = "0.1.0" +dependencies = [ + "advisory-lock", + "bytes", + "crc32fast", + "criterion", + "nix", + "rand", + "tempfile", + "testdir", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.36.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "oorandom" +version = "11.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustix" +version = "0.38.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +dependencies = [ + "serde", +] + +[[package]] +name = "serde" +version = "1.0.215" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.215" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "2.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d46482f1c1c87acd84dea20c1bf5ebff4c757009ed6bf19cfd36fb10e92c4e" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sysinfo" +version = "0.26.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c18a6156d1f27a9592ee18c1a846ca8dd5c258b7179fc193ae87c74ebb666f5" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "winapi", +] + +[[package]] +name = "tempfile" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "testdir" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee79e927b64d193f5abb60d20a0eb56be0ee5a242fdeb8ce3bf054177006de52" +dependencies = [ + "anyhow", + "backtrace", + "cargo_metadata", + "once_cell", + "sysinfo", + "whoami", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" + +[[package]] +name = "web-sys" +version = "0.3.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "whoami" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" +dependencies = [ + "redox_syscall", + "wasite", + "web-sys", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/code/crates/wal/Cargo.toml b/code/crates/wal/Cargo.toml new file mode 100644 index 000000000..20c350870 --- /dev/null +++ b/code/crates/wal/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "malachite-wal" +version = "0.1.0" +edition = "2021" +rust-version = "1.82.0" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[[bench]] +name = "wal" +harness = false + +[features] +compression = ["dep:lz4_flex"] +force-compression = ["compression"] + +[dependencies] +cfg-if = "1" +advisory-lock = "0.3.0" +bytes = "1.5.0" +crc32fast = "1.4.0" +lz4_flex = { version = "0.11.0", optional = true } + +[dev-dependencies] +criterion = "0.5.1" +nix = { version = "0.29.0", features = ["signal"] } +rand = "0.8.5" +tempfile = "3.14.0" +testdir = "0.9.1" diff --git a/code/crates/wal/benches/wal.rs b/code/crates/wal/benches/wal.rs new file mode 100644 index 000000000..b1b91f84d --- /dev/null +++ b/code/crates/wal/benches/wal.rs @@ -0,0 +1,242 @@ +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use std::fs; +use std::path::PathBuf; +use tempfile::tempdir; + +use malachite_wal::Log; + +/// Benchmark configuration +struct BenchConfig { + entry_size: usize, + batch_size: usize, + sync_interval: usize, +} + +impl BenchConfig { + fn total_size(&self) -> usize { + self.entry_size * self.batch_size + } +} + +fn wal_benchmarks(c: &mut Criterion) { + let dir = tempdir().unwrap(); + + // Different entry sizes to test + let entry_sizes = vec![ + 64, // Small entries + 1024, // 1 KB + 16 * 1024, // 16 KB + 256 * 1024, // 256 KB + 1024 * 1024, // 1 MB + ]; + + // Read benchmarks + let mut read_group = c.benchmark_group("wal_read"); + + // Benchmark reading different entry sizes + for size in &entry_sizes { + let config = BenchConfig { + entry_size: *size, + batch_size: 1000, + sync_interval: 100, + }; + + read_group.throughput(Throughput::Bytes(config.total_size() as u64)); + read_group.bench_with_input(BenchmarkId::new("sequential_read", size), size, |b, _| { + let path = get_temp_wal_path(&dir); + setup_wal_for_reading(&path, &config); + b.iter(|| bench_sequential_read(&path)); + fs::remove_file(path).unwrap(); + }); + } + read_group.finish(); + + // Write benchmarks + let mut write_group = c.benchmark_group("wal_write"); + + // Benchmark writing different entry sizes + for size in &entry_sizes { + let config = BenchConfig { + entry_size: *size, + batch_size: 1000, + sync_interval: 100, + }; + + write_group.throughput(Throughput::Bytes(config.total_size() as u64)); + write_group.bench_with_input(BenchmarkId::new("sequential_write", size), size, |b, _| { + let path = get_temp_wal_path(&dir); + b.iter(|| bench_sequential_write(&path, &config)); + }); + } + + // Benchmark different batch sizes + let batch_sizes = vec![1, 10, 100, 1000, 10000]; + + for batch_size in &batch_sizes { + let config = BenchConfig { + entry_size: 1024, + batch_size: *batch_size, + sync_interval: *batch_size, + }; + + write_group.throughput(Throughput::Bytes(config.total_size() as u64)); + write_group.bench_with_input( + BenchmarkId::new("batch_write", batch_size), + batch_size, + |b, _| { + let path = get_temp_wal_path(&dir); + b.iter(|| bench_batch_write(&path, &config)); + }, + ); + } + + // Benchmark different sync intervals + let sync_intervals = vec![1, 10, 100, 1000]; + + for interval in &sync_intervals { + let config = BenchConfig { + entry_size: 1024, + batch_size: 1000, + sync_interval: *interval, + }; + + write_group.throughput(Throughput::Bytes(config.total_size() as u64)); + write_group.bench_with_input( + BenchmarkId::new("sync_interval", interval), + interval, + |b, &_| { + let path = get_temp_wal_path(&dir); + b.iter(|| bench_sync_interval(&path, &config)); + }, + ); + } + + // // Mixed read/write benchmarks + // let mut mixed_group = c.benchmark_group("wal_mixed"); + // + // mixed_group.finish(); +} + +/// Helper function to get a unique WAL path +fn get_temp_wal_path(dir: &tempfile::TempDir) -> PathBuf { + use std::sync::atomic::{AtomicUsize, Ordering}; + static COUNTER: AtomicUsize = AtomicUsize::new(0); + let id = COUNTER.fetch_add(1, Ordering::SeqCst); + dir.path().join(format!("bench_{}.wal", id)) +} + +/// Benchmark sequential writes +fn bench_sequential_write(path: &PathBuf, config: &BenchConfig) { + let mut wal = Log::open(path).unwrap(); + let data = vec![0u8; config.entry_size]; + + for i in 0..config.batch_size { + wal.write(&data).unwrap(); + if i % config.sync_interval == 0 { + wal.sync().unwrap(); + } + } + + fs::remove_file(path).unwrap() +} + +/// Benchmark sequential reads +fn bench_sequential_read(path: &PathBuf) { + let mut wal = Log::open(path).unwrap(); + black_box(wal.iter().unwrap().collect::, _>>().unwrap()); +} + +/// Setup WAL with data for reading benchmarks +fn setup_wal_for_reading(path: &PathBuf, config: &BenchConfig) { + let mut wal = Log::open(path).unwrap(); + let data = vec![0u8; config.entry_size]; + + for i in 0..config.batch_size { + wal.write(&data).unwrap(); + if i % config.sync_interval == 0 { + wal.sync().unwrap(); + } + } +} + +/// Benchmark batch writes +fn bench_batch_write(path: &PathBuf, config: &BenchConfig) { + let mut wal = Log::open(path).unwrap(); + let data = vec![0u8; config.entry_size]; + + for _ in 0..config.batch_size { + wal.write(&data).unwrap(); + } + wal.sync().unwrap(); + fs::remove_file(path).unwrap(); +} + +/// Benchmark different sync intervals +fn bench_sync_interval(path: &PathBuf, config: &BenchConfig) { + let mut wal = Log::open(path).unwrap(); + let data = vec![0u8; config.entry_size]; + + for i in 0..config.batch_size { + wal.write(&data).unwrap(); + if i % config.sync_interval == 0 { + wal.sync().unwrap(); + } + } +} + +/// Benchmark small writes with frequent syncs +fn bench_small_writes_frequent_sync(c: &mut Criterion) { + let mut group = c.benchmark_group("small_writes_frequent_sync"); + let dir = tempdir().unwrap(); + + group.throughput(Throughput::Bytes(64 * 100)); + group.bench_function("write_sync_every", |b| { + b.iter(|| { + let path = get_temp_wal_path(&dir); + let mut wal = Log::open(&path).unwrap(); + let data = vec![0u8; 64]; + + for _ in 0..100 { + wal.write(&data).unwrap(); + wal.sync().unwrap(); + } + fs::remove_file(path).unwrap(); + }); + }); + + group.finish(); +} + +/// Benchmark random access patterns +fn bench_random_access(c: &mut Criterion) { + use rand::Rng; + + let mut group = c.benchmark_group("random_access"); + let dir = tempdir().unwrap(); + + group.bench_function("random_sized_writes", |b| { + b.iter(|| { + let path = get_temp_wal_path(&dir); + let mut wal = Log::open(&path).unwrap(); + let mut rng = rand::thread_rng(); + + for _ in 0..100 { + let size = rng.gen_range(64..4096); + let data = vec![0u8; size]; + wal.write(&data).unwrap(); + } + wal.sync().unwrap(); + fs::remove_file(path).unwrap(); + }); + }); + + group.finish(); +} + +criterion_group!( + benches, + wal_benchmarks, + bench_small_writes_frequent_sync, + bench_random_access +); +criterion_main!(benches); diff --git a/code/crates/wal/src/ext.rs b/code/crates/wal/src/ext.rs new file mode 100644 index 000000000..2a9bf68af --- /dev/null +++ b/code/crates/wal/src/ext.rs @@ -0,0 +1,37 @@ +use std::io::{self, Read, Write}; + +#[inline(always)] +pub fn read_u8(reader: &mut R) -> io::Result { + let mut buf = [0; 1]; + reader.read_exact(&mut buf)?; + Ok(buf[0]) +} + +#[inline(always)] +pub fn write_u8(writer: &mut W, value: u8) -> io::Result<()> { + writer.write_all(&[value]) +} + +#[inline(always)] +pub fn read_u32(reader: &mut R) -> io::Result { + let mut buf = [0; 4]; + reader.read_exact(&mut buf)?; + Ok(u32::from_be_bytes(buf)) +} + +#[inline(always)] +pub fn write_u32(writer: &mut W, value: u32) -> io::Result<()> { + writer.write_all(&value.to_be_bytes()) +} + +#[inline(always)] +pub fn read_u64(reader: &mut R) -> io::Result { + let mut buf = [0; 8]; + reader.read_exact(&mut buf)?; + Ok(u64::from_be_bytes(buf)) +} + +#[inline(always)] +pub fn write_u64(writer: &mut W, value: u64) -> io::Result<()> { + writer.write_all(&value.to_be_bytes()) +} diff --git a/code/crates/wal/src/file.rs b/code/crates/wal/src/file.rs new file mode 100644 index 000000000..bd942e424 --- /dev/null +++ b/code/crates/wal/src/file.rs @@ -0,0 +1,51 @@ +use std::fs::{self, File}; +use std::io; +use std::path::Path; + +use advisory_lock::{AdvisoryFileLock, FileLockMode}; + +use crate::storage::Storage; + +/// Write-Ahead Log (WAL) backed by a [`File`](std::fs::File) +pub type Log = crate::log::Log; + +/// Write-Ahead Log (WAL) entry, backed by a [`File`](std::fs::File) +pub type LogEntry<'a> = crate::log::LogEntry<'a, File>; + +/// Iterator over the WAL entries, backed by a [`File`](std::fs::File) +pub type LogIter<'a> = crate::log::LogIter<'a, File>; + +impl Storage for File { + type OpenOptions = (); + + fn open_with(path: impl AsRef, _: ()) -> io::Result { + // Open file with read+write access, create if doesn't exist + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) // Don't truncate existing file + .open(&path)?; + + AdvisoryFileLock::try_lock(&file, FileLockMode::Exclusive).map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("Failed to acquire exclusive advisory lock: {e}"), + ) + })?; + + Ok(file) + } + + fn size_bytes(&self) -> io::Result { + File::metadata(self).map(|m| m.len()) + } + + fn truncate_to(&mut self, size: u64) -> io::Result<()> { + File::set_len(self, size) + } + + fn sync_all(&mut self) -> io::Result<()> { + File::sync_all(self) + } +} diff --git a/code/crates/wal/src/lib.rs b/code/crates/wal/src/lib.rs new file mode 100644 index 000000000..9374307a3 --- /dev/null +++ b/code/crates/wal/src/lib.rs @@ -0,0 +1,14 @@ +#![cfg_attr(docsrs, feature(doc_cfg))] + +//! Write-Ahead Log (WAL) implementation + +mod ext; +mod file; +mod storage; +mod version; + +pub mod log; + +pub use file::{Log, LogEntry, LogIter}; +pub use storage::Storage; +pub use version::Version; diff --git a/code/crates/wal/src/log.rs b/code/crates/wal/src/log.rs new file mode 100644 index 000000000..05f9b91b4 --- /dev/null +++ b/code/crates/wal/src/log.rs @@ -0,0 +1,590 @@ +//! Write-Ahead Log (WAL) implementation, generic over its backing storage. +//! +//! # Warning +//! Not for regular use, use [`crate::Log`] instead. + +use std::io::{self, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +use cfg_if::cfg_if; + +use crate::ext::{read_u32, read_u64, read_u8, write_u32, write_u64, write_u8}; +use crate::{Storage, Version}; + +/// Represents a single entry in the Write-Ahead Log (WAL). +/// +/// Each entry has the following format on disk: +/// +/// ```text +/// +-----------------|-----------------+----------------+-----------------+ +/// | Is compressed | Length | CRC | Data | +/// | (1 byte) | (4 bytes) | (4 bytes) | ($length bytes) | +/// +-----------------|-----------------+----------------+-----------------+ +/// ``` +pub struct LogEntry<'a, S> { + /// Reference to the parent WAL + log: &'a mut Log, +} + +impl LogEntry<'_, S> +where + S: Storage, +{ + /// Reads the compression flag of the current entry + fn read_compression_flag(&mut self) -> io::Result { + read_u8(&mut self.log.storage).map(|byte| byte != 0) + } + + /// Reads the length field of the current entry + fn read_length(&mut self) -> io::Result { + read_u64(&mut self.log.storage) + } + + /// Reads the CRC field of the current entry + fn read_crc(&mut self) -> io::Result { + read_u32(&mut self.log.storage) + } + + /// Reads the current entry's data and advances to the next entry. + /// The entry data is written to the provided writer. + /// + /// # Arguments + /// * `writer` - The writer to output the entry data to + /// + /// # Returns + /// * `Ok(Some(self))` - If there are more entries to read + /// * `Ok(None)` - If this was the last entry + /// * `Err` - If an I/O error occurs or the CRC check fails + pub fn read_to_next(mut self, writer: &mut W) -> io::Result> { + let is_compressed = self.read_compression_flag()?; + let length = self.read_length()? as usize; + let expected_crc = self.read_crc()?; + + let mut data = vec![0; length]; + self.log.storage.read_exact(&mut data)?; + + #[cfg(not(feature = "compression"))] + if is_compressed { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Entry is compressed but compression is disabled", + )); + } + + #[cfg(feature = "compression")] + if is_compressed { + data = lz4_flex::decompress_size_prepended(&data).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Failed to decompress entry: {e}"), + ) + })?; + } + + let actual_crc = compute_crc(&data); + + if expected_crc != actual_crc { + return Err(io::Error::new(io::ErrorKind::InvalidData, "CRC mismatch")); + } + + writer.write_all(&data)?; + + let pos = self.log.storage.stream_position()?; + let len = self.log.storage.size_bytes()?; + + if pos < len { + Ok(Some(self)) + } else { + Ok(None) + } + } +} + +/// Write-Ahead Log (WAL) +/// +/// A Write-Ahead Log is a sequential log of records that provides durability and atomicity +/// guarantees by writing changes to disk before they are applied to the main database. +/// +/// # Format on disk +/// +/// ```text +/// +-----------------+-----------------+-----------------+-----------------+-----------------+ +/// | Version | Sequence | Entry #1 | ... | Entry #n | +/// | (4 bytes) | (8 bytes) | (variable) | | (variable) | +/// +-----------------+-----------------+-----------------+-----------------+-----------------+ +/// ``` +#[derive(Debug)] +pub struct Log { + storage: S, + path: PathBuf, + version: Version, + sequence: u64, + len: usize, +} + +const VERSION_SIZE: u64 = size_of::() as u64; +const SEQUENCE_SIZE: u64 = size_of::() as u64; +const HEADER_SIZE: u64 = VERSION_SIZE + SEQUENCE_SIZE; + +const VERSION_OFFSET: u64 = 0; +const SEQUENCE_OFFSET: u64 = VERSION_OFFSET + VERSION_SIZE; +const FIRST_ENTRY_OFFSET: u64 = HEADER_SIZE; + +const ENTRY_LENGTH_SIZE: u64 = size_of::() as u64; +const ENTRY_CRC_SIZE: u64 = size_of::() as u64; +const ENTRY_COMPRESSION_FLAG_SIZE: u64 = size_of::() as u64; +const ENTRY_HEADER_SIZE: u64 = ENTRY_COMPRESSION_FLAG_SIZE + ENTRY_LENGTH_SIZE + ENTRY_CRC_SIZE; + +enum WriteEntry<'a> { + Raw(&'a [u8]), + + #[cfg(feature = "compression")] + Compressed { + compressed: &'a [u8], + uncompressed: &'a [u8], + }, +} + +impl WriteEntry<'_> { + fn data(&self) -> &[u8] { + match self { + WriteEntry::Raw(data) => data, + + #[cfg(feature = "compression")] + WriteEntry::Compressed { compressed, .. } => compressed, + } + } + + fn len(&self) -> usize { + match self { + WriteEntry::Raw(data) => data.len(), + + #[cfg(feature = "compression")] + WriteEntry::Compressed { compressed, .. } => compressed.len(), + } + } + + fn uncompressed_crc(&self) -> u32 { + match self { + WriteEntry::Raw(data) => compute_crc(data), + + #[cfg(feature = "compression")] + WriteEntry::Compressed { uncompressed, .. } => compute_crc(uncompressed), + } + } + + fn is_compressed(&self) -> bool { + cfg_if! { + if #[cfg(feature = "compression")] { + matches!(self, WriteEntry::Compressed { .. }) + } else { + false + } + } + } +} + +impl Log +where + S: Storage, +{ + /// Opens a Write-Ahead Log file at the specified path. + /// + /// If the file already exists, it will be opened and validated. + /// If the file does not exist, a new one will be created. + /// + /// # Arguments + /// * `path` - Path where the WAL file should be created/opened + /// + /// # Returns + /// * `Ok(Wal)` - Successfully opened/created WAL + /// * `Err` - If file operations fail or existing WAL is invalid + pub fn open(path: impl AsRef) -> io::Result { + Self::open_with(path, ()) + } +} + +impl Log +where + S: Storage, +{ + /// Opens a Write-Ahead Log file at the specified path. + /// + /// If the file already exists, it will be opened and validated. + /// If the file does not exist, a new one will be created. + /// + /// # Arguments + /// * `path` - Path where the WAL file should be created/opened + /// + /// # Returns + /// * `Ok(Wal)` - Successfully opened/created WAL + /// * `Err` - If file operations fail or existing WAL is invalid + pub fn open_with(path: impl AsRef, options: S::OpenOptions) -> io::Result { + let path = path.as_ref().to_owned(); + + let mut storage = S::open_with(&path, options)?; + + let size = storage.size_bytes()?; + + // If file exists and has content + if size > 0 { + // Read and validate version number + let version = Version::try_from(read_u32(&mut storage)?) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Invalid WAL version"))?; + + // Read sequence number + let sequence = read_u64(&mut storage).map_err(|_| { + io::Error::new( + io::ErrorKind::UnexpectedEof, + "Failed to read sequence number", + ) + })?; + + // Track current position and entry count + let mut pos = FIRST_ENTRY_OFFSET; // Start after header + let mut len = 0; + + // Scan through entries to validate and count them + while size.saturating_sub(pos) > ENTRY_HEADER_SIZE - ENTRY_CRC_SIZE { + // Skip over compression flag + read_u8(&mut storage)?; + + // Read entry length + let data_length = read_u64(&mut storage)?; + + // Calculate total entry size including CRC + let Some(entry_length) = data_length.checked_add(ENTRY_CRC_SIZE) else { + break; // Integer overflow, file is corrupt + }; + + // Check if enough bytes remain for full entry + if size.saturating_sub(pos) < entry_length { + break; // Partial/corrupt entry + } + + // Skip to next entry + pos = storage.seek(SeekFrom::Current(entry_length.try_into().unwrap()))?; + len += 1; + } + + // Truncate any partial entries at the end + storage.truncate_to(pos)?; + storage.sync_all()?; + + return Ok(Self { + version, + storage, + path, + sequence, + len, + }); + } + + // Creating new WAL file + let version = Version::V1; + + // Write header: version (4 bytes) + write_u32(&mut storage, version as u32)?; + + // Write header: sequence (8 bytes) + write_u64(&mut storage, 0)?; + + // Ensure file is exactly header size + storage.truncate_to(HEADER_SIZE)?; + + // Ensure header is persisted to disk + storage.sync_all()?; + + Ok(Self { + version, + storage, + path, + sequence: 0, + len: 0, + }) + } + + /// Writes a new entry to the WAL. + /// + /// The entry is appended to the end of the log with length, CRC and data. + /// If writing fails, the WAL is truncated to remove the partial write. + /// + /// If the `force-compression` feature is enabled, all entries will be compressed. + /// + /// # Arguments + /// * `data` - The data to write as a new WAL entry + /// + /// # Returns + /// * `Ok(())` - Entry was successfully written + /// * `Err` - If writing fails + pub fn write(&mut self, data: impl AsRef<[u8]>) -> io::Result<()> { + cfg_if! { + if #[cfg(feature = "force-compression")] { + self.write_compressed(data) + } else { + self.write_raw(data) + } + } + } + + /// Writes a new entry to the WAL, without compressing it. + /// + /// The entry is appended to the end of the log with length, CRC and data. + /// If writing fails, the WAL is truncated to remove the partial write. + /// + /// # Arguments + /// * `data` - The data to write as a new WAL entry + /// + /// # Returns + /// * `Ok(())` - Entry was successfully written + /// * `Err` - If writing fails + pub fn write_raw(&mut self, data: impl AsRef<[u8]>) -> io::Result<()> { + self.write_entry(WriteEntry::Raw(data.as_ref())) + } + + /// Writes a new entry to the WAL, compressing it with the LZ4 algorithm. + /// + /// The entry is appended to the end of the log with length, CRC and data. + /// If writing fails, the WAL is truncated to remove the partial write. + /// + /// # Arguments + /// * `data` - The data to write as a new WAL entry + /// + /// # Returns + /// * `Ok(())` - Entry was successfully written + /// * `Err` - If writing fails + #[cfg(feature = "compression")] + #[cfg_attr(docsrs, doc(cfg(feature = "compression")))] + pub fn write_compressed(&mut self, data: impl AsRef<[u8]>) -> io::Result<()> { + let data = data.as_ref(); + let compressed = lz4_flex::compress_prepend_size(data); + + // Only use compression if it actually helps + let entry = if compressed.len() < data.len() { + WriteEntry::Compressed { + compressed: &compressed, + uncompressed: data, + } + } else { + WriteEntry::Raw(data) + }; + + // Rest of write logic... + self.write_entry(entry) + } + + fn write_entry(&mut self, entry: WriteEntry<'_>) -> io::Result<()> { + let pos = self.storage.seek(SeekFrom::End(0))?; + + let result = || -> io::Result<()> { + // Write compression flag + write_u8(&mut self.storage, entry.is_compressed() as u8)?; + + // Write length of (compressed) data + write_u64(&mut self.storage, entry.len() as u64)?; + + // Write CRC of (uncompressed) data + write_u32(&mut self.storage, entry.uncompressed_crc())?; + + // Write (compressed) entry data + self.storage.write_all(entry.data())?; + + Ok(()) + }(); + + match result { + Ok(()) => { + self.len += 1; + Ok(()) + } + Err(e) => { + self.storage.truncate_to(pos)?; + Err(e) + } + } + } + + /// Returns an the first entry in the WAL if it exists. + /// + /// # Returns + /// * `Ok(Some(WalEntry))` - First entry exists and was retrieved + /// * `Ok(None)` - WAL is empty + /// * `Err` - If reading fails or WAL is invalid + pub fn first_entry(&mut self) -> io::Result>> { + // IF the file is empty, return an error + if self.storage.size_bytes()? == 0 { + return Err(io::Error::new(io::ErrorKind::NotFound, "Empty WAL")); + } + + // If there are no entries, return None + if self.len == 0 { + return Ok(None); + } + + // Seek to the first entry after the header + self.storage.seek(SeekFrom::Start(FIRST_ENTRY_OFFSET))?; + + Ok(Some(LogEntry { log: self })) + } + + /// Returns an iterator over all entries in the WAL. + /// + /// # Returns + /// * `Ok(LogIter)` - Iterator over WAL entries + /// * `Err` - If reading fails + pub fn iter(&mut self) -> io::Result> { + Ok(LogIter { + next: self.first_entry()?, + }) + } + + /// Restarts the WAL with a new sequence number. + /// + /// This truncates all existing entries and resets the WAL to an empty state + /// with the specified sequence number. + /// + /// # Arguments + /// * `sequence` - New sequence number to start from + /// + /// # Returns + /// * `Ok(())` - WAL was successfully restarted + /// * `Err` - If file operations fail + pub fn restart(&mut self, sequence: u64) -> io::Result<()> { + // Reset sequence number and entry count + self.sequence = sequence; + self.len = 0; + + // Seek to start of sequence number + self.storage.seek(SeekFrom::Start(SEQUENCE_OFFSET))?; + + // Write new sequence number + write_u64(&mut self.storage, sequence)?; + + // Truncate all entries + self.storage.truncate_to(HEADER_SIZE)?; + + // Sync changes to disk + self.storage.sync_all()?; + + Ok(()) + } + + /// Syncs all written data to disk. + /// + /// On UNIX systems, this will call `fsync` to ensure all data is written to disk. + /// + /// # Returns + /// * `Ok(())` - Successfully synced to disk + /// * `Err` - If sync fails + pub fn sync(&mut self) -> io::Result<()> { + self.storage.sync_all() + } + + /// Build a Write-Ahead Log (WAL) from its raw components. + /// + /// # Safety + /// This is a dangerous function that should not be used directly. + /// It bypasses important initialization and validation checks. + /// Instead, use `malachite_wal::file::Log::open` which properly initializes the WAL. + /// + /// This function exists primarily for internal use and testing purposes. + pub fn from_raw_parts( + file: S, + path: PathBuf, + version: Version, + sequence: u64, + len: usize, + ) -> Self { + Self { + storage: file, + path, + version, + sequence, + len, + } + } + + /// Returns the size in bytes of the underlying storage + pub fn size_bytes(&self) -> io::Result { + self.storage.size_bytes() + } +} + +impl Log { + /// Returns the version of the WAL format. + pub fn version(&self) -> Version { + self.version + } + + /// Returns the current sequence number. + pub fn sequence(&self) -> u64 { + self.sequence + } + + /// Returns the path to the WAL file. + pub fn path(&self) -> &Path { + &self.path + } + + /// Returns the number of entries in the WAL. + pub fn len(&self) -> usize { + self.len + } + + /// Returns whether the WAL is empty. + pub fn is_empty(&self) -> bool { + self.len == 0 + } +} + +/// Iterator over entries in a Write-Ahead Log (WAL) +pub struct LogIter<'a, F> { + /// The next entry to be read from the WAL + next: Option>, +} + +/// Iterator over entries in a Write-Ahead Log (WAL) +/// +/// Provides sequential access to entries stored in the WAL. +/// Each iteration returns the data contained in the next entry. +impl Iterator for LogIter<'_, F> +where + F: Storage, +{ + /// Each iteration returns a Result containing either the entry data as a `Vec` + /// or an IO error if reading fails + type Item = io::Result>; + + /// Advances the iterator and returns the next entry's data + /// + /// # Returns + /// * `Some(Ok(Vec))` - Successfully read entry data + /// * `Some(Err(e))` - Error occurred while reading entry + /// * `None` - No more entries to read + fn next(&mut self) -> Option { + let mut buf = Vec::new(); + let next = self.next.take()?; + + match next.read_to_next(&mut buf) { + Ok(Some(entry)) => { + self.next = Some(entry); + Some(Ok(buf)) + } + Ok(None) => Some(Ok(buf)), + Err(e) => Some(Err(e)), + } + } +} + +/// Computes the CRC32 checksum of the provided data +/// +/// # Arguments +/// * `data` - The bytes to compute the checksum for +/// +/// # Returns +/// The CRC32 checksum as a u32 in big-endian byte order +fn compute_crc(data: &[u8]) -> u32 { + let mut hasher = crc32fast::Hasher::new(); + hasher.update(data); + u32::from_be_bytes(hasher.finalize().to_be_bytes()) +} diff --git a/code/crates/wal/src/ops.rs b/code/crates/wal/src/ops.rs new file mode 100644 index 000000000..15391a6e2 --- /dev/null +++ b/code/crates/wal/src/ops.rs @@ -0,0 +1,9 @@ +use std::io::{Read, Seek, Write}; + +pub trait FileOps: Read + Write + Seek { + fn metadata(&self) -> std::io::Result; + + fn set_len(&mut self, size: u64) -> std::io::Result<()>; + + fn sync_all(&mut self) -> std::io::Result<()>; +} diff --git a/code/crates/wal/src/storage.rs b/code/crates/wal/src/storage.rs new file mode 100644 index 000000000..52a4de516 --- /dev/null +++ b/code/crates/wal/src/storage.rs @@ -0,0 +1,25 @@ +use std::io::{self, Read, Seek, Write}; +use std::path::Path; + +/// Operations that the backing storage for the Write-Ahead Log must implement. +/// +/// This is mainly used to exercise various failure scenarios in tests, +/// and should otherwise not be used in production code. +/// +/// Users are instead encouraged to to use the default [`File`](std::fs::File)-based +/// implementation at [`crate::Log`]. +pub trait Storage: Read + Write + Seek + Sized { + type OpenOptions; + + /// Open the backing storage for the Write-Ahead Log at the given path. + fn open_with(path: impl AsRef, options: Self::OpenOptions) -> io::Result; + + /// Returns the size of the file in bytes. + fn size_bytes(&self) -> io::Result; + + /// Truncates the file to the specified size. + fn truncate_to(&mut self, size: u64) -> io::Result<()>; + + /// Synchronizes all in-memory data to the underlying storage device. + fn sync_all(&mut self) -> io::Result<()>; +} diff --git a/code/crates/wal/src/version.rs b/code/crates/wal/src/version.rs new file mode 100644 index 000000000..87a7854db --- /dev/null +++ b/code/crates/wal/src/version.rs @@ -0,0 +1,28 @@ +/// Version identifier for the Write-Ahead Log (WAL) format +/// +/// Currently only supports version 1 (V1) of the format. +#[repr(u32)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Version { + /// Version 1 of the WAL format + V1 = 1, +} + +impl TryFrom for Version { + type Error = (); + + /// Attempts to convert a u32 into a WalVersion + /// + /// # Arguments + /// * `value` - The u32 value to convert + /// + /// # Returns + /// * `Ok(WalVersion)` - If the value represents a valid version + /// * `Err(())` - If the value does not correspond to any known version + fn try_from(value: u32) -> Result { + match value { + 1 => Ok(Self::V1), + _ => Err(()), + } + } +} diff --git a/code/crates/wal/tests/basic.rs b/code/crates/wal/tests/basic.rs new file mode 100644 index 000000000..9d7fe97ad --- /dev/null +++ b/code/crates/wal/tests/basic.rs @@ -0,0 +1,202 @@ +#![allow(clippy::bool_assert_comparison)] + +use std::fs::OpenOptions; +use std::path::Path; +use std::time::Duration; +use std::{fs, io, str}; + +use malachite_wal::{Log, Version}; + +use testdir::testdir; + +const ENTRIES_1: &[&str] = &[ + "Hello, world!", + "Wheeee!", + "1234567890", + "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "Done!", +]; + +const ENTRIES_2: &[&str] = &[ + "Something new", + "Another thing", + "And another", + "Yet another", +]; + +fn setup_wal(path: &Path, entries: &[&str]) -> io::Result { + let mut wal = Log::open(path)?; + println!("Path: {}", wal.path().display()); + + let version = wal.version(); + let sequence = wal.sequence(); + assert_eq!(version, Version::V1); + assert_eq!(sequence, 0); + + for entry in entries { + wal.write(entry)?; + } + + assert_eq!(wal.len(), entries.len()); + assert_eq!(wal.is_empty(), entries.is_empty()); + + wal.sync()?; + + Ok(wal) +} + +#[test] +fn new_wal() -> io::Result<()> { + let path = testdir!().join("test.wal"); + let wal = Log::open(path)?; + println!("Path: {}", wal.path().display()); + + assert_eq!(wal.version(), Version::V1); + assert_eq!(wal.sequence(), 0); + assert_eq!(wal.len(), 0); + assert_eq!(wal.is_empty(), true); + + Ok(()) +} + +#[test] +fn open_empty_wal() -> io::Result<()> { + let path = testdir!().join("test.wal"); + + let wal = setup_wal(&path, &[])?; + let version = wal.version(); + let sequence = wal.sequence(); + drop(wal); + + let wal = Log::open(&path)?; + assert_eq!(wal.version(), version); + assert_eq!(wal.sequence(), sequence); + assert_eq!(wal.len(), 0); + assert_eq!(wal.is_empty(), true); + + Ok(()) +} + +#[test] +fn write_entries() -> io::Result<()> { + let path = testdir!().join("test.wal"); + + setup_wal(&path, ENTRIES_1)?; + + let mut wal = Log::open(&path)?; + assert_eq!(wal.sequence(), 0); + assert_eq!(wal.len(), ENTRIES_1.len()); + assert_eq!(wal.is_empty(), false); + + for (actual, &expected) in wal.iter()?.zip(ENTRIES_1) { + let actual = actual?; + + let text = str::from_utf8(&actual).unwrap(); + println!("Entry: {text}"); + assert_eq!(text, expected); + } + + Ok(()) +} + +#[test] +fn restart() -> io::Result<()> { + let path = testdir!().join("test.wal"); + + { + let mut wal = setup_wal(&path, ENTRIES_1)?; + wal.restart(1)?; + + assert_eq!(wal.sequence(), 1); + assert_eq!(wal.len(), 0); + + for entry in ENTRIES_2 { + wal.write(entry)?; + } + + wal.sync()?; + } + + let mut wal = Log::open(&path)?; + assert_eq!(wal.sequence(), 1); + + for (actual, &expected) in wal.iter()?.zip(ENTRIES_2) { + let actual = actual?; + + let text = str::from_utf8(&actual).unwrap(); + println!("Entry: {text}"); + assert_eq!(text, expected); + } + + Ok(()) +} +#[test] +fn corrupted_wal() -> io::Result<()> { + let path = testdir!().join("test.wal"); + + // Create and write some entries + { + let mut wal = Log::open(&path)?; + wal.write(b"entry1")?; + wal.write(b"entry2")?; + wal.sync()?; + } + + // Corrupt the file by truncating it in the middle + { + let metadata = fs::metadata(&path)?; + let truncate_len = metadata.len() / 2; + let file = OpenOptions::new().write(true).open(&path)?; + file.set_len(truncate_len)?; + } + + // Reopen and verify it handles corruption gracefully + let wal = Log::open(&path)?; + + // Should have fewer entries due to corruption + assert!(wal.len() < 2); + + Ok(()) +} + +#[test] +fn empty_wal_operations() -> io::Result<()> { + let path = testdir!().join("test.wal"); + let mut wal = Log::open(&path)?; + + assert!(matches!(wal.first_entry(), Ok(None))); + assert!(wal.iter()?.next().is_none()); + + Ok(()) +} + +#[test] +fn concurrent_access() -> io::Result<()> { + use std::thread; + + let path = testdir!().join("test.wal"); + + // Write in one thread + let path_clone = path.clone(); + let write_thread = thread::spawn(move || -> io::Result<()> { + let mut wal = Log::open(&path_clone)?; + wal.write(b"thread1")?; + wal.sync()?; + std::thread::sleep(Duration::from_millis(100)); + + Ok(()) + }); + + thread::sleep(std::time::Duration::from_millis(50)); + + let wal = Log::open(path); + + assert!(wal + .unwrap_err() + .to_string() + .contains("Failed to acquire exclusive advisory lock")); + + write_thread.join().unwrap()?; + + Ok(()) +} diff --git a/code/crates/wal/tests/compression.rs b/code/crates/wal/tests/compression.rs new file mode 100644 index 000000000..552cad8e7 --- /dev/null +++ b/code/crates/wal/tests/compression.rs @@ -0,0 +1,58 @@ +#![cfg(all(feature = "compression", not(feature = "force-compression")))] + +use std::io; + +use malachite_wal::Log; +use testdir::testdir; + +const ENTRIES: &[&[u8]] = &[ + &[0; 1000], + &[1; 2000], + &[2; 3000], + &[3; 4000], + &[4; 5000], + &[5; 6000], + &[6; 7000], + &[7; 8000], + &[8; 9000], + &[9; 10000], +]; + +#[test] +fn large_entries() -> io::Result<()> { + let temp = testdir!(); + + let mut no_compression = Log::open(temp.join("no-compression.wal"))?; + for entry in ENTRIES { + no_compression.write(entry)?; + } + + verify_entries(&mut no_compression, ENTRIES)?; + + let mut compression = Log::open(temp.join("compression.wal"))?; + for entry in ENTRIES { + compression.write_compressed(entry)?; + } + + verify_entries(&mut compression, ENTRIES)?; + + let noncompressed_size = no_compression.size_bytes()?; + let compressed_size = compression.size_bytes()?; + + println!("Non-compressed size: {:>5} bytes", noncompressed_size); + println!(" Compressed size: {:>5} bytes", compressed_size); + + assert!(compressed_size < noncompressed_size); + + Ok(()) +} + +fn verify_entries(wal: &mut Log, entries: &[&[u8]]) -> io::Result<()> { + assert_eq!(wal.len(), entries.len()); + + for (actual, &expected) in wal.iter()?.zip(entries) { + assert_eq!(actual?, expected); + } + + Ok(()) +} diff --git a/code/crates/wal/tests/corruption.rs b/code/crates/wal/tests/corruption.rs new file mode 100644 index 000000000..37fe4b015 --- /dev/null +++ b/code/crates/wal/tests/corruption.rs @@ -0,0 +1,250 @@ +use std::fs::OpenOptions; +use std::io::{self, Seek, SeekFrom, Write}; + +use malachite_wal::{Log, Version}; +use testdir::testdir; + +#[allow(dead_code)] +#[path = "../src/ext.rs"] +mod ext; + +use ext::*; + +#[test] +fn corrupted_crc() -> io::Result<()> { + let path = testdir!().join("test.wal"); + + // Write initial entries + { + let mut wal = Log::open(&path)?; + wal.write(b"entry1")?; + wal.write(b"entry2")?; + wal.sync()?; + } + + // Corrupt the CRC of the second entry + { + let mut file = OpenOptions::new().read(true).write(true).open(&path)?; + + // Skip version (4 bytes) + sequence (8 bytes) + first entry + file.seek(SeekFrom::Start(12))?; + read_u8(&mut file)?; // Skip compression flag + let first_entry_len = read_u64(&mut file)?; + file.seek(SeekFrom::Current(first_entry_len as i64 + 4))?; // +4 for CRC + + // Now at the start of second entry, skip compression flag and length + file.seek(SeekFrom::Current(1 + 8))?; + + // Write incorrect CRC + write_u32(&mut file, 0xdeadbeef)?; + } + + // Reopen and verify + { + let mut wal = Log::open(&path)?; + let mut entries = wal.iter()?; + + // First entry should be readable + assert!(entries.next().is_some()); + + // Second entry should fail CRC check + match entries.next() { + Some(Err(e)) => assert_eq!(e.kind(), io::ErrorKind::InvalidData), + _ => panic!("Expected CRC error for corrupted entry"), + } + } + + Ok(()) +} + +#[test] +fn incomplete_entries() -> io::Result<()> { + let path = testdir!().join("test.wal"); + + // Write initial entries + { + let mut wal = Log::open(&path)?; + wal.write(b"entry1")?; + wal.write(b"entry2")?; + wal.sync()?; + } + + // Truncate file in the middle of the second entry + { + let mut file = OpenOptions::new().read(true).write(true).open(&path)?; + + // Skip header + file.seek(SeekFrom::Start(12))?; + + read_u8(&mut file)?; // Skip compression flag + let first_entry_len = read_u64(&mut file)?; + + // header + compression flag + length + CRC + data + partial second entry + let truncate_pos = 12 + 1 + 8 + 4 + first_entry_len + 3; + + // Seek to middle of second entry + file.set_len(truncate_pos)?; + } + + // Reopen and verify + { + let mut wal = Log::open(&path)?; + let entries: Vec<_> = wal.iter()?.collect::, _>>()?; + + // Should only have the first entry + assert_eq!(entries.len(), 1); + assert_eq!(&entries[0], b"entry1"); + } + + Ok(()) +} + +#[test] +fn invalid_version() -> io::Result<()> { + let path = testdir!().join("test.wal"); + + // Create WAL file with invalid version + { + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&path)?; + + // Write invalid version + write_u32(&mut file, 0xFFFFFFFF)?; + write_u64(&mut file, 0)?; // sequence + } + + // Attempt to open WAL + match Log::open(&path) { + Err(e) => { + assert_eq!(e.kind(), io::ErrorKind::InvalidData); + // Verify error message contains version information + assert!(e.to_string().contains("version")); + } + Ok(_) => panic!("Expected error when opening WAL with invalid version"), + } + + Ok(()) +} + +#[test] +fn invalid_sequence() -> io::Result<()> { + let path = testdir!().join("test.wal"); + + // Create WAL with valid version but corrupted sequence + { + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&path)?; + + write_u32(&mut file, Version::V1 as u32)?; + + // Write partial/corrupted sequence number + file.write_all(&[0xFF, 0xFF])?; // Only write 2 bytes instead of 8 + } + + // Attempt to open WAL + match Log::open(&path) { + Err(e) => { + assert_eq!(e.kind(), io::ErrorKind::UnexpectedEof); + } + Ok(_) => panic!("Expected error when opening WAL with invalid sequence"), + } + + Ok(()) +} + +#[test] +fn multiple_corruptions() -> io::Result<()> { + let path = testdir!().join("test.wal"); + + // Create initial WAL with entries + { + let mut wal = Log::open(&path)?; + wal.write(b"entry1")?; + wal.write(b"entry2")?; + wal.write(b"entry3")?; + wal.sync()?; + } + + // Introduce multiple types of corruption + { + let mut file = OpenOptions::new().read(true).write(true).open(&path)?; + + // Corrupt sequence number + file.seek(SeekFrom::Start(4))?; + write_u64(&mut file, u64::MAX)?; + + // Corrupt entry length + file.seek(SeekFrom::Start(12 + 1))?; + write_u64(&mut file, u64::MAX - 1)?; + + // Corrupt CRC of another entry + file.seek(SeekFrom::Start(50 + 1))?; + write_u32(&mut file, 0xdeadbeef)?; + } + + // Attempt to open and read + { + let mut wal = Log::open(&path)?; + let entries: Vec<_> = wal + .iter()? + .take_while(|r| r.is_ok()) + .collect::, _>>()?; + + // Should have recovered what it could + assert!(entries.len() < 3); + } + + Ok(()) +} + +#[test] +fn zero_length_entries() -> io::Result<()> { + let path = testdir!().join("test.wal"); + + // Create WAL with zero-length entry + { + let mut wal = Log::open(&path)?; + wal.write(b"")?; + wal.write(b"normal entry")?; + wal.sync()?; + } + + // Verify reading + { + let mut wal = Log::open(&path)?; + let entries: Vec<_> = wal.iter()?.collect::, _>>()?; + + assert_eq!(entries.len(), 2); + assert_eq!(&entries[0], b""); + assert_eq!(&entries[1], b"normal entry"); + } + + Ok(()) +} + +#[test] +#[ignore] +fn max_entry_size() -> io::Result<()> { + let path = testdir!().join("test.wal"); + + let mut wal = Log::open(&path)?; + + // Try to write an entry that's too large + let large_entry = vec![0u8; usize::MAX / 2]; + assert!(wal.write(&large_entry).is_err()); + + // Verify WAL is still usable + wal.write(b"normal entry")?; + + let entries: Vec<_> = wal.iter()?.collect::, _>>()?; + assert_eq!(entries.len(), 1); + assert_eq!(&entries[0], b"normal entry"); + + Ok(()) +} diff --git a/code/crates/wal/tests/crashes.rs b/code/crates/wal/tests/crashes.rs new file mode 100644 index 000000000..f44165506 --- /dev/null +++ b/code/crates/wal/tests/crashes.rs @@ -0,0 +1,322 @@ +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Seek, Write}; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +use nix::sys::signal::{self, Signal}; +use nix::unistd::Pid; +use testdir::testdir; + +use malachite_wal::log::Log; +use malachite_wal::Log as FileLog; +use malachite_wal::*; + +/// Helper struct to simulate failures during writes +#[derive(Debug)] +pub struct FailingFile { + inner: File, + fail_after: usize, + bytes_written: usize, +} + +impl FailingFile { + pub fn new(file: File, fail_after: usize) -> Self { + Self { + inner: file, + fail_after, + bytes_written: 0, + } + } +} + +impl Seek for FailingFile { + fn seek(&mut self, pos: io::SeekFrom) -> io::Result { + self.inner.seek(pos) + } +} + +impl Read for FailingFile { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.inner.read(buf) + } +} + +impl Write for FailingFile { + fn write(&mut self, buf: &[u8]) -> io::Result { + if self.bytes_written + buf.len() > self.fail_after { + return Err(io::Error::new( + io::ErrorKind::Other, + "Simulated system failure", + )); + } + let written = self.inner.write(buf)?; + self.bytes_written += written; + Ok(written) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +impl Storage for FailingFile { + type OpenOptions = usize; + + fn open_with(path: impl AsRef, fail_after: usize) -> io::Result { + let inner = ::open_with(path, ())?; + Ok(Self::new(inner, fail_after)) + } + + fn size_bytes(&self) -> io::Result { + self.inner.size_bytes() + } + + fn truncate_to(&mut self, size: u64) -> io::Result<()> { + self.inner.truncate_to(size) + } + + fn sync_all(&mut self) -> io::Result<()> { + self.inner.sync_all() + } +} + +type FailingLog = Log; + +/// Helper function to verify WAL integrity +fn verify_wal_integrity(path: &Path) -> io::Result>> { + let mut wal = FileLog::open(path)?; + let entries = wal.iter()?.collect::>>()?; + Ok(entries) +} + +#[test] +fn system_crash_during_write() -> io::Result<()> { + let temp_dir = testdir!(); + + // Test different crash points + let crash_points = vec![ + // During header write + 4, // During version write + 8, // During sequence number write + // During entry write + 14, // During entry length write + 18, // During CRC write + 22, // During data write + ]; + + for crash_point in crash_points { + let path = temp_dir.join(format!("crash-{crash_point}.wal")); + + // Create an empty normal WAL + FileLog::open(&path)?; + + // Open WAL with failing file + let storage = FailingFile::open_with(&path, crash_point)?; + let mut wal = FailingLog::from_raw_parts(storage, path.clone(), Version::V1, 0, 0); + + // Attempt to write entries + let result = (|| -> io::Result<()> { + wal.write(b"entry1")?; + wal.write(b"entry2")?; + Ok(()) + })(); + + assert!(result.is_err()); + + // Drop the WAL to unlock the backing file + drop(wal); + + // Verify WAL integrity after crash + let entries = verify_wal_integrity(&path)?; + + // The number of successful entries should be consistent with the crash point + assert!(entries.len() <= 1); + } + + Ok(()) +} + +// Simulate power failure during fsync +struct FailingSync { + inner: File, + should_fail: bool, +} + +impl Write for FailingSync { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.inner.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +impl Read for FailingSync { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.inner.read(buf) + } +} + +impl Seek for FailingSync { + fn seek(&mut self, pos: io::SeekFrom) -> io::Result { + self.inner.seek(pos) + } +} + +impl Storage for FailingSync { + type OpenOptions = bool; + + fn open_with(path: impl AsRef, should_fail: bool) -> io::Result { + let inner = ::open_with(path, ())?; + Ok(Self { inner, should_fail }) + } + + fn size_bytes(&self) -> io::Result { + self.inner.size_bytes() + } + + fn truncate_to(&mut self, size: u64) -> io::Result<()> { + self.inner.truncate_to(size) + } + + fn sync_all(&mut self) -> io::Result<()> { + if self.should_fail { + return Err(io::Error::new( + io::ErrorKind::Other, + "Simulated power failure during fsync", + )); + } + + self.inner.sync_all() + } +} + +type FailingSyncLog = Log; + +#[test] +fn power_failure_simulation() -> io::Result<()> { + let path = testdir!().join("test.wal"); + + // Create an empty normal WAL + FileLog::open(&path)?; + + // Test power failure during sync + { + // Use `from_raw_parts` to avoid calling `sync` during initialization + let storage = FailingSync::open_with(&path, true)?; + let mut wal = FailingSyncLog::from_raw_parts(storage, path.to_owned(), Version::V1, 0, 0); + + wal.write(b"entry1")?; + + assert!(wal.sync().is_err()); + } + + // Verify recovery after power failure + let entries = verify_wal_integrity(&path)?; + assert!(entries.is_empty() || entries.len() == 1); + + Ok(()) +} + +#[test] +fn process_termination() -> io::Result<()> { + let path = testdir!().join("test.wal"); + let path_str = path.to_str().unwrap(); + + // Create a separate process that will be terminated + let child = Command::new(std::env::current_exe()?) + .arg("--test") + .arg("wal_write_test") + .arg(path_str) + .stdout(Stdio::piped()) + .spawn()?; + + // Give the child process time to start writing + thread::sleep(Duration::from_millis(100)); + + // Terminate the process + signal::kill(Pid::from_raw(child.id() as i32), Signal::SIGKILL)?; + + // Wait for the process to exit + let _ = child.wait_with_output(); + + // Verify WAL integrity after process termination + let entries = verify_wal_integrity(&path)?; + + // The WAL should either be empty or contain complete entries + for entry in entries { + assert!(!entry.is_empty()); + } + + Ok(()) +} + +// Helper binary for process termination test +#[test] +fn wal_write_test() { + if std::env::args().any(|arg| arg == "--test") { + if let Some(path) = std::env::args().nth(3) { + let mut wal = FileLog::open(path).unwrap(); + loop { + // Continuously write entries until terminated + wal.write(b"test entry").unwrap(); + wal.sync().unwrap(); + thread::sleep(Duration::from_millis(10)); + } + } + } +} + +#[test] +fn concurrent_crash_recovery() -> io::Result<()> { + let path = testdir!().join("test.wal"); + let path_clone = path.clone(); + + let running = Arc::new(AtomicBool::new(true)); + let running_clone = running.clone(); + + // Writer thread + let writer_handle = thread::spawn(move || -> io::Result<()> { + let mut wal = FileLog::open(&path_clone)?; + + while running_clone.load(Ordering::SeqCst) { + wal.write(b"test entry")?; + wal.sync()?; + thread::sleep(Duration::from_millis(10)); + } + + Ok(()) + }); + + // Crasher thread + let path2 = path.clone(); + let crasher_handle = thread::spawn(move || { + for _ in 0..5 { + thread::sleep(Duration::from_millis(50)); + + // Simulate crash by truncating file + if let Ok(file) = OpenOptions::new().write(true).open(&path2) { + let _ = file.set_len(12); // Truncate to header size + } + } + running.store(false, Ordering::SeqCst); + }); + + writer_handle.join().unwrap()?; + crasher_handle.join().unwrap(); + + // Verify final WAL integrity + let entries = verify_wal_integrity(&path)?; + + // The WAL should be in a consistent state + for entry in entries { + assert_eq!(&entry, b"test entry"); + } + + Ok(()) +} diff --git a/code/crates/wal/tests/stress.rs b/code/crates/wal/tests/stress.rs new file mode 100644 index 000000000..fb2b08cd1 --- /dev/null +++ b/code/crates/wal/tests/stress.rs @@ -0,0 +1,141 @@ +use rand::{thread_rng, Rng}; +use std::fs; +use std::io; +use std::time::Instant; +use testdir::testdir; + +use malachite_wal::*; + +const KB: usize = 1024; +const MB: usize = 1024 * KB; + +/// Test configuration for customizable stress tests +struct StressTestConfig { + num_entries: usize, + sync_interval: usize, +} + +#[test] +fn large_number_of_entries() -> io::Result<()> { + let dir = testdir!(); + let path = dir.join("large_entries.wal"); + + let config = StressTestConfig { + num_entries: 1_000_000, // 1 million entries + sync_interval: 1000, // Sync every 1000 entries + }; + + let start = Instant::now(); + let mut wal = Log::open(&path)?; + + println!("Starting large number of entries test..."); + + // Write entries + for i in 0..config.num_entries { + let entry = format!("entry-{}", i); + wal.write(entry.as_bytes())?; + + if i % config.sync_interval == 0 { + wal.sync()?; + if i % 100_000 == 0 { + println!("Wrote {} entries...", i); + } + } + } + + wal.sync()?; + + let write_duration = start.elapsed(); + println!("Write phase completed in {:?}", write_duration); + + // Verify entries + let start = Instant::now(); + let entries: Vec> = wal.iter()?.collect::>>()?; + + assert_eq!(entries.len(), config.num_entries); + + for (i, entry) in entries.iter().enumerate() { + let expected = format!("entry-{}", i); + assert_eq!(entry, expected.as_bytes()); + } + + let read_duration = start.elapsed(); + println!("Read phase completed in {:?}", read_duration); + + // Report statistics + let file_size = fs::metadata(&path)?.len(); + println!("WAL Statistics:"); + println!(" Total entries: {}", config.num_entries); + println!(" File size: {:.2} MB", file_size as f64 / MB as f64); + println!( + " Write throughput: {:.2} entries/sec ({:.2} MB/sec)", + config.num_entries as f64 / write_duration.as_secs_f64(), + file_size as f64 / MB as f64 / write_duration.as_secs_f64() + ); + println!( + " Read throughput: {:.2} entries/sec ({:.2} MB/sec)", + config.num_entries as f64 / read_duration.as_secs_f64(), + file_size as f64 / MB as f64 / read_duration.as_secs_f64() + ); + + Ok(()) +} + +#[test] +#[ignore] +fn entry_sizes() -> io::Result<()> { + let dir = testdir!(); + let path = dir.join("large_sizes.wal"); + + #[allow(clippy::identity_op)] + let entry_sizes = vec![ + 1 * KB, // 1 KiB + 10 * KB, // 10 KiB + 100 * KB, // 100 KiB + 500 * KB, // 500 KiB + 1 * MB, // 1 MiB + 10 * MB, // 10 MiB + ]; + + let mut wal = Log::open(&path)?; + + println!("Starting entry sizes test..."); + + for &size in &entry_sizes { + // Create entry with random data + let entry: Vec = (0..size).map(|_| thread_rng().gen::()).collect(); + + let start = Instant::now(); + + if size >= MB { + println!("Writing entry of size: {} MB", size / MB); + } else { + println!("Writing entry of size: {} KB", size / KB); + } + + // Write entry + wal.write(&entry)?; + wal.sync()?; + + let duration = start.elapsed(); + println!(" Write completed in {:?}", duration); + println!( + " Throughput: {:.2} MB/s", + size as f64 / MB as f64 / duration.as_secs_f64() + ); + } + + // Verify entries + let start = Instant::now(); + let entries: Vec> = wal.iter()?.collect::>>()?; + + assert_eq!(entries.len(), entry_sizes.len()); + + for (entry, &expected_size) in entries.iter().zip(entry_sizes.iter()) { + assert_eq!(entry.len(), expected_size); + } + + println!("Read verification completed in {:?}", start.elapsed()); + + Ok(()) +}